seq = SequenceGen(args.task, vec_size, args.hi, args.lo)
# An object that keeps the optimizer state during training
optimizer = RMSProp(model.W)
n = 0 # counts the number of sequences trained on
bpc = None # keeps track of trailing bpc (cost)
# train forever
while True:
i, t, seq_len = seq.make()
inputs = np.matrix(i)
targets = np.matrix(t)
loss, deltas, outputs, r, w, a, e = model.lossFun(inputs, targets)
newbpc = np.sum(loss) / ((seq_len*2 + 2) * vec_size)
if bpc is not None:
bpc = 0.99 * bpc + 0.01 * newbpc
else:
bpc = newbpc
# sometimes print out diagnostic info
if ((n % args.log_freq) == 0) or args.test_mode:
print 'iter %d' % (n)
visualize(inputs, outputs, r, w, a, e)
# log ratio of delta l2 norm to weight l2 norm
print "update/weight ratios:"
for k in model.W.keys():
vec_size = model.vec_size # vec size comes from model
seq = SequenceGen(args.task, vec_size, args.hi, args.lo)
# An object that keeps the optimizer state during training
optimizer = RMSProp(model.W)
n = 0 # counts the number of sequences trained on
bpc = None # keeps track of trailing bpc (cost)
while n < 100:
i, t, seq_len = seq.make()
inputs = np.matrix(i)
targets = np.matrix(t)
loss, deltas, outputs, r, w, a, e = model.lossFun(inputs, targets, args.manual_grad)
newbpc = np.sum(loss) / ((seq_len*2 + 2) * vec_size)
if bpc is not None:
bpc = 0.99 * bpc + 0.01 * newbpc
else:
bpc = newbpc
# sometimes print out diagnostic info
if ((n % args.log_freq) == 0) or args.test_mode:
print 'iter %d' % (n)
visualize(inputs, outputs, r, w, a, e)
# log ratio of delta l2 norm to weight l2 norm
print "update/weight ratios:"
for k in model.W.keys():
vec_size = model.vec_size # vec size comes from model
seq = SequenceGen(args.task, vec_size, args.hi, args.lo)
# An object that keeps the optimizer state during training
optimizer = RMSProp(model.W)
n = 0 # counts the number of sequences trained on
bpc = None # keeps track of trailing bpc (cost)
while n < 100:
i, t, seq_len = seq.make()
inputs = np.matrix(i)
targets = np.matrix(t)
loss, deltas, outputs, r, w, a, e = model.lossFun(inputs, targets,
args.manual_grad)
newbpc = np.sum(loss) / ((seq_len * 2 + 2) * vec_size)
if bpc is not None:
bpc = 0.99 * bpc + 0.01 * newbpc
else:
bpc = newbpc
# sometimes print out diagnostic info
if ((n % args.log_freq) == 0) or args.test_mode:
print 'iter %d' % (n)
visualize(inputs, outputs, r, w, a, e)
# log ratio of delta l2 norm to weight l2 norm
print "update/weight ratios:"
for k in model.W.keys():
