masks.append( np.vstack(utter_mask) )
'''
for dim in range(0,len(batched_inputs)):
for vec in range(0,batched_inputs[0].shape[1]):
batched_inputs[vec][dim] = (batched_inputs[vec][dim]-0.5)*2
'''
x_seq = np.dstack(batched_inputs)
x_seq = np.asarray([ x.T for x in x_seq ],'float32')
y_hat_seq = np.dstack(batched_outputs)
y_hat_seq = np.asarray([ y.T for y in y_hat_seq ],'float32')
mask_seq = np.dstack(masks)
mask_seq = np.asarray([ m.T for m in mask_seq ],'float32')
cost += nn.train(x_seq, y_hat_seq, mask_seq,
LEARNING_RATE,
RMS_ALPHA,
CLIP,
MOMENTUM)
tEnd = time.time()
totaltime += tEnd - tStart
err_range = float('inf')
if cal_dev == 0:
#sec_m = (prev_3**2 + prev_2**2 + prev_err**2) / 3
m = [prev_3, prev_2, prev_err]
err_range = max(m) - min(m)
print "err_range : ", err_range
if cal_dev > 0:
cal_dev -= 1
y_hat_seq = np.dstack(batched_outputs)
y_hat_seq = np.asarray([ y.T for y in y_hat_seq ],'float32')
mask_seq = np.dstack(masks)
mask_seq = np.asarray([ m.T for m in mask_seq ],'float32')
i_cost = np.sum((x_seq-y_hat_seq)**2)
print "init cost = {0}".format(i_cost)
totaltime = 0
print "Start training......"
for epoch in range(MAX_EPOCH):
tStart = time.time()
#cost = 0
cost = nn.train(x_seq, y_hat_seq, mask_seq,
LEARNING_RATE,
MOMENTUM,
CLIP)
tEnd = time.time()
totaltime += tEnd - tStart
#if (epoch+1 != MAX_EPOCH) and ((epoch+1) % L_RATE_DECAY_STEP == 0):
# print "learning rate annealed at epoch {0}".format(epoch+1)
# LEARNING_RATE*=0.9
if epoch+1 != MAX_EPOCH and (epoch+1) % SAVE_MODEL_EPOCH == 0:
fh = open(MODEL_ROOT+MODEL+"_at_{0}".format(epoch+1),'wb')
saved_params = (nn.layers, nn.W, nn.Wh, nn.b)
pickle.dump(saved_params, fh)
fh.close()
batched_inputs.append(np.vstack(extended_in))
batched_outputs.append(np.vstack(extended_out))
masks.append(np.vstack(utter_mask))
'''
for dim in range(0,len(batched_inputs)):
for vec in range(0,batched_inputs[0].shape[1]):
batched_inputs[vec][dim] = (batched_inputs[vec][dim]-0.5)*2
'''
x_seq = np.dstack(batched_inputs)
x_seq = np.asarray([x.T for x in x_seq], 'float32')
y_hat_seq = np.dstack(batched_outputs)
y_hat_seq = np.asarray([y.T for y in y_hat_seq], 'float32')
mask_seq = np.dstack(masks)
mask_seq = np.asarray([m.T for m in mask_seq], 'float32')
cost += nn.train(x_seq, y_hat_seq, mask_seq, LEARNING_RATE, RMS_ALPHA,
CLIP, MOMENTUM)
tEnd = time.time()
totaltime += tEnd - tStart
err_range = float('inf')
if cal_dev == 0:
#sec_m = (prev_3**2 + prev_2**2 + prev_err**2) / 3
m = [prev_3, prev_2, prev_err]
err_range = max(m) - min(m)
print "err_range : ", err_range
if cal_dev > 0:
cal_dev -= 1
if (epoch + 1 != MAX_EPOCH) and (
err_range < 0.2): #epoch+1 != 1) and (prev_cost<cost):
masks.append( np.vstack(utter_mask) )
'''
for dim in range(0,len(batched_inputs)):
for vec in range(0,batched_inputs[0].shape[1]):
batched_inputs[vec][dim] = (batched_inputs[vec][dim]-0.5)*2
'''
x_seq = np.dstack(batched_inputs)
x_seq = np.asarray([ x.T for x in x_seq ],'float32')
y_hat_seq = np.dstack(batched_outputs)
y_hat_seq = np.asarray([ y.T for y in y_hat_seq ],'float32')
mask_seq = np.dstack(masks)
mask_seq = np.asarray([ m.T for m in mask_seq ],'float32')
tempcost.append( float( nn.train(x_seq, y_hat_seq, mask_seq,
LEARNING_RATE,
RMS_ALPHA,
CLIP,
MOMENTUM) ) )
cost += tempcost[-1]
tEnd = time.time()
totaltime += tEnd - tStart
err_range = float('inf')
if cal_dev == 0:
#sec_m = (prev_3**2 + prev_2**2 + prev_err**2) / 3
m = [prev_3, prev_2, prev_err]
err_range = max(m) - min(m)
print "err_range : ", err_range
if cal_dev > 0:
cal_dev -= 1