def train(self):
print 'Total name :'
nb_epochs = self.args['epochs']
time_spent = [0 for _ in range(10)]
with tf.Graph().as_default() as graph:
config_ = tf.ConfigProto()
config_.gpu_options.allow_growth = True
config_.allow_soft_placement = True
with tf.Session(graph=graph, config=config_).as_default() as sess:
tfds = TFDataset(**self.args)
additional_args = {
"mix": tfds.next_mix,
"non_mix": tfds.next_non_mix,
"ind": tfds.next_ind,
"pipeline": True,
"tot_speakers" : 251
}
self.args.update(additional_args)
self.build()
nb_batches_train = tfds.length(tfds.TRAIN)#894 # tfds.length('train')
nb_batches_test = tfds.length(tfds.TEST)#50 #tfds.length('test')
nb_batches_valid = tfds.length(tfds.VALID)#50#tfds.length('valid')
print 'BATCHES'
print nb_batches_train, nb_batches_test, nb_batches_valid
feed_dict_train = {tfds.handle: tfds.get_handle(tfds.TRAIN)}
feed_dict_valid = {tfds.handle: tfds.get_handle(tfds.VALID)}
feed_dict_test = {tfds.handle: tfds.get_handle(tfds.TEST)}
best_validation_cost = 1e100
t1 = time.time()
step = 0
for epoch in range(nb_epochs):
sess.run(tfds.training_initializer)
for b in range(nb_batches_train):
t = time.time()
# m = self.model.test(feed_dict_train)
# print m
c = self.model.train(feed_dict_train, step)
if (step+1)%self.args['validation_step'] == 0:
t = time.time()
sess.run(tfds.validation_initializer)
# Compute validation mean cost with batches to avoid memory problems
costs = []
for b_v in range(nb_batches_valid):
cost = self.model.valid_batch(feed_dict_valid,step)
costs.append(cost)
valid_cost = np.mean(costs)
self.model.add_valid_summary(valid_cost, step)
# Save the model if it is better:
if valid_cost < best_validation_cost:
best_validation_cost = valid_cost # Save as new lowest cost
best_path = self.model.save(step)
print 'Save best model with :', best_validation_cost
t_f = time.time()
print 'Validation set tested in ', t_f - t, ' seconds'
print 'Validation set: ', valid_cost
time_spent = time_spent[1:] +[time.time()-t1]
avg = sum(time_spent)/len(time_spent)
print 'Epoch #', epoch+1,'/', nb_epochs,' Batch #', b+1,'/',nb_batches_train,'in', avg,'sec loss=', c \
, ' ETA = ', getETA(avg, nb_batches_train, b+1, nb_epochs, epoch+1)
t1 = time.time()
step += 1
print 'Best model with Validation: ', best_validation_cost
print 'Path = ', best_path
# Load the best model on validation set and test it
self.model.restore_last_checkpoint()
sess.run(tfds.test_initializer)
for b_t in range(nb_batches_test):
cost = self.model.test_batch(feed_dict_test)
costs.append(cost)
print 'Test cost = ', np.mean(costs)
init = model.non_initialized_variables() model.sess.run(init) print 'Total name :' print model.runID # nb_iterations = 500 mixed_data.adjust_split_size_to_batchsize(batch_size) nb_batches = mixed_data.nb_batches(batch_size) nb_epochs = 1 time_spent = [ 0 for _ in range(5)] for epoch in range(nb_epochs): for b in range(nb_batches): X_non_mix, X_mix, _ = mixed_data.get_batch(batch_size) t = time.time() c = model.train(X_mix, X_non_mix, learning_rate, b) t_f = time.time() time_spent = time_spent[1:] +[t_f-t] print 'Step #' ,b,' loss=', c ,' ETA = ', getETA(sum(time_spent)/float(np.count_nonzero(time_spent)) , nb_batches, b, nb_epochs, epoch) # print 'length of data =', X_non_mix.shape ,'step ', b+1, mixed_data.datasets[0].index_item_split, mixed_data.selected_split_size(),getETA(sum(time_spent)/float(np.count_nonzero(time_spent)), nb_batches, b, nb_epochs, epoch) if b%20 == 0: #cost_valid < cost_valid_min: print 'DAS model saved at iteration number ', nb_batches*epoch + b,' with cost = ', c model.save(b)
def train(self):
print 'Total name :'
print self.model.runID
nb_epochs = self.args['epochs']
batch_size = self.args['batch_size']
time_spent = [0 for _ in range(5)]
nb_batches = self.dataset.nb_batch(batch_size)
best_validation_cost = 1e100
step = 0
for epoch in range(nb_epochs):
for b, (x_mix, x_non_mix, I) in enumerate(
self.dataset.get_batch(self.dataset.TRAIN, batch_size)):
x_mix, x_non_mix, _, _ = normalize_mix(x_mix, x_non_mix)
t = time.time()
c = self.model.train(x_mix, x_non_mix, I, step)
t_f = time.time()
time_spent = time_spent[1:] + [t_f - t]
print 'Epoch #', epoch+1,'Step #', step+1,' loss=', c \
, ' ETA = ', getETA(sum(time_spent)/float(np.count_nonzero(time_spent)) \
, nb_batches, b, nb_epochs, epoch)
if step % self.args['validation_step'] == 0:
t = time.time()
# Compute validation mean cost with batches to avoid memory problems
costs = []
for x_mix_v, x_non_mix_v, I_v in self.dataset.get_batch(
self.dataset.VALID, batch_size):
x_mix_v, x_non_mix_v, _, _ = normalize_mix(
x_mix_v, x_non_mix_v)
cost = self.model.valid_batch(x_mix_v, x_non_mix_v,
I_v)
costs.append(cost)
valid_cost = np.mean(costs)
self.model.add_valid_summary(valid_cost, step)
# Save the model if it is better:
if valid_cost < best_validation_cost:
best_validation_cost = valid_cost # Save as new lowest cost
best_path = self.model.save(step)
print 'Save best model with :', best_validation_cost
t_f = time.time()
print 'Validation set tested in ', t_f - t, ' seconds'
print 'Validation set: ', valid_cost
step += 1
print 'Best model with Validation: ', best_validation_cost
print 'Path = ', best_path
# Load the best model on validation set and test it
self.model.restore_last_checkpoint()
for x_mix_t, x_non_mix_t, I_t in self.dataset.get_batch(
self.dataset.TEST, batch_size):
x_mix_t, x_non_mix_t, _, _ = normalize_mix(x_mix_t, x_non_mix_t)
cost = self.model.valid_batch(x_mix_t, x_non_mix_t, I_t)
costs.append(cost)
print 'Test cost = ', np.mean(costs)
