def setup(self):
#================[ check output dim with target size ]================#
assert self.model.input_dim == self.dataset.feature_size(), \
'input dim: ' + str(self.model.input_dim) + \
', is not equal to feature size: ' + str(self.dataset.feature_size())
assert self.model.layers[-1].dim == self.dataset.target_size(), \
'output dim: ' + str(self.model.layers[-1].dim) + \
', is not equal to target size: ' + str(self.dataset.target_size())
#===================[ build params and deltas list ]==================#
def is_shared_var(var):
return var.__class__.__name__ == 'TensorSharedVariable' or \
var.__class__.__name__ == 'CudaNdarraySharedVariable'
params = []
deltas = []
prev_layer_dim = self.model.input_dim
for layer in self.model.layers:
# append layer params that will be updated during training
if len(layer.params) > 0:
for param in layer.params:
params += [param]
deltas += [
theano.shared(
np.zeros(param.shape.eval(), dtype=floatX))
]
if is_shared_var(layer.W):
assert layer.W.dtype == floatX
params += [layer.W]
deltas += [
theano.shared(
np.zeros((prev_layer_dim, layer.dim), dtype=floatX))
]
else:
self.log.info(layer.W.name + ' is ' +
layer.W.__class__.__name__ +
' but not SharedVariable.')
if is_shared_var(layer.b):
assert layer.b.dtype == floatX
params += [layer.b]
deltas += [theano.shared(np.zeros(layer.dim, dtype=floatX))]
else:
self.log.info(layer.b.name + ' is ' +
layer.b.__class__.__name__ +
' but not SharedVariable.')
prev_layer_dim = layer.dim
#=====================[ training params updates ]=====================#
self.log.info("..update params: " + str(params))
train_x = T.matrix('train_x', dtype=floatX)
train_y = T.matrix('train_y', dtype=floatX)
train_y_pred, train_layers_stats = self.model.train_fprop(train_x)
train_cost = self.learning_rule.cost.get_cost(train_y, train_y_pred)
if self.learning_rule.L1_lambda:
self.log.info('..applying L1_lambda: %f' %
self.learning_rule.L1_lambda)
L1 = theano.shared(0.)
for layer in self.model.layers:
if is_shared_var(layer.W):
L1 += T.sqrt((layer.W**2).sum(axis=0)).sum()
else:
self.log.info(layer.W.name + ' is ' +
layer.W.__class__.__name__ +
' is not used in L1 regularization')
train_cost += self.learning_rule.L1_lambda * L1
if self.learning_rule.L2_lambda:
self.log.info('..applying L2_lambda: %f' %
self.learning_rule.L2_lambda)
L2 = theano.shared(0.)
for layer in self.model.layers:
if is_shared_var(layer.W):
L2 += ((layer.W**2).sum(axis=0)).sum()
else:
self.log.info(layer.W.name + ' is ' +
layer.W.__class__.__name__ +
' is not used in L2 regularization')
train_cost += self.learning_rule.L2_lambda * L2
train_updates = []
gparams = T.grad(train_cost, params)
for delta, param, gparam in zip(deltas, params, gparams):
train_updates += self.learning_method.update(delta, gparam)
# applying max_col_norm regularisation
if param.name[0] == 'W' and self.learning_rule.max_col_norm:
W_update = param + delta
w_len = T.sqrt((W_update**2).sum(axis=0))
divisor = (w_len <= self.learning_rule.max_col_norm) + \
(w_len > self.learning_rule.max_col_norm) * w_len / \
self.learning_rule.max_col_norm
W_update = W_update / divisor.reshape((1, divisor.shape[0]))
train_updates += [(param, W_update)]
else:
train_updates += [(param, param + delta)]
#----[ append updates of stats from each layer to train updates ]-----#
self.train_stats_names, train_stats_vars = split_list(
train_layers_stats)
train_stats_vars = [var.astype(floatX) for var in train_stats_vars]
self.train_stats_shared = generate_shared_list(train_stats_vars)
train_stats_updates = merge_lists(self.train_stats_shared,
train_stats_vars)
train_updates += train_stats_updates
#-------------------------[ train functions ]-------------------------#
self.log.info('..begin compiling functions')
train_stopping_cost = self.learning_rule.stopping_criteria[
'cost'].get_cost(train_y, train_y_pred)
self.training = theano.function(inputs=[train_x, train_y],
outputs=(train_stopping_cost,
train_cost),
updates=train_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..training function compiled')
#======================[ testing params updates ]=====================#
test_x = T.matrix('test_x', dtype=floatX)
test_y = T.matrix('test_y', dtype=floatX)
test_y_pred, test_layers_stats = self.model.test_fprop(test_x)
#-----[ append updates of stats from each layer to test updates ]-----#
self.test_stats_names, test_stats_vars = split_list(test_layers_stats)
test_stats_vars = [var.astype(floatX) for var in test_stats_vars]
self.test_stats_shared = generate_shared_list(test_stats_vars)
test_stats_updates = merge_lists(self.test_stats_shared,
test_stats_vars)
#-------------------------[ test functions ]--------------------------#
test_stopping_cost = self.learning_rule.stopping_criteria[
'cost'].get_cost(test_y, test_y_pred)
test_cost = self.learning_rule.cost.get_cost(test_y, test_y_pred)
self.testing = theano.function(inputs=[test_x, test_y],
outputs=(test_stopping_cost, test_cost),
updates=test_stats_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..testing function compiled')
def setup(self):
#================[ check output dim with target size ]================#
assert self.model.input_dim == self.dataset.feature_size(), \
'input dim: ' + str(self.model.input_dim) + \
', is not equal to feature size: ' + str(self.dataset.feature_size())
assert self.model.layers[-1].dim == self.dataset.target_size(), \
'output dim: ' + str(self.model.layers[-1].dim) + \
', is not equal to target size: ' + str(self.dataset.target_size())
#===================[ build params and deltas list ]==================#
def is_shared_var(var):
return var.__class__.__name__ == 'TensorSharedVariable' or \
var.__class__.__name__ == 'CudaNdarraySharedVariable'
params = []
deltas = []
prev_layer_dim = self.model.input_dim
for layer in self.model.layers:
# append layer params that will be updated during training
if len(layer.params) > 0:
for param in layer.params:
params += [param]
deltas += [theano.shared(np.zeros(param.shape.eval(), dtype=floatX))]
if is_shared_var(layer.W):
assert layer.W.dtype == floatX
params += [layer.W]
deltas += [theano.shared(np.zeros((prev_layer_dim, layer.dim), dtype=floatX))]
else:
self.log.info(layer.W.name + ' is ' + layer.W.__class__.__name__ +
' but not SharedVariable.')
if is_shared_var(layer.b):
assert layer.b.dtype == floatX
params += [layer.b]
deltas += [theano.shared(np.zeros(layer.dim, dtype=floatX))]
else:
self.log.info(layer.b.name + ' is ' + layer.b.__class__.__name__ +
' but not SharedVariable.')
prev_layer_dim = layer.dim
#=====================[ training params updates ]=====================#
self.log.info("..update params: " + str(params))
train_x = T.matrix('train_x', dtype=floatX)
train_y = T.matrix('train_y', dtype=floatX)
train_y_pred, train_layers_stats = self.model.train_fprop(train_x)
train_cost = self.learning_rule.cost.get_cost(train_y, train_y_pred)
if self.learning_rule.L1_lambda:
self.log.info('..applying L1_lambda: %f'%self.learning_rule.L1_lambda)
L1 = theano.shared(0.)
for layer in self.model.layers:
if is_shared_var(layer.W):
L1 += T.sqrt((layer.W ** 2).sum(axis=0)).sum()
else:
self.log.info(layer.W.name + ' is ' + layer.W.__class__.__name__ +
' is not used in L1 regularization')
train_cost += self.learning_rule.L1_lambda * L1
if self.learning_rule.L2_lambda:
self.log.info('..applying L2_lambda: %f'%self.learning_rule.L2_lambda)
L2 = theano.shared(0.)
for layer in self.model.layers:
if is_shared_var(layer.W):
L2 += ((layer.W ** 2).sum(axis=0)).sum()
else:
self.log.info(layer.W.name + ' is ' + layer.W.__class__.__name__ +
' is not used in L2 regularization')
train_cost += self.learning_rule.L2_lambda * L2
train_updates = []
gparams = T.grad(train_cost, params)
for delta, param, gparam in zip(deltas, params, gparams):
train_updates += self.learning_method.update(delta, gparam)
# applying max_col_norm regularisation
if param.name[0] == 'W' and self.learning_rule.max_col_norm:
W_update = param + delta
w_len = T.sqrt((W_update ** 2).sum(axis=0))
divisor = (w_len <= self.learning_rule.max_col_norm) + \
(w_len > self.learning_rule.max_col_norm) * w_len / \
self.learning_rule.max_col_norm
W_update = W_update / divisor.reshape((1, divisor.shape[0]))
train_updates += [(param, W_update)]
else:
train_updates += [(param, param + delta)]
#----[ append updates of stats from each layer to train updates ]-----#
self.train_stats_names, train_stats_vars = split_list(train_layers_stats)
train_stats_vars = [var.astype(floatX) for var in train_stats_vars]
self.train_stats_shared = generate_shared_list(train_stats_vars)
train_stats_updates = merge_lists(self.train_stats_shared, train_stats_vars)
train_updates += train_stats_updates
#-------------------------[ train functions ]-------------------------#
self.log.info('..begin compiling functions')
train_stopping_cost = self.learning_rule.stopping_criteria['cost'].get_cost(train_y, train_y_pred)
self.training = theano.function(inputs=[train_x, train_y],
outputs=(train_stopping_cost, train_cost),
updates=train_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..training function compiled')
#======================[ testing params updates ]=====================#
test_x = T.matrix('test_x', dtype=floatX)
test_y = T.matrix('test_y', dtype=floatX)
test_y_pred, test_layers_stats = self.model.test_fprop(test_x)
#-----[ append updates of stats from each layer to test updates ]-----#
self.test_stats_names, test_stats_vars = split_list(test_layers_stats)
test_stats_vars = [var.astype(floatX) for var in test_stats_vars]
self.test_stats_shared = generate_shared_list(test_stats_vars)
test_stats_updates = merge_lists(self.test_stats_shared, test_stats_vars)
#-------------------------[ test functions ]--------------------------#
test_stopping_cost = self.learning_rule.stopping_criteria['cost'].get_cost(test_y, test_y_pred)
test_cost = self.learning_rule.cost.get_cost(test_y, test_y_pred)
self.testing = theano.function(inputs=[test_x, test_y],
outputs=(test_stopping_cost, test_cost),
updates=test_stats_updates,
on_unused_input='warn',
allow_input_downcast=True)
self.log.info('..testing function compiled')
def run(self):
best_train_error = float(sys.maxint)
best_valid_error = float(sys.maxint)
# best_test_error = float(sys.maxint)
mean_train_error = float(sys.maxint)
mean_valid_error = float(sys.maxint)
mean_test_error = float(sys.maxint)
mean_train_cost = float(sys.maxint)
mean_valid_cost = float(sys.maxint)
mean_test_cost = float(sys.maxint)
best_train_cost = float(sys.maxint)
train_stats_values = []
valid_stats_values = []
# test_stats_values = []
epoch = 0
error_dcr = 0
self.best_epoch_last_update = 0
self.best_valid_last_update = float(sys.maxint)
train_stats_names = [
'train_' + name for name in self.train_stats_names
]
valid_stats_names = ['valid_' + name for name in self.test_stats_names]
# test_stats_names = ['test_' + name for name in self.test_stats_names]
job_start = time.time()
while (self.continue_learning(epoch, error_dcr, best_valid_error)):
if epoch > 0:
self.log.info("best_epoch_last_update: %d" %
self.best_epoch_last_update)
self.log.info("valid_error_decrease: %f" % error_dcr)
self.log.info("best_valid_last_update: %f" %
self.best_valid_last_update)
self.log.info("========[ End of Epoch ]========\n\n")
epoch += 1
start_time = time.time()
num_train_examples = 0
total_train_cost = 0.
total_train_stopping_cost = 0.
train_stats_values = np.zeros(len(train_stats_names), dtype=floatX)
num_valid_examples = 0
total_valid_cost = 0.
total_valid_stopping_cost = 0.
valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX)
num_test_examples = 0
total_test_cost = 0.
total_test_stopping_cost = 0.
# test_stats_values = np.zeros(len(test_stats_names), dtype=floatX)
blk = 0
for block in self.dataset:
block_time = time.time()
blk += 1
train_set = block.get_train()
valid_set = block.get_valid()
test_set = block.get_test()
#====================[ Training Progress ]====================#
if train_set.dataset_size() > 0:
self.log.info('..training ' +
self.dataset.__class__.__name__ +
' block %s/%s' %
(blk, self.dataset.nblocks()))
for idx in train_set:
stopping_cost, cost = self.training(
train_set.X[idx], train_set.y[idx])
total_train_cost += cost * len(idx)
total_train_stopping_cost += stopping_cost * len(idx)
num_train_examples += len(idx)
train_stats_values += len(idx) * get_shared_values(
self.train_stats_shared)
#-------[ Update train best cost and error values ]-------#
mean_train_error = total_train_stopping_cost / num_train_examples
mean_train_cost = total_train_cost / num_train_examples
train_stats_values /= num_train_examples
if mean_train_error < best_train_error:
best_train_error = mean_train_error
if mean_train_cost < 0.999 * best_train_cost:
best_train_cost = mean_train_cost
else:
self.log.info(
'training cost is not improving after epoch %d, ' %
epoch)
#---[ Use learning rate decay if the error does not improve after 1 epoch ]---#
if self.learning_rule.learning_rate_decay_factor and hasattr(
self.learning_method, 'learning_rate'):
self.log.info(
'decay learning_rate by factor %.3f' %
self.learning_rule.learning_rate_decay_factor)
new_lr = self.learning_method.learning_rate.get_value(return_internal_type=True) \
/ self.learning_rule.learning_rate_decay_factor
self.log.info('new learning rate %.3f' % new_lr)
self.learning_method.learning_rate.set_value(
new_lr)
#===================[ Validating Progress ]===================#
if valid_set.dataset_size() > 0:
self.log.info('..validating ' +
self.dataset.__class__.__name__ +
' block %s/%s' %
(blk, self.dataset.nblocks()))
for idx in valid_set:
stopping_cost, cost = self.testing(
valid_set.X[idx], valid_set.y[idx])
total_valid_cost += cost * len(idx)
total_valid_stopping_cost += stopping_cost * len(idx)
num_valid_examples += len(idx)
valid_stats_values += len(idx) * get_shared_values(
self.test_stats_shared)
#-------[ Update valid best cost and error values ]-------#
mean_valid_error = total_valid_stopping_cost / num_valid_examples
mean_valid_cost = total_valid_cost / num_valid_examples
valid_stats_values /= num_valid_examples
if mean_valid_error < best_valid_error:
best_valid_error = mean_valid_error
self.log.info('..best validation error so far')
if self.log.save_model:
self.log._save_model(self.model)
self.log.info('..model saved')
if self.log.save_learning_rule:
self.log._save_learning_rule(self.learning_rule)
self.log.info('..learning rule saved')
if mean_valid_error < self.best_valid_last_update:
error_dcr = self.best_valid_last_update - mean_valid_error
else:
error_dcr = 0
#====================[ Testing Progress ]=====================#
if test_set.dataset_size() > 0:
self.log.info('..testing ' +
self.dataset.__class__.__name__ +
' block %s/%s' %
(blk, self.dataset.nblocks()))
for idx in test_set:
stopping_cost, cost = self.testing(
test_set.X[idx], test_set.y[idx])
total_test_cost += cost * len(idx)
total_test_stopping_cost += stopping_cost * len(idx)
num_test_examples += len(idx)
#-------[ Update test best cost and error values ]--------#
mean_test_error = total_test_stopping_cost / num_test_examples
mean_test_cost = total_test_cost / num_test_examples
self.log.info('block time: %0.2fs' %
(time.time() - block_time))
self.log.info(get_mem_usage())
#==============[ save to database, save epoch error]==============#
if self.log.save_to_database:
self.log._save_to_database(epoch, best_train_error,
best_valid_error, mean_test_error)
self.log.info('..sent to database: %s:%s' %
(self.log.save_to_database['name'],
self.log.experiment_name))
if self.log.save_epoch_error:
self.log._save_epoch_error(epoch, mean_train_error,
mean_valid_error, mean_test_error)
self.log.info('..epoch error saved')
end_time = time.time()
#=====================[ log outputs to file ]=====================#
merged_train = merge_lists(train_stats_names, train_stats_values)
merged_valid = merge_lists(valid_stats_names, valid_stats_values)
# merged_test = merge_lists(test_stats_names, test_stats_values)
stopping_cost_type = self.learning_rule.stopping_criteria[
'cost'].type
outputs = [
('epoch', epoch), ('runtime(s)', int(end_time - start_time)),
('mean_train_cost_' + self.learning_rule.cost.type,
mean_train_cost),
('mean_train_error_' + stopping_cost_type, mean_train_error),
('best_train_error_' + stopping_cost_type, best_train_error),
('mean_valid_cost_' + self.learning_rule.cost.type,
mean_valid_cost),
('mean_valid_error_' + stopping_cost_type, mean_valid_error),
('best_valid_error_' + stopping_cost_type, best_valid_error),
('mean_test_cost_' + self.learning_rule.cost.type,
mean_test_cost),
('mean_test_error_' + stopping_cost_type, mean_test_error)
]
# outputs += merged_train + merged_valid + merged_test
outputs += merged_train + merged_valid
self.log._log_outputs(outputs)
job_end = time.time()
self.log.info(
'Job Completed on %s' %
time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end)))
ttl_time = int(job_end - job_start)
dt = datetime.timedelta(seconds=ttl_time)
self.log.info('Total Time Taken: %s' % str(dt))
self.log.info("========[ End of Job ]========\n\n")
def run(self):
best_train_error = float(sys.maxint)
best_valid_error = float(sys.maxint)
# best_test_error = float(sys.maxint)
mean_train_error = float(sys.maxint)
mean_valid_error = float(sys.maxint)
mean_test_error = float(sys.maxint)
mean_train_cost = float(sys.maxint)
mean_valid_cost = float(sys.maxint)
mean_test_cost = float(sys.maxint)
best_train_cost = float(sys.maxint)
train_stats_values = []
valid_stats_values = []
# test_stats_values = []
epoch = 0
error_dcr = 0
self.best_epoch_last_update = 0
self.best_valid_last_update = float(sys.maxint)
train_stats_names = ['train_' + name for name in self.train_stats_names]
valid_stats_names = ['valid_' + name for name in self.test_stats_names]
# test_stats_names = ['test_' + name for name in self.test_stats_names]
job_start = time.time()
while (self.continue_learning(epoch, error_dcr, best_valid_error)):
if epoch > 0:
self.log.info("best_epoch_last_update: %d"%self.best_epoch_last_update)
self.log.info("valid_error_decrease: %f"%error_dcr)
self.log.info("best_valid_last_update: %f"%self.best_valid_last_update)
self.log.info("========[ End of Epoch ]========\n\n")
epoch += 1
start_time = time.time()
num_train_examples = 0
total_train_cost = 0.
total_train_stopping_cost = 0.
train_stats_values = np.zeros(len(train_stats_names), dtype=floatX)
num_valid_examples = 0
total_valid_cost = 0.
total_valid_stopping_cost = 0.
valid_stats_values = np.zeros(len(valid_stats_names), dtype=floatX)
num_test_examples = 0
total_test_cost = 0.
total_test_stopping_cost = 0.
# test_stats_values = np.zeros(len(test_stats_names), dtype=floatX)
blk = 0
for block in self.dataset:
block_time = time.time()
blk += 1
train_set = block.get_train()
valid_set = block.get_valid()
test_set = block.get_test()
#====================[ Training Progress ]====================#
if train_set.dataset_size() > 0:
self.log.info('..training '+ self.dataset.__class__.__name__
+ ' block %s/%s'%(blk, self.dataset.nblocks()))
for idx in train_set:
stopping_cost, cost = self.training(train_set.X[idx], train_set.y[idx])
total_train_cost += cost * len(idx)
total_train_stopping_cost += stopping_cost * len(idx)
num_train_examples += len(idx)
train_stats_values += len(idx) * get_shared_values(self.train_stats_shared)
#-------[ Update train best cost and error values ]-------#
mean_train_error = total_train_stopping_cost / num_train_examples
mean_train_cost = total_train_cost / num_train_examples
train_stats_values /= num_train_examples
if mean_train_error < best_train_error:
best_train_error = mean_train_error
if mean_train_cost < 0.999 * best_train_cost:
best_train_cost = mean_train_cost
else:
self.log.info('training cost is not improving after epoch %d, '%epoch)
#---[ Use learning rate decay if the error does not improve after 1 epoch ]---#
if self.learning_rule.learning_rate_decay_factor and hasattr(self.learning_method, 'learning_rate'):
self.log.info('decay learning_rate by factor %.3f'%self.learning_rule.learning_rate_decay_factor)
new_lr = self.learning_method.learning_rate.get_value(return_internal_type=True) \
/ self.learning_rule.learning_rate_decay_factor
self.log.info('new learning rate %.3f'%new_lr)
self.learning_method.learning_rate.set_value(new_lr)
#===================[ Validating Progress ]===================#
if valid_set.dataset_size() > 0:
self.log.info('..validating ' + self.dataset.__class__.__name__
+ ' block %s/%s'%(blk, self.dataset.nblocks()))
for idx in valid_set:
stopping_cost, cost = self.testing(valid_set.X[idx], valid_set.y[idx])
total_valid_cost += cost * len(idx)
total_valid_stopping_cost += stopping_cost * len(idx)
num_valid_examples += len(idx)
valid_stats_values += len(idx) * get_shared_values(self.test_stats_shared)
#-------[ Update valid best cost and error values ]-------#
mean_valid_error = total_valid_stopping_cost / num_valid_examples
mean_valid_cost = total_valid_cost / num_valid_examples
valid_stats_values /= num_valid_examples
if mean_valid_error < best_valid_error:
best_valid_error = mean_valid_error
self.log.info('..best validation error so far')
if self.log.save_model:
self.log._save_model(self.model)
self.log.info('..model saved')
if self.log.save_learning_rule:
self.log._save_learning_rule(self.learning_rule)
self.log.info('..learning rule saved')
if mean_valid_error < self.best_valid_last_update:
error_dcr = self.best_valid_last_update - mean_valid_error
else:
error_dcr = 0
#====================[ Testing Progress ]=====================#
if test_set.dataset_size() > 0:
self.log.info('..testing ' + self.dataset.__class__.__name__
+ ' block %s/%s'%(blk, self.dataset.nblocks()))
for idx in test_set:
stopping_cost, cost = self.testing(test_set.X[idx], test_set.y[idx])
total_test_cost += cost * len(idx)
total_test_stopping_cost += stopping_cost * len(idx)
num_test_examples += len(idx)
#-------[ Update test best cost and error values ]--------#
mean_test_error = total_test_stopping_cost / num_test_examples
mean_test_cost = total_test_cost / num_test_examples
self.log.info('block time: %0.2fs'%(time.time()-block_time))
self.log.info(get_mem_usage())
#==============[ save to database, save epoch error]==============#
if self.log.save_to_database:
self.log._save_to_database(epoch, best_train_error, best_valid_error, mean_test_error)
self.log.info('..sent to database: %s:%s' % (self.log.save_to_database['name'],
self.log.experiment_name))
if self.log.save_epoch_error:
self.log._save_epoch_error(epoch, mean_train_error, mean_valid_error, mean_test_error)
self.log.info('..epoch error saved')
end_time = time.time()
#=====================[ log outputs to file ]=====================#
merged_train = merge_lists(train_stats_names, train_stats_values)
merged_valid = merge_lists(valid_stats_names, valid_stats_values)
# merged_test = merge_lists(test_stats_names, test_stats_values)
stopping_cost_type = self.learning_rule.stopping_criteria['cost'].type
outputs = [('epoch', epoch),
('runtime(s)', int(end_time-start_time)),
('mean_train_cost_' + self.learning_rule.cost.type, mean_train_cost),
('mean_train_error_' + stopping_cost_type, mean_train_error),
('best_train_error_' + stopping_cost_type, best_train_error),
('mean_valid_cost_' + self.learning_rule.cost.type, mean_valid_cost),
('mean_valid_error_' + stopping_cost_type, mean_valid_error),
('best_valid_error_' + stopping_cost_type, best_valid_error),
('mean_test_cost_' + self.learning_rule.cost.type, mean_test_cost),
('mean_test_error_' + stopping_cost_type, mean_test_error)]
# outputs += merged_train + merged_valid + merged_test
outputs += merged_train + merged_valid
self.log._log_outputs(outputs)
job_end = time.time()
self.log.info('Job Completed on %s'%time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime(job_end)))
ttl_time = int(job_end - job_start)
dt = datetime.timedelta(seconds=ttl_time)
self.log.info('Total Time Taken: %s'%str(dt))
self.log.info("========[ End of Job ]========\n\n")
