def on_epoch_end(self, epoch, logs={}):
#seconds = str(int(time.time()))
# my_plots.save_model_as_image(self.model, save_path=self.path_to_save,
# filename_prefix='local_',
# normalize='local',
# mono=True)
my_plots.save_model_as_image(self.model, save_path=self.path_to_save,
filename_prefix='global_',
normalize='global',
mono=True)
average_change_per_layer = self.get_weights_change()
print 'average change per layer:'
print [('%5.3f' % change) for change in average_change_per_layer]
save_best_only=False)
weight_image_saver = my_keras_utils.Weight_Image_Saver(PATH_RESULTS + model_name_dir + 'images/')
if TR_CONST["isRegre"]:
history = my_keras_utils.History_Regression_Val()
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss',
patience=8,
verbose=0)
else:
history = my_keras_utils.History_Classification_Val()
early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc',
patience=8,
verbose=0)
#train!
my_plots.save_model_as_image(model, save_path=PATH_RESULTS + model_name_dir + 'images/',
filename_prefix='INIT_',
normalize='local',
mono=True)
predicted = model.predict(train_x, batch_size=16)
np.save(PATH_RESULTS + model_name_dir+ 'predicted_and_truths_init.npy', [predicted, train_y])
if TR_CONST["tf_type"] == 'cqt':
batch_size = 24
elif TR_CONST["tf_type"] == 'stft':
batch_size = 16
else:
raise RuntimeError('batch size for this? %s' % TF_CONST["tf_type"])
print '--- train starts ---'
if TR_CONST["isRegre"]:
model.fit(train_x, train_y, validation_data=(valid_x, valid_y),
def run_with_setting(hyperparams, argv=None, batch_size=None):
# f = open('will_stop.keunwoo', 'w')
# f.close()
if os.path.exists('stop_asap.keunwoo'):
os.remove('stop_asap.keunwoo')
# pick top-N from label matrix
dim_labels = hyperparams['dim_labels']
if hyperparams['is_LDA']:
best_result = 1.0 # mse
criteria = 'mse'
else:
best_result = 0.0 # auc
criteria = 'auc'
is_getting_better = get_getting_beter_func(hyperparams)
# label_matrix = np.load(PATH_DATA + FILE_DICT['sorted_merged_label_matrix'])
# label_matrix = label_matrix[:, :dim_labels]
hdf_xs = io.load_x(hyperparams['tf_type'], is_test=hyperparams['is_test'])
hdf_ys = io.load_y(dim_labels,
is_test=hyperparams['is_test'],
merged=hyperparams['merged'],
is_LDA=hyperparams['is_LDA'],
is_LDA_normalised=hyperparams['is_LDA_normalised'])
hdf_train_xs = hdf_xs[:12]
hdf_valid_xs = hdf_xs[12:13]
hdf_test_xs = hdf_xs[13:]
hdf_train_ys = hdf_ys[:12]
hdf_valid_ys = hdf_ys[12:13]
hdf_test_ys = hdf_ys[13:]
# train_x, valid_x, test_x = io.load_x(hyperparams['tf_type'])
# train_y, valid_y, test_y = io.load_y(dim_labels)
if hyperparams['is_test']:
pdb.set_trace()
# num_data_in_test = 256
# train_x = train_x[:num_data_in_test]
# valid_x = valid_x[:num_data_in_test]
# test_x = test_x[:num_data_in_test]
# train_y = train_y[:num_data_in_test]
# valid_y = valid_y[:num_data_in_test]
# test_y = test_y[:num_data_in_test]
# shuffle = False
# num_sub_epoch = 1
hyperparams['height_image'] = hdf_train_xs[0].shape[2]
hyperparams["width_image"] = hdf_train_xs[0].shape[3]
if hyperparams['model_type'] == 'multi_input':
mfcc_hdf_xs = io.load_x('mfcc', is_test=hyperparams['is_test'])
mfcc_hdf_train_xs = mfcc_hdf_xs[:12]
mfcc_hdf_valid_xs = mfcc_hdf_xs[12:13]
mfcc_hdf_test_xs = mfcc_hdf_xs[13:]
hyperparams['mfcc_height_image'] = mfcc_hdf_train_xs[0].shape[2]
hyperparams['mfcc_width_image'] = mfcc_hdf_train_xs[0].shape[3]
hp_manager = hyperparams_manager.Hyperparams_Manager()
# name, path, ...
nickname = hp_manager.get_name(hyperparams)
timename = time.strftime('%m-%d-%Hh%M')
if hyperparams["is_test"]:
model_name = 'test_' + nickname
else:
model_name = timename + '_' + nickname
if hyperparams['resume'] != '':
model_name = model_name + '_from_' + hyperparams['resume']
hp_manager.save_new_setting(hyperparams)
print '-' * 60
print 'model name: %s' % model_name
model_name_dir = model_name + '/'
model_weight_name_dir = 'w_' + model_name + '/'
fileout = model_name + '_results'
# build model
model = my_keras_models.build_convnet_model(setting_dict=hyperparams)
if not os.path.exists(PATH_RESULTS + model_name_dir):
os.mkdir(PATH_RESULTS + model_name_dir)
os.mkdir(PATH_RESULTS + model_name_dir + 'images/')
os.mkdir(PATH_RESULTS + model_name_dir + 'plots/')
os.mkdir(PATH_RESULTS_W + model_weight_name_dir)
hp_manager.write_setting_as_texts(PATH_RESULTS + model_name_dir,
hyperparams)
hp_manager.print_setting(hyperparams)
model.summary()
# prepare callbacks
keras_plot(model,
to_file=PATH_RESULTS + model_name_dir + 'images/' +
'graph_of_model_' + hyperparams["!memo"] + '.png')
# checkpointer = keras.callbacks.ModelCheckpoint(filepath=PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5",
# monitor='val_acc',
# verbose=1,
# save_best_only=True)
weight_image_monitor = my_keras_utils.Weight_Image_Saver(PATH_RESULTS +
model_name_dir +
'images/')
patience = 100
if hyperparams["is_test"] is True:
patience = 99999999
early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc',
patience=patience,
verbose=0)
if batch_size == None:
batch_size = 16
if hyperparams['model_type'] == 'vgg_original':
batch_size = (batch_size * 3) / 5
# ready to run
if hyperparams['debug'] == True:
pdb.set_trace()
print '--- %s train starts. Remove will_stop.keunwoo to continue learning after %d epochs ---' % (
model_name, hyperparams["num_epoch"])
num_epoch = hyperparams["num_epoch"]
total_epoch = 0
callbacks = [weight_image_monitor]
total_history = {'loss': [], 'val_loss': [], 'acc': [], 'val_acc': []}
# total_label_count = np.sum([hdf_train.shape[0]*hdf_train.shape[1] for hdf_train in hdf_train_ys])
# total_zeros =
# print 'With predicting all zero, acc is %0.6f' % ((total_label_count - np.sum(train_y))/float(total_label_count))
if hyperparams['resume'] != '':
if os.path.exists(PATH_RESULTS_W + 'w_' + hyperparams['resume']):
model.load_weights(PATH_RESULTS_W + 'w_' + hyperparams['resume'] +
'/weights_best.hdf5')
if os.path.exists(PATH_RESULTS + hyperparams['resume'] +
'/total_history.cP'):
previous_history = cP.load(
open(
PATH_RESULTS + hyperparams['resume'] + '/total_history.cP',
'r'))
print 'previously learned weight: %s is loaded ' % hyperparams[
'resume']
append_history(total_history, previous_history)
best_result = min(total_history[criteria])
if not hyperparams['do_not_learn']:
my_plots.save_model_as_image(model,
save_path=PATH_RESULTS + model_name_dir +
'images/',
filename_prefix='local_INIT',
normalize='local',
mono=True)
my_plots.save_model_as_image(model,
save_path=PATH_RESULTS + model_name_dir +
'images/',
filename_prefix='global_INIT',
normalize='global',
mono=True)
# run
print '--TEST FLIGHT--'
if hyperparams['model_type'] in ['multi_task', 'multi_input'
]: # multi_input assumes multi_task.
if hyperparams['model_type'] == 'multi_task':
fit_dict = get_fit_dict(hdf_train_xs[-1][-256:],
hdf_train_ys[-1][-256:],
hyperparams['dim_labels'])
else:
fit_dict = get_fit_dict(
hdf_train_xs[-1][-256:],
hdf_train_ys[-1][-256:],
hyperparams['dim_labels'],
mfcc_train_x=mfcc_hdf_train_xs[-1][-256:])
# pdb.set_trace()
model.fit(fit_dict,
batch_size=batch_size,
nb_epoch=1,
shuffle='batch')
else:
model.fit(hdf_train_xs[-1][-256:],
hdf_train_ys[-1][-256:],
validation_data=(hdf_valid_xs[0][:512],
hdf_valid_ys[0][:512]),
batch_size=batch_size,
nb_epoch=1,
show_accuracy=hyperparams['isClass'],
callbacks=callbacks,
shuffle='batch')
print '--TEST FLIGHT DONE: %s--' % model_name
total_epoch_count = 0
while True:
for sub_epoch_idx, (train_x, train_y) in enumerate(
zip(hdf_train_xs, hdf_train_ys)):
total_epoch_count += 1
print ' --- I will check stop_asap.keunwoo'
if os.path.exists(
'stop_asap.keunwoo') and total_epoch_count > 1:
print ' --- stop_asap.keunwoo found. will stop now.'
break
print ' --- stop_asap.keunwoo NOT found. keep going on..'
if hyperparams['model_type'] == 'multi_input':
mfcc_train_x = mfcc_hdf_train_xs[sub_epoch_idx]
else:
mfcc_train_x = None
# early_stop should watch overall AUC rather than val_loss or val_acc
# [run]
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(train_x,
train_y,
hyperparams['dim_labels'],
mfcc_train_x=mfcc_train_x)
loss_history = model.fit(fit_dict,
batch_size=batch_size,
nb_epoch=1,
shuffle='batch')
else:
loss_history = model.fit(
train_x,
train_y,
validation_data=(hdf_valid_xs[0][:2048],
hdf_valid_ys[0][:2048]),
batch_size=batch_size,
nb_epoch=1,
show_accuracy=hyperparams['isClass'],
verbose=1,
callbacks=callbacks,
shuffle='batch')
# [validation]
if not sub_epoch_idx in [0, 6]: # validation with subset
if hyperparams['model_type'] in [
'multi_task', 'multi_input'
]:
fit_dict = get_fit_dict(
hdf_valid_xs[-1],
hdf_valid_ys[-1],
hyperparams['dim_labels'],
mfcc_train_x=mfcc_hdf_valid_xs[-1])
predicted_dict = model.predict(fit_dict,
batch_size=batch_size)
predicted = merge_multi_outputs(predicted_dict)
val_loss_here = model.evaluate(fit_dict,
batch_size=batch_size)
print 'val_loss:%f' % val_loss_here
else:
valid_x, valid_y = (hdf_valid_xs[0][:8092],
hdf_valid_ys[0][:8092])
predicted = model.predict(valid_x,
batch_size=batch_size)
else: # validation with all
print ' * Compute AUC with full validation data for model: %s.' % model_name
if hyperparams['model_type'] in [
'multi_task', 'multi_input'
]:
valid_y = hdf_valid_ys[
0][:] # I know I'm using only one set for validation.
fit_dict = get_fit_dict(
hdf_valid_xs[-1],
hdf_valid_ys[-1],
hyperparams['dim_labels'],
mfcc_train_x=mfcc_hdf_valid_xs[-1])
predicted_dict = model.predict(fit_dict,
batch_size=batch_size)
predicted = merge_multi_outputs(predicted_dict)
val_loss_here = model.evaluate(fit_dict,
batch_size=batch_size)
print 'val_loss:%f' % val_loss_here
else:
predicted = np.zeros((0, dim_labels))
valid_y = np.zeros((0, dim_labels))
for valid_x_partial, valid_y_partial in zip(
hdf_valid_xs, hdf_valid_ys):
predicted = np.vstack(
(predicted,
model.predict(valid_x_partial,
batch_size=batch_size)))
valid_y = np.vstack((valid_y, valid_y_partial))
# [check if should stop]
val_result = evaluate_result(valid_y, predicted, hyperparams)
history = {}
history[criteria] = [val_result[criteria]]
print '[%d] %s: %f' % (total_epoch_count, criteria,
val_result[criteria])
# history['coverage_error'] = [val_result['coverage_error']]
# history['label_ranking_average_precision_score'] = [val_result['label_ranking_average_precision_score']]
# history['label_ranking_loss'] = [val_result['label_ranking_loss']]
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
history['val_loss'] = [val_loss_here]
if is_getting_better(best_result, val_result[criteria]):
print ', which is new record! it was %f btw (%s)' % (
best_result, model_name)
best_result = val_result[criteria]
model.save_weights(filepath=PATH_RESULTS_W +
model_weight_name_dir +
"weights_best.hdf5",
overwrite=True)
else:
print 'Keep old auc record, %f' % best_result
append_history(total_history, history)
append_history(total_history, loss_history.history)
my_plots.export_list_png(
total_history[criteria],
out_filename=PATH_RESULTS + model_name_dir + 'plots/' +
('plot_%s.png' % criteria),
title=model_name + criteria + '\n' + hyperparams['!memo'])
my_plots.export_history(total_history['loss'],
total_history['val_loss'],
acc=total_history['acc'],
val_acc=total_history['val_acc'],
out_filename=PATH_RESULTS +
model_name_dir + 'plots/' +
'loss_plots.png')
print '[%d], %d-th of %d epoch is complete, %s:%f' % (
total_epoch_count, total_epoch, num_epoch, criteria,
val_result[criteria])
total_epoch += 1
if os.path.exists('stop_asap.keunwoo'):
os.remove('stop_asap.keunwoo')
break
if os.path.exists('will_stop.keunwoo'):
if total_epoch > num_epoch:
break
else:
print ' *** will go for %d epochs' % (num_epoch -
total_epoch)
else:
print ' *** will go for another one epoch. '
print ' *** $ touch will_stop.keunwoo to stop at the end of this, otherwise it will be endless.'
# [summarise]
if hyperparams["debug"] == True:
pdb.set_trace()
##################################
# test with last weights
predicted = np.zeros((0, dim_labels))
test_y = np.zeros((0, dim_labels))
for test_idx, (test_x_partial,
test_y_partial) in enumerate(zip(hdf_test_xs, hdf_test_ys)):
if hyperparams['model_type'] == 'multi_input':
mfcc_test_x_partial = mfcc_hdf_test_xs[test_idx]
else:
mfcc_test_x_partial = None
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(test_x_partial,
test_y_partial,
hyperparams['dim_labels'],
mfcc_train_x=mfcc_test_x_partial)
predicted_dict = model.predict(fit_dict, batch_size=batch_size)
predicted = np.vstack(
(predicted, merge_multi_outputs(predicted_dict)))
else:
predicted = np.vstack(
(predicted, model.predict(test_x_partial,
batch_size=batch_size)))
test_y = np.vstack((test_y, test_y_partial))
eval_result_final = evaluate_result(test_y, predicted, hyperparams)
print '.' * 60
for key in sorted(eval_result_final.keys()):
print key, eval_result_final[key]
print '.' * 60
#####################
if not hyperparams['is_test']:
if not best_result == val_result[
criteria]: # load weights only it's necessary
print 'Load best weight for test sets'
model.load_weights(PATH_RESULTS_W + model_weight_name_dir +
"weights_best.hdf5")
predicted = np.zeros((0, dim_labels))
test_y = np.zeros((0, dim_labels))
for test_idx, (test_x_partial,
test_y_partial) in enumerate(zip(hdf_test_xs, hdf_test_ys)):
if hyperparams['model_type'] == 'multi_input':
mfcc_test_x_partial = mfcc_hdf_test_xs[test_idx]
else:
mfcc_test_x_partial = None
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(test_x_partial,
test_y_partial,
hyperparams['dim_labels'],
mfcc_train_x=mfcc_test_x_partial)
predicted_dict = model.predict(fit_dict, batch_size=batch_size)
predicted = np.vstack(
(predicted, merge_multi_outputs(predicted_dict)))
else:
predicted = np.vstack(
(predicted, model.predict(test_x_partial,
batch_size=batch_size)))
test_y = np.vstack((test_y, test_y_partial))
eval_result_final = evaluate_result(test_y, predicted, hyperparams)
print '.' * 60
for key in sorted(eval_result_final.keys()):
print key, eval_result_final[key]
print '.' * 60
#save results
cP.dump(total_history,
open(PATH_RESULTS + model_name_dir + 'total_history.cP', 'w'))
# np.save(PATH_RESULTS + model_name_dir + 'loss_testset.npy', loss_testset)
np.save(PATH_RESULTS + model_name_dir + 'predicted_and_truths_result.npy',
[predicted, test_y])
np.save(PATH_RESULTS + model_name_dir + 'weights_changes.npy',
np.array(weight_image_monitor.weights_changes))
# ADD weight change saving code
if total_history != {}:
# max_auc = np.max(total_history['auc'])
best_batch = np.argmax(total_history[criteria]) + 1
num_run_epoch = len(total_history[criteria])
oneline_result = '%6.4f, %s %d_of_%d, %s' % (
best_result, criteria, best_batch, num_run_epoch, model_name)
with open(PATH_RESULTS + model_name_dir + oneline_result, 'w') as f:
pass
f = open( (PATH_RESULTS + '%s_%s_%s_%06.4f_at_(%d_of_%d)_%s' % \
(timename, hyperparams["loss_function"], criteria, best_result, best_batch, num_run_epoch, nickname)), 'w')
f.close()
with open('one_line_log.txt', 'a') as f:
f.write(oneline_result)
f.write(' ' + ' '.join(argv) + '\n')
else:
max_auc = 0.0
print '========== DONE: %s ==========' % model_name
return best_result
def run_with_setting(hyperparams, argv=None, batch_size=None):
# f = open('will_stop.keunwoo', 'w')
# f.close()
if os.path.exists('stop_asap.keunwoo'):
os.remove('stop_asap.keunwoo')
# pick top-N from label matrix
dim_labels = hyperparams['dim_labels']
if hyperparams['is_LDA']:
best_result = 1.0 # mse
criteria = 'mse'
else:
best_result = 0.0 # auc
criteria = 'auc'
is_getting_better = get_getting_beter_func(hyperparams)
# label_matrix = np.load(PATH_DATA + FILE_DICT['sorted_merged_label_matrix'])
# label_matrix = label_matrix[:, :dim_labels]
hdf_xs = io.load_x(hyperparams['tf_type'], is_test=hyperparams['is_test'])
hdf_ys = io.load_y(dim_labels, is_test=hyperparams['is_test'], merged=hyperparams['merged'], is_LDA=hyperparams['is_LDA'], is_LDA_normalised=hyperparams['is_LDA_normalised'])
hdf_train_xs = hdf_xs[:12]
hdf_valid_xs = hdf_xs[12:13]
hdf_test_xs = hdf_xs[13:]
hdf_train_ys = hdf_ys[:12]
hdf_valid_ys = hdf_ys[12:13]
hdf_test_ys = hdf_ys[13:]
# train_x, valid_x, test_x = io.load_x(hyperparams['tf_type'])
# train_y, valid_y, test_y = io.load_y(dim_labels)
if hyperparams['is_test']:
pdb.set_trace()
# num_data_in_test = 256
# train_x = train_x[:num_data_in_test]
# valid_x = valid_x[:num_data_in_test]
# test_x = test_x[:num_data_in_test]
# train_y = train_y[:num_data_in_test]
# valid_y = valid_y[:num_data_in_test]
# test_y = test_y[:num_data_in_test]
# shuffle = False
# num_sub_epoch = 1
hyperparams['height_image'] = hdf_train_xs[0].shape[2]
hyperparams["width_image"] = hdf_train_xs[0].shape[3]
if hyperparams['model_type'] == 'multi_input':
mfcc_hdf_xs = io.load_x('mfcc', is_test=hyperparams['is_test'])
mfcc_hdf_train_xs = mfcc_hdf_xs[:12]
mfcc_hdf_valid_xs = mfcc_hdf_xs[12:13]
mfcc_hdf_test_xs = mfcc_hdf_xs[13:]
hyperparams['mfcc_height_image'] = mfcc_hdf_train_xs[0].shape[2]
hyperparams['mfcc_width_image'] = mfcc_hdf_train_xs[0].shape[3]
hp_manager = hyperparams_manager.Hyperparams_Manager()
# name, path, ...
nickname = hp_manager.get_name(hyperparams)
timename = time.strftime('%m-%d-%Hh%M')
if hyperparams["is_test"]:
model_name = 'test_' + nickname
else:
model_name = timename + '_' + nickname
if hyperparams['resume'] != '':
model_name = model_name + '_from_' + hyperparams['resume']
hp_manager.save_new_setting(hyperparams)
print '-'*60
print 'model name: %s' % model_name
model_name_dir = model_name + '/'
model_weight_name_dir = 'w_' + model_name + '/'
fileout = model_name + '_results'
# build model
model = my_keras_models.build_convnet_model(setting_dict=hyperparams)
if not os.path.exists(PATH_RESULTS + model_name_dir):
os.mkdir(PATH_RESULTS + model_name_dir)
os.mkdir(PATH_RESULTS + model_name_dir + 'images/')
os.mkdir(PATH_RESULTS + model_name_dir + 'plots/')
os.mkdir(PATH_RESULTS_W + model_weight_name_dir)
hp_manager.write_setting_as_texts(PATH_RESULTS + model_name_dir, hyperparams)
hp_manager.print_setting(hyperparams)
model.summary()
# prepare callbacks
keras_plot(model, to_file=PATH_RESULTS + model_name_dir + 'images/'+'graph_of_model_'+hyperparams["!memo"]+'.png')
# checkpointer = keras.callbacks.ModelCheckpoint(filepath=PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5",
# monitor='val_acc',
# verbose=1,
# save_best_only=True)
weight_image_monitor = my_keras_utils.Weight_Image_Saver(PATH_RESULTS + model_name_dir + 'images/')
patience = 100
if hyperparams["is_test"] is True:
patience = 99999999
early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc',
patience=patience,
verbose=0)
if batch_size == None:
batch_size = 16
if hyperparams['model_type'] == 'vgg_original':
batch_size = (batch_size * 3)/5
# ready to run
if hyperparams['debug'] == True:
pdb.set_trace()
print '--- %s train starts. Remove will_stop.keunwoo to continue learning after %d epochs ---' % (model_name, hyperparams["num_epoch"])
num_epoch = hyperparams["num_epoch"]
total_epoch = 0
callbacks = [weight_image_monitor]
total_history = {'loss':[], 'val_loss':[], 'acc':[], 'val_acc':[]}
# total_label_count = np.sum([hdf_train.shape[0]*hdf_train.shape[1] for hdf_train in hdf_train_ys])
# total_zeros =
# print 'With predicting all zero, acc is %0.6f' % ((total_label_count - np.sum(train_y))/float(total_label_count))
if hyperparams['resume'] != '':
if os.path.exists(PATH_RESULTS_W + 'w_' + hyperparams['resume']):
model.load_weights(PATH_RESULTS_W + 'w_' + hyperparams['resume'] + '/weights_best.hdf5')
if os.path.exists(PATH_RESULTS + hyperparams['resume'] + '/total_history.cP'):
previous_history = cP.load(open(PATH_RESULTS + hyperparams['resume'] + '/total_history.cP', 'r'))
print 'previously learned weight: %s is loaded ' % hyperparams['resume']
append_history(total_history, previous_history)
best_result = min(total_history[criteria])
if not hyperparams['do_not_learn']:
my_plots.save_model_as_image(model, save_path=PATH_RESULTS + model_name_dir + 'images/',
filename_prefix='local_INIT',
normalize='local',
mono=True)
my_plots.save_model_as_image(model, save_path=PATH_RESULTS + model_name_dir + 'images/',
filename_prefix='global_INIT',
normalize='global',
mono=True)
# run
print '--TEST FLIGHT--'
if hyperparams['model_type'] in ['multi_task', 'multi_input']: # multi_input assumes multi_task.
if hyperparams['model_type'] == 'multi_task':
fit_dict = get_fit_dict(hdf_train_xs[-1][-256:], hdf_train_ys[-1][-256:], hyperparams['dim_labels'])
else:
fit_dict = get_fit_dict(hdf_train_xs[-1][-256:], hdf_train_ys[-1][-256:], hyperparams['dim_labels'], mfcc_train_x=mfcc_hdf_train_xs[-1][-256:])
# pdb.set_trace()
model.fit(fit_dict, batch_size=batch_size, nb_epoch=1, shuffle='batch')
else:
model.fit(hdf_train_xs[-1][-256:], hdf_train_ys[-1][-256:],
validation_data=(hdf_valid_xs[0][:512], hdf_valid_ys[0][:512]),
batch_size=batch_size,
nb_epoch=1,
show_accuracy=hyperparams['isClass'],
callbacks=callbacks,
shuffle='batch')
print '--TEST FLIGHT DONE: %s--' % model_name
total_epoch_count = 0
while True:
for sub_epoch_idx, (train_x, train_y) in enumerate(zip(hdf_train_xs, hdf_train_ys)):
total_epoch_count += 1
print ' --- I will check stop_asap.keunwoo'
if os.path.exists('stop_asap.keunwoo') and total_epoch_count > 1:
print ' --- stop_asap.keunwoo found. will stop now.'
break
print ' --- stop_asap.keunwoo NOT found. keep going on..'
if hyperparams['model_type'] == 'multi_input':
mfcc_train_x = mfcc_hdf_train_xs[sub_epoch_idx]
else:
mfcc_train_x = None
# early_stop should watch overall AUC rather than val_loss or val_acc
# [run]
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(train_x, train_y, hyperparams['dim_labels'], mfcc_train_x=mfcc_train_x)
loss_history = model.fit(fit_dict,
batch_size=batch_size,
nb_epoch=1,
shuffle='batch')
else:
loss_history = model.fit(train_x, train_y, validation_data=(hdf_valid_xs[0][:2048], hdf_valid_ys[0][:2048]),
batch_size=batch_size,
nb_epoch=1,
show_accuracy=hyperparams['isClass'],
verbose=1,
callbacks=callbacks,
shuffle='batch')
# [validation]
if not sub_epoch_idx in [0, 6]: # validation with subset
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(hdf_valid_xs[-1], hdf_valid_ys[-1], hyperparams['dim_labels'], mfcc_train_x=mfcc_hdf_valid_xs[-1])
predicted_dict = model.predict(fit_dict, batch_size=batch_size)
predicted = merge_multi_outputs(predicted_dict)
val_loss_here = model.evaluate(fit_dict, batch_size=batch_size)
print 'val_loss:%f' % val_loss_here
else:
valid_x, valid_y = (hdf_valid_xs[0][:8092], hdf_valid_ys[0][:8092])
predicted = model.predict(valid_x, batch_size=batch_size)
else: # validation with all
print ' * Compute AUC with full validation data for model: %s.' % model_name
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
valid_y = hdf_valid_ys[0][:] # I know I'm using only one set for validation.
fit_dict = get_fit_dict(hdf_valid_xs[-1], hdf_valid_ys[-1], hyperparams['dim_labels'], mfcc_train_x=mfcc_hdf_valid_xs[-1])
predicted_dict = model.predict(fit_dict, batch_size=batch_size)
predicted = merge_multi_outputs(predicted_dict)
val_loss_here = model.evaluate(fit_dict, batch_size=batch_size)
print 'val_loss:%f' % val_loss_here
else:
predicted = np.zeros((0, dim_labels))
valid_y = np.zeros((0, dim_labels))
for valid_x_partial, valid_y_partial in zip(hdf_valid_xs, hdf_valid_ys):
predicted = np.vstack((predicted, model.predict(valid_x_partial, batch_size=batch_size)))
valid_y = np.vstack((valid_y, valid_y_partial))
# [check if should stop]
val_result = evaluate_result(valid_y, predicted, hyperparams)
history = {}
history[criteria] = [val_result[criteria]]
print '[%d] %s: %f' % (total_epoch_count, criteria, val_result[criteria])
# history['coverage_error'] = [val_result['coverage_error']]
# history['label_ranking_average_precision_score'] = [val_result['label_ranking_average_precision_score']]
# history['label_ranking_loss'] = [val_result['label_ranking_loss']]
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
history['val_loss'] = [val_loss_here]
if is_getting_better(best_result, val_result[criteria]):
print ', which is new record! it was %f btw (%s)' % (best_result, model_name)
best_result = val_result[criteria]
model.save_weights(filepath=PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5",
overwrite=True)
else:
print 'Keep old auc record, %f' % best_result
append_history(total_history, history)
append_history(total_history, loss_history.history)
my_plots.export_list_png(total_history[criteria], out_filename=PATH_RESULTS + model_name_dir + 'plots/' + ('plot_%s.png' % criteria), title=model_name + criteria + '\n'+hyperparams['!memo'] )
my_plots.export_history(total_history['loss'], total_history['val_loss'],
acc=total_history['acc'],
val_acc=total_history['val_acc'],
out_filename=PATH_RESULTS + model_name_dir + 'plots/' + 'loss_plots.png')
print '[%d], %d-th of %d epoch is complete, %s:%f' % (total_epoch_count, total_epoch, num_epoch, criteria, val_result[criteria])
total_epoch += 1
if os.path.exists('stop_asap.keunwoo'):
os.remove('stop_asap.keunwoo')
break
if os.path.exists('will_stop.keunwoo'):
if total_epoch > num_epoch:
break
else:
print ' *** will go for %d epochs' % (num_epoch - total_epoch)
else:
print ' *** will go for another one epoch. '
print ' *** $ touch will_stop.keunwoo to stop at the end of this, otherwise it will be endless.'
# [summarise]
if hyperparams["debug"] == True:
pdb.set_trace()
##################################
# test with last weights
predicted = np.zeros((0, dim_labels))
test_y = np.zeros((0, dim_labels))
for test_idx, (test_x_partial, test_y_partial) in enumerate(zip(hdf_test_xs, hdf_test_ys)):
if hyperparams['model_type'] == 'multi_input':
mfcc_test_x_partial = mfcc_hdf_test_xs[test_idx]
else:
mfcc_test_x_partial = None
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(test_x_partial, test_y_partial, hyperparams['dim_labels'], mfcc_train_x=mfcc_test_x_partial)
predicted_dict = model.predict(fit_dict, batch_size=batch_size)
predicted = np.vstack((predicted, merge_multi_outputs(predicted_dict)))
else:
predicted = np.vstack((predicted, model.predict(test_x_partial, batch_size=batch_size)))
test_y = np.vstack((test_y, test_y_partial))
eval_result_final = evaluate_result(test_y, predicted, hyperparams)
print '.'*60
for key in sorted(eval_result_final.keys()):
print key, eval_result_final[key]
print '.'*60
#####################
if not hyperparams['is_test']:
if not best_result == val_result[criteria]: # load weights only it's necessary
print 'Load best weight for test sets'
model.load_weights(PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5")
predicted = np.zeros((0, dim_labels))
test_y = np.zeros((0, dim_labels))
for test_idx, (test_x_partial, test_y_partial) in enumerate(zip(hdf_test_xs, hdf_test_ys)):
if hyperparams['model_type'] == 'multi_input':
mfcc_test_x_partial = mfcc_hdf_test_xs[test_idx]
else:
mfcc_test_x_partial = None
if hyperparams['model_type'] in ['multi_task', 'multi_input']:
fit_dict = get_fit_dict(test_x_partial, test_y_partial, hyperparams['dim_labels'], mfcc_train_x=mfcc_test_x_partial)
predicted_dict = model.predict(fit_dict, batch_size=batch_size)
predicted = np.vstack((predicted, merge_multi_outputs(predicted_dict)))
else:
predicted = np.vstack((predicted, model.predict(test_x_partial, batch_size=batch_size)))
test_y = np.vstack((test_y, test_y_partial))
eval_result_final = evaluate_result(test_y, predicted, hyperparams)
print '.'*60
for key in sorted(eval_result_final.keys()):
print key, eval_result_final[key]
print '.'*60
#save results
cP.dump(total_history, open(PATH_RESULTS + model_name_dir + 'total_history.cP', 'w'))
# np.save(PATH_RESULTS + model_name_dir + 'loss_testset.npy', loss_testset)
np.save(PATH_RESULTS + model_name_dir + 'predicted_and_truths_result.npy', [predicted, test_y])
np.save(PATH_RESULTS + model_name_dir + 'weights_changes.npy', np.array(weight_image_monitor.weights_changes))
# ADD weight change saving code
if total_history != {}:
# max_auc = np.max(total_history['auc'])
best_batch = np.argmax(total_history[criteria])+1
num_run_epoch = len(total_history[criteria])
oneline_result = '%6.4f, %s %d_of_%d, %s' % (best_result, criteria, best_batch, num_run_epoch, model_name)
with open(PATH_RESULTS + model_name_dir + oneline_result, 'w') as f:
pass
f = open( (PATH_RESULTS + '%s_%s_%s_%06.4f_at_(%d_of_%d)_%s' % \
(timename, hyperparams["loss_function"], criteria, best_result, best_batch, num_run_epoch, nickname)), 'w')
f.close()
with open('one_line_log.txt', 'a') as f:
f.write(oneline_result)
f.write(' ' + ' '.join(argv) + '\n')
else:
max_auc = 0.0
print '========== DONE: %s ==========' % model_name
return best_result