ret = {}

if not hyperparams['is_LDA']:

ret['auc'] = metrics.roc_auc_score(y_true, y_pred, average='macro')

# ret['coverage_error'] = metrics.coverage_error(y_true, y_pred)

# ret['label_ranking_average_precision_score'] = metrics.label_ranking_average_precision_score(y_true, y_pred)

# ret['label_ranking_loss'] = metrics.label_ranking_loss(y_true, y_pred)

ret['mse'] = metrics.mean_squared_error(y_true, y_pred)

print '.'*60

for key in ret:

print key, ret[key]

print '.'*60

fit_dict = {}

fit_dict[name_x] = train_x

if mfcc_train_x is not None:

fit_dict[mfcc_name_x] = mfcc_train_x

for dense_idx in xrange(dim_labels):

output_node_name = 'output_%d' % dense_idx

fit_dict[output_node_name] = train_y[:, dense_idx:dense_idx+1]

dim_label = len(predicted_dict.keys())

num_data = predicted_dict[predicted_dict.keys()[0]].shape[0]

predicted = np.zeros((num_data, dim_label))

for i in range(dim_label):

predicted[:, i] = predicted_dict['output_%d'%i][:, 0]

setting_dict["num_feat_maps"] = [setting_dict["num_feat_maps"][0]]*setting_dict["num_layers"]

setting_dict["activations"] = [setting_dict["activations"][0]] *setting_dict["num_layers"]

setting_dict["dropouts"] = [setting_dict["dropouts"][0]]*setting_dict["num_layers"]

setting_dict["regulariser"] = [setting_dict["regulariser"][0]]*setting_dict["num_layers"]

if setting_dict['num_fc_layers'] == 0:

setting_dict["dropouts_fc_layers"] = []

setting_dict["nums_units_fc_layers"] = []

setting_dict["activations_fc_layers"] = []

setting_dict["regulariser_fc_layers"] = []

setting_dict["act_regulariser_fc_layers"] = []

else:

setting_dict["dropouts_fc_layers"] = [setting_dict["dropouts_fc_layers"][0]]*setting_dict["num_fc_layers"]

setting_dict["nums_units_fc_layers"] = [setting_dict["nums_units_fc_layers"][0]]*setting_dict["num_fc_layers"]

setting_dict["activations_fc_layers"] = [setting_dict["activations_fc_layers"][0]]*setting_dict["num_fc_layers"]

setting_dict["regulariser_fc_layers"] = [setting_dict["regulariser_fc_layers"][0]]*setting_dict["num_fc_layers"]

setting_dict["act_regulariser_fc_layers"] = [setting_dict["act_regulariser_fc_layers"][0]]*setting_dict["num_fc_layers"]

'''local history is a dictionary,

key:value == string:dictionary.

key: loss, vall_loss, batch, size

Therefore total_history has the same keys and append the values.

'''

for key in local_history:

if key not in total_history:

total_history[key] = []

if hyperparams['is_LDA']:

return is_smaller # MSE

else:

# f = open('will_stop.keunwoo', 'w')

# f.close()

if os.path.exists('stop_asap.keunwoo'):

os.remove('stop_asap.keunwoo')

# pick top-N from label matrix

dim_labels = hyperparams['dim_labels']

if hyperparams['is_LDA']:

best_result = 1.0 # mse

criteria = 'mse'

else:

best_result = 0.0 # auc

criteria = 'auc'

is_getting_better = get_getting_beter_func(hyperparams)

# label_matrix = np.load(PATH_DATA + FILE_DICT['sorted_merged_label_matrix'])

# label_matrix = label_matrix[:, :dim_labels]

hdf_xs = io.load_x(hyperparams['tf_type'], is_test=hyperparams['is_test'])

hdf_ys = io.load_y(dim_labels, is_test=hyperparams['is_test'], merged=hyperparams['merged'], is_LDA=hyperparams['is_LDA'], is_LDA_normalised=hyperparams['is_LDA_normalised'])

hdf_train_xs = hdf_xs[:12]

hdf_valid_xs = hdf_xs[12:13]

hdf_test_xs = hdf_xs[13:]

hdf_train_ys = hdf_ys[:12]

hdf_valid_ys = hdf_ys[12:13]

hdf_test_ys = hdf_ys[13:]

# train_x, valid_x, test_x = io.load_x(hyperparams['tf_type'])

# train_y, valid_y, test_y = io.load_y(dim_labels)

if hyperparams['is_test']:

pdb.set_trace()

# num_data_in_test = 256

# train_x = train_x[:num_data_in_test]

# valid_x = valid_x[:num_data_in_test]

# test_x = test_x[:num_data_in_test]

# train_y = train_y[:num_data_in_test]

# valid_y = valid_y[:num_data_in_test]

# test_y = test_y[:num_data_in_test]

# shuffle = False

# num_sub_epoch = 1

hyperparams['height_image'] = hdf_train_xs[0].shape[2]

hyperparams["width_image"] = hdf_train_xs[0].shape[3]

if hyperparams['model_type'] == 'multi_input':

mfcc_hdf_xs = io.load_x('mfcc', is_test=hyperparams['is_test'])

mfcc_hdf_train_xs = mfcc_hdf_xs[:12]

mfcc_hdf_valid_xs = mfcc_hdf_xs[12:13]

mfcc_hdf_test_xs = mfcc_hdf_xs[13:]

hyperparams['mfcc_height_image'] = mfcc_hdf_train_xs[0].shape[2]

hyperparams['mfcc_width_image'] = mfcc_hdf_train_xs[0].shape[3]

hp_manager = hyperparams_manager.Hyperparams_Manager()

# name, path, ...

nickname = hp_manager.get_name(hyperparams)

timename = time.strftime('%m-%d-%Hh%M')

if hyperparams["is_test"]:

model_name = 'test_' + nickname

else:

model_name = timename + '_' + nickname

if hyperparams['resume'] != '':

model_name = model_name + '_from_' + hyperparams['resume']

hp_manager.save_new_setting(hyperparams)

print '-'*60

print 'model name: %s' % model_name

model_name_dir = model_name + '/'

model_weight_name_dir = 'w_' + model_name + '/'

fileout = model_name + '_results'

# build model

model = my_keras_models.build_convnet_model(setting_dict=hyperparams)

if not os.path.exists(PATH_RESULTS + model_name_dir):

os.mkdir(PATH_RESULTS + model_name_dir)

os.mkdir(PATH_RESULTS + model_name_dir + 'images/')

os.mkdir(PATH_RESULTS + model_name_dir + 'plots/')

os.mkdir(PATH_RESULTS_W + model_weight_name_dir)

hp_manager.write_setting_as_texts(PATH_RESULTS + model_name_dir, hyperparams)

hp_manager.print_setting(hyperparams)

model.summary()

# prepare callbacks

keras_plot(model, to_file=PATH_RESULTS + model_name_dir + 'images/'+'graph_of_model_'+hyperparams["!memo"]+'.png')

# checkpointer = keras.callbacks.ModelCheckpoint(filepath=PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5",

# monitor='val_acc',

# verbose=1,

# save_best_only=True)

weight_image_monitor = my_keras_utils.Weight_Image_Saver(PATH_RESULTS + model_name_dir + 'images/')

patience = 100

if hyperparams["is_test"] is True:

patience = 99999999

early_stopping = keras.callbacks.EarlyStopping(monitor='val_acc',

patience=patience,

verbose=0)

if batch_size == None:

batch_size = 16

if hyperparams['model_type'] == 'vgg_original':

batch_size = (batch_size * 3)/5

# ready to run

if hyperparams['debug'] == True:

pdb.set_trace()

print '--- %s train starts. Remove will_stop.keunwoo to continue learning after %d epochs ---' % (model_name, hyperparams["num_epoch"])

num_epoch = hyperparams["num_epoch"]

total_epoch = 0

callbacks = [weight_image_monitor]

total_history = {'loss':[], 'val_loss':[], 'acc':[], 'val_acc':[]}

# total_label_count = np.sum([hdf_train.shape[0]*hdf_train.shape[1] for hdf_train in hdf_train_ys])

# total_zeros =

# print 'With predicting all zero, acc is %0.6f' % ((total_label_count - np.sum(train_y))/float(total_label_count))

if hyperparams['resume'] != '':

if os.path.exists(PATH_RESULTS_W + 'w_' + hyperparams['resume']):

model.load_weights(PATH_RESULTS_W + 'w_' + hyperparams['resume'] + '/weights_best.hdf5')

if os.path.exists(PATH_RESULTS + hyperparams['resume'] + '/total_history.cP'):

previous_history = cP.load(open(PATH_RESULTS + hyperparams['resume'] + '/total_history.cP', 'r'))

print 'previously learned weight: %s is loaded ' % hyperparams['resume']

append_history(total_history, previous_history)

best_result = min(total_history[criteria])

if not hyperparams['do_not_learn']:

my_plots.save_model_as_image(model, save_path=PATH_RESULTS + model_name_dir + 'images/',

filename_prefix='local_INIT',

normalize='local',

mono=True)

my_plots.save_model_as_image(model, save_path=PATH_RESULTS + model_name_dir + 'images/',

filename_prefix='global_INIT',

normalize='global',

mono=True)

# run

print '--TEST FLIGHT--'

if hyperparams['model_type'] in ['multi_task', 'multi_input']: # multi_input assumes multi_task.

if hyperparams['model_type'] == 'multi_task':

fit_dict = get_fit_dict(hdf_train_xs[-1][-256:], hdf_train_ys[-1][-256:], hyperparams['dim_labels'])

else:

fit_dict = get_fit_dict(hdf_train_xs[-1][-256:], hdf_train_ys[-1][-256:], hyperparams['dim_labels'], mfcc_train_x=mfcc_hdf_train_xs[-1][-256:])

# pdb.set_trace()

model.fit(fit_dict, batch_size=batch_size, nb_epoch=1, shuffle='batch')

else:

model.fit(hdf_train_xs[-1][-256:], hdf_train_ys[-1][-256:],

validation_data=(hdf_valid_xs[0][:512], hdf_valid_ys[0][:512]),

batch_size=batch_size,

nb_epoch=1,

show_accuracy=hyperparams['isClass'],

callbacks=callbacks,

shuffle='batch')

print '--TEST FLIGHT DONE: %s--' % model_name

total_epoch_count = 0

while True:

for sub_epoch_idx, (train_x, train_y) in enumerate(zip(hdf_train_xs, hdf_train_ys)):

total_epoch_count += 1

print ' --- I will check stop_asap.keunwoo'

if os.path.exists('stop_asap.keunwoo') and total_epoch_count > 1:

print ' --- stop_asap.keunwoo found. will stop now.'

break

print ' --- stop_asap.keunwoo NOT found. keep going on..'

if hyperparams['model_type'] == 'multi_input':

mfcc_train_x = mfcc_hdf_train_xs[sub_epoch_idx]

else:

mfcc_train_x = None

# early_stop should watch overall AUC rather than val_loss or val_acc

# [run]

if hyperparams['model_type'] in ['multi_task', 'multi_input']:

fit_dict = get_fit_dict(train_x, train_y, hyperparams['dim_labels'], mfcc_train_x=mfcc_train_x)

loss_history = model.fit(fit_dict,

batch_size=batch_size,

nb_epoch=1,

shuffle='batch')

else:

loss_history = model.fit(train_x, train_y, validation_data=(hdf_valid_xs[0][:2048], hdf_valid_ys[0][:2048]),

batch_size=batch_size,

nb_epoch=1,

show_accuracy=hyperparams['isClass'],

verbose=1,

callbacks=callbacks,

shuffle='batch')

# [validation]

if not sub_epoch_idx in [0, 6]: # validation with subset

if hyperparams['model_type'] in ['multi_task', 'multi_input']:

fit_dict = get_fit_dict(hdf_valid_xs[-1], hdf_valid_ys[-1], hyperparams['dim_labels'], mfcc_train_x=mfcc_hdf_valid_xs[-1])

predicted_dict = model.predict(fit_dict, batch_size=batch_size)

predicted = merge_multi_outputs(predicted_dict)

val_loss_here = model.evaluate(fit_dict, batch_size=batch_size)

print 'val_loss:%f' % val_loss_here

else:

valid_x, valid_y = (hdf_valid_xs[0][:8092], hdf_valid_ys[0][:8092])

predicted = model.predict(valid_x, batch_size=batch_size)

else: # validation with all

print ' * Compute AUC with full validation data for model: %s.' % model_name

if hyperparams['model_type'] in ['multi_task', 'multi_input']:

valid_y = hdf_valid_ys[0][:] # I know I'm using only one set for validation.

fit_dict = get_fit_dict(hdf_valid_xs[-1], hdf_valid_ys[-1], hyperparams['dim_labels'], mfcc_train_x=mfcc_hdf_valid_xs[-1])

predicted_dict = model.predict(fit_dict, batch_size=batch_size)

predicted = merge_multi_outputs(predicted_dict)

val_loss_here = model.evaluate(fit_dict, batch_size=batch_size)

print 'val_loss:%f' % val_loss_here

else:

predicted = np.zeros((0, dim_labels))

valid_y = np.zeros((0, dim_labels))

for valid_x_partial, valid_y_partial in zip(hdf_valid_xs, hdf_valid_ys):

predicted = np.vstack((predicted, model.predict(valid_x_partial, batch_size=batch_size)))

valid_y = np.vstack((valid_y, valid_y_partial))

# [check if should stop]

val_result = evaluate_result(valid_y, predicted, hyperparams)

history = {}

history[criteria] = [val_result[criteria]]

print '[%d] %s: %f' % (total_epoch_count, criteria, val_result[criteria])

# history['coverage_error'] = [val_result['coverage_error']]

# history['label_ranking_average_precision_score'] = [val_result['label_ranking_average_precision_score']]

# history['label_ranking_loss'] = [val_result['label_ranking_loss']]

if hyperparams['model_type'] in ['multi_task', 'multi_input']:

history['val_loss'] = [val_loss_here]

if is_getting_better(best_result, val_result[criteria]):

print ', which is new record! it was %f btw (%s)' % (best_result, model_name)

best_result = val_result[criteria]

model.save_weights(filepath=PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5",

overwrite=True)

else:

print 'Keep old auc record, %f' % best_result

append_history(total_history, history)

append_history(total_history, loss_history.history)

my_plots.export_list_png(total_history[criteria], out_filename=PATH_RESULTS + model_name_dir + 'plots/' + ('plot_%s.png' % criteria), title=model_name + criteria + '\n'+hyperparams['!memo'] )

my_plots.export_history(total_history['loss'], total_history['val_loss'],

acc=total_history['acc'],

val_acc=total_history['val_acc'],

out_filename=PATH_RESULTS + model_name_dir + 'plots/' + 'loss_plots.png')

print '[%d], %d-th of %d epoch is complete, %s:%f' % (total_epoch_count, total_epoch, num_epoch, criteria, val_result[criteria])

total_epoch += 1

if os.path.exists('stop_asap.keunwoo'):

os.remove('stop_asap.keunwoo')

break

if os.path.exists('will_stop.keunwoo'):

if total_epoch > num_epoch:

break

else:

print ' *** will go for %d epochs' % (num_epoch - total_epoch)

else:

print ' *** will go for another one epoch. '

print ' *** $ touch will_stop.keunwoo to stop at the end of this, otherwise it will be endless.'

# [summarise]

if hyperparams["debug"] == True:

pdb.set_trace()

##################################

# test with last weights

predicted = np.zeros((0, dim_labels))

test_y = np.zeros((0, dim_labels))

for test_idx, (test_x_partial, test_y_partial) in enumerate(zip(hdf_test_xs, hdf_test_ys)):

if hyperparams['model_type'] == 'multi_input':

mfcc_test_x_partial = mfcc_hdf_test_xs[test_idx]

else:

mfcc_test_x_partial = None

if hyperparams['model_type'] in ['multi_task', 'multi_input']:

fit_dict = get_fit_dict(test_x_partial, test_y_partial, hyperparams['dim_labels'], mfcc_train_x=mfcc_test_x_partial)

predicted_dict = model.predict(fit_dict, batch_size=batch_size)

predicted = np.vstack((predicted, merge_multi_outputs(predicted_dict)))

else:

predicted = np.vstack((predicted, model.predict(test_x_partial, batch_size=batch_size)))

test_y = np.vstack((test_y, test_y_partial))

eval_result_final = evaluate_result(test_y, predicted, hyperparams)

print '.'*60

for key in sorted(eval_result_final.keys()):

print key, eval_result_final[key]

print '.'*60

#####################

if not hyperparams['is_test']:

if not best_result == val_result[criteria]: # load weights only it's necessary

print 'Load best weight for test sets'

model.load_weights(PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5")

predicted = np.zeros((0, dim_labels))

test_y = np.zeros((0, dim_labels))

for test_idx, (test_x_partial, test_y_partial) in enumerate(zip(hdf_test_xs, hdf_test_ys)):

if hyperparams['model_type'] == 'multi_input':

mfcc_test_x_partial = mfcc_hdf_test_xs[test_idx]

else:

mfcc_test_x_partial = None

if hyperparams['model_type'] in ['multi_task', 'multi_input']:

fit_dict = get_fit_dict(test_x_partial, test_y_partial, hyperparams['dim_labels'], mfcc_train_x=mfcc_test_x_partial)

predicted_dict = model.predict(fit_dict, batch_size=batch_size)

predicted = np.vstack((predicted, merge_multi_outputs(predicted_dict)))

else:

predicted = np.vstack((predicted, model.predict(test_x_partial, batch_size=batch_size)))

test_y = np.vstack((test_y, test_y_partial))

eval_result_final = evaluate_result(test_y, predicted, hyperparams)

print '.'*60

for key in sorted(eval_result_final.keys()):

print key, eval_result_final[key]

print '.'*60

#save results

cP.dump(total_history, open(PATH_RESULTS + model_name_dir + 'total_history.cP', 'w'))

# np.save(PATH_RESULTS + model_name_dir + 'loss_testset.npy', loss_testset)

np.save(PATH_RESULTS + model_name_dir + 'predicted_and_truths_result.npy', [predicted, test_y])

np.save(PATH_RESULTS + model_name_dir + 'weights_changes.npy', np.array(weight_image_monitor.weights_changes))

# ADD weight change saving code

if total_history != {}:

# max_auc = np.max(total_history['auc'])

best_batch = np.argmax(total_history[criteria])+1

num_run_epoch = len(total_history[criteria])

oneline_result = '%6.4f, %s %d_of_%d, %s' % (best_result, criteria, best_batch, num_run_epoch, model_name)

with open(PATH_RESULTS + model_name_dir + oneline_result, 'w') as f:

pass

f = open( (PATH_RESULTS + '%s_%s_%s_%06.4f_at_(%d_of_%d)_%s' % \

(timename, hyperparams["loss_function"], criteria, best_result, best_batch, num_run_epoch, nickname)), 'w')

f.close()

with open('one_line_log.txt', 'a') as f:

f.write(oneline_result)

f.write(' ' + ' '.join(argv) + '\n')

else:

max_auc = 0.0

print '========== DONE: %s ==========' % model_name