print ' --- num_layers: ', TR_CONST["num_layers"] print ' --- task: %s' % args.task print ' --- usage ratio: %f' % usage_ratio # label matrix dim_latent_feature = TR_CONST["dim_labels"] # label_matrix_filename = (FILE_DICT["mood_latent_matrix"] % dim_latent_feature) label_matrix_filename = (FILE_DICT["mood_latent_tfidf_matrix"] % dim_latent_feature) # tfidf is better! if os.path.exists(PATH_DATA + label_matrix_filename): label_matrix = np.load(PATH_DATA + label_matrix_filename) #np matrix, 9320-by-100 else: "print let's cook the mood-latent feature matrix" import main_prepare mood_tags_matrix = np.load(PATH_DATA + label_matrix_filename) #np matrix, 9320-by-100 label_matrix = main_prepare.get_LDA(X=mood_tags_matrix, num_components=k, show_topics=False) np.save(PATH_DATA + label_matrix_filename, W) print 'size of mood tag matrix:' print label_matrix.shape # load dataset for tf_type in tf_types: TR_CONST["tf_type"] = tf_type print '='*60 print 'tf type: %s' % tf_type print '='*60 print "I'll take %d clips for each song." % TR_CONST["clips_per_song"] # train_x, train_y, valid_x, valid_y, test_x, test_y = my_utils.load_all_sets(label_matrix=label_matrix, # clips_per_song=TR_CONST["clips_per_song"], # num_train_songs=TR_CONST["num_songs"], # tf_type=TR_CONST["tf_type"])
def run_with_setting(hyperparams, argv):
print '#'*60
#function: input args: TR_CONST, sys.argv.
# -------------------------------
if os.path.exists('stop_asap.keunwoo'):
os.remove('stop_asap.keunwoo')
if hyperparams["is_test"]:
print '==== This is a test, to quickly check the code. ===='
print 'excuted by $ ' + ' '.join(argv)
mse_history = []
# label matrix
dim_latent_feature = hyperparams["dim_labels"]
# label_matrix_filename = (FILE_DICT["mood_latent_matrix"] % dim_latent_feature)
label_matrix_filename = (FILE_DICT["mood_latent_tfidf_matrix"] % dim_latent_feature) # tfidf is better!
if os.path.exists(PATH_DATA + label_matrix_filename):
label_matrix = np.load(PATH_DATA + label_matrix_filename) #np matrix, 9320-by-100
else:
"print let's create a new mood-latent feature matrix"
import main_prepare
mood_tags_matrix = np.load(PATH_DATA + label_matrix_filename) #np matrix, 9320-by-100
label_matrix = main_prepare.get_LDA(X=mood_tags_matrix,
num_components=k,
show_topics=False)
np.save(PATH_DATA + label_matrix_filename, W)
# print 'size of mood tag matrix:'
print label_matrix.shape
# load dataset
train_x, valid_x, test_x, = my_utils.load_all_sets_from_hdf(tf_type=hyperparams["tf_type"],
n_dim=dim_latent_feature,
task_cla=hyperparams['isClass'])
# *_y is not correct - 01 Jan 2016. Use numpy files directly.
train_y, valid_y, test_y = my_utils.load_all_labels(n_dim=dim_latent_feature,
num_fold=10,
clips_per_song=3)
if hyperparams["is_test"]:
train_x, valid_x, test_x, train_y, valid_y, test_y = [ele[:64] for ele in [train_x, valid_x, test_x, train_y, valid_y, test_y]]
threshold_label = 1.0
if hyperparams['isClass']:
train_y = (train_y>=threshold_label).astype(int)
valid_y = (valid_y>=threshold_label).astype(int)
test_y = (test_y>=threshold_label).astype(int)
# print 'temporary came back with numpy loading'
# if hyperparams["debug"]:
# num_train_songs = 30
# else:
# num_train_songs = 1000
# train_x, train_y, valid_x, valid_y, test_x, test_y = my_utils.load_all_sets(label_matrix,
# hyperparams=hyperparams)
hyperparams["height_image"] = train_x.shape[2]
hyperparams["width_image"] = train_x.shape[3]
if hyperparams["debug"]:
pdb.set_trace()
moodnames = cP.load(open(PATH_DATA + FILE_DICT["moodnames"], 'r')) #list, 100
# train_x : (num_samples, num_channel, height, width)
hp_manager = hyperparams_manager.Hyperparams_Manager()
nickname = hp_manager.get_name(hyperparams)
timename = time.strftime('%m-%d-%Hh%M')
if hyperparams["is_test"]:
model_name = 'test_' + nickname
else:
model_name = timename + '_' + nickname
hp_manager.save_new_setting(hyperparams)
print '-'*60
print 'model name: %s' % model_name
model_name_dir = model_name + '/'
model_weight_name_dir = 'w_' + model_name + '/'
fileout = model_name + '_results'
model = my_keras_models.build_convnet_model(setting_dict=hyperparams)
model.summary()
if not os.path.exists(PATH_RESULTS + model_name_dir):
os.mkdir(PATH_RESULTS + model_name_dir)
os.mkdir(PATH_RESULTS + model_name_dir + 'images/')
os.mkdir(PATH_RESULTS + model_name_dir + 'plots/')
os.mkdir(PATH_RESULTS_W + model_weight_name_dir)
hp_manager.write_setting_as_texts(PATH_RESULTS + model_name_dir, hyperparams)
hp_manager.print_setting(hyperparams)
keras_plot(model, to_file=PATH_RESULTS + model_name_dir + 'images/'+'graph_of_model_'+hyperparams["!memo"]+'.png')
#prepare callbacks
weight_image_monitor = my_keras_utils.Weight_Image_Saver(PATH_RESULTS + model_name_dir + 'images/')
patience = 3
if hyperparams["is_test"] is True:
patience = 99999999
if hyperparams["isRegre"]:
value_to_monitor = 'val_loss'
else:
value_to_monitor = 'val_acc'
#history = my_keras_utils.History_Regression_Val()
# early_stopping = keras.callbacks.EarlyStopping(monitor=value_to_monitor,
# patience=patience,
# verbose=0)
# other constants
batch_size = 16
# if hyperparams['model_type'] == 'vgg_original':
# batch_size = (batch_size * 3)/5
predicted = model.predict(test_x, batch_size=batch_size)
if hyperparams['debug'] == True:
pdb.set_trace()
print 'mean of target value:'
if hyperparams['isRegre']:
print np.mean(test_y, axis=0)
else:
print np.sum(test_y, axis=0)
print 'mean of predicted value:'
if hyperparams['isRegre']:
print np.mean(predicted, axis=0)
else:
print np.sum(predicted, axis=0)
print 'mse with just predicting average is %f' % np.mean((test_y - np.mean(test_y, axis=0))**2)
np.save(PATH_RESULTS + model_name_dir + 'predicted_and_truths_init.npy', [predicted[:len(test_y)], test_y[:len(test_y)]])
#train!
print '--- train starts. Remove will_stop.keunwoo to continue learning after %d epochs ---' % hyperparams["num_epoch"]
f = open('will_stop.keunwoo', 'w')
f.close()
total_history = {}
num_epoch = hyperparams["num_epoch"]
total_epoch = 0
callbacks = [weight_image_monitor]
best_mse = 0.5
while True:
# [run]
if os.path.exists('stop_asap.keunwoo'):
print ' stop by stop_asap.keunwoo file'
break
history = model.fit(train_x, train_y, validation_data=(valid_x, valid_y),
batch_size=batch_size,
nb_epoch=1,
show_accuracy=hyperparams['isClass'],
verbose=1,
callbacks=callbacks,
shuffle='batch')
my_utils.append_history(total_history, history.history)
# [validation]
val_result = evaluate_result(valid_y, predicted) # mse
if val_result['mse'] < best_mse:
model.save_weights(PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5", overwrite=True)
best_mse = val_result['mse']
mse_history.append(val_result['mse'])
print '%d-th of %d epoch is complete' % (total_epoch, num_epoch)
total_epoch += 1
if os.path.exists('will_stop.keunwoo'):
loss_testset = model.evaluate(test_x, test_y, show_accuracy=False, batch_size=batch_size)
else:
print ' *** will go for another one epoch. '
print ' *** $ touch will_stop.keunwoo to stop at the end of this, otherwise it will be endless.'
#
best_batch = np.argmin(mse_history)+1
model.load_weights(PATH_RESULTS_W + model_weight_name_dir + "weights_best.hdf5")
predicted = model.predict(test_x, batch_size=batch_size)
print 'predicted example using best model'
print predicted[:10]
print 'and truths'
print test_y[:10]
#save results
np.save(PATH_RESULTS + model_name_dir + fileout + '_history.npy', [total_history['loss'], total_history['val_loss']])
np.save(PATH_RESULTS + model_name_dir + fileout + '_loss_testset.npy', loss_testset)
np.save(PATH_RESULTS + model_name_dir + 'predicted_and_truths_result.npy', [predicted, test_y])
# np.save(PATH_RESULTS + model_name_dir + 'weights_changes.npy', np.array(weight_image_monitor.weights_changes))
# ADD weight change saving code
my_plots.export_history(total_history['loss'], total_history['val_loss'],
acc=None,
val_acc=None,
out_filename=PATH_RESULTS + model_name_dir + 'plots/' + 'plots.png')
min_loss = np.min(total_history[value_to_monitor])
best_batch = np.argmin(total_history[value_to_monitor])+1
num_run_epoch = len(total_history[value_to_monitor])
oneline_result = '%s, %6.4f, %d_of_%d, %s' % (value_to_monitor, min_loss, best_batch, num_run_epoch, model_name)
with open(PATH_RESULTS + model_name_dir + oneline_result, 'w') as f:
pass
f = open( (PATH_RESULTS + '%s_%s_%s_%06.4f_at_(%d_of_%d)_mse_%06.4f_%s' % \
(timename, hyperparams["loss_function"], value_to_monitor, min_loss, best_batch, num_run_epoch, best_mse, nickname)), 'w')
f.close()
with open('one_line_log.txt', 'a') as f:
f.write(oneline_result)
f.write(' ' + ' '.join(argv) + '\n')
print '========== DONE: %s ==========' % model_name
return min_loss
