model_checkpoint = ModelCheckpoint(model_name, verbose=1, monitor='val_loss',save_best_only=True, mode='auto')
early_stopping = EarlyStopping(monitor="val_loss", verbose=1, patience=8)
log_file_name = os.path.join(config['plotting']['MEASURE_FOLDER'], 'evals-{}.json'.format(curr_time))
csv_logger = CSVLogger(filename=log_file_name, append = True)
# generate a model by training
history = model.fit_generator(train_generator,
epochs=num_epochs,
steps_per_epoch=36690//batch_size,
validation_data= val_generator,
validation_steps=12227//batch_size,
verbose=1,
callbacks=[model_checkpoint, early_stopping, csv_logger])
with open('{}-config_{}.ini'.format(config[parser_args.task]['MODEL_FOLDER'], curr_time), 'w') as cfgfile:
newConfig = configparser.ConfigParser()
newConfig[parser_args.task] = config[parser_args.task]
#print(config[parser_args.task])
newConfig.write(cfgfile)
print("Model is saved at: {}".format(model_name))
# model history generate plot
visualize.visualize_curves(n_epochs=num_epochs, tr=history.history['loss'], val=history.history['val_loss'], filename='{}/loss_{}.png'.format(plot_path, curr_time), network='temporal', optimizer='adam', learning_rate=learning_rate, epsilon=epsilon, clf_type=parser_args.task, batch_size=batch_size, viz_type="loss", early_stopping=True)
visualize.visualize_curves(n_epochs=num_epochs, tr=history.history['acc'], val=history.history['val_acc'], filename='{}/acc_{}.png'.format(plot_path, curr_time), network='temporal', optimizer='adam', learning_rate=learning_rate, epsilon=epsilon, clf_type=parser_args.task, batch_size=batch_size, viz_type="accuracy", early_stopping=True)
# write grid to disk during current iteration to ensure that temporary results are stored
grid_df.to_csv(grid_file, index=False)
def directory_curves_vis(curve_dirs,
line_dirs,
output='./',
fig_size=(9, 8),
dataset_names=DATASET_LIST,
test_filename=TEST_LL_FILE,
curve_filename=CURVE_FILE):
"""
WRITEME
ex:
directory_curves_vis(['exp/mix-learnspn-f/',
'exp/mix-learnspn-f-clt'],
['../SLSPN_Release/exp/spn/',
'../libra-tk-1.0.1/exp/best-idspn/',
'../libra-tk-1.0.1/exp/mt/'],
output='exp/curves2/')
"""
print('\n\n')
print('**********************************************************')
print('Drawing curves for different experiments')
print('**********************************************************\n')
curve_contents = []
for curve_dir in curve_dirs:
curve_contents.append([elems[0] for elems in os.walk(curve_dir)])
line_contents = []
for line_dir in line_dirs:
line_contents.append([elems[0] for elems in os.walk(line_dir)])
x_axis = numpy.arange(1, N_COMPONENTS + 1)
#
# precomputing folder contents
#
# better to loop through experiments first
for dataset_name in dataset_names:
#
# cheking if it is present for all of them
#
# curves first
curve_exps = []
not_found = False
for i, curve_dir in enumerate(curve_dirs):
#
# composing path
curves_log_path = None
curve_dataset_exps = matching_dirs(curve_contents[i], dataset_name)
if len(curve_dataset_exps) > 1:
raise ValueError('More than one folder for dataset',
dataset_name, curve_dir, curve_dataset_exps)
if len(curve_dataset_exps) < 1:
not_found = True
print('Skipping dataset', dataset_name, 'Exp not present in',
curve_dir)
break
else:
curves_log_path = curve_dataset_exps[0] + '/' + curve_filename
if os.path.exists(curves_log_path):
#
# loading it
curves_array = numpy.loadtxt(curves_log_path,
delimiter=',')
curve_exps.append((x_axis, curves_array[CURVE_INDEX]))
else:
not_found = True
print('Curves file not found for dataset', dataset_name,
'and experiment', curve_dir)
#
# lol
if not not_found:
line_exps = []
not_found = False
for i, line_dir in enumerate(line_dirs):
test_lls_file_path = None
line_dataset_exps = matching_dirs(line_contents[i],
dataset_name)
if len(line_dataset_exps) > 1:
raise ValueError('More than one folder for dataset',
dataset_name, line_dir, line_dataset_exps)
if len(line_dataset_exps) < 1:
not_found = True
print('Skipping dataset', dataset_name,
'Exp not present in', line_dir)
break
else:
test_lls_file_path = line_dataset_exps[0] + \
'/' + test_filename
if os.path.exists(test_lls_file_path):
test_lls = numpy.loadtxt(test_lls_file_path)
test_ll = test_lls.mean()
print('Test LL', test_ll, 'for exp', curve_dir,
'for dataset', dataset_name)
line_exps.append(test_ll)
else:
not_found = True
print('Test.lls file not found for dataset',
dataset_name, 'and experiment', line_dir)
if not not_found:
#
# if we have all the values we can plot them
output_path = output + dataset_name + '.png'
print('Printint to', output_path)
visualize.visualize_curves(
curve_exps,
output=output_path,
labels=['SPN-BB', 'SPN-BTB', 'LearnSPN', 'ID-SPN', 'MT'],
lines=line_exps,
loc=7,
fig_size=fig_size)
(n_edges, n_levels, n_weights, n_leaves)),
fmt='%d')
#
# now visualizing
visualize.visualize_curves(
[(x_axis, train_ll_m_history),
(x_axis, valid_ll_m_history), (x_axis, test_ll_m_history),
(x_axis, -train_ll_sgd_m_history),
(x_axis, -valid_ll_sgd_m_history),
(x_axis, -test_ll_sgd_m_history)
# ,
# (x_axis, train_ll_wm_history),
# (x_axis, valid_ll_wm_history),
# (x_axis, test_ll_wm_history)
],
labels=[
'train', 'valid', 'test', 'train-sgd', 'valid-sgd',
'test-sgd'
# , 'train-wm',
# 'valid-wm', 'test-wm'
],
linestyles=[
'-', '-', '-', ':', ':', ':'
# ,
# '-', '-', '-'
],
output=out_png_path)
#
# visualizing the number of components
visualize.visualize_curves([(x_axis, n_components_history)],
with open(
'{}-config_{}.ini'.format(
config[parser_args.task]['MODEL_FOLDER'], curr_time),
'w') as cfgfile:
newConfig = configparser.ConfigParser()
newConfig[parser_args.task] = config[parser_args.task]
#print(config[parser_args.task])
newConfig.write(cfgfile)
#print(f"Hostname: {socket.gethostname()}, Number of classes = {num_classes}\n Learning rate = {learning_rate}\n dropout rate = {dropout_rate}\n timestamp = {curr_time}")
print("Model is saved at: {}".format(model_name))
# model history generate plot
visualize.visualize_curves(
n_epochs=num_epochs,
tr=history.history['loss'],
val=history.history['val_loss'],
filename='{}/loss_{}.png'.format(plot_path, curr_time),
network=str(config[parser_args.task]['NETWORK']),
optimizer='adam',
learning_rate=learning_rate,
epsilon=epsilon,
clf_type=parser_args.task,
batch_size=batch_size,
viz_type="loss",
early_stopping=True)
# write grid to disk during current iteration to ensure that temporary results are stored
grid_df.to_csv(grid_file, index=False)
def directory_curves_vis(curve_dirs,
line_dirs,
output='./',
dataset_names=DATASET_LIST,
test_filename=TEST_LL_FILE,
curve_filename=CURVE_FILE):
"""
WRITEME
"""
print('\n\n')
print('**********************************************************')
print('Drawing curves for different experiments')
print('**********************************************************\n')
curve_contents = []
for curve_dir in curve_dirs:
curve_contents.append([elems[0] for elems in os.walk(curve_dir)])
line_contents = []
for line_dir in line_dirs:
line_contents.append([elems[0] for elems in os.walk(line_dir)])
x_axis = numpy.arange(1, N_COMPONENTS + 1)
#
# precomputing folder contents
#
# better to loop through experiments first
for dataset_name in dataset_names:
#
# cheking if it is present for all of them
#
# curves first
curve_exps = []
not_found = False
for i, curve_dir in enumerate(curve_dirs):
#
# composing path
curves_log_path = None
curve_dataset_exps = matching_dirs(curve_contents[i], dataset_name)
if len(curve_dataset_exps) > 1:
raise ValueError('More than one folder for dataset',
dataset_name, curve_dir,
curve_dataset_exps)
if len(curve_dataset_exps) < 1:
not_found = True
print('Skipping dataset', dataset_name,
'Exp not present in', curve_dir)
break
else:
curves_log_path = curve_dataset_exps[0] + '/' + curve_filename
if os.path.exists(curves_log_path):
#
# loading it
curves_array = numpy.loadtxt(
curves_log_path, delimiter=',')
curve_exps.append((x_axis, curves_array[CURVE_INDEX]))
else:
not_found = True
print('Curves file not found for dataset', dataset_name,
'and experiment', curve_dir)
#
# lol
if not not_found:
line_exps = []
not_found = False
for i, line_dir in enumerate(line_dirs):
test_lls_file_path = None
line_dataset_exps = matching_dirs(
line_contents[i], dataset_name)
if len(line_dataset_exps) > 1:
raise ValueError('More than one folder for dataset',
dataset_name, line_dir,
line_dataset_exps)
if len(line_dataset_exps) < 1:
not_found = True
print('Skipping dataset', dataset_name,
'Exp not present in', line_dir)
break
else:
test_lls_file_path = line_dataset_exps[0] + \
'/' + test_filename
if os.path.exists(test_lls_file_path):
test_lls = numpy.loadtxt(test_lls_file_path)
test_ll = test_lls.mean()
print('Test LL', test_ll,
'for exp', curve_dir,
'for dataset', dataset_name)
line_exps.append(test_ll)
else:
not_found = True
print('Test.lls file not found for dataset',
dataset_name,
'and experiment', line_dir)
if not not_found:
#
# if we have all the values we can plot them
output_path = output + dataset_name + '.png'
print('Printint to', output_path)
visualize.visualize_curves(curve_exps,
output=output_path,
labels=['SPN-BB',
'SPN-BTB',
'LearnSPN',
'ID-SPN',
'MT'],
lines=line_exps,
loc=7)