def compare_experiment_yaml_strings(experiments_file_name, dir_name,
file_name):
assert (not ((dir_name is None) and (file_name is None)))
old_train_strs = extract_old_train_strings(dir_name, file_name)
# from http://stackoverflow.com/a/845432/1469195
new_train_strs = create_experiment_yaml_strings_from_files(
experiments_file_name,
main_template_filename='configs/eegnet_template.yaml')
for i_str, (old_str, new_str) in enumerate(
zip(old_train_strs, new_train_strs)):
new_lines = new_str.splitlines()
old_lines = old_str.splitlines()
# ignore inputlayer line
new_lines = [l for l in new_lines if (not "InputLayer" in l)]
old_lines = [l for l in old_lines if (not "InputLayer" in l)]
diff_str = '\n'.join(difflib.unified_diff(old_lines, new_lines))
if diff_str != '':
print("Experiment {:d}".format(i_str + 1))
print(diff_str)
print('\n')
else:
print("Experiment {:2d} ok.".format(i_str + 1))
if (len(old_train_strs) != len(new_train_strs)):
print("Could not compare all experiments.")
print("Old experiments: {:d}".format(len(old_train_strs)))
print("New experiments: {:d}".format(len(new_train_strs)))
def store_experiment_yaml_strings(experiments_file_name):
train_strs = create_experiment_yaml_strings_from_files(
experiments_file_name,
main_template_filename='configs/eegnet_template.yaml')
train_str_file_name = experiments_file_name + ".old_train_strs"
with open(train_str_file_name, 'w') as tmp_file:
tmp_file.write(_train_str_separator.join(train_strs))
def test_extended_filenames():
config_str = """
{
templates: {
a_layer: {
weight_init: $weight_init,
},
glorotuniform: 'funny glorot uniform template $range',
},
variants: [[
{
weight_init: [$glorotuniform],
range: [2],
layer:[$a_layer],
},
]]
}
"""
config_str_sub = """
{
extends: ['extendsfilename'],
variants: [[{
range: [8,10],
}]]
}
"""
main_template_str = """layer: $layer"""
with NamedTemporaryFile() as (config_top_file), NamedTemporaryFile() as (
config_bottom_file), NamedTemporaryFile() as main_template_file:
config_str_sub = config_str_sub.replace('extendsfilename',
config_top_file.name)
config_top_file.write(config_str)
config_bottom_file.write(config_str_sub)
main_template_file.write(main_template_str)
config_top_file.flush()
config_bottom_file.flush()
main_template_file.flush()
train_strings = create_experiment_yaml_strings_from_files(
config_bottom_file.name, main_template_file.name)
assert train_strings == [
"layer: {weight_init: 'funny glorot uniform template 8'\n}\n",
"layer: {weight_init: 'funny glorot uniform template 10'\n}\n"
]
def test_extended_filenames():
config_str = """
{
templates: {
a_layer: {
weight_init: $weight_init,
},
glorotuniform: 'funny glorot uniform template $range',
},
variants: [[
{
weight_init: [$glorotuniform],
range: [2],
layer:[$a_layer],
},
]]
}
"""
config_str_sub = """
{
extends: ['extendsfilename'],
variants: [[{
range: [8,10],
}]]
}
"""
main_template_str = """layer: $layer"""
with NamedTemporaryFile() as (config_top_file
), NamedTemporaryFile() as (config_bottom_file
), NamedTemporaryFile() as main_template_file:
config_str_sub = config_str_sub.replace('extendsfilename',
config_top_file.name)
config_top_file.write(config_str)
config_bottom_file.write(config_str_sub)
main_template_file.write(main_template_str)
config_top_file.flush()
config_bottom_file.flush()
main_template_file.flush()
train_strings = create_experiment_yaml_strings_from_files(
config_bottom_file.name, main_template_file.name)
assert train_strings == [
"layer: {weight_init: 'funny glorot uniform template 8'\n}\n",
"layer: {weight_init: 'funny glorot uniform template 10'\n}\n"]
if args.startid is not None:
# model ids printed are 1-based, python is zerobased
# stop id can remain same as stop implies exlusive index
# and 1-based inclusive == 0-based exclusive
args.startid = args.startid - 1
return args
if __name__ == "__main__":
setup_logging()
args = parse_command_line_arguments()
all_train_strs = create_experiment_yaml_strings_from_files(
args.experiments_file_name,
args.template_file_name,
args.debug,
command_line_params=args.params,
only_first_n_sets=args.firstsets,
filter_params=args.filters,
)
exp_runner = ExperimentsRunner(
quiet=args.quiet,
start_id=args.startid,
stop_id=args.stopid,
cross_validation=args.cv,
shuffle=args.shuffle,
debug=args.debug,
dry_run=args.dryrun,
only_first_n_sets=args.firstsets,
batch_test=args.batchtest,
skip_existing=args.skipexisting,
pred_loss_hack=not args.nopredlosshack,
if (args.startid is not None):
# model ids printed are 1-based, python is zerobased
# stop id can remain same as stop implies exlusive index
# and 1-based inclusive == 0-based exclusive
args.startid = args.startid - 1
return args
if __name__ == "__main__":
setup_logging()
args = parse_command_line_arguments()
all_train_strs = create_experiment_yaml_strings_from_files(
args.experiments_file_name,
args.template_file_name,
args.debug,
command_line_params=args.params,
only_first_n_sets=args.firstsets,
filter_params=args.filters)
exp_runner = ExperimentsRunner(quiet=args.quiet,
start_id=args.startid,
stop_id=args.stopid,
cross_validation=args.cv,
shuffle=args.shuffle,
debug=args.debug,
dry_run=args.dryrun,
only_first_n_sets=args.firstsets,
batch_test=args.batchtest,
skip_existing=args.skipexisting,
pred_loss_hack=not args.nopredlosshack)
exp_runner.run(all_train_strs)
