def test_read_parameters_from_command_line(self):
# Legal call
sys.argv = ["test.py", "--folds", "10", "--fold", "0", "--params", "-x",
"3"]
args, params = benchmark_util.parse_cli()
self.assertEqual(params, {'x': '3'})
self.assertEqual(args, {'folds': '10', 'fold': '0'})
# illegal call, arguments with one minus before --params
sys.argv = ["test.py", "-folds", "10", "--fold", "0", "--params", "-x",
"3"]
with self.assertRaises(ValueError) as cm1:
benchmark_util.parse_cli()
self.assertEqual(cm1.exception.message, "You either try to use arguments"
" with only one leading minus or try to specify a "
"hyperparameter before the --params argument. test.py"
" -folds 10 --fold 0 --params -x 3")
# illegal call, trying to specify an arguments after --params
sys.argv = ["test.py", "--folds", "10", "--params", "-x",
"'3'", "--fold", "0"]
with self.assertRaises(ValueError) as cm5:
benchmark_util.parse_cli()
self.assertEqual(cm5.exception.message, "You are trying to specify an argument after the "
"--params argument. Please change the order.")
# illegal call, no - in front of parameter name
sys.argv = ["test_cv.py", "--params", "x", "'5'"]
with self.assertRaises(ValueError) as cm2:
benchmark_util.parse_cli()
self.assertEqual(cm2.exception.message, "Illegal command line string, expected a hyperpara"
"meter starting with - but found x")
def main():
starttime = time.time()
args, params = benchmark_util.parse_cli()
assert args["folds"] == "1", args
params_hash = DictHash(params)
sys.stderr.write("Run hash: {}\n".format(params_hash))
with open(params_hash + ".out", "w") as stdout_file:
with open(params_hash + ".err", "w") as stderr_file:
exit_code = subprocess.call(["../tpe3.py", json.dumps(params), params_hash],
stdout=stdout_file, stderr=stderr_file)
if exit_code != 0:
raise subprocess.CalledProcessError(exit_code)
output = open(params_hash + ".out").readlines()[-1]
result = float(output.strip().split()[-1])
duration = time.time() - starttime
print "Result for ParamILS: SAT, {}, 1, {}, -1, {}".format(
abs(duration), result, str(__file__))
def test_read_parameters_from_command_line(self):
# Legal call
sys.argv = [
"test.py", "--folds", "10", "--fold", "0", "--params", "-x", "3"
]
args, params = benchmark_util.parse_cli()
self.assertEqual(params, {'x': '3'})
self.assertEqual(args, {'folds': '10', 'fold': '0'})
# illegal call, arguments with one minus before --params
sys.argv = [
"test.py", "-folds", "10", "--fold", "0", "--params", "-x", "3"
]
with self.assertRaises(ValueError) as cm1:
benchmark_util.parse_cli()
self.assertEqual(
cm1.exception.message, "You either try to use arguments"
" with only one leading minus or try to specify a "
"hyperparameter before the --params argument. test.py"
" -folds 10 --fold 0 --params -x 3")
# illegal call, trying to specify an arguments after --params
sys.argv = [
"test.py", "--folds", "10", "--params", "-x", "'3'", "--fold", "0"
]
with self.assertRaises(ValueError) as cm5:
benchmark_util.parse_cli()
self.assertEqual(
cm5.exception.message,
"You are trying to specify an argument after the "
"--params argument. Please change the order.")
# illegal call, no - in front of parameter name
sys.argv = ["test_cv.py", "--params", "x", "'5'"]
with self.assertRaises(ValueError) as cm2:
benchmark_util.parse_cli()
self.assertEqual(
cm2.exception.message,
"Illegal command line string, expected a hyperpara"
"meter starting with - but found x")
"""For a given problem train the metric function and return its loss
value. Arguments:
* task_files_list
* experiment_files_list
* metalearning_directory
Sample call: python kNDEvaluateSurrogate.py --task_files_list
/mhome/feurerm/thesis/experiments/AutoSklearn/metalearning_experiments/2014_09_10_test/tasks.txt
--experiments_list /mhome/feurerm/thesis/experiments/AutoSklearn
/metalearning_experiments/2014_09_10_test/experiments.txt
--metalearning_directory /mhome/feurerm/thesis/experiments/AutoSklearn/ --params -random_state 5
"""
starttime = time.time()
args, params = benchmark_util.parse_cli()
os.chdir(args['metalearning_directory'])
pyMetaLearn.directory_manager.set_local_directory(
args['metalearning_directory'])
with open(args["task_files_list"]) as fh:
task_files_list = fh.readlines()
with open(args["experiments_list"]) as fh:
experiments_list = fh.readlines()
if 'keep_configurations' in args:
keep_configurations = args['keep_configurations']
keep_configurations = keep_configurations.split(',')
keep_configurations = tuple(
[tuple(kc.split('=')) for kc in keep_configurations])
else:
TerminationCriterionMaxEpoch(200),
TerminationCriterionDivergenceDetection(),
TerminationCriterionExternalInBackground(
external_cmd="python -m pylrpredictor.terminationcriterion --nthreads 3",
run_every_x_epochs=30)],
test_every_x_epoch=0.5,
num_valid=5000,
snapshot_prefix="caffnet-run%d" % get_run_num(),
batch_size_valid=100,
snapshot_on_exit=1,
device="GPU",
device_id=0)
return caffe
def main(params, **kwargs):
experiment_dir = os.path.dirname(os.path.realpath(__file__))
working_dir = os.getcwd()
return hpolib_experiment_main(params, construct_caffeconvnet,
experiment_dir=experiment_dir,
working_dir=working_dir,
mean_performance_on_last=MEAN_PERFORMANCE_ON_LAST)
if __name__ == "__main__":
starttime = time.time()
args, params = benchmark_util.parse_cli()
result = main(params, **args)
duration = time.time() - starttime
print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % \
("SAT", abs(duration), result, -1, str(__file__))
