def main(config, options, experiment_dir, **kwargs):
# config: Loaded .cfg file
# options: Options containing seed, restore_dir,
# experiment_dir: Experiment directory/Benchmark_directory
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
cmd = ""
SYSTEM_WIDE = 0
AUGUST_2013_MOD = 1
try:
import hyperopt
version = SYSTEM_WIDE
except ImportError:
cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \
"/optimizers/hyperopt_august2013_mod/hyperopt"
import optimizers.spearmint_april2013_mod.hyperopt as hyperopt
version = AUGUST_2013_MOD
path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__))
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory does not exist")
optimizer_dir = options.restore
else:
optimizer_dir = os.path.join(
experiment_dir, "tree_" + str(options.seed) + "_" + time_string)
# Build call
cmd += build_tree_call(config, options, optimizer_dir)
# Set up experiment directory
if not os.path.exists(optimizer_dir):
os.mkdir(optimizer_dir)
space = config.get('TREE', 'space')
# Copy the hyperopt search space
if not os.path.exists(os.path.join(optimizer_dir, space)):
os.symlink(os.path.join(experiment_dir, "tree", space),
os.path.join(optimizer_dir, space))
sys.stdout.write(
"### INFORMATION ################################################################\n"
)
sys.stdout.write("# You're running %40s #\n" %
path_to_optimizer)
if version == SYSTEM_WIDE:
pass
else:
sys.stdout.write(
"# To reproduce our results you need version 0.0.3.dev, which can be found here:#\n"
)
sys.stdout.write("%s" % version_info)
sys.stdout.write(
"# A newer version might be available, but not yet built in. #\n"
)
sys.stdout.write(
"################################################################################\n"
)
return cmd, optimizer_dir
def main(config, options, experiment_dir, **kwargs):
# config: Loaded .cfg file
# options: Options containing seed, restore_dir,
# experiment_dir: Experiment directory/Benchmark_directory
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory does not exist")
optimizer_dir = options.restore
else:
optimizer_dir = os.path.join(experiment_dir, "spearmint_" \
+ str(options.seed) + "_" + time_string)
# Build call
cmd = build_spearmint_call(config, options, optimizer_dir)
# Set up experiment directory
if not os.path.exists(optimizer_dir):
os.mkdir(optimizer_dir)
# Make a link to the Protocol-Buffer config file
configpb = config.get('SPEARMINT', 'config')
if not os.path.exists(os.path.join(optimizer_dir, configpb)):
os.symlink(os.path.join(experiment_dir, "spearmint", configpb),
os.path.join(optimizer_dir, configpb))
sys.stdout.write("### INFORMATION ################################################################\n")
sys.stdout.write("# You're running %40s #\n" % path_to_optimizer)
sys.stdout.write("%s" % version_info)
sys.stdout.write("# A newer version might be available, but not yet built in. #\n")
sys.stdout.write("# Please use this version only to reproduce our results on automl.org #\n")
sys.stdout.write("################################################################################\n")
return cmd, optimizer_dir
def main(config, options, experiment_dir, **kwargs):
# config: Loaded .cfg file
# options: Options containing seed, restore,
# experiment_dir: Experiment directory/Benchmarkdirectory
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory %s does not exist" %
(options.restore, ))
optimizer_dir = options.restore
else:
optimizer_dir = os.path.join(experiment_dir, "randomtpe_" + \
str(options.seed) + "_" + time_string)
# Build call
cmd = buildRandomCall(config, options, optimizer_dir)
# Set up experiment directory
if not os.path.exists(optimizer_dir):
os.mkdir(optimizer_dir)
space = config.get('TPE', 'space')
# Copy the hyperopt search space
if not os.path.exists(os.path.join(optimizer_dir, space)):
os.symlink(os.path.join(experiment_dir, "tpe", space),
os.path.join(optimizer_dir, space))
return cmd, optimizer_dir
def main(config, options, experiment_dir, **kwargs):
# config: Loaded .cfg file
# options: Options containing seed, restore,
# experiment_dir: Experiment directory/Benchmarkdirectory
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory %s does not exist" % (options.restore,))
optimizer_dir = options.restore
else:
optimizer_dir = os.path.join(experiment_dir, "randomtpe_" + \
str(options.seed) + "_" + time_string)
# Build call
cmd = buildRandomCall(config, options, optimizer_dir)
# Set up experiment directory
if not os.path.exists(optimizer_dir):
os.mkdir(optimizer_dir)
space = config.get('TPE', 'space')
# Copy the hyperopt search space
if not os.path.exists(os.path.join(optimizer_dir, space)):
os.symlink(os.path.join(experiment_dir, "tpe", space),
os.path.join(optimizer_dir, space))
return cmd, optimizer_dir
def main(config, options, experiment_dir, **kwargs):
# config: Loaded .cfg file
# options: Options containing seed, restore_dir,
# experiment_dir: Experiment directory/Benchmark_directory
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory does not exist")
optimizer_dir = options.restore
else:
optimizer_dir = os.path.join(
experiment_dir, "smac_" + str(options.seed) + "_" + time_string)
# Build call
cmd = build_smac_call(config, options, optimizer_dir)
# Set up experiment directory
if not os.path.exists(optimizer_dir):
os.mkdir(optimizer_dir)
# TODO: This can cause huge problems when the files are located
# somewhere else?
params = os.path.split(config.get('SMAC', "p"))[1]
# Copy the smac search space and create the instance information
fh = open(os.path.join(optimizer_dir, 'train.txt'), "w")
for i in range(config.getint('DEFAULT', 'numberCV')):
fh.write(str(i) + "\n")
fh.close()
fh = open(os.path.join(optimizer_dir, 'test.txt'), "w")
for i in range(config.getint('DEFAULT', 'numberCV')):
fh.write(str(i) + "\n")
fh.close()
fh = open(os.path.join(optimizer_dir, "scenario.txt"), "w")
fh.close()
if not os.path.exists(os.path.join(optimizer_dir, params)):
os.symlink(os.path.join(experiment_dir, "smac", params),
os.path.join(optimizer_dir, params))
sys.stdout.write(
"### INFORMATION ################################################################\n"
)
sys.stdout.write("# You're running %40s #\n" %
path_to_optimizer)
sys.stdout.write("%s" % version_info)
sys.stdout.write(
"# A newer version might be available, but not yet built in. #\n"
)
sys.stdout.write(
"# Please use this version only to reproduce our results on automl.org #\n"
)
sys.stdout.write(
"################################################################################\n"
)
return cmd, optimizer_dir
def main():
# Parse options and arguments
usage = "Coming soon"
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 1:
raise Exception("You must specify a directory.")
# Then load config.cfg and function
cfg = parse_config("config.cfg", allow_no_value=True)
module = cfg.get("DEFAULT", "function")
os.chdir(args[0])
# Load the experiment pickle and extract best configuration according to the
# validation loss
experiment = loadExperimentFile()
optimizer = experiment.optimizer
if optimizer == "smac":
tmp_results = experiment.instance_results
tmp_results = np.ma.masked_array(tmp_results, np.isnan(tmp_results))
results = np.mean(tmp_results, axis=1)
else:
tmp_results = experiment.results
tmp_results[np.isnan(tmp_results)] = cfg.get('DEFAULT', 'result_on_terminate')
results = tmp_results
best_config_idx = np.argmin(results)
print "Found best config #%d with validation loss %f" % (best_config_idx, results[best_config_idx])
params = experiment.params[best_config_idx]
del experiment #release lock
# Create a param pickle
time_string = wrapping_util.get_time_string()
params_filename = os.path.join(os.getcwd(), "params" + time_string)
params_fh = open(params_filename, 'w')
print "Pickling param dict", params_filename, params
cPickle.dump(params, params_fh)
params_fh.close()
#Call run_instance.py
fh = open(args[0][0:-1] + "_test_run.out", "w")
leading_runsolver_info = cfg.get('DEFAULT', 'leading_runsolver_info')
leading_algo_info = cfg.get('DEFAULT', 'leading_algo_info')
leading_algo_info = "optirun"
cmd = "%s %s python %s --fold 1 --config ../config.cfg --test %s" % (leading_runsolver_info, leading_algo_info, os.path.join(os.path.dirname(os.path.realpath(__file__)) + "/run_instance.py"), params_filename)
process = subprocess.Popen(cmd, stdout=fh, stderr=fh,
shell=True, executable="/bin/bash")
print
print cmd
print "-----------------------RUNNING TEST----------------------------"
ret = process.wait()
fh.close()
os.remove(params_filename)
print ret
def main(config, options, experiment_dir, **kwargs):
# config: Loaded .cfg file
# options: Options containing seed, restore_dir,
# experiment_dir: Experiment directory/Benchmark_directory
# **kwargs: Nothing so far
time_string = wrapping_util.get_time_string()
cmd = ""
SYSTEM_WIDE = 0
AUGUST_2013_MOD = 1
try:
import hyperopt
version = SYSTEM_WIDE
except ImportError:
cmd += "export PYTHONPATH=$PYTHONPATH:" + os.path.dirname(os.path.abspath(__file__)) + \
"/optimizers/hyperopt_august2013_mod/hyperopt"
import optimizers.spearmint_april2013_mod.hyperopt as hyperopt
version = AUGUST_2013_MOD
path_to_optimizer = os.path.abspath(os.path.dirname(hyperopt.__file__))
# Find experiment directory
if options.restore:
if not os.path.exists(options.restore):
raise Exception("The restore directory does not exist")
optimizer_dir = options.restore
else:
optimizer_dir = os.path.join(experiment_dir, "anneal_" +
str(options.seed) + "_" +
time_string)
# Build call
cmd += build_anneal_call(config, options, optimizer_dir)
# Set up experiment directory
if not os.path.exists(optimizer_dir):
os.mkdir(optimizer_dir)
space = config.get('ANNEAL', 'space')
# Copy the hyperopt search space
if not os.path.exists(os.path.join(optimizer_dir, space)):
os.symlink(os.path.join(experiment_dir, "anneal", space),
os.path.join(optimizer_dir, space))
sys.stdout.write("### INFORMATION ################################################################\n")
sys.stdout.write("# You're running %40s #\n" % path_to_optimizer)
if version == SYSTEM_WIDE:
pass
else:
sys.stdout.write("# To reproduce our results you need version 0.0.3.dev, which can be found here:#\n")
sys.stdout.write("%s" % version_info)
sys.stdout.write("# A newer version might be available, but not yet built in. #\n")
sys.stdout.write("################################################################################\n")
return cmd, optimizer_dir
def main():
# Parse options and arguments
usage = "This script pickles the params and runs the runsolver with " +\
"run_instance and extract the output for the optimizer \n" + \
"The output is printed im a SMACish way: \n\n" + \
"'Result for ParamILS: <solved>, <runtime>, <runlength>, " + \
"<quality>, <seed>, <additional rundata>' \n\n" + \
"Usage: runsolver_wrapper <instancename> " + \
"<instancespecificinformation> <cutofftime> <cutofflength> " + \
"<seed> <param> <param> <param>\n" + \
"<instancename> might be the optimizer name if not" + \
" called by smac\n"
if len(sys.argv) < 7:
sys.stdout.write(usage)
exit(1)
optimizer = os.getcwd().split("/")[-1].split("_")[0]
# This has to be done here for SMAC, since smac does not call cv.py
if optimizer == 'smac':
cv_starttime = time.time()
experiment = load_experiment_file()
experiment.start_cv(cv_starttime)
del experiment
# Again we need to find the config.cfg
cfg, cfg_filename = get_config()
# Ignore smac cutofftime
time_limit = cfg.getint('DEFAULT', 'runsolver_time_limit')
memory_limit = cfg.getint('DEFAULT', 'memory_limit')
cpu_limit = cfg.getint('DEFAULT', 'cpu_limit')
# Then get some information for run_instance
fold = int(sys.argv[1])
seed = int(sys.argv[5])
# Now build param dict
param_list = sys.argv[6:]
params = dict()
for idx, i in enumerate(param_list[0::2]):
params[param_list[idx*2][1:]] = (param_list[idx*2+1].strip("'"))
# TODO: remove overhead of pickling the param dict
time_string = wrapping_util.get_time_string()
params_filename = os.path.join(os.getcwd(), "params" + time_string)
params_fh = open(params_filename, 'w')
print 'Pickling param dict', params_filename, params
cPickle.dump(params, params_fh)
params_fh.close()
# Timestamp and use-pty are options are used so that output of the "solver"
# is flushed to the solver output file directly
run_instance_output = os.path.join(os.getcwd(), time_string + "_run_instance.out")
sys.stdout.write("Using optimizer: " + str(optimizer) + "\n")
python_cmd = cfg.get("DEFAULT", "leading_algo_info") + " python " + \
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"run_instance.py") + \
" --fold %d --config %s --seed %d --pkl %s" % \
(fold, cfg_filename, seed, optimizer + ".pkl")
# Do not write the actual task in quotes because runsolver will not work
# then
delay = 0
cmd = cfg.get("DEFAULT", "leading_runsolver_info") + \
" runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s %s" \
% (run_instance_output, time_limit, cpu_limit, memory_limit, delay,
python_cmd, params_filename)
runsolver_output_file = os.path.join(os.getcwd(),
time_string + "_runsolver.out")
fh = open(runsolver_output_file, "w")
process = subprocess.Popen(cmd, stdout=fh,
stderr=fh, shell=True, executable="/bin/bash")
print
print cmd
print "-----------------------RUNNING RUNSOLVER----------------------------"
process.wait()
fh.close()
limit_exceeded, error_time = read_runsolver_output(runsolver_output_file)
# <solved>, <runtime>, <runlength>, <quality>, <seed>, <additional rundata>
error_string = "Result for ParamILS: %s, %d, 0, %f, %d, %s"
result_array, result_string = read_run_instance_output(run_instance_output)
# Write the SMACish output to the command line and remove temporary files
# Additionaly, we also have to replace the NaN-value in the result file by
# the worst possible value
if result_array is not None and result_array[3].strip(",") not in ("SAT", "CRASHED"):
raise Exception("Unknown return status %s" % result_array[3].strip(","))
if (limit_exceeded is None and result_string is None) or\
(result_string is not None and "CRASHED" in result_string and not limit_exceeded):
replace_nan_in_last_trial(cfg.getfloat("DEFAULT", "result_on_terminate"))
return_string = error_string % ("SAT", error_time,
cfg.getfloat('DEFAULT', "result_on_terminate"),
seed,
"Please have a look at " + run_instance_output)
print return_string
os.remove(runsolver_output_file)
elif limit_exceeded is None:
result_float = float(result_array[6].strip(","))
if not np.isfinite(result_float):
replace_nan_in_last_trial()
experiment = load_experiment_file()
last_experiment = experiment.instance_order[-1]
assert last_experiment is not None
experiment.instance_results[last_experiment[0]][last_experiment[1]]\
= cfg.getfloat('DEFAULT', "result_on_terminate")
del experiment # This also saves the experiment
result_array[6] = (cfg.getfloat("DEFAULT", "result_on_terminate")) + ","
result_string = " ".join(result_array)
else:
# Remove the run_instance_output only if there is a valid result
os.remove(run_instance_output)
os.remove(runsolver_output_file)
return_string = result_string
print result_string
else:
return_string = error_string % ("SAT", error_time, cfg.getfloat
('DEFAULT', "result_on_terminate"), seed,
limit_exceeded)
print return_string
# It is useful to have the run_instance_output for debugging
#os.remove(run_instance_output)
os.remove(params_filename)
# Remove param pkl and runsolver files
#os.remove(run_instance_output)
#os.remove(runsolver_output_file)
#os.remove(params_filename)
if optimizer == 'smac':
experiment = load_experiment_file()
experiment.end_cv(time.time())
del experiment
return return_string
def main():
# Parse options and arguments
usage = "This script pickles the params and runs the runsolver with " +\
"run_instance and extract the output for the optimizer \n" + \
"The output is printed im a SMACish way: \n\n" + \
"'Result for ParamILS: <solved>, <runtime>, <runlength>, " + \
"<quality>, <seed>, <additional rundata>' \n\n" + \
"Usage: runsolver_wrapper <instancename> " + \
"<instancespecificinformation> <cutofftime> <cutofflength> " + \
"<seed> <param> <param> <param>\n" + \
"<instancename> might be the optimizer name if not" + \
" called by smac\n"
if len(sys.argv) < 7:
sys.stdout.write(usage)
exit(1)
optimizer = os.getcwd().split("/")[-1].split("_")[0]
# This has to be done here for SMAC, since smac does not call cv.py
if optimizer == 'smac':
cv_starttime = time.time()
experiment = load_experiment_file()
experiment.start_cv(cv_starttime)
del experiment
# Again we need to find the config.cfg
cfg, cfg_filename = get_config()
# Ignore smac cutofftime
time_limit = cfg.getint('DEFAULT', 'runsolver_time_limit')
memory_limit = cfg.getint('DEFAULT', 'memory_limit')
cpu_limit = cfg.getint('DEFAULT', 'cpu_limit')
# Then get some information for run_instance
fold = int(sys.argv[1])
seed = int(sys.argv[5])
# Now build param dict
param_list = sys.argv[6:]
params = dict()
for idx, i in enumerate(param_list[0::2]):
params[param_list[idx * 2][1:]] = (param_list[idx * 2 + 1].strip("'"))
# TODO: remove overhead of pickling the param dict
time_string = wrapping_util.get_time_string()
params_filename = os.path.join(os.getcwd(), "params" + time_string)
params_fh = open(params_filename, 'w')
print 'Pickling param dict', params_filename, params
cPickle.dump(params, params_fh)
params_fh.close()
# Timestamp and use-pty are options are used so that output of the "solver"
# is flushed to the solver output file directly
run_instance_output = os.path.join(os.getcwd(),
time_string + "_run_instance.out")
sys.stdout.write("Using optimizer: " + str(optimizer) + "\n")
python_cmd = cfg.get("DEFAULT", "leading_algo_info") + " python " + \
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"run_instance.py") + \
" --fold %d --config %s --seed %d --pkl %s" % \
(fold, cfg_filename, seed, optimizer + ".pkl")
# Do not write the actual task in quotes because runsolver will not work
# then
delay = 0
cmd = cfg.get("DEFAULT", "leading_runsolver_info") + \
" runsolver -o %s --timestamp --use-pty -W %d -C %d -M %d -d %d %s %s" \
% (run_instance_output, time_limit, cpu_limit, memory_limit, delay,
python_cmd, params_filename)
runsolver_output_file = os.path.join(os.getcwd(),
time_string + "_runsolver.out")
fh = open(runsolver_output_file, "w")
process = subprocess.Popen(cmd,
stdout=fh,
stderr=fh,
shell=True,
executable="/bin/bash")
print
print cmd
print "-----------------------RUNNING RUNSOLVER----------------------------"
process.wait()
fh.close()
limit_exceeded, error_time = read_runsolver_output(runsolver_output_file)
# <solved>, <runtime>, <runlength>, <quality>, <seed>, <additional rundata>
error_string = "Result for ParamILS: %s, %d, 0, %f, %d, %s"
result_array, result_string = read_run_instance_output(run_instance_output)
# Write the SMACish output to the command line and remove temporary files
# Additionaly, we also have to replace the NaN-value in the result file by
# the worst possible value
if result_array is not None and result_array[3].strip(",") not in (
"SAT", "CRASHED"):
raise Exception("Unknown return status %s" %
result_array[3].strip(","))
if (limit_exceeded is None and result_string is None) or\
(result_string is not None and "CRASHED" in result_string and not limit_exceeded):
replace_nan_in_last_trial(
cfg.getfloat("DEFAULT", "result_on_terminate"))
return_string = error_string % (
"SAT", error_time, cfg.getfloat('DEFAULT', "result_on_terminate"),
seed, "Please have a look at " + run_instance_output)
print return_string
os.remove(runsolver_output_file)
elif limit_exceeded is None:
result_float = float(result_array[6].strip(","))
if not np.isfinite(result_float):
replace_nan_in_last_trial()
experiment = load_experiment_file()
last_experiment = experiment.instance_order[-1]
assert last_experiment is not None
experiment.instance_results[last_experiment[0]][last_experiment[1]]\
= cfg.getfloat('DEFAULT', "result_on_terminate")
del experiment # This also saves the experiment
result_array[6] = (cfg.getfloat("DEFAULT",
"result_on_terminate")) + ","
result_string = " ".join(result_array)
else:
# Remove the run_instance_output only if there is a valid result
os.remove(run_instance_output)
os.remove(runsolver_output_file)
return_string = result_string
print result_string
else:
return_string = error_string % (
"SAT", error_time, cfg.getfloat(
'DEFAULT', "result_on_terminate"), seed, limit_exceeded)
print return_string
# It is useful to have the run_instance_output for debugging
#os.remove(run_instance_output)
os.remove(params_filename)
# Remove param pkl and runsolver files
#os.remove(run_instance_output)
#os.remove(runsolver_output_file)
#os.remove(params_filename)
if optimizer == 'smac':
experiment = load_experiment_file()
experiment.end_cv(time.time())
del experiment
return return_string