def main(basename, input_dir, params, time_limit=sys.maxint):
for key in params:
try:
params[key] = float(params[key])
except:
pass
output_dir = os.getcwd()
D = store_and_or_load_data(data_dir=input_dir,
dataset=basename,
outputdir=output_dir)
if D.info["task"].lower() == "regression":
cs = AutoSklearnRegressor.get_hyperparameter_search_space()
else:
cs = AutoSklearnClassifier.get_hyperparameter_search_space()
configuration = configuration_space.Configuration(cs, **params)
global evaluator
evaluator = models.evaluate.Evaluator(Datamanager=D,
configuration=configuration,
with_predictions=True,
all_scoring_functions=True)
evaluator.output_dir = output_dir
evaluator.basename = basename
evaluator.D = D
evaluator.fit()
evaluator.finish_up()
def main(basename, input_dir, params):
output_dir = os.getcwd()
D = store_and_or_load_data(data_dir=input_dir, dataset=basename,
outputdir=output_dir)
cs = get_class(D.info).get_hyperparameter_search_space()
configuration = configuration_space.Configuration(cs, **params)
global evaluator
evaluator = autosklearn.models.holdout_evaluator.HoldoutEvaluator(
Datamanager=D, configuration=configuration, with_predictions=True,
all_scoring_functions=True, output_dir=output_dir)
evaluator.fit()
evaluator.finish_up()
def main(dataset_info, mode, seed, params, mode_args=None):
"""This command line interface has three different operation modes:
* CV: useful for the Tweakathon
* 1/3 test split: useful to evaluate a configuration
* cv on 2/3 train split: useful to optimize hyperparameters in a training
mode before testing a configuration on the 1/3 test split.
It must by no means be used for the Auto part of the competition!
"""
if mode != "test":
num_run = get_new_run_num()
for key in params:
try:
params[key] = int(params[key])
except:
try:
params[key] = float(params[key])
except:
pass
if seed is not None:
seed = int(float(seed))
else:
seed = 1
output_dir = os.getcwd()
D = store_and_or_load_data(dataset_info=dataset_info, outputdir=output_dir)
cs = get_configuration_space(D.info)
configuration = configuration_space.Configuration(cs, params)
metric = D.info['metric']
global evaluator
# Train/test split
if mode == 'holdout':
evaluator = HoldoutEvaluator(D,
configuration,
with_predictions=True,
all_scoring_functions=True,
output_y_test=True,
seed=seed,
num_run=num_run)
evaluator.fit()
signal.signal(15, empty_signal_handler)
evaluator.finish_up()
model_directory = os.path.join(os.getcwd(), "models_%d" % seed)
if os.path.exists(model_directory):
model_filename = os.path.join(model_directory,
"%s.model" % num_run)
with open(model_filename, "w") as fh:
pickle.dump(evaluator.model, fh, -1)
elif mode == 'test':
evaluator = TestEvaluator(D,
configuration,
all_scoring_functions=True,
seed=seed)
evaluator.fit()
scores = evaluator.predict()
duration = time.time() - evaluator.starttime
score = scores[metric]
additional_run_info = ";".join(
["%s: %s" % (m_, value) for m_, value in scores.items()])
additional_run_info += ";" + "duration: " + str(duration)
print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % (
"SAT", abs(duration), score, evaluator.seed, additional_run_info)
# CV on the whole training set
elif mode == 'cv':
evaluator = CVEvaluator(D,
configuration,
with_predictions=True,
all_scoring_functions=True,
output_y_test=True,
cv_folds=mode_args['folds'],
seed=seed,
num_run=num_run)
evaluator.fit()
signal.signal(15, empty_signal_handler)
evaluator.finish_up()
elif mode == 'partial_cv':
evaluator = CVEvaluator(D,
configuration,
all_scoring_functions=True,
cv_folds=mode_args['folds'],
seed=seed,
num_run=num_run)
evaluator.partial_fit(mode_args['fold'])
scores = evaluator.predict()
duration = time.time() - evaluator.starttime
score = scores[metric]
additional_run_info = ";".join(
["%s: %s" % (m_, value) for m_, value in scores.items()])
additional_run_info += ";" + "duration: " + str(duration)
print "Result for ParamILS: %s, %f, 1, %f, %d, %s" % (
"SAT", abs(duration), score, evaluator.seed, additional_run_info)
elif mode == 'nested-cv':
evaluator = NestedCVEvaluator(D,
configuration,
with_predictions=True,
inner_cv_folds=mode_args['inner_folds'],
outer_cv_folds=mode_args['outer_folds'],
all_scoring_functions=True,
output_y_test=True,
seed=seed,
num_run=num_run)
evaluator.fit()
signal.signal(15, empty_signal_handler)
evaluator.finish_up()
else:
raise ValueError("Must choose a legal mode.")
def main(dataset_info, mode, seed, params, mode_args=None):
"""This command line interface has three different operation modes:
* CV: useful for the Tweakathon
* 1/3 test split: useful to evaluate a configuration
* cv on 2/3 train split: useful to optimize hyperparameters in a training
mode before testing a configuration on the 1/3 test split.
It must by no means be used for the Auto part of the competition!
"""
if mode_args is None:
mode_args = {}
output_dir = os.getcwd()
if not isinstance(dataset_info, AbstractDataManager):
D = store_and_or_load_data(dataset_info=dataset_info,
outputdir=output_dir)
else:
D = dataset_info
metric = D.info['metric']
num_run = None
if mode != 'test':
num_run = get_new_run_num()
if params is not None:
for key in params:
try:
params[key] = int(params[key])
except Exception:
try:
params[key] = float(params[key])
except Exception:
pass
cs = get_configuration_space(D.info)
configuration = configuration_space.Configuration(cs, params)
else:
configuration = None
if seed is not None:
seed = int(float(seed))
else:
seed = 1
global evaluator
if mode == 'holdout':
make_mode_holdout(D, seed, configuration, num_run)
elif mode == 'holdout-iterative-fit':
make_mode_holdout_iterative_fit(D, seed, configuration, num_run)
elif mode == 'test':
make_mode_test(D, seed, configuration, metric)
elif mode == 'cv':
make_mode_cv(D, seed, configuration, num_run, mode_args['folds'])
elif mode == 'partial-cv':
make_mode_partial_cv(D, seed, configuration, num_run, metric,
mode_args['fold'], mode_args['folds'])
elif mode == 'nested-cv':
make_mode_nested_cv(D, seed, configuration, num_run,
mode_args['inner_folds'], mode_args['outer_folds'])
else:
raise ValueError('Must choose a legal mode.')