def test_framework_definition_raises_error_if_no_matching_framework():
res = ns(config=ns(frameworks=ns(definition_file="none")),
_frameworks={default_tag: ns(present=ns(name="present"))})
# binding `framework_definition` method to our resource mock: use pytest-mock instead?
res.framework_definition = Resources.framework_definition.__get__(res)
assert res.framework_definition("present")
with pytest.raises(ValueError, match=r"Incorrect framework `missing`"):
res.framework_definition("missing")
def oml_config():
return from_config(
ns(input_dir="my_input",
output_dir="my_output",
user_dir="my_user_dir",
root_dir="my_root_dir",
openml=ns(apikey="c1994bdb7ecb3c6f3c8f3b35f4b47f1f",
infer_dtypes=False))).config
def test_framework_definition_raises_error_if_the_framework_is_abstract():
res = ns(config=ns(frameworks=ns(definition_file="none")),
_frameworks={
default_tag: ns(present=ns(name="present", abstract=True))
})
# binding `framework_definition` method to our resource mock: use pytest-mock instead?
res.framework_definition = Resources.framework_definition.__get__(res)
with pytest.raises(
ValueError,
match=
r"Framework definition `present` is abstract and cannot be run directly"
):
res.framework_definition("present")
def test_framework_definition_lookup_is_case_insensitive(
frameworks, lookup, expected):
res = ns(_frameworks={default_tag: frameworks})
# binding `framework_definition` method to our resource mock: use pytest-mock instead?
res.framework_definition = Resources.framework_definition.__get__(res)
assert res.framework_definition(lookup) == (frameworks[expected],
frameworks[expected].name)
def run(dataset, config):
log.info("\n**** Random Forest (sklearn %s) ****\n", sklearn.__version__)
is_classification = config.type == 'classification'
# Impute any missing data (can test using -t 146606)
X_train, X_test = impute(dataset.train.X_enc, dataset.test.X_enc)
y_train, y_test = dataset.train.y, dataset.test.y
log.info(
"Running RandomForest with a maximum time of {}s on {} cores.".format(
config.max_runtime_seconds, config.cores))
log.warning(
"We completely ignore the requirement to stay within the time limit.")
log.warning(
"We completely ignore the advice to optimize towards metric: {}.".
format(config.metric))
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
rfc = estimator(n_jobs=config.cores, **config.framework_params)
rfc.fit(X_train, y_train)
predictions = rfc.predict(X_test)
probabilities = rfc.predict_proba(X_test) if is_classification else None
return ns(output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=False)
def file_config():
return from_config(
ns(
input_dir="my_input",
output_dir="my_output",
user_dir="my_user_dir",
root_dir="my_root_dir",
)
).config
def test_load_multiclass_task_arff(file_loader):
ds_def = ns(
train=os.path.join(res, "iris_train.arff"),
test=os.path.join(res, "iris_test.arff"),
target="class"
)
ds = file_loader.load(ds_def)
assert ds.type is DatasetType.multiclass
_assert_X_y_types(ds.train)
_assert_data_consistency(ds)
_assert_data_paths(ds, ds_def)
_assert_iris_features(ds, ds_def)
def test_load_binary_task_arff(file_loader):
ds_def = ns(
train=os.path.join(res, "kc2_train.arff"),
test=os.path.join(res, "kc2_test.arff"),
target="problems"
)
ds = file_loader.load(ds_def)
assert ds.type is DatasetType.binary
_assert_X_y_types(ds.train)
_assert_data_consistency(ds)
_assert_data_paths(ds, ds_def)
_assert_kc2_features(ds, ds_def)
def run(dataset: Dataset, config: TaskConfig):
with TmpDir() as tmpdir:
ds = ns(
train=ns(
X_enc=os.path.join(tmpdir, 'train.X_enc'),
y=os.path.join(tmpdir, 'train.y')
),
test=ns(
X_enc=os.path.join(tmpdir, 'test.X_enc'),
y=os.path.join(tmpdir, 'test.y')
)
)
write_csv(dataset.train.X_enc, ds.train.X_enc),
write_csv(dataset.train.y.reshape(-1, 1), ds.train.y),
write_csv(dataset.test.X_enc, ds.test.X_enc),
write_csv(dataset.test.y.reshape(-1, 1), ds.test.y),
dataset.release()
config.result_token = str(uuid.uuid1())
config.result_dir = tmpdir
params = json_dumps(dict(dataset=ds, config=config), style='compact')
output, err = run_cmd('{python} {here}/exec_proc.py'.format(python=PYTHON, here=dir_of(__file__)), _input_str_=params)
out = io.StringIO(output)
res = ns()
for line in out:
li = line.rstrip()
if li == config.result_token:
res = json_loads(out.readline(), as_namespace=True)
break
def load_data(path):
return read_csv(path, as_data_frame=False, header=False)
log.debug("Result from subprocess:\n%s", res)
save_predictions_to_file(dataset=dataset,
output_file=res.output_file,
probabilities=load_data(res.probabilities) if res.probabilities is not None else None,
predictions=load_data(res.predictions).squeeze(),
truth=load_data(res.truth).squeeze(),
target_is_encoded=res.target_is_encoded)
def test_load_regression_task_arff(file_loader):
ds_def = ns(
train=os.path.join(res, "cholesterol_train.arff"),
test=os.path.join(res, "cholesterol_test.arff"),
target="chol"
)
ds = file_loader.load(ds_def)
assert ds.type is DatasetType.regression
print(ds.train.X.dtypes)
_assert_X_y_types(ds.train)
_assert_data_consistency(ds)
_assert_data_paths(ds, ds_def)
_assert_cholesterol_features(ds, ds_def, 'arff')
def run_cmd_in_venv(caller_file, cmd, *args, **kwargs):
params = ns(python_exec='python')
for k, v in params:
kk = '_' + k + '_'
if kk in kwargs:
params[k] = kwargs[kk]
del kwargs[kk]
here = dir_of(caller_file)
venv_bin_path = os.path.join(here, 'venv', 'bin')
if os.path.isdir(venv_bin_path):
py = os.path.join(venv_bin_path, 'python -W ignore')
pip = os.path.join(venv_bin_path, 'python -m pip')
else:
py = f"{params.python_exec} -W ignore"
pip = f"{params.python_exec} -m pip"
cmd = cmd.format(py=py, pip=pip)
return run_cmd(cmd, *args, **kwargs)
config_user = config_load(os.path.join(args.userdir if args.userdir is not None else config.user_dir, "config.yaml"))
# config listing properties set by command line
config_args = ns.parse(
{'results.save': args.keep_scores},
input_dir=args.indir,
output_dir=args.outdir,
user_dir=args.userdir,
root_dir=root_dir,
script=os.path.basename(__file__),
run_mode=args.mode,
parallel_jobs=args.parallel,
sid=sid,
) + ns.parse(extras)
if args.mode != 'local':
config_args + ns.parse({'monitoring.frequency_seconds': 0})
config_args = ns({k: v for k, v in config_args if v is not None})
log.debug("Config args: %s.", config_args)
# merging all configuration files
amlb.resources.from_configs(config, config_user, config_args)
try:
if args.mode == 'local':
bench = amlb.Benchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'docker':
bench = amlb.DockerBenchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'singularity':
bench = amlb.SingularityBenchmark(args.framework, args.benchmark, args.constraint)
elif args.mode == 'aws':
bench = amlb.AWSBenchmark(args.framework, args.benchmark, args.constraint)
# bench = amlb.AWSBenchmark(args.framework, args.benchmark, args.constraint, region=args.region)
# elif args.mode == "aws-remote":
def run_in_venv(caller_file,
script_file: str,
*args,
input_data: Union[dict, ns],
dataset: Dataset,
config: TaskConfig,
process_results=None,
python_exec=None):
here = dir_of(caller_file)
venv_bin_path = os.path.join(here, 'venv', 'bin')
if python_exec is None: # use local virtual env by default
python_exec = os.path.join(venv_bin_path, 'python -W ignore')
script_path = os.path.join(here, script_file)
cmd = f"{python_exec} {script_path}"
input_data = ns.from_dict(input_data)
with TemporaryDirectory() as tmpdir:
def make_path(k, v, parents=None):
if isinstance(v, np.ndarray):
path = os.path.join(tmpdir, '.'.join(parents + [k, 'npy']))
if vector_keys.match(k):
v = v.reshape(-1, 1)
np.save(path, v, allow_pickle=True)
return k, path
return k, v
ds = ns.walk(input_data, make_path)
dataset.release()
config.result_dir = tmpdir
config.result_file = mktemp(dir=tmpdir)
params = json_dumps(dict(dataset=ds, config=config), style='compact')
with Timer() as proc_timer:
output, err = run_cmd(
cmd,
*args,
_input_str_=params,
_live_output_=True,
_error_level_=logging.DEBUG,
_env_=dict(PATH=os.pathsep.join(
[venv_bin_path, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([
rconfig().root_dir,
]),
AMLB_PATH=os.path.join(rconfig().root_dir, "amlb")),
)
res = ns(lambda: None)
if os.path.exists(config.result_file):
res = json_load(config.result_file, as_namespace=True)
log.debug("Result from subprocess:\n%s", res)
if not res:
raise NoResultError(f"Process crashed:\n{err}")
if res.error_message is not None:
raise NoResultError(res.error_message)
for name in ['predictions', 'truth', 'probabilities']:
res[name] = np.load(
res[name],
allow_pickle=True) if res[name] is not None else None
if callable(process_results):
res = process_results(res)
if res.output_file:
save_predictions(
dataset=dataset,
output_file=res.output_file,
predictions=res.predictions.reshape(-1)
if res.predictions is not None else None,
truth=res.truth.reshape(-1) if res.truth is not None else None,
probabilities=res.probabilities,
probabilities_labels=res.probabilities_labels,
target_is_encoded=res.target_is_encoded)
return dict(models_count=res.models_count
if res.models_count is not None else 1,
training_duration=res.training_duration if
res.training_duration is not None else proc_timer.duration,
predict_duration=res.predict_duration,
**res.others.__dict__)
def run_in_venv(caller_file,
script_file: str,
*args,
input_data: Union[dict, ns],
dataset: Dataset,
config: TaskConfig,
process_results=None,
python_exec=None):
here = dir_of(caller_file)
if python_exec is None: # use local virtual env by default
python_exec = os.path.join(here, 'venv/bin/python -W ignore')
script_path = os.path.join(here, script_file)
cmd = f"{python_exec} {script_path}"
input_data = ns.from_dict(input_data)
with TmpDir() as tmpdir:
def make_path(k, v, parents=None):
if isinstance(v, np.ndarray):
path = os.path.join(tmpdir, '.'.join(parents + [k, 'npy']))
if vector_keys.match(k):
v = v.reshape(-1, 1)
np.save(path, v, allow_pickle=True)
return k, path
return k, v
ds = ns.walk(input_data, make_path)
dataset.release()
config.result_token = str(uuid.uuid1())
config.result_dir = tmpdir
params = json_dumps(dict(dataset=ds, config=config), style='compact')
with Timer() as proc_timer:
output, err = run_cmd(cmd,
*args,
_input_str_=params,
_live_output_=True,
_env_=dict(PYTHONPATH=os.pathsep.join([
rconfig().root_dir,
os.path.join(rconfig().root_dir, "amlb"),
])))
out = io.StringIO(output)
res = ns()
for line in out:
li = line.rstrip()
if li == config.result_token:
res = json_loads(out.readline(), as_namespace=True)
break
if res.error_message is not None:
raise NoResultError(res.error_message)
for name in ['predictions', 'truth', 'probabilities']:
res[name] = np.load(
res[name],
allow_pickle=True) if res[name] is not None else None
log.debug("Result from subprocess:\n%s", res)
if callable(process_results):
res = process_results(res)
save_predictions_to_file(
dataset=dataset,
output_file=res.output_file,
predictions=res.predictions.reshape(-1)
if res.predictions is not None else None,
truth=res.truth.reshape(-1) if res.truth is not None else None,
probabilities=res.probabilities,
target_is_encoded=res.target_is_encoded)
return dict(models_count=res.models_count
if res.models_count is not None else 1,
training_duration=res.training_duration if
res.training_duration is not None else proc_timer.duration)
import pytest
from amlb.frameworks import default_tag
from amlb.resources import Resources
from amlb.utils import Namespace as ns
@pytest.mark.parametrize("frameworks, lookup, expected", [
(ns(MixedCase=ns(name="MixedCase")), "MixedCase", "MixedCase"),
(ns(MixedCase=ns(name="MixedCase")), "mixedcase", "MixedCase"),
(ns(MixedCase=ns(name="MixedCase")), "MIXEDCASE", "MixedCase"),
(ns(MixedCase=ns(name="MixedCase")), "mIxEdCasE", "MixedCase"),
])
def test_framework_definition_lookup_is_case_insensitive(
frameworks, lookup, expected):
res = ns(_frameworks={default_tag: frameworks})
# binding `framework_definition` method to our resource mock: use pytest-mock instead?
res.framework_definition = Resources.framework_definition.__get__(res)
assert res.framework_definition(lookup) == (frameworks[expected],
frameworks[expected].name)
def test_framework_definition_raises_error_if_no_matching_framework():
res = ns(config=ns(frameworks=ns(definition_file="none")),
_frameworks={default_tag: ns(present=ns(name="present"))})
# binding `framework_definition` method to our resource mock: use pytest-mock instead?
res.framework_definition = Resources.framework_definition.__get__(res)
assert res.framework_definition("present")
with pytest.raises(ValueError, match=r"Incorrect framework `missing`"):
res.framework_definition("missing")
return ns(output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test,
target_is_encoded=False)
if __name__ == '__main__':
params = json_loads(sys.stdin.read(), as_namespace=True)
def load_data(path):
return read_csv(path, as_data_frame=False, header=False)
ds = ns(train=ns(X_enc=load_data(params.dataset.train.X_enc),
y=load_data(params.dataset.train.y).squeeze()),
test=ns(
X_enc=load_data(params.dataset.test.X_enc),
y=load_data(params.dataset.test.y).squeeze(),
))
config = params.config
config.framework_params = ns.dict(config.framework_params)
result = run(ds, config)
res = copy.copy(result)
res.predictions = os.path.join(config.result_dir, 'predictions')
res.truth = os.path.join(config.result_dir, 'truth')
write_csv(result.predictions.reshape(-1, 1), res.predictions)
write_csv(result.truth.reshape(-1, 1), res.truth)
if result.probabilities is not None:
res.probabilities = os.path.join(config.result_dir, 'probabilities')
write_csv(result.probabilities, res.probabilities)
import os
import re
import shutil
import sys
# prevent asap other modules from defining the root logger using basicConfig
import amlb.logger
from openml.config import get_cache_directory
import amlb
from amlb.utils import Namespace as ns, config_load, datetime_iso, str2bool, str_sanitize, zip_path
from amlb import log, AutoMLError
default_dirs = ns(input_dir=get_cache_directory(),
output_dir="./results",
user_dir="~/.config/automlbenchmark",
root_dir=os.path.dirname(__file__))
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
'framework',
type=str,
help=
"The framework to evaluate as defined by default in resources/frameworks.yaml."
"\nTo use a labelled framework (i.e. a framework defined in resources/frameworks-{label}.yaml),"
"\nuse the syntax {framework}:{label}.")
parser.add_argument(
'benchmark',
type=str,
nargs='?',
default='test',
