def call_run(run_fn):
import numpy as np
params = NS.from_dict(json.loads(sys.stdin.read()))
def load_data(name, path, **ignored):
if isinstance(path, str) and data_keys.match(name):
return name, np.load(path, allow_pickle=True)
return name, path
print(params.dataset)
ds = NS.walk(params.dataset, load_data)
config = params.config
config.framework_params = NS.dict(config.framework_params)
try:
result = run_fn(ds, config)
res = dict(result)
for name in ['predictions', 'truth', 'probabilities']:
arr = result[name]
if arr is not None:
res[name] = os.path.join(config.result_dir, '.'.join([name, 'npy']))
np.save(res[name], arr, allow_pickle=True)
except Exception as e:
log.exception(e)
res = dict(
error_message=str(e),
models_count=0
)
print(config.result_token)
print(json.dumps(res, separators=(',', ':')))
def run_in_venv(caller_file,
script_file: str,
*args,
input_data: Union[dict, ns],
dataset: Dataset,
config: TaskConfig,
process_results=None,
python_exec=None):
here = dir_of(caller_file)
venv_bin_path = os.path.join(here, 'venv', 'bin')
if python_exec is None: # use local virtual env by default
python_exec = os.path.join(venv_bin_path, 'python -W ignore')
script_path = os.path.join(here, script_file)
cmd = f"{python_exec} {script_path}"
input_data = ns.from_dict(input_data)
with TemporaryDirectory() as tmpdir:
def make_path(k, v, parents=None):
if isinstance(v, np.ndarray):
path = os.path.join(tmpdir, '.'.join(parents + [k, 'npy']))
if vector_keys.match(k):
v = v.reshape(-1, 1)
np.save(path, v, allow_pickle=True)
return k, path
return k, v
ds = ns.walk(input_data, make_path)
dataset.release()
config.result_dir = tmpdir
config.result_file = mktemp(dir=tmpdir)
params = json_dumps(dict(dataset=ds, config=config), style='compact')
with Timer() as proc_timer:
output, err = run_cmd(
cmd,
*args,
_input_str_=params,
_live_output_=True,
_error_level_=logging.DEBUG,
_env_=dict(PATH=os.pathsep.join(
[venv_bin_path, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([
rconfig().root_dir,
]),
AMLB_PATH=os.path.join(rconfig().root_dir, "amlb")),
)
res = ns(lambda: None)
if os.path.exists(config.result_file):
res = json_load(config.result_file, as_namespace=True)
log.debug("Result from subprocess:\n%s", res)
if not res:
raise NoResultError(f"Process crashed:\n{err}")
if res.error_message is not None:
raise NoResultError(res.error_message)
for name in ['predictions', 'truth', 'probabilities']:
res[name] = np.load(
res[name],
allow_pickle=True) if res[name] is not None else None
if callable(process_results):
res = process_results(res)
if res.output_file:
save_predictions(
dataset=dataset,
output_file=res.output_file,
predictions=res.predictions.reshape(-1)
if res.predictions is not None else None,
truth=res.truth.reshape(-1) if res.truth is not None else None,
probabilities=res.probabilities,
probabilities_labels=res.probabilities_labels,
target_is_encoded=res.target_is_encoded)
return dict(models_count=res.models_count
if res.models_count is not None else 1,
training_duration=res.training_duration if
res.training_duration is not None else proc_timer.duration,
predict_duration=res.predict_duration,
**res.others.__dict__)
def run_in_venv(caller_file,
script_file: str,
*args,
input_data: Union[dict, ns],
dataset: Dataset,
config: TaskConfig,
process_results=None,
python_exec=None):
here = dir_of(caller_file)
if python_exec is None: # use local virtual env by default
python_exec = os.path.join(here, 'venv/bin/python -W ignore')
script_path = os.path.join(here, script_file)
cmd = f"{python_exec} {script_path}"
input_data = ns.from_dict(input_data)
with TmpDir() as tmpdir:
def make_path(k, v, parents=None):
if isinstance(v, np.ndarray):
path = os.path.join(tmpdir, '.'.join(parents + [k, 'npy']))
if vector_keys.match(k):
v = v.reshape(-1, 1)
np.save(path, v, allow_pickle=True)
return k, path
return k, v
ds = ns.walk(input_data, make_path)
dataset.release()
config.result_token = str(uuid.uuid1())
config.result_dir = tmpdir
params = json_dumps(dict(dataset=ds, config=config), style='compact')
with Timer() as proc_timer:
output, err = run_cmd(cmd,
*args,
_input_str_=params,
_live_output_=True,
_env_=dict(PYTHONPATH=os.pathsep.join([
rconfig().root_dir,
os.path.join(rconfig().root_dir, "amlb"),
])))
out = io.StringIO(output)
res = ns()
for line in out:
li = line.rstrip()
if li == config.result_token:
res = json_loads(out.readline(), as_namespace=True)
break
if res.error_message is not None:
raise NoResultError(res.error_message)
for name in ['predictions', 'truth', 'probabilities']:
res[name] = np.load(
res[name],
allow_pickle=True) if res[name] is not None else None
log.debug("Result from subprocess:\n%s", res)
if callable(process_results):
res = process_results(res)
save_predictions_to_file(
dataset=dataset,
output_file=res.output_file,
predictions=res.predictions.reshape(-1)
if res.predictions is not None else None,
truth=res.truth.reshape(-1) if res.truth is not None else None,
probabilities=res.probabilities,
target_is_encoded=res.target_is_encoded)
return dict(models_count=res.models_count
if res.models_count is not None else 1,
training_duration=res.training_duration if
res.training_duration is not None else proc_timer.duration)
def call_run(run_fn):
import numpy as np
params = NS.from_dict({
"dataset": {
"train": {
"X_enc": "/tmp/dress/train.X_enc.npy",
"y_enc": "/tmp/dress/train.y_enc.npy"
},
"test": {
"X_enc": "/tmp/dress/test.X_enc.npy",
"y_enc": "/tmp/dress/test.y_enc.npy"
}
},
"config": {
"framewor k": "FEDOT",
"framework_params": {},
"type": "classification",
"name": "Australian",
"fold": 0,
"metrics": ["auc", "logloss", "acc"],
"metric": "auc",
"seed": 3029240368,
"max_runtime_seconds": 600,
"cores": 4,
"max_mem_size_mb": 91763,
"min_vol_size_mb": -1,
"input_dir": "/home/rosneft_user_2500/.openml/cache",
"output_dir":
"/home/rosneft_user_2500/bench/automlbenchmark/results/fedot.small.test.local.20201225T163641",
"output_predictions_file":
"/home/rosneft_user_2500/bench/automlbenchmark/results/fedot.small.test.local.20201225T163641/predictions/fedot.Australian.0.csv",
"result_token": "5e433616-46cf-11eb-a671-7957e32fc18d",
"result_dir": "/tmp/iris"
}
})
def load_data(name, path, **ignored):
if isinstance(path, str) and data_keys.match(name):
return name, np.load(path, allow_pickle=True)
return name, path
print(params.dataset)
ds = NS.walk(params.dataset, load_data)
config = params.config
config.framework_params = NS.dict(config.framework_params)
try:
result = run_fn(ds, config)
res = dict(result)
for name in ['predictions', 'truth', 'probabilities']:
arr = result[name]
if arr is not None:
res[name] = os.path.join(config.result_dir,
'.'.join([name, 'npy']))
np.save(res[name], arr, allow_pickle=True)
except Exception as e:
log.exception(e)
res = dict(error_message=str(e), models_count=0)
print(config.result_token)
print(json.dumps(res, separators=(',', ':')))
