def run_one_config(args=None,
sys_argv=None,
return_0_if_ok=True,
config_dict=None,
client=None,
started=None):
started = started or datetime.datetime.now()
args = args or cli(args=None, sys_argv=sys_argv)
config_dict = config_dict or args.config
with try_finally_log_etime(started) as _:
config = ConfigParser(config_dict, cmd_args=args)
if client is None:
if args.echo_config:
logger.info(str(config))
dask_client = getattr(config, 'DASK_CLIENT', 'SERIAL')
dask_scheduler = getattr(config, 'DASK_SCHEDULER', None)
with warnings.catch_warnings():
# scikit-learn has a number
# of deprecation warnings for kmeans
warnings.simplefilter("ignore")
with client_context(dask_client, dask_scheduler) as client:
return_values = parse_run_config(config, client)
else:
return_values = parse_run_config(config, client)
if return_0_if_ok:
return 0
return return_values
def run_many_configs(args=None,
sys_argv=None,
return_0_if_ok=True,
started=None):
started = started or datetime.datetime.now()
env_cmd_line = Namespace(
**{k: v
for d in (vars(args), parse_env_vars()) for k, v in d.items()})
logger.info(
'With --config-dir, DASK_CLIENT and DASK_SCHEDULER in config files are ignored'
)
dask_client = getattr(env_cmd_line, 'DASK_CLIENT', 'SERIAL')
dask_scheduler = getattr(env_cmd_line, 'DASK_SCHEDULER', None)
ret_val = 1
with try_finally_log_etime(started) as _:
with warnings.catch_warnings():
# scikit-learn has a number
# of deprecation warnings for kmeans
warnings.simplefilter("ignore")
results = [1]
with client_context(dask_client, dask_scheduler) as client:
kw = {
'args': args,
'sys_argv': sys_argv,
'return_0_if_ok': True,
'client': client,
}
pipe = partial(_run_one_config_of_many, **kw)
fnames = glob.glob(os.path.join(args.config_dir, '*.yaml'))
ret_val = max(map(pipe, fnames))
return ret_val
def dist_test(func):
if bool(int(os.environ.get('IS_TRAVIS', 1))):
def new(*a, **kw):
return func(*a, **kw)
return new
else:
with client_context() as client: # taking dask env variables
def new(*a, **kw):
kw = dict(client=client, **kw)
return func(*a, **kw)
return new
def dist_test(func):
with client_context() as client: # taking dask env variables
def new(*a, **kw):
kw = dict(client=client, **kw)
return func(*a, **kw)
return new