def test_format_maximal():
file_obj = utils.MockPath('config.yml',
is_file=True,
content=example_file_maximal)
output = format_config_yaml(file_obj)
# handy for updating
# import json
# print(json.dumps(output, indent=4))
assert edict.diff(output, expected_output_maximal) == {}
def local_pathlib():
test_folder = os.path.join(os.path.dirname(__file__), 'test_tmp')
if os.path.exists(test_folder):
shutil.rmtree(test_folder)
os.mkdir(test_folder)
configpath = os.path.join(test_folder, "config.yml")
with open(configpath, "w") as f:
f.write(example_run.format(host="null", port=1))
inpath = os.path.join(test_folder, "input")
os.mkdir(inpath)
with open(os.path.join(inpath, 'script.in'), 'w') as f:
f.write('test @v{var1} @f{frag1}')
with open(os.path.join(inpath, 'frag.in'), 'w') as f:
f.write('replace frag')
with open(os.path.join(inpath, 'other.in'), 'w') as f:
f.write('another file')
yield format_config_yaml(configpath), test_folder
def remote():
test_folder = os.path.join(os.path.dirname(__file__), 'test_tmp')
if os.path.exists(test_folder):
shutil.rmtree(test_folder)
os.mkdir(test_folder)
configpath = os.path.abspath(os.path.join(test_folder, "config.yml"))
inpath = os.path.join(test_folder, "input")
os.mkdir(inpath)
with open(os.path.join(inpath, 'script.in'), 'w') as f:
f.write('test @v{var1} @f{frag1}')
with open(os.path.join(inpath, 'frag.in'), 'w') as f:
f.write('replace frag')
with open(os.path.join(inpath, 'other.in'), 'w') as f:
f.write('another file')
with mockserver.Server({"user": {
"password": "******"
}}, test_folder) as server:
with open(configpath, "w") as f:
f.write(example_run.format(host=server.host, port=server.port))
yield format_config_yaml(configpath), test_folder
def local_mock():
config_file = MockPath('config.yml',
is_file=True,
content=example_run.format(host="null", port=1))
test_folder = MockPath("test_tmp",
structure=[
config_file, {
"input": [
MockPath(
'script.in',
is_file=True,
content='test @v{var1} @f{frag1}'),
MockPath('frag.in',
is_file=True,
content='replace frag'),
MockPath('other.in',
is_file=True,
content='another file'),
]
}
])
yield format_config_yaml(config_file), test_folder
def run(fpath, runs=None, basepath="", log_level='INFO',
ignore_fail=False, if_exists="abort", test_run=False):
"""
Parameters
----------
fpath: str
runs: list of ints or None
basepath: str
log_level: str
ignore_fail: bool
if True; if a command line execution fails continue the run
if_exists: ["abort", "remove", "use"]
either; raise an IOError if the output path already exists,
remove the output path or use it without change
test_run: bool
if True don't run any executables
Returns
-------
"""
if log_level.upper() == "DEBUG_FULL":
log_level = "DEBUG"
filter_ext = False
else:
filter_ext = True
root = logging.getLogger()
root.handlers = [] # remove any existing handlers
root.setLevel(logging.DEBUG)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(getattr(logging, log_level.upper()))
# TODO align messages
formatter = logging.Formatter('%(levelname)8s: %(module)10s: %(message)s')
stream_handler.setFormatter(formatter)
stream_handler.propogate = False
if filter_ext:
stream_handler.addFilter(logging.Filter('atomic_hpc'))
root.addHandler(stream_handler)
# TODO add option for file logger
# file_handler = logging.FileHandler(
# os.path.join(outdir, ipynb_name + '.config.log'), 'w')
# file_handler.setLevel(getattr(logging, log_level.upper()))
# file_handler.setFormatter(formatter)
# file_handler.propogate = False
# file_handler.addFilter(logging.Filter('atomic_hpc'))
# root.addHandler(file_handler)
fpath = os.path.abspath(fpath)
basepath = os.path.abspath(basepath)
try:
runs_to_deploy = format_config_yaml(fpath, errormsg_only=True)
except ValidationError as err:
logger.critical(err)
return
if runs is not None:
runs_to_deploy = [r for r in runs_to_deploy if r["id"] in runs]
exec_errors = not ignore_fail
try:
deploy_runs(runs_to_deploy, basepath, if_exists=if_exists,
exec_errors=exec_errors, test_run=test_run)
except RuntimeError as err:
logger.critical(err)
return
def run(fpath,
runs=None,
outpath="",
basepath="",
log_level='INFO',
if_exists="abort",
path_regex="*",
ignore_regex=None):
"""
Parameters
----------
fpath: str
runs: list of ints or None
outpath: str
basepath: str
log_level: str
if_exists: ["abort", "remove", "use"]
either; raise an IOError if the output path already exists,
remove the output path or use it without change
path_regex: str
regex to search for files
ignore_regex: None or list of str
file regexes to ignore (not copy)
Returns
-------
"""
if log_level.upper() == "DEBUG_FULL":
log_level = "DEBUG"
filter_ext = False
else:
filter_ext = True
root = logging.getLogger()
root.handlers = [] # remove any existing handlers
root.setLevel(logging.DEBUG)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setLevel(getattr(logging, log_level.upper()))
# TODO align messages
formatter = logging.Formatter('%(levelname)8s: %(module)10s: %(message)s')
stream_handler.setFormatter(formatter)
stream_handler.propogate = False
if filter_ext:
stream_handler.addFilter(logging.Filter('atomic_hpc'))
root.addHandler(stream_handler)
# TODO add option for file logger
# file_handler = logging.FileHandler(
# os.path.join(outdir, ipynb_name + '.config.log'), 'w')
# file_handler.setLevel(getattr(logging, log_level.upper()))
# file_handler.setFormatter(formatter)
# file_handler.propogate = False
# file_handler.addFilter(logging.Filter('atomic_hpc'))
# root.addHandler(file_handler)
fpath = os.path.abspath(fpath)
basepath = os.path.abspath(basepath)
try:
runs_to_deploy = format_config_yaml(fpath, errormsg_only=True)
except ValidationError as err:
logger.critical(err)
return
if runs is not None:
runs_to_deploy = [r for r in runs_to_deploy if r["id"] in runs]
try:
retrieve_outputs(runs_to_deploy,
outpath,
basepath,
if_exists=if_exists,
path_regex=path_regex,
ignore_regex=ignore_regex)
except RuntimeError as err:
logger.critical(err)