def check_write_template_error_case(template_path: str, data: dict,
exp_error: Exception):
with TemporaryDirectory() as tmp_dir:
output_path = join(tmp_dir, 'my_exe.sh')
with pytest.raises(MlVToolException) as e:
write_template(output_path, template_path, **data)
assert isinstance(e.value.__cause__, exp_error)
def gen_dvc_command(input_path: str,
dvc_output_path: str,
conf: MlVToolConf,
docstring_conf: dict = None):
logging.info(
f'Generate DVC command "{dvc_output_path}" from "{input_path}"')
logging.debug(f'Global configuration {conf}')
logging.debug(f'Docstring configuration {docstring_conf}')
docstring_info = extract_docstring_from_file(input_path, docstring_conf)
python_cmd_rel_path = relpath(input_path, conf.top_directory)
extra_var = {
conf.dvc_var_python_cmd_path: python_cmd_rel_path,
conf.dvc_var_python_cmd_name: basename(python_cmd_rel_path)
}
info = get_dvc_template_data(
docstring_info, conf.top_directory, python_cmd_rel_path,
conf.dvc_var_meta_filename,
conf.path.dvc_metadata_root_dir if conf.path else '', extra_var)
templates_path = join(CURRENT_DIR, 'templates', DVC_CMD_TEMPLATE_NAME)
write_template(dvc_output_path, templates_path, info=info)
logging.log(
logging.WARNING + 1,
f'DVC bash command successfully generated in {dvc_output_path}')
def test_should_create_directory_to_write_template(work_dir,
valid_template_path):
"""
Test create directory and write template
"""
output_path = join(work_dir, 'a_dir', 'my_exe.sh')
write_template(output_path, valid_template_path, given_data='test')
assert exists(join(work_dir, 'a_dir'))
assert exists(output_path)
def test_write_template_should_raise_if_can_not_write_executable_output(
work_dir, mocker, valid_template_path):
"""
Test write_template raise an MlVToolException if the executable output cannot be written
"""
output_path = join(work_dir, 'output.sh')
mocker.patch('builtins.open', side_effect=IOError)
with pytest.raises(MlVToolException) as e:
write_template(output_path, valid_template_path, given_data='test')
assert isinstance(e.value.__cause__, IOError)
def test_should_write_template(work_dir, valid_template_path):
"""
Test write an executable file from a given template and data
"""
output_path = join(work_dir, 'my_exe.sh')
write_template(output_path, valid_template_path, given_data='test')
assert exists(output_path)
assert stat.S_IMODE(os_stat(output_path).st_mode) == 0o755
with open(output_path, 'r') as fd:
assert fd.read() == 'a_value=test'
def export_pipeline(dvc_meta_file: str, output: str, work_dir: str):
"""
Generate an executable script to run a whole pipeline
"""
logging.info(f'Export pipeline from step {dvc_meta_file} to {output}')
logging.debug(f'Work directory {work_dir}')
ordered_dvc_metas = get_dvc_dependencies(dvc_meta_file,
get_dvc_files(dvc_meta_file))
template_data = {
'work_dir': work_dir,
'cmds': [dvc_meta.cmd for dvc_meta in ordered_dvc_metas]
}
logging.debug(f'Template data: {template_data}')
template_path = join(CURRENT_DIR, '..', 'template',
PIPELINE_EXPORT_TEMPLATE_NAME)
write_template(output, template_path, info=template_data)
logging.log(logging.WARNING + 1,
f'Pipeline successfully exported in {abspath(output)}')