def get_dvc_template_data(docstring_info: DocstringInfo,
python_cmd_path: str,
meta_file_variable_name: str,
extra_variables: dict = None):
"""
Format data from docstring for dvc bash command template
"""
logging.info('Build data for DVC command generation using template')
dvc_params = get_dvc_params(docstring_info.docstring)
variables = [] if not extra_variables else [
f'{name}="{value}"' for name, value in extra_variables.items()
]
meta_file_name = dvc_params.meta_file_name or to_dvc_meta_filename(
python_cmd_path)
info = {
'variables': variables,
'meta_file_name_var_assign':
f'{meta_file_variable_name}="{meta_file_name}"',
'meta_file_name_var': meta_file_variable_name,
'whole_command': None,
'python_script': python_cmd_path,
'dvc_inputs': [],
'dvc_outputs': [],
'python_params': ''
}
if dvc_params.dvc_cmd:
logging.info('DVC mode: whole command provided')
info['whole_command'] = dvc_params.dvc_cmd.cmd.replace('\n', ' \\\n')
logging.debug(f'Custom command {info["whole_command"]}')
return info
logging.info('DVC mode: generate command from parameters')
python_params = []
def handle_params(dvc_docstring_params: List[DocstringDvc], label: str):
for dvc_param in dvc_docstring_params:
if dvc_param.related_param:
variable_name = to_bash_variable(dvc_param.related_param)
py_cmd_param = to_cmd_param(dvc_param.related_param)
info['variables'].append(
f'{variable_name}="{dvc_param.file_path}"')
python_params.append(f'--{py_cmd_param} ${variable_name}')
info[label].append(f'${variable_name}')
else:
info[label].append(dvc_param.file_path)
for extra_param in dvc_params.dvc_extra:
python_params.append(extra_param.extra)
handle_params(dvc_params.dvc_in, 'dvc_inputs')
handle_params(dvc_params.dvc_out, 'dvc_outputs')
info['python_params'] = ' '.join(python_params)
logging.debug(f'Template info: {info}')
return info
def get_dvc_metadata_output_path(script_path: str, conf: MlVToolConf) -> str:
""" Generate dvc metadata path according to conf and python script file name """
file_name = to_dvc_meta_filename(basename(script_path))
return join(conf.top_directory, conf.path.dvc_metadata_root_dir, file_name)
def test_should_convert_to_dvc_meta_filename():
"""
Test convert notebook path to dvc meta filename
"""
assert to_dvc_meta_filename(
'./toto/truc/My notebook.ipynb') == 'my_notebook.dvc'