def get_dvc_template_data(docstring_info: DocstringInfo, python_cmd_path: str, meta_file_variable_name: str, extra_variables: dict = None): """ Format data from docstring for dvc bash command template """ logging.info('Build data for DVC command generation using template') dvc_params = get_dvc_params(docstring_info.docstring) variables = [] if not extra_variables else [ f'{name}="{value}"' for name, value in extra_variables.items() ] meta_file_name = dvc_params.meta_file_name or to_dvc_meta_filename( python_cmd_path) info = { 'variables': variables, 'meta_file_name_var_assign': f'{meta_file_variable_name}="{meta_file_name}"', 'meta_file_name_var': meta_file_variable_name, 'whole_command': None, 'python_script': python_cmd_path, 'dvc_inputs': [], 'dvc_outputs': [], 'python_params': '' } if dvc_params.dvc_cmd: logging.info('DVC mode: whole command provided') info['whole_command'] = dvc_params.dvc_cmd.cmd.replace('\n', ' \\\n') logging.debug(f'Custom command {info["whole_command"]}') return info logging.info('DVC mode: generate command from parameters') python_params = [] def handle_params(dvc_docstring_params: List[DocstringDvc], label: str): for dvc_param in dvc_docstring_params: if dvc_param.related_param: variable_name = to_bash_variable(dvc_param.related_param) py_cmd_param = to_cmd_param(dvc_param.related_param) info['variables'].append( f'{variable_name}="{dvc_param.file_path}"') python_params.append(f'--{py_cmd_param} ${variable_name}') info[label].append(f'${variable_name}') else: info[label].append(dvc_param.file_path) for extra_param in dvc_params.dvc_extra: python_params.append(extra_param.extra) handle_params(dvc_params.dvc_in, 'dvc_inputs') handle_params(dvc_params.dvc_out, 'dvc_outputs') info['python_params'] = ' '.join(python_params) logging.debug(f'Template info: {info}') return info
def get_dvc_metadata_output_path(script_path: str, conf: MlVToolConf) -> str: """ Generate dvc metadata path according to conf and python script file name """ file_name = to_dvc_meta_filename(basename(script_path)) return join(conf.top_directory, conf.path.dvc_metadata_root_dir, file_name)
def test_should_convert_to_dvc_meta_filename(): """ Test convert notebook path to dvc meta filename """ assert to_dvc_meta_filename( './toto/truc/My notebook.ipynb') == 'my_notebook.dvc'