def test_spec_invalid_glob_pattern(tmp_nbs_no_yaml): Path('some_invalid_script.sh').touch() with pytest.raises(ValueError) as excinfo: DAGSpec.from_files('*') assert ('Cannot instantiate DAGSpec from files with invalid extensions' in str(excinfo.value))
def test_spec_glob_pattern(tmp_nbs_no_yaml): # directory should be ignored Path('output').mkdir() # if passed a string, it's interpreted as a glob-like pattern dag = DAGSpec.from_files('load.py').to_dag() assert list(dag) == ['load']
def _process_file_dir_or_glob(parser, dagspec_arg=None): """ Process a file entry point file, directory or glob-like pattern, the initialized dag and parsed args Parameters ---------- parser : CustomParser CLI arg parser """ # NOTE: we must use parser.parse_entry_point_value() instead or # args.parse_args because calling the latter wont allow us to add more # cli parameters, but we want that to expose parms from env entry_point_value = dagspec_arg or parser.parse_entry_point_value() entry = EntryPoint(entry_point_value) if entry.type in {EntryPoint.Directory, EntryPoint.Pattern}: # pipelines initialized from directories or patterns cannot be # parametrized path_to_env = None # file else: path_to_env = default.path_to_env_from_spec(entry_point_value) if path_to_env: env_dict = EnvDict(path_to_env, path_to_here=Path(entry_point_value).parent if entry.type == EntryPoint.File else None) _add_cli_args_from_env_dict_keys(parser, env_dict) args = parser.parse_args() dagspec_arg = dagspec_arg or args.entry_point if hasattr(args, 'log'): if args.log is not None: logging.basicConfig(level=args.log.upper()) entry_point = EntryPoint(dagspec_arg) # directory if entry_point.type == EntryPoint.Directory: dag = DAGSpec.from_directory(dagspec_arg).to_dag() # pattern elif entry_point.type == EntryPoint.Pattern: dag = DAGSpec.from_files(dagspec_arg).to_dag() # file else: if path_to_env: # and replace keys depending on passed cli args replaced = _env_keys_to_override(args, parser.static_args) env = env_dict._replace_flatten_keys(replaced) dag = DAGSpec(dagspec_arg, env=env).to_dag() else: dag = DAGSpec(dagspec_arg).to_dag() return dag, args
def _process_file_dir_or_glob(parser): """ Process a file entry point file or directory), returns the initialized dag and parsed args Parameters ---------- parser : CustomParser CLI arg parser """ # look for env.yaml by searching in default locations path_to_env = default.path_to_env( Path(parser.parse_entry_point_value()).parent) if path_to_env: env_dict = EnvDict(path_to_env) _add_cli_args_from_env_dict_keys(parser, env_dict) args = parser.parse_args() if hasattr(args, 'log'): if args.log is not None: logging.basicConfig(level=args.log.upper()) entry_point = EntryPoint(args.entry_point) if entry_point.type == EntryPoint.Directory: dag = DAGSpec.from_directory(args.entry_point).to_dag() elif entry_point.type == EntryPoint.Pattern: dag = DAGSpec.from_files(args.entry_point).to_dag() else: if path_to_env: # and replace keys depending on passed cli args replaced = _env_keys_to_override(args, parser.static_args) env = env_dict._replace_flatten_keys(replaced) dag = DAGSpec(args.entry_point, env=env).to_dag() else: dag = DAGSpec(args.entry_point).to_dag() return dag, args
def test_spec_from_files(tmp_nbs_no_yaml): dag = DAGSpec.from_files(['load.py', 'clean.py', 'plot.py']).to_dag() assert list(dag) == ['load', 'clean', 'plot']