def test_spec_from_directory(chdir, dir_, tmp_nbs_no_yaml): os.chdir(chdir) Path('output').mkdir() dag = DAGSpec.from_directory(dir_).to_dag() assert list(dag) == ['load', 'clean', 'plot']
def _process_file_dir_or_glob(parser, dagspec_arg=None): """ Process a file entry point file, directory or glob-like pattern, the initialized dag and parsed args Parameters ---------- parser : CustomParser CLI arg parser """ # NOTE: we must use parser.parse_entry_point_value() instead or # args.parse_args because calling the latter wont allow us to add more # cli parameters, but we want that to expose parms from env entry_point_value = dagspec_arg or parser.parse_entry_point_value() entry = EntryPoint(entry_point_value) if entry.type in {EntryPoint.Directory, EntryPoint.Pattern}: # pipelines initialized from directories or patterns cannot be # parametrized path_to_env = None # file else: path_to_env = default.path_to_env_from_spec(entry_point_value) if path_to_env: env_dict = EnvDict(path_to_env, path_to_here=Path(entry_point_value).parent if entry.type == EntryPoint.File else None) _add_cli_args_from_env_dict_keys(parser, env_dict) args = parser.parse_args() dagspec_arg = dagspec_arg or args.entry_point if hasattr(args, 'log'): if args.log is not None: logging.basicConfig(level=args.log.upper()) entry_point = EntryPoint(dagspec_arg) # directory if entry_point.type == EntryPoint.Directory: dag = DAGSpec.from_directory(dagspec_arg).to_dag() # pattern elif entry_point.type == EntryPoint.Pattern: dag = DAGSpec.from_files(dagspec_arg).to_dag() # file else: if path_to_env: # and replace keys depending on passed cli args replaced = _env_keys_to_override(args, parser.static_args) env = env_dict._replace_flatten_keys(replaced) dag = DAGSpec(dagspec_arg, env=env).to_dag() else: dag = DAGSpec(dagspec_arg).to_dag() return dag, args
def load_entry_point(entry_point): type_ = find_entry_point_type(entry_point) if type_ == EntryPoint.Directory: spec = DAGSpec.from_directory(entry_point) path = Path(entry_point) elif type_ == EntryPoint.File: spec = DAGSpec(entry_point) path = Path(entry_point).parent else: raise NotImplementedError( f'loading entry point type {type_!r} is unsupported') return spec, spec.to_dag(), path
def _process_file_dir_or_glob(parser): """ Process a file entry point file or directory), returns the initialized dag and parsed args Parameters ---------- parser : CustomParser CLI arg parser """ # look for env.yaml by searching in default locations path_to_env = default.path_to_env( Path(parser.parse_entry_point_value()).parent) if path_to_env: env_dict = EnvDict(path_to_env) _add_cli_args_from_env_dict_keys(parser, env_dict) args = parser.parse_args() if hasattr(args, 'log'): if args.log is not None: logging.basicConfig(level=args.log.upper()) entry_point = EntryPoint(args.entry_point) if entry_point.type == EntryPoint.Directory: dag = DAGSpec.from_directory(args.entry_point).to_dag() elif entry_point.type == EntryPoint.Pattern: dag = DAGSpec.from_files(args.entry_point).to_dag() else: if path_to_env: # and replace keys depending on passed cli args replaced = _env_keys_to_override(args, parser.static_args) env = env_dict._replace_flatten_keys(replaced) dag = DAGSpec(args.entry_point, env=env).to_dag() else: dag = DAGSpec(args.entry_point).to_dag() return dag, args
def test_spec_from_non_existing_directory(touch, tmp_directory): if touch: Path('not_a_directory').touch() with pytest.raises(NotADirectoryError): DAGSpec.from_directory('not_a_directory')