def test_spec_invalid_glob_pattern(tmp_nbs_no_yaml):
Path('some_invalid_script.sh').touch()
with pytest.raises(ValueError) as excinfo:
DAGSpec.from_files('*')
assert ('Cannot instantiate DAGSpec from files with invalid extensions'
in str(excinfo.value))
def test_spec_glob_pattern(tmp_nbs_no_yaml):
# directory should be ignored
Path('output').mkdir()
# if passed a string, it's interpreted as a glob-like pattern
dag = DAGSpec.from_files('load.py').to_dag()
assert list(dag) == ['load']
def _process_file_dir_or_glob(parser, dagspec_arg=None):
"""
Process a file entry point file, directory or glob-like pattern,
the initialized dag and parsed args
Parameters
----------
parser : CustomParser
CLI arg parser
"""
# NOTE: we must use parser.parse_entry_point_value() instead or
# args.parse_args because calling the latter wont allow us to add more
# cli parameters, but we want that to expose parms from env
entry_point_value = dagspec_arg or parser.parse_entry_point_value()
entry = EntryPoint(entry_point_value)
if entry.type in {EntryPoint.Directory, EntryPoint.Pattern}:
# pipelines initialized from directories or patterns cannot be
# parametrized
path_to_env = None
# file
else:
path_to_env = default.path_to_env_from_spec(entry_point_value)
if path_to_env:
env_dict = EnvDict(path_to_env,
path_to_here=Path(entry_point_value).parent
if entry.type == EntryPoint.File else None)
_add_cli_args_from_env_dict_keys(parser, env_dict)
args = parser.parse_args()
dagspec_arg = dagspec_arg or args.entry_point
if hasattr(args, 'log'):
if args.log is not None:
logging.basicConfig(level=args.log.upper())
entry_point = EntryPoint(dagspec_arg)
# directory
if entry_point.type == EntryPoint.Directory:
dag = DAGSpec.from_directory(dagspec_arg).to_dag()
# pattern
elif entry_point.type == EntryPoint.Pattern:
dag = DAGSpec.from_files(dagspec_arg).to_dag()
# file
else:
if path_to_env:
# and replace keys depending on passed cli args
replaced = _env_keys_to_override(args, parser.static_args)
env = env_dict._replace_flatten_keys(replaced)
dag = DAGSpec(dagspec_arg, env=env).to_dag()
else:
dag = DAGSpec(dagspec_arg).to_dag()
return dag, args
def _process_file_dir_or_glob(parser):
"""
Process a file entry point file or directory), returns the initialized dag
and parsed args
Parameters
----------
parser : CustomParser
CLI arg parser
"""
# look for env.yaml by searching in default locations
path_to_env = default.path_to_env(
Path(parser.parse_entry_point_value()).parent)
if path_to_env:
env_dict = EnvDict(path_to_env)
_add_cli_args_from_env_dict_keys(parser, env_dict)
args = parser.parse_args()
if hasattr(args, 'log'):
if args.log is not None:
logging.basicConfig(level=args.log.upper())
entry_point = EntryPoint(args.entry_point)
if entry_point.type == EntryPoint.Directory:
dag = DAGSpec.from_directory(args.entry_point).to_dag()
elif entry_point.type == EntryPoint.Pattern:
dag = DAGSpec.from_files(args.entry_point).to_dag()
else:
if path_to_env:
# and replace keys depending on passed cli args
replaced = _env_keys_to_override(args, parser.static_args)
env = env_dict._replace_flatten_keys(replaced)
dag = DAGSpec(args.entry_point, env=env).to_dag()
else:
dag = DAGSpec(args.entry_point).to_dag()
return dag, args
def test_spec_from_files(tmp_nbs_no_yaml):
dag = DAGSpec.from_files(['load.py', 'clean.py', 'plot.py']).to_dag()
assert list(dag) == ['load', 'clean', 'plot']
