def test_env_dict_initialized_with_replaced_env_dict():
a = EnvDict({'a': {'b': 1}})
a_mod = a._replace_flatten_keys({'env__a__b': 2})
b = EnvDict(a_mod)
# make sure the new object has the updated values
assert b['a']['b'] == 2
def _process_file_dir_or_glob(parser, dagspec_arg=None):
"""
Process a file entry point file, directory or glob-like pattern,
the initialized dag and parsed args
Parameters
----------
parser : CustomParser
CLI arg parser
"""
# NOTE: we must use parser.parse_entry_point_value() instead or
# args.parse_args because calling the latter wont allow us to add more
# cli parameters, but we want that to expose parms from env
entry_point_value = dagspec_arg or parser.parse_entry_point_value()
entry = EntryPoint(entry_point_value)
if entry.type in {EntryPoint.Directory, EntryPoint.Pattern}:
# pipelines initialized from directories or patterns cannot be
# parametrized
path_to_env = None
# file
else:
path_to_env = default.path_to_env_from_spec(entry_point_value)
if path_to_env:
env_dict = EnvDict(path_to_env,
path_to_here=Path(entry_point_value).parent
if entry.type == EntryPoint.File else None)
_add_cli_args_from_env_dict_keys(parser, env_dict)
args = parser.parse_args()
dagspec_arg = dagspec_arg or args.entry_point
if hasattr(args, 'log'):
if args.log is not None:
logging.basicConfig(level=args.log.upper())
entry_point = EntryPoint(dagspec_arg)
# directory
if entry_point.type == EntryPoint.Directory:
dag = DAGSpec.from_directory(dagspec_arg).to_dag()
# pattern
elif entry_point.type == EntryPoint.Pattern:
dag = DAGSpec.from_files(dagspec_arg).to_dag()
# file
else:
if path_to_env:
# and replace keys depending on passed cli args
replaced = _env_keys_to_override(args, parser.static_args)
env = env_dict._replace_flatten_keys(replaced)
dag = DAGSpec(dagspec_arg, env=env).to_dag()
else:
dag = DAGSpec(dagspec_arg).to_dag()
return dag, args
def test_replace_nested_flatten_keys_env_dict():
env = EnvDict({'a': {'b': 1, 'c': 1}})
new_env = env._replace_flatten_keys({'env__a__b': 2, 'env__a__c': 2})
assert (new_env.a.b == 2 and new_env.a.c == 2
and env is not new_env) # must return a copy