def test_env_dict_initialized_with_replaced_env_dict(): a = EnvDict({'a': {'b': 1}}) a_mod = a._replace_flatten_keys({'env__a__b': 2}) b = EnvDict(a_mod) # make sure the new object has the updated values assert b['a']['b'] == 2
def test_env_dict_initialized_with_env_dict(data, keys): env = EnvDict(EnvDict(data)) # ensure we initialized the object correctly assert repr(env) assert str(env) # check we can access nested keys for key in keys: env = env[key] assert env == 1
def test_replace_value_casts_if_possible(): env = EnvDict({'a': False, 'b': 1, 'c': 1.1}) env._replace_value('True', ['a']) env._replace_value('2', ['b']) env._replace_value('2.2', ['c']) assert env.a is True assert env.b == 2 assert env.c == 2.2
def test_dagspec_initialization_from_yaml_and_env(tmp_nbs, monkeypatch): """ DAGSpec can be initialized with a path to a spec or a dictionary, but they have a slightly different behavior. This ensure the cli passes the path, instead of a dictionary """ mock_DAGSpec = Mock(wraps=parsers.DAGSpec) mock_default_path_to_env = Mock(wraps=parsers.default.path_to_env) mock_EnvDict = Mock(wraps=parsers.EnvDict) monkeypatch.setattr(sys, 'argv', ['python']) monkeypatch.setattr(parsers, 'DAGSpec', mock_DAGSpec) monkeypatch.setattr(parsers.default, 'path_to_env', mock_default_path_to_env) monkeypatch.setattr(parsers, 'EnvDict', mock_EnvDict) parser = CustomParser() with parser: pass dag, args = _custom_command(parser) # ensure called using the path to the yaml spec mock_DAGSpec.assert_called_once_with('pipeline.yaml', env=EnvDict({'sample': False})) # and EnvDict initialized from env.yaml mock_EnvDict.assert_called_once_with(str(Path('env.yaml').resolve()))
def test_default_with_root(monkeypatch): mock = Mock(return_value='some_value') monkeypatch.setattr(default, 'find_root_recursively', mock) env = EnvDict(dict()) assert env.root == 'some_value'
def test_default(monkeypatch): monkeypatch.setattr(getpass, 'getuser', Mock(return_value='User')) monkeypatch.setattr(os, 'getcwd', Mock(return_value='/some_path')) env = EnvDict(dict()) assert env.cwd == str(Path('/some_path').resolve()) assert env.user == 'User'
def test_error_when_loaded_obj_is_not_dict(content, type_, tmp_directory): path = Path(tmp_directory, 'file.yaml') path.write_text(content) with pytest.raises(ValueError) as excinfo: EnvDict('file.yaml') expected = ("Expected object loaded from 'file.yaml' to be " "a dict but got '{}' instead, " "verify the content").format(type_) assert str(excinfo.value) == expected
def test_attribute_error_message(): env = EnvDict({'user': '******', 'cwd': 'cwd', 'root': 'root'}) with pytest.raises(AttributeError) as excinfo_attr: env.aa with pytest.raises(KeyError) as excinfo_key: env['aa'] assert str(excinfo_attr.value) == f"{env!r} object has no atttribute 'aa'" assert str(excinfo_key.value) == f'"{env!r} object has no key \'aa\'"'
def _process_file_dir_or_glob(parser): """ Process a file entry point file or directory), returns the initialized dag and parsed args Parameters ---------- parser : CustomParser CLI arg parser """ # look for env.yaml by searching in default locations path_to_env = default.path_to_env( Path(parser.parse_entry_point_value()).parent) if path_to_env: env_dict = EnvDict(path_to_env) _add_cli_args_from_env_dict_keys(parser, env_dict) args = parser.parse_args() if hasattr(args, 'log'): if args.log is not None: logging.basicConfig(level=args.log.upper()) entry_point = EntryPoint(args.entry_point) if entry_point.type == EntryPoint.Directory: dag = DAGSpec.from_directory(args.entry_point).to_dag() elif entry_point.type == EntryPoint.Pattern: dag = DAGSpec.from_files(args.entry_point).to_dag() else: if path_to_env: # and replace keys depending on passed cli args replaced = _env_keys_to_override(args, parser.static_args) env = env_dict._replace_flatten_keys(replaced) dag = DAGSpec(args.entry_point, env=env).to_dag() else: dag = DAGSpec(args.entry_point).to_dag() return dag, args
def decorator(fn): _validate_and_modify_signature(fn) try: env_dict = EnvDict(source) except Exception as e: raise RuntimeError('Failed to resolve environment using ' '@with_env decorator in function "{}". ' 'Tried to call Env with argument: {}'.format( _get_function_name_w_module(fn), source)) from e fn._env_dict = env_dict @wraps(fn) def wrapper(*args, **kwargs): to_replace = { k: v for k, v in kwargs.items() if k.startswith('env__') } for key in to_replace.keys(): kwargs.pop(key) env_dict_new = env_dict._replace_flatten_keys(to_replace) try: Env._init_from_decorator(env_dict_new, _get_function_name_w_module(fn)) except Exception as e: current = Env.load() raise RuntimeError('Failed to initialize environment using ' '@with_env decorator in function "{}". ' 'Current environment: {}'.format( _get_function_name_w_module(fn), repr(current))) from e Env._ref = _get_function_name_w_module(fn) try: res = fn(Env.load(), *args, **kwargs) except Exception as e: Env.end() raise e Env.end() return res return wrapper
def test_adds_default_keys_if_they_dont_exist(monkeypatch): monkeypatch.setattr(getpass, 'getuser', Mock(return_value='User')) monkeypatch.setattr(os, 'getcwd', Mock(return_value='/some_path')) mock = Mock(return_value='some_value') monkeypatch.setattr(default, 'find_root_recursively', mock) monkeypatch.setattr(expand.default, 'find_root_recursively', mock) env = EnvDict({'a': 1}, path_to_here='/dir') assert env.cwd == str(Path('/some_path').resolve()) assert env.here == str(Path('/dir').resolve()) assert env.user == 'User' assert env.root == 'some_value' assert env.default_keys == {'cwd', 'here', 'user', 'root'}
def test_serialize_env_dict(): # this tests an edge case due to EnvDict's implementation: to enable # accessing values in the underlying dictionary as attributes, we are # customizing __getattr__, however, when an object is unserialized, # Python tries to look for __getstate__ (which triggers calling # __getattr__), since it cannot find it, it will go to __getitem__ # (given the current implementation of __getattr__). But __getitem__ # uses self.preprocessed. At unserialization time, this attribute does # not exist yet!, which will cause another call to __getattr__. To avoid # this recursive loop, we have to prevent special methods to call # __getitem__ if they do not exist - EnvDict and Env objects are not # expected to be serialized but we have fix it anyway env = EnvDict({'a': 1}) assert pickle.loads(pickle.dumps(env))
def __init__(self, source=None): """Start the environment Parameters ---------- source: dict, pathlib.Path, str, optional If dict, loads it directly, if pathlib.Path, reads the file (assumes yaml format), if str, looks for a file named that way in the current directory and their parents. If None, it first looks for a file named env.{host}.yaml where host is replaced by the hostname (by calling platform.node()), if it fails, it looks for a file called env.yaml Raises ------ FileNotFoundError If source is None and an environment file cannot be found automatically RuntimeError If one environment has already started Returns ------- ploomber.Env An environment object """ if not isinstance(source, EnvDict): # try to initialize an EnvDict to perform validation, if any # errors occur, discard object try: source = EnvDict(source) except Exception: Env.__instance = None raise self._data = source self._fn_name = None
def __init__(self, data, env=None, lazy_import=False, reload=False, parent_path=None): if isinstance(data, (str, Path)): if parent_path is not None: raise ValueError('parent_path must be None when ' f'initializing {type(self).__name__} with ' 'a path to a YAML spec') # this is only used to display an error message with the path # to the loaded file path_for_errors = data # resolve the parent path to make sources and products unambiguous # even if the current working directory changes path_to_entry_point = Path(data).resolve() self._parent_path = str(path_to_entry_point.parent) content = Path(data).read_text() try: data = yaml.safe_load(content) except (yaml.parser.ParserError, yaml.constructor.ConstructorError) as e: error = e else: error = None if error: if '{{' in content or '}}' in content: raise DAGSpecInitializationError( 'Failed to initialize spec. It looks like ' 'you\'re using placeholders (i.e. {{placeholder}}). ' 'Make sure values are enclosed in parentheses ' '(e.g. key: "{{placeholder}}"). Original ' 'parser error:\n\n' f'{error}') else: raise error else: path_for_errors = None # FIXME: add test cases, some of those features wont work if # _parent_path is None. We should make sure that we either raise # an error if _parent_path is needed or use the current working # directory if it's appropriate - this is mostly to make relative # paths consistent: they should be relative to the file that # contains them self._parent_path = (None if not parent_path else str( Path(parent_path).resolve())) # try to look env.yaml in default locations env_default_path = default.path_to_env(self._parent_path) self.data = data if isinstance(self.data, list): self.data = {'tasks': self.data} # validate keys defined at the top (nested keys are not validated here) self._validate_top_keys(self.data, path_for_errors) logger.debug('DAGSpec enviroment:\n%s', pp.pformat(env)) env = env or dict() # NOTE: when loading from a path, EnvDict recursively looks # at parent folders, this is useful when loading envs # in nested directories where scripts/functions need the env # but here, since we just need this for the spec, we might # want to turn it off. should we add a parameter to EnvDict # to control this? if env_default_path: defaults = yaml.safe_load(Path(env_default_path).read_text()) self.env = EnvDict(env, path_to_here=self._parent_path, defaults=defaults) else: self.env = EnvDict(env, path_to_here=self._parent_path) self.data = expand_raw_dictionary(self.data, self.env) logger.debug('Expanded DAGSpec:\n%s', pp.pformat(data)) # if there is a "location" top key, we don't have to do anything else # as we will just load the dotted path when .to_dag() is called if 'location' not in self.data: Meta.initialize_inplace(self.data) import_tasks_from = self.data['meta']['import_tasks_from'] if import_tasks_from is not None: # when using a relative path in "import_tasks_from", we must # make it absolute... if not Path(import_tasks_from).is_absolute(): # use _parent_path if there is one if self._parent_path: self.data['meta']['import_tasks_from'] = str( Path(self._parent_path, import_tasks_from)) # otherwise just make it absolute else: self.data['meta']['import_tasks_from'] = str( Path(import_tasks_from).resolve()) imported = yaml.safe_load( Path(self.data['meta']['import_tasks_from']).read_text()) if self.env is not None: imported = expand_raw_dictionaries(imported, self.env) # relative paths here are relative to the file where they # are declared base_path = Path(self.data['meta']['import_tasks_from']).parent for task in imported: add_base_path_to_source_if_relative(task, base_path=base_path) self.data['tasks'].extend(imported) self.data['tasks'] = [ normalize_task(task) for task in self.data['tasks'] ] # make sure the folder where the pipeline is located is in sys.path # otherwise dynamic imports needed by TaskSpec will fail with add_to_sys_path(self._parent_path, chdir=False): self.data['tasks'] = [ TaskSpec(t, self.data['meta'], project_root=self._parent_path, lazy_import=lazy_import, reload=reload) for t in self.data['tasks'] ] else: self.data['meta'] = Meta.empty()
def test_add_cli_args_from_env_dict_keys(): parser = ArgumentParser() _add_cli_args_from_env_dict_keys(parser, EnvDict({'a': 1})) assert {action.dest for action in parser._actions} == {'env__a', 'help'}
def test_replace_nested_flatten_key_env_dict(): env = EnvDict({'a': {'b': 1}}) new_env = env._replace_flatten_key(2, 'env__a__b') assert new_env.a.b == 2 and env is not new_env # must return a copy
def test_replace_flatten_key_env_dict(): env = EnvDict({'a': 1}) new_env = env._replace_flatten_key(2, 'env__a') assert new_env.a == 2 and env is not new_env # must return a copy
def test_default_keys(kwargs, expected): assert EnvDict(**kwargs).default_keys == expected
def test_replace_nested_flatten_keys_env_dict(): env = EnvDict({'a': {'b': 1, 'c': 1}}) new_env = env._replace_flatten_keys({'env__a__b': 2, 'env__a__c': 2}) assert (new_env.a.b == 2 and new_env.a.c == 2 and env is not new_env) # must return a copy
def test_default_with_here_absolute(tmp_directory): here = str(Path(tmp_directory, 'dir').resolve()) env = EnvDict(dict(), path_to_here=here) assert env.here == here
def test_default_with_here_relative(tmp_directory): Path('dir').mkdir() env = EnvDict(dict(), path_to_here='dir') assert env.here == str(Path(tmp_directory, 'dir').resolve())
def test_error_when_flatten_key_doesnt_exist(): env = EnvDict({'a': 1}) with pytest.raises(KeyError): env._replace_flatten_key(2, 'env__b')