def get_root(self): root = default.find_root_recursively() if root is None: raise ValueError('Failed to expand {{root}}, could not ' 'find a setup.py in a parent folder') return root
def _default_dict(include_here): placeholders = { 'user': '******', 'cwd': '{{cwd}}', } if default.find_root_recursively() is not None: placeholders['root'] = '{{root}}' if include_here: placeholders['here'] = '{{here}}' return placeholders
def __init__(self, path_to_backup_dir, path_to_project_root=None): self._path_to_backup_dir = Path(path_to_backup_dir) self._path_to_backup_dir.mkdir(exist_ok=True, parents=True) if path_to_project_root: project_root = path_to_project_root else: try: project_root = find_root_recursively() except Exception as e: raise DAGSpecInvalidError( f'Cannot initialize {self!r} because there ' 'is not project root. Set one or explicitly pass ' 'a value in the path_to_project_root argument') from e self._path_to_project_root = Path(project_root).resolve()
def __init__(self, bucket_name, parent, json_credentials_path=None, path_to_project_root=None, credentials_relative_to_project_root=True, **kwargs): if path_to_project_root: project_root = path_to_project_root else: try: project_root = find_root_recursively() except Exception as e: raise DAGSpecInvalidError( f'Cannot initialize {type(self).__name__} because there ' 'is not project root. Set one or explicitly pass ' 'a value in the path_to_project_root argument') from e self._path_to_project_root = Path(project_root).resolve() if (credentials_relative_to_project_root and json_credentials_path and not Path(json_credentials_path).is_absolute()): json_credentials_path = Path(self._path_to_project_root, json_credentials_path) self._client_kwargs = kwargs if json_credentials_path: c = json.loads(Path(json_credentials_path).read_text()) self._client_kwargs = { 'aws_access_key_id': c['aws_access_key_id'], 'aws_secret_access_key': c['aws_secret_access_key'], **kwargs } self._client = self._init_client() self._parent = parent self._bucket_name = bucket_name
def __init__(self, bucket_name, parent, json_credentials_path=None, path_to_project_root=None, credentials_relative_to_project_root=True, **kwargs): if path_to_project_root: project_root = path_to_project_root else: try: project_root = find_root_recursively() except Exception as e: raise DAGSpecInvalidError( f'Cannot initialize {type(self).__name__} because there ' 'is not project root. Set one or explicitly pass ' 'a value in the path_to_project_root argument') from e self._path_to_project_root = Path(project_root).resolve() if (credentials_relative_to_project_root and json_credentials_path and not Path(json_credentials_path).is_absolute()): json_credentials_path = Path(self._path_to_project_root, json_credentials_path) self._from_json = json_credentials_path is not None if not self._from_json: self._client_kwargs = kwargs else: self._client_kwargs = { 'json_credentials_path': json_credentials_path, **kwargs } storage_client = self._init_client() self._parent = parent self._bucket_name = bucket_name self._bucket = storage_client.bucket(bucket_name)
def _init(self, data, env, lazy_import, reload, parent_path, look_up_project_root_recursively): self._lazy_import = lazy_import # initialized with a path to a yaml file... if isinstance(data, (str, Path)): # TODO: test this if parent_path is not None: raise ValueError('parent_path must be None when ' f'initializing {type(self).__name__} with ' 'a path to a YAML spec') # resolve the parent path to make sources and products unambiguous # even if the current working directory changes self._path = Path(data).resolve() self._parent_path = str(self._path.parent) if not Path(data).is_file(): raise FileNotFoundError( 'Error initializing DAGSpec with argument ' f'{data!r}: Expected it to be a path to a YAML file, but ' 'such file does not exist') content = Path(data).read_text() try: data = yaml.safe_load(content) except (yaml.parser.ParserError, yaml.constructor.ConstructorError) as e: error = e else: error = None if error: if '{{' in content or '}}' in content: raise DAGSpecInitializationError( 'Failed to initialize spec. It looks like ' 'you\'re using placeholders (i.e. {{placeholder}}). ' 'Make sure values are enclosed in parentheses ' '(e.g. key: "{{placeholder}}"). Original ' 'parser error:\n\n' f'{error}') else: raise error # initialized with a dictionary... else: self._path = None # FIXME: add test cases, some of those features wont work if # _parent_path is None. We should make sure that we either raise # an error if _parent_path is needed or use the current working # directory if it's appropriate - this is mostly to make relative # paths consistent: they should be relative to the file that # contains them self._parent_path = (None if not parent_path else str( Path(parent_path).resolve())) self.data = data if isinstance(self.data, list): self.data = {'tasks': self.data} # validate keys defined at the top (nested keys are not validated here) self._validate_top_keys(self.data, self._path) logger.debug('DAGSpec enviroment:\n%s', pp.pformat(env)) env = env or dict() path_to_defaults = default.path_to_env_from_spec( path_to_spec=self._path) if path_to_defaults: defaults = yaml.safe_load(Path(path_to_defaults).read_text()) self.env = EnvDict(env, path_to_here=self._parent_path, defaults=defaults) else: self.env = EnvDict(env, path_to_here=self._parent_path) self.data, tags = expand_raw_dictionary_and_extract_tags( self.data, self.env) logger.debug('Expanded DAGSpec:\n%s', pp.pformat(data)) # if there is a "location" top key, we don't have to do anything else # as we will just load the dotted path when .to_dag() is called if 'location' not in self.data: Meta.initialize_inplace(self.data) import_tasks_from = self.data['meta']['import_tasks_from'] if import_tasks_from is not None: # when using a relative path in "import_tasks_from", we must # make it absolute... if not Path(import_tasks_from).is_absolute(): # use _parent_path if there is one if self._parent_path: self.data['meta']['import_tasks_from'] = str( Path(self._parent_path, import_tasks_from)) # otherwise just make it absolute else: self.data['meta']['import_tasks_from'] = str( Path(import_tasks_from).resolve()) imported = yaml.safe_load( Path(self.data['meta']['import_tasks_from']).read_text()) if self.env is not None: (imported, tags_other) = expand_raw_dictionaries_and_extract_tags( imported, self.env) tags = tags | tags_other # relative paths here are relative to the file where they # are declared base_path = Path(self.data['meta']['import_tasks_from']).parent for task in imported: add_base_path_to_source_if_relative(task, base_path=base_path) self.data['tasks'].extend(imported) # check if there are any params declared in env, not used in # in the pipeline extra = set(self.env) - self.env.default_keys - tags if extra: warnings.warn('The following placeholders are declared in the ' 'environment but ' f'unused in the spec: {extra}') self.data['tasks'] = [ normalize_task(task) for task in self.data['tasks'] ] # NOTE: for simple projects, project root is the parent folder # of pipeline.yaml, for package projects is the parent folder # of setup.py if look_up_project_root_recursively: project_root = ( None if not self._parent_path else default.find_root_recursively( starting_dir=self._parent_path, filename=None if not self._path else self._path.name)) else: project_root = self._parent_path # make sure the folder where the pipeline is located is in sys.path # otherwise dynamic imports needed by TaskSpec will fail with add_to_sys_path(self._parent_path, chdir=False): self.data['tasks'] = [ TaskSpec(t, self.data['meta'], project_root=project_root, lazy_import=lazy_import, reload=reload) for t in self.data['tasks'] ] else: self.data['meta'] = Meta.empty()