def _get_config(path: Union[pathlib.Path, os.PathLike]) -> Dict[str, Any]: if not os.path.exists(path): utils.log_and_raise(logger.error, f"Path {str(path)} does not exist.", ValueError, errors.GP_PATH_DOES_NOT_EXIST) with open(path) as fh: return yaml.load(fh, Loader=yaml.BaseLoader)
def get_topological_sort(self) -> List[GraphNode[Action]]: """Get's the topological sort to account for dependencies. Uses a modified version of Kahn's algorithm. """ active_list = [self._root] sorted_list: List[GraphNode[Action]] = [] known_node_count = self._get_num_nodes() key = str(uuid.uuid1()) while len(active_list) > 0: node = active_list.pop() node.set_visited(key) sorted_list.append(node) for child in node.children: if any(not parent.was_visited(key) for parent in child.parents): continue else: active_list.append(child) if len(sorted_list) < known_node_count: utils.log_and_raise( logger.error, "Found circular dependency in dependency graph.", CircularDependencyException, errors.EG_CIRCULAR_DEPENDENCY) elif len(sorted_list) > known_node_count: utils.log_and_raise( logger.error, "Hit an impossible state, you've found a bug!", errors.ImpossibleStateException, errors.EG_IMPOSSIBLE_STATE) return sorted_list
def _get_klass_for_node(node: str) -> Type: """Gets the class associated with a top-level configuration node like "file_syncs".""" klass = KLASS_MAP.get(node, None) if klass is None: utils.log_and_raise(logger.error, f"Couldn't map yaml node {node} to a class.", ValueError, errors.GP_NO_CLASS_MAP) return klass
def execute(self, exec_context: ExecutionContext) -> None: """Attempts to copy the source file to the destination. Gets a source file from self.file_source, an example of the Adapter pattern. Returns: True of the execution was successful. Raises: ActionFailureException: An error occurred executing this action. """ source_path = self.file_source.get_file_path() if pathlib.Path(self.dest_path).resolve().is_file(): if not self.overwrite: utils.log_and_raise( logger.error, f"File exists at {self.dest_path} and overwrite is not set to true.", ActionFailureException, ) else: if not self._handle_modified_date(exec_context): return shutil.copy(source_path, self.dest_path)
def _handle_modified_date(self, exec_context: ExecutionContext) -> bool: """Based on the ModifiedDateDecisionEnum returned by compare_modified date, take an action. Args: exec_context: The execution context Returns: True if the action should finish executing, False if not. """ choice = self.file_source.compare_modified_date(self.dest_path) if choice is not None: if choice == ModifiedDateDecisionEnum.stop_execution: utils.log_and_raise( logger.error, f"User stopped execution at action with key {self.key}", UserStoppedExecutionException, errors.AC_USER_STOPPED_EXECUTION, ) elif choice == ModifiedDateDecisionEnum.skip_this_action: logger.info(f"Skipped action with key {self.key}") return False elif choice == ModifiedDateDecisionEnum.proceed_once: pass elif choice == ModifiedDateDecisionEnum.ignore_in_future: exec_context.skip_modified_date_warning = True return True return True
def _set_repo_name(self) -> None: if not validators.url(self.repo_url): utils.log_and_raise( logger.error, f"Invalid URL {self.repo_url}.", ValueError, errors.AC_BAD_GITHUB_URL, ) self._repo_name = ".".join(self.repo_url.split("/")[-2:])
def map_location_type(location_type: str) -> actions.FileSyncBackendType: if location_type == const.LOCATION_TYPE_LOCAL: return actions.FileSyncBackendType.local elif location_type == const.LOCATION_TYPE_GITHUB: return actions.FileSyncBackendType.github else: utils.log_and_raise( logger.error, f"Invalid location_type {location_type}.", NotImplementedError, errors.NP_INVALID_LOCATION_TYPE, )
def _get_func_for_klass(klass: Type) -> Callable: """Gets the parser function for a given class `klass`. This strategy intends to better encapsulate changes to the input file format, avoiding changes to the underlying action class's __init__ as a result. Args: klass: The class to map Returns: A parser function that returns an instance of `klass`. """ func = PARSER_MAP.get(klass, None) if func is None: utils.log_and_raise(logger.error, f"Couldn't map class {klass} to parser function", ValueError, errors.GP_NO_PARSER_FUNC_MAP) return func
def _build_dependency_graph( action_list: List[Action], object_mapping: Dict[str, object] ) -> None: """Builds the Dependency objects from Action.dependency_keys for each action. Args: action_list: The list of all actions. object_mapping: Mapping from action.key to action instance. """ for action in action_list: if len(action.dependency_keys) > 0: try: any( action.add_dependency(actions.Dependency(obj)) for obj in [object_mapping[key] for key in action.dependency_keys] ) except KeyError as err: utils.log_and_raise(logger.error, f"Dependency key {err.args[0]} does not refer to an object that exists.", KeyError, errors.GP_BAD_DEPENDENCY_REF)
def parse_installation( file_sync_config: Dict[str, Union[str, bool, int]], exec_context: Optional[ExecutionContext] = None, ) -> actions.Installation: """Creates an Installation object from configuration.""" if any(x not in file_sync_config for x in REQUIRED_INSTALLATION_NODES): missing_nodes = filter( lambda x: x not in file_sync_config, [node for node in REQUIRED_INSTALLATION_NODES], ) utils.log_and_raise( logger.error, f"File sync config missing nodes {','.join(missing_nodes)}", ValueError, errors.NP_MISSING_FILE_SYNC_CONFIG, ) return actions.Installation( install_command=file_sync_config.get(const.INSTALL_COMMAND, None), check_command=file_sync_config[const.CHECK_COMMAND], key=file_sync_config[const.NODE_KEY], dependency_keys=file_sync_config.get(const.DEPENDENCY, None))
def _build_nodes( action_list: List[Action], object_node_mapping: Dict[Action, GraphNode[Action]] ) -> None: """Builds GraphNodes based on actions with dependencies. Args: action_list: The list of all actions. object_node_mapping: A mapping from action instance to the GraphNode instance that contains it. """ for action in action_list: if len(action.dependencies) > 0: try: node = object_node_mapping[action] any( node.add_child(dep_node) for dep_node in [ object_node_mapping[dep_obj] for dep_obj in [dep.value for dep in action.dependencies] ] ) except KeyError as err: utils.log_and_raise(logger.error, f"Coudldn't find execution graph node for object with key {err.args[0]}", KeyError)
def execute(self, exec_context: Optional[ExecutionContext] = None): """See base class.""" if self._is_installed(): return if not self._install(): utils.log_and_raise( logger.error, f"Failed to install Installation with key {self.key}", ActionFailureException, errors.AC_FAILED_TO_INSTALL, ) if not self._is_installed(): if self.install_command is not None: logger.warning( f"Installation with key {self.key} failed install check after" f"successful installation.") return utils.log_and_raise( logger.error, f"'Check-only' installation with key {self.key} is not installed.", ActionFailureException, errors.AC_CHECK_ONLY_INSTALLATION_NOT_INSTALLED)
def _clone(self) -> None: try: dir_to_save = os.path.join(self._app_dir, self._repo_name) try: shutil.rmtree(dir_to_save) except FileNotFoundError: pass self._repo = git.Repo.clone_from(self.repo_url, dir_to_save) self._path = os.path.join(dir_to_save, self.relative_path) except git.exc.GitCommandError as err: utils.log_and_raise( logger.error, f"There was a problem downloading from the git repo at {self.repo_url} to {self._app_dir}", err, errors.AC_FAILED_TO_CLONE, ) if not pathlib.Path(self._path).resolve().is_file(): utils.log_and_raise( logger.error, f"Github file to sync at path {self._path} does not exist.", ValueError, errors.AC_BAD_FINAL_FILE_PATH, )
def parse_file_sync( file_sync_config: Dict[str, Union[str, bool, int]], exec_context: Optional[ExecutionContext] = None, ) -> actions.FileSync: """Creates an Action object from configuration.""" if (const.LOCATION_TYPE_NODE not in file_sync_config or file_sync_config[const.LOCATION_TYPE_NODE] not in LOCATION_TYPES): utils.log_and_raise( logger.error, f"File sync config missing {const.LOCATION_TYPE_NODE}", ValueError, errors.NP_MISSING_LOCATION_TYPE, ) if (exec_context is None and file_sync_config[const.LOCATION_TYPE_NODE] == const.LOCATION_TYPE_GITHUB): utils.log_and_raise( logger.error, f"Execution contet must be passed in when file sync is of type {const.LOCATION_TYPE_GITHUB}.", ValueError, errors.NP_MISSING_EXEC_CONTEXT, ) dependency_keys = file_sync_config.get(const.DEPENDENCY, []) backend = map_location_type(file_sync_config[const.LOCATION_TYPE_NODE]) if any(node not in file_sync_config for node in REQUIRED_FS_NODES[backend]): utils.log_and_raise( logger.error, "File sync config missing some required elements", ValueError, errors.NP_MISSING_FILE_SYNC_CONFIG, ) file_source = None if backend == actions.FileSyncBackendType.local: file_source = actions.LocalFileLocation( pathlib.Path(file_sync_config[const.SOURCE_FILE_PATH])) elif backend == actions.FileSyncBackendType.github: file_source = actions.GithubFileLocation( file_sync_config[const.REPOSITORY], pathlib.PurePath(file_sync_config[const.SOURCE_FILE_PATH]), exec_context.pyrsonalizer_directory, ) return actions.FileSync( backend=backend, file_source=file_source, local_path=pathlib.Path(file_sync_config[const.DEST_FILE_PATH]), overwrite=file_sync_config.get(const.OVERWRITE, False), key=file_sync_config[const.NODE_KEY], dependency_keys=dependency_keys, )