def _get_config(path: Union[pathlib.Path, os.PathLike]) -> Dict[str, Any]:
if not os.path.exists(path):
utils.log_and_raise(logger.error, f"Path {str(path)} does not exist.", ValueError,
errors.GP_PATH_DOES_NOT_EXIST)
with open(path) as fh:
return yaml.load(fh, Loader=yaml.BaseLoader)
def get_topological_sort(self) -> List[GraphNode[Action]]:
"""Get's the topological sort to account for dependencies.
Uses a modified version of Kahn's algorithm.
"""
active_list = [self._root]
sorted_list: List[GraphNode[Action]] = []
known_node_count = self._get_num_nodes()
key = str(uuid.uuid1())
while len(active_list) > 0:
node = active_list.pop()
node.set_visited(key)
sorted_list.append(node)
for child in node.children:
if any(not parent.was_visited(key)
for parent in child.parents):
continue
else:
active_list.append(child)
if len(sorted_list) < known_node_count:
utils.log_and_raise(
logger.error, "Found circular dependency in dependency graph.",
CircularDependencyException, errors.EG_CIRCULAR_DEPENDENCY)
elif len(sorted_list) > known_node_count:
utils.log_and_raise(
logger.error, "Hit an impossible state, you've found a bug!",
errors.ImpossibleStateException, errors.EG_IMPOSSIBLE_STATE)
return sorted_list
def _get_klass_for_node(node: str) -> Type:
"""Gets the class associated with a top-level configuration node like "file_syncs"."""
klass = KLASS_MAP.get(node, None)
if klass is None:
utils.log_and_raise(logger.error, f"Couldn't map yaml node {node} to a class.", ValueError,
errors.GP_NO_CLASS_MAP)
return klass
def execute(self, exec_context: ExecutionContext) -> None:
"""Attempts to copy the source file to the destination.
Gets a source file from self.file_source, an example of the Adapter pattern.
Returns:
True of the execution was successful.
Raises:
ActionFailureException: An error occurred executing this action.
"""
source_path = self.file_source.get_file_path()
if pathlib.Path(self.dest_path).resolve().is_file():
if not self.overwrite:
utils.log_and_raise(
logger.error,
f"File exists at {self.dest_path} and overwrite is not set to true.",
ActionFailureException,
)
else:
if not self._handle_modified_date(exec_context):
return
shutil.copy(source_path, self.dest_path)
def _handle_modified_date(self, exec_context: ExecutionContext) -> bool:
"""Based on the ModifiedDateDecisionEnum returned by compare_modified date, take
an action.
Args:
exec_context: The execution context
Returns:
True if the action should finish executing, False if not.
"""
choice = self.file_source.compare_modified_date(self.dest_path)
if choice is not None:
if choice == ModifiedDateDecisionEnum.stop_execution:
utils.log_and_raise(
logger.error,
f"User stopped execution at action with key {self.key}",
UserStoppedExecutionException,
errors.AC_USER_STOPPED_EXECUTION,
)
elif choice == ModifiedDateDecisionEnum.skip_this_action:
logger.info(f"Skipped action with key {self.key}")
return False
elif choice == ModifiedDateDecisionEnum.proceed_once:
pass
elif choice == ModifiedDateDecisionEnum.ignore_in_future:
exec_context.skip_modified_date_warning = True
return True
return True
def _set_repo_name(self) -> None:
if not validators.url(self.repo_url):
utils.log_and_raise(
logger.error,
f"Invalid URL {self.repo_url}.",
ValueError,
errors.AC_BAD_GITHUB_URL,
)
self._repo_name = ".".join(self.repo_url.split("/")[-2:])
def map_location_type(location_type: str) -> actions.FileSyncBackendType:
if location_type == const.LOCATION_TYPE_LOCAL:
return actions.FileSyncBackendType.local
elif location_type == const.LOCATION_TYPE_GITHUB:
return actions.FileSyncBackendType.github
else:
utils.log_and_raise(
logger.error,
f"Invalid location_type {location_type}.",
NotImplementedError,
errors.NP_INVALID_LOCATION_TYPE,
)
def _get_func_for_klass(klass: Type) -> Callable:
"""Gets the parser function for a given class `klass`.
This strategy intends to better encapsulate changes to the input file format,
avoiding changes to the underlying action class's __init__ as a result.
Args:
klass: The class to map
Returns:
A parser function that returns an instance of `klass`.
"""
func = PARSER_MAP.get(klass, None)
if func is None:
utils.log_and_raise(logger.error, f"Couldn't map class {klass} to parser function", ValueError,
errors.GP_NO_PARSER_FUNC_MAP)
return func
def _build_dependency_graph(
action_list: List[Action], object_mapping: Dict[str, object]
) -> None:
"""Builds the Dependency objects from Action.dependency_keys for each action.
Args:
action_list: The list of all actions.
object_mapping: Mapping from action.key to action instance.
"""
for action in action_list:
if len(action.dependency_keys) > 0:
try:
any(
action.add_dependency(actions.Dependency(obj))
for obj in [object_mapping[key] for key in action.dependency_keys]
)
except KeyError as err:
utils.log_and_raise(logger.error,
f"Dependency key {err.args[0]} does not refer to an object that exists.", KeyError,
errors.GP_BAD_DEPENDENCY_REF)
def parse_installation(
file_sync_config: Dict[str, Union[str, bool, int]],
exec_context: Optional[ExecutionContext] = None,
) -> actions.Installation:
"""Creates an Installation object from configuration."""
if any(x not in file_sync_config for x in REQUIRED_INSTALLATION_NODES):
missing_nodes = filter(
lambda x: x not in file_sync_config,
[node for node in REQUIRED_INSTALLATION_NODES],
)
utils.log_and_raise(
logger.error,
f"File sync config missing nodes {','.join(missing_nodes)}",
ValueError,
errors.NP_MISSING_FILE_SYNC_CONFIG,
)
return actions.Installation(
install_command=file_sync_config.get(const.INSTALL_COMMAND, None),
check_command=file_sync_config[const.CHECK_COMMAND],
key=file_sync_config[const.NODE_KEY],
dependency_keys=file_sync_config.get(const.DEPENDENCY, None))
def _build_nodes(
action_list: List[Action], object_node_mapping: Dict[Action, GraphNode[Action]]
) -> None:
"""Builds GraphNodes based on actions with dependencies.
Args:
action_list: The list of all actions.
object_node_mapping: A mapping from action instance to the GraphNode instance that contains it.
"""
for action in action_list:
if len(action.dependencies) > 0:
try:
node = object_node_mapping[action]
any(
node.add_child(dep_node)
for dep_node in [
object_node_mapping[dep_obj]
for dep_obj in [dep.value for dep in action.dependencies]
]
)
except KeyError as err:
utils.log_and_raise(logger.error,
f"Coudldn't find execution graph node for object with key {err.args[0]}", KeyError)
def execute(self, exec_context: Optional[ExecutionContext] = None):
"""See base class."""
if self._is_installed():
return
if not self._install():
utils.log_and_raise(
logger.error,
f"Failed to install Installation with key {self.key}",
ActionFailureException,
errors.AC_FAILED_TO_INSTALL,
)
if not self._is_installed():
if self.install_command is not None:
logger.warning(
f"Installation with key {self.key} failed install check after"
f"successful installation.")
return
utils.log_and_raise(
logger.error,
f"'Check-only' installation with key {self.key} is not installed.",
ActionFailureException,
errors.AC_CHECK_ONLY_INSTALLATION_NOT_INSTALLED)
def _clone(self) -> None:
try:
dir_to_save = os.path.join(self._app_dir, self._repo_name)
try:
shutil.rmtree(dir_to_save)
except FileNotFoundError:
pass
self._repo = git.Repo.clone_from(self.repo_url, dir_to_save)
self._path = os.path.join(dir_to_save, self.relative_path)
except git.exc.GitCommandError as err:
utils.log_and_raise(
logger.error,
f"There was a problem downloading from the git repo at {self.repo_url} to {self._app_dir}",
err,
errors.AC_FAILED_TO_CLONE,
)
if not pathlib.Path(self._path).resolve().is_file():
utils.log_and_raise(
logger.error,
f"Github file to sync at path {self._path} does not exist.",
ValueError,
errors.AC_BAD_FINAL_FILE_PATH,
)
def parse_file_sync(
file_sync_config: Dict[str, Union[str, bool, int]],
exec_context: Optional[ExecutionContext] = None,
) -> actions.FileSync:
"""Creates an Action object from configuration."""
if (const.LOCATION_TYPE_NODE not in file_sync_config or
file_sync_config[const.LOCATION_TYPE_NODE] not in LOCATION_TYPES):
utils.log_and_raise(
logger.error,
f"File sync config missing {const.LOCATION_TYPE_NODE}",
ValueError,
errors.NP_MISSING_LOCATION_TYPE,
)
if (exec_context is None and file_sync_config[const.LOCATION_TYPE_NODE]
== const.LOCATION_TYPE_GITHUB):
utils.log_and_raise(
logger.error,
f"Execution contet must be passed in when file sync is of type {const.LOCATION_TYPE_GITHUB}.",
ValueError,
errors.NP_MISSING_EXEC_CONTEXT,
)
dependency_keys = file_sync_config.get(const.DEPENDENCY, [])
backend = map_location_type(file_sync_config[const.LOCATION_TYPE_NODE])
if any(node not in file_sync_config
for node in REQUIRED_FS_NODES[backend]):
utils.log_and_raise(
logger.error,
"File sync config missing some required elements",
ValueError,
errors.NP_MISSING_FILE_SYNC_CONFIG,
)
file_source = None
if backend == actions.FileSyncBackendType.local:
file_source = actions.LocalFileLocation(
pathlib.Path(file_sync_config[const.SOURCE_FILE_PATH]))
elif backend == actions.FileSyncBackendType.github:
file_source = actions.GithubFileLocation(
file_sync_config[const.REPOSITORY],
pathlib.PurePath(file_sync_config[const.SOURCE_FILE_PATH]),
exec_context.pyrsonalizer_directory,
)
return actions.FileSync(
backend=backend,
file_source=file_source,
local_path=pathlib.Path(file_sync_config[const.DEST_FILE_PATH]),
overwrite=file_sync_config.get(const.OVERWRITE, False),
key=file_sync_config[const.NODE_KEY],
dependency_keys=dependency_keys,
)