def __load_plugins_from_installed_package(self, ip: InstalledPackage) -> list: out = [] for _, plugindef in ip.plugins.items(): out.append(plugindef) try: # Build the plugin package name module_name = LeafPluginManager.__get_plugin_module_name( ip.identifier, plugindef.location) # Load the module modules = self.__load_modules_from_path( plugindef.source_file, module_name) # Search for the class to instanciate cls = self.__find_class(modules, LeafPluginCommand, plugindef.classname) if cls is not None: # If the class is found, instantiate the command plugindef.command = cls(plugindef.name, plugindef.description, ip=plugindef.installed_package) except BaseException: print_trace( "Cannot load plugin {pd.location} from {ip.folder}".format( pd=plugindef, ip=ip)) return out
def init_leaf_settings(self): userenvmap = self.read_user_configuration()._getenvmap() for s in LeafSettings.values(): if s.key in userenvmap: try: user_value = userenvmap[s.key] s.value = user_value except ValueError: print_trace("Invalid value in user scope for setting {s.identifier}: {s.key}={value}".format(s=s, value=user_value))
def __load_plugins(self, ipmap: dict) -> dict: out = OrderedDict() for ip in ipmap.values(): for plugindef in self.__load_plugins_from_installed_package(ip): if plugindef.location in out: # Plugin already defined, deactivate it print_trace("Disable plugin {pd.location} declared more than once".format(pd=plugindef)) out[plugindef.location] = None else: # New plugin out[plugindef.location] = plugindef return out
def provision_profile(self, profile): if not profile.folder.is_dir(): # Create folder if needed profile.folder.mkdir(parents=True) else: # Clean folder content for item in profile.folder.glob("*"): if item.is_symlink(): item.unlink() else: shutil.rmtree(str(item)) # Check if all needed packages are installed missing_packages = DependencyUtils.install( profile.packages, self.list_available_packages(), self.list_installed_packages(), env=self.build_pf_environment(profile)) if len(missing_packages) == 0: self.logger.print_verbose("All packages are already installed") else: self.logger.print_default("Profile is out of sync") try: self.install_packages(profile.packages, env=self.build_pf_environment(profile)) except Exception as e: raise ProfileProvisioningException(e) # Do all needed links errors = 0 for ip in self.get_profile_dependencies(profile): pi_folder = profile.folder / ip.identifier.name if pi_folder.exists(): pi_folder = profile.folder / str(ip.identifier) try: env = self.build_pf_environment(profile) self.sync_packages([ip.identifier], env=env) pi_folder.symlink_to(ip.folder) except Exception as e: errors += 1 self.logger.print_error( "Error while sync operation on {ip.identifier}".format( ip=ip)) self.logger.print_error(str(e)) print_trace() # Touch folder when provisionning is done without error if errors == 0: profile.folder.touch(exist_ok=True)
def __load_modules_from_path(self, source: Path, module_name: str) -> type: out = [] print_trace("Load {module} from {path}".format(module=module_name, path=source)) if source.is_file() and source.suffix == ".py": # If source is py file, load it directly out.append(self.__load_spec(source, module_name)) elif source.is_dir() and (source / LeafPluginManager.__INIT__PY).is_file(): # If source is a py folder: # Load the __init__.py out.append(self.__load_spec(source / LeafPluginManager.__INIT__PY, module_name)) # The reccursively load content for item in source.iterdir(): # Do not load __* files if not item.name.startswith("__"): submodule_name = "{parent}.{name}".format(parent=module_name, name=re.sub(r"\.py$", "", item.name)) out += self.__load_modules_from_path(item, submodule_name) return out
def _list_installed_packages(self, root_folder: Path, read_only: bool) -> dict: """ Return all installed packages in given folder @return: PackageIdentifier/InstalledPackage dict """ out = {} if root_folder is not None and root_folder.is_dir(): for folder in root_folder.iterdir(): # iterate over non ignored sub folders if folder.is_dir() and not is_folder_ignored(folder): # test if a manifest exists mffile = folder / LeafFiles.MANIFEST if mffile.is_file(): try: ip = InstalledPackage(mffile, read_only=read_only) out[ip.identifier] = ip except BaseException: print_trace("Invalid manifest found: {mf}".format(mf=mffile)) return out
def _check_model(self): """ Method used to check the model and do mandatory migration """ # Check leaf min version if CURRENT_LEAF_VERSION < self.leaf_min_version: raise LeafOutOfDateException( "Leaf has to be updated to version {min_ver}".format( min_ver=self.leaf_min_version)) # perform upgrade if needed if CURRENT_LEAF_VERSION > self.leaf_min_version: # Perform migration for update_version, updater in self._get_updaters(): if update_version is None or self.leaf_min_version < update_version: print_trace( "Perform config update: {fnc.__name__} ({update_version})" .format(fnc=updater, update_version=update_version)) updater(self)
def print_renderer(self): out = str(self) if len(out) > 0: print(out, file=sys.stderr) print_trace()
def _download_file_http(url: str, output: Path, logger: TextLogger, resume: bool = None, retry: int = None, buffer_size: int = 262144): # Handle default values if retry is None: retry = LeafSettings.DOWNLOAD_RETRY.as_int() if resume is None: resume = not LeafSettings.DOWNLOAD_NORESUME.as_boolean() _display_progress(logger, "Downloading {0.name}".format(output)) iteration = 0 while True: try: headers = {} size_current = 0 if output.exists(): if resume: size_current = output.stat().st_size headers = {"Range": "bytes={0}-".format(size_current)} else: output.unlink() with output.open("ab" if resume else "wb") as fp: req = requests.get( url, stream=True, headers=headers, timeout=LeafSettings.DOWNLOAD_TIMEOUT.as_int()) # Get total size on first request size_total = int(req.headers.get("content-length", -1)) + size_current # Read remote data and write to output file for data in req.iter_content(buffer_size): size_current += fp.write(data) _display_progress(logger, "Downloading {0.name}".format(output), size_current, size_total) # Rare case when no exception raised and download is not finished if 0 < size_current < size_total: raise ValueError("Incomplete download") # End the progress display _display_progress(logger, "Downloading {0.name}".format(output), 1, 1, end="\n") return size_current except (ValueError, requests.RequestException, requests.ConnectionError, requests.HTTPError, requests.Timeout) as e: iteration += 1 # Check retry if iteration > retry: raise e # Log the retry attempt if logger: logger.print_default( "\nError while downloading, retry {0}/{1}".format( iteration, retry)) print_trace() # Prevent imediate retry time.sleep(1)