def __load_plugins_from_installed_package(self,
ip: InstalledPackage) -> list:
out = []
for _, plugindef in ip.plugins.items():
out.append(plugindef)
try:
# Build the plugin package name
module_name = LeafPluginManager.__get_plugin_module_name(
ip.identifier, plugindef.location)
# Load the module
modules = self.__load_modules_from_path(
plugindef.source_file, module_name)
# Search for the class to instanciate
cls = self.__find_class(modules, LeafPluginCommand,
plugindef.classname)
if cls is not None:
# If the class is found, instantiate the command
plugindef.command = cls(plugindef.name,
plugindef.description,
ip=plugindef.installed_package)
except BaseException:
print_trace(
"Cannot load plugin {pd.location} from {ip.folder}".format(
pd=plugindef, ip=ip))
return out
def init_leaf_settings(self):
userenvmap = self.read_user_configuration()._getenvmap()
for s in LeafSettings.values():
if s.key in userenvmap:
try:
user_value = userenvmap[s.key]
s.value = user_value
except ValueError:
print_trace("Invalid value in user scope for setting {s.identifier}: {s.key}={value}".format(s=s, value=user_value))
def __load_plugins(self, ipmap: dict) -> dict:
out = OrderedDict()
for ip in ipmap.values():
for plugindef in self.__load_plugins_from_installed_package(ip):
if plugindef.location in out:
# Plugin already defined, deactivate it
print_trace("Disable plugin {pd.location} declared more than once".format(pd=plugindef))
out[plugindef.location] = None
else:
# New plugin
out[plugindef.location] = plugindef
return out
def provision_profile(self, profile):
if not profile.folder.is_dir():
# Create folder if needed
profile.folder.mkdir(parents=True)
else:
# Clean folder content
for item in profile.folder.glob("*"):
if item.is_symlink():
item.unlink()
else:
shutil.rmtree(str(item))
# Check if all needed packages are installed
missing_packages = DependencyUtils.install(
profile.packages,
self.list_available_packages(),
self.list_installed_packages(),
env=self.build_pf_environment(profile))
if len(missing_packages) == 0:
self.logger.print_verbose("All packages are already installed")
else:
self.logger.print_default("Profile is out of sync")
try:
self.install_packages(profile.packages,
env=self.build_pf_environment(profile))
except Exception as e:
raise ProfileProvisioningException(e)
# Do all needed links
errors = 0
for ip in self.get_profile_dependencies(profile):
pi_folder = profile.folder / ip.identifier.name
if pi_folder.exists():
pi_folder = profile.folder / str(ip.identifier)
try:
env = self.build_pf_environment(profile)
self.sync_packages([ip.identifier], env=env)
pi_folder.symlink_to(ip.folder)
except Exception as e:
errors += 1
self.logger.print_error(
"Error while sync operation on {ip.identifier}".format(
ip=ip))
self.logger.print_error(str(e))
print_trace()
# Touch folder when provisionning is done without error
if errors == 0:
profile.folder.touch(exist_ok=True)
def __load_modules_from_path(self, source: Path, module_name: str) -> type:
out = []
print_trace("Load {module} from {path}".format(module=module_name, path=source))
if source.is_file() and source.suffix == ".py":
# If source is py file, load it directly
out.append(self.__load_spec(source, module_name))
elif source.is_dir() and (source / LeafPluginManager.__INIT__PY).is_file():
# If source is a py folder:
# Load the __init__.py
out.append(self.__load_spec(source / LeafPluginManager.__INIT__PY, module_name))
# The reccursively load content
for item in source.iterdir():
# Do not load __* files
if not item.name.startswith("__"):
submodule_name = "{parent}.{name}".format(parent=module_name, name=re.sub(r"\.py$", "", item.name))
out += self.__load_modules_from_path(item, submodule_name)
return out
def _list_installed_packages(self, root_folder: Path, read_only: bool) -> dict:
"""
Return all installed packages in given folder
@return: PackageIdentifier/InstalledPackage dict
"""
out = {}
if root_folder is not None and root_folder.is_dir():
for folder in root_folder.iterdir():
# iterate over non ignored sub folders
if folder.is_dir() and not is_folder_ignored(folder):
# test if a manifest exists
mffile = folder / LeafFiles.MANIFEST
if mffile.is_file():
try:
ip = InstalledPackage(mffile, read_only=read_only)
out[ip.identifier] = ip
except BaseException:
print_trace("Invalid manifest found: {mf}".format(mf=mffile))
return out
def _check_model(self):
"""
Method used to check the model and do mandatory migration
"""
# Check leaf min version
if CURRENT_LEAF_VERSION < self.leaf_min_version:
raise LeafOutOfDateException(
"Leaf has to be updated to version {min_ver}".format(
min_ver=self.leaf_min_version))
# perform upgrade if needed
if CURRENT_LEAF_VERSION > self.leaf_min_version:
# Perform migration
for update_version, updater in self._get_updaters():
if update_version is None or self.leaf_min_version < update_version:
print_trace(
"Perform config update: {fnc.__name__} ({update_version})"
.format(fnc=updater, update_version=update_version))
updater(self)
def print_renderer(self):
out = str(self)
if len(out) > 0:
print(out, file=sys.stderr)
print_trace()
def _download_file_http(url: str,
output: Path,
logger: TextLogger,
resume: bool = None,
retry: int = None,
buffer_size: int = 262144):
# Handle default values
if retry is None:
retry = LeafSettings.DOWNLOAD_RETRY.as_int()
if resume is None:
resume = not LeafSettings.DOWNLOAD_NORESUME.as_boolean()
_display_progress(logger, "Downloading {0.name}".format(output))
iteration = 0
while True:
try:
headers = {}
size_current = 0
if output.exists():
if resume:
size_current = output.stat().st_size
headers = {"Range": "bytes={0}-".format(size_current)}
else:
output.unlink()
with output.open("ab" if resume else "wb") as fp:
req = requests.get(
url,
stream=True,
headers=headers,
timeout=LeafSettings.DOWNLOAD_TIMEOUT.as_int())
# Get total size on first request
size_total = int(req.headers.get("content-length",
-1)) + size_current
# Read remote data and write to output file
for data in req.iter_content(buffer_size):
size_current += fp.write(data)
_display_progress(logger,
"Downloading {0.name}".format(output),
size_current, size_total)
# Rare case when no exception raised and download is not finished
if 0 < size_current < size_total:
raise ValueError("Incomplete download")
# End the progress display
_display_progress(logger,
"Downloading {0.name}".format(output),
1,
1,
end="\n")
return size_current
except (ValueError, requests.RequestException,
requests.ConnectionError, requests.HTTPError,
requests.Timeout) as e:
iteration += 1
# Check retry
if iteration > retry:
raise e
# Log the retry attempt
if logger:
logger.print_default(
"\nError while downloading, retry {0}/{1}".format(
iteration, retry))
print_trace()
# Prevent imediate retry
time.sleep(1)
