def create_package(self, input_folder: Path, output_file: Path, store_extenal_info: bool = True, tar_extra_args: list = None, validate_only: bool = False):
"""
Create a leaf artifact from given folder containing a manifest.json
"""
mffile = input_folder / LeafFiles.MANIFEST
infofile = self.find_external_info_file(output_file)
if not mffile.exists():
raise LeafException("Cannot find manifest: {file}".format(file=mffile))
manifest = Manifest.parse(mffile)
manifest.validate_model()
if not validate_only:
if is_latest_package(manifest.identifier):
raise LeafException("Invalid version for manifest {mf} ({kw} is a reserved keyword)".format(mf=mffile, kw=LeafConstants.LATEST))
self.logger.print_default("Found package {mf.identifier} in {folder}".format(mf=manifest, folder=input_folder))
# Check if external info file exists
if not store_extenal_info and infofile.exists():
raise LeafException(
"A previous info file ({file}) exists for your package".format(file=infofile),
hints="You should remove it with 'rm {file}'".format(file=infofile),
)
self.__exec_tar(output_file, input_folder, extra_args=tar_extra_args)
self.logger.print_default("Leaf package created: {file}".format(file=output_file))
if store_extenal_info:
self.logger.print_default("Write info to {file}".format(file=infofile))
jwritefile(infofile, self.__build_pkg_node(output_file, manifest=manifest), pp=True)
def __exec_tar(self, output: Path, workdir: Path, extra_args: list = None):
tar = "tar"
if LeafSettings.CUSTOM_TAR.is_set():
tar = LeafSettings.CUSTOM_TAR.value
command = [tar, "-c"]
command += ["-f", output]
command += ["-C", workdir]
if extra_args is not None and len(extra_args) > 0:
forbidden_args = set(
extra_args) & RelengManager.__TAR_FORBIDDEN_ARGS
if len(forbidden_args) > 0:
raise LeafException(
"You should not use tar extra arguments: {invalid_args}".
format(invalid_args=" ".join(forbidden_args)))
command += extra_args
else:
command.append(".")
command_text = " ".join(map(str, command))
self.logger.print_default(
"Executing command: {cmd}".format(cmd=command_text))
rc = subprocess.call(list(map(str, command)),
stdout=None,
stderr=subprocess.STDOUT)
if rc != 0:
raise LeafException(
"Error executing: {cmd}".format(cmd=command_text))
def is_profile_sync(self, profile: Profile, raise_if_not_sync=False):
"""
Check if a profile contains all needed links to all contained packages
"""
try:
linked_pi_list = [
ip.identifier for ip in profile.list_linked_packages()
]
needed_pi_list = [
ip.identifier for ip in self.get_profile_dependencies(profile)
]
for pi in needed_pi_list:
if pi not in linked_pi_list:
raise LeafException(
"Missing package link for {pi}".format(pi=pi))
for pi in linked_pi_list:
if pi not in needed_pi_list:
raise LeafException(
"Package should not be linked: {pi}".format(pi=pi))
except Exception as e:
if raise_if_not_sync:
raise ProfileOutOfSyncException(profile, cause=e)
self.logger.print_verbose(str(e))
return False
return True
def create_remote(self,
alias: str,
url: str,
enabled: bool = True,
insecure: bool = False,
gpgkey: str = None):
with self.open_user_configuration() as usrc:
remotes = usrc.remotes
if alias in remotes:
raise LeafException(
"Remote {alias} already exists".format(alias=alias))
if insecure:
remotes[alias] = {
JsonConstants.CONFIG_REMOTE_URL: str(url),
JsonConstants.CONFIG_REMOTE_ENABLED: enabled
}
elif gpgkey is not None:
remotes[alias] = {
JsonConstants.CONFIG_REMOTE_URL: str(url),
JsonConstants.CONFIG_REMOTE_ENABLED: enabled,
JsonConstants.CONFIG_REMOTE_GPGKEY: gpgkey,
}
else:
raise LeafException(
"Invalid security for remote {alias}".format(alias=alias))
self.__clean_remote_files(alias)
def __extract_artifact(
self,
la: LeafArtifact,
env: Environment,
install_folder: Path,
ipmap: dict = None,
keep_folder_on_error: bool = False) -> InstalledPackage:
"""
Install a leaf artifact
@return InstalledPackage
"""
target_folder = install_folder / str(la.identifier)
if target_folder.is_dir():
raise LeafException(
"Folder already exists: {folder}".format(folder=target_folder))
# Check already installed
ipmap = ipmap or self.list_installed_packages()
if la.identifier in ipmap:
raise LeafException(
"Package is already installed: {la.identifier}".format(la=la))
# Check leaf min version
min_version = check_leaf_min_version([la])
if min_version:
raise LeafOutOfDateException(
"You need to upgrade leaf to v{version} to install {la.identifier}"
.format(version=min_version, la=la))
# Create folder
target_folder.mkdir(parents=True)
try:
# Extract content
self.logger.print_verbose("Extract {la.path} in {dest}".format(
la=la, dest=target_folder))
with TarFile.open(str(la.path)) as tf:
tf.extractall(str(target_folder))
# Execute post install steps
out = InstalledPackage(target_folder / LeafFiles.MANIFEST)
ipmap[out.identifier] = out
self.__execute_steps(out.identifier,
ipmap,
StepExecutor.install,
env=env)
return out
except Exception as e:
self.logger.print_error("Error during installation:", e)
if keep_folder_on_error:
target_folder = mark_folder_as_ignored(target_folder)
self.logger.print_verbose(
"Mark folder as ignored: {folder}".format(
folder=target_folder))
else:
self.logger.print_verbose(
"Remove folder: {folder}".format(folder=target_folder))
rmtree_force(target_folder)
raise e
def check_valid_name(name):
if not isinstance(name, str):
raise LeafException("Profile name must be a string")
if name in Profile.__RESERVED_NAMES:
raise LeafException(
"'{name}' is not a valid profile name".format(name=name))
if " " in name:
raise LeafException("Profile cannot contain space")
return name
def gpg_verify_file(self, data: Path, sig: Path, expected_key=None):
self.logger.print_verbose("Known GPG keys: {count}".format(count=len(self.gpg.list_keys())))
with sig.open("rb") as sigfile:
verif = self.gpg.verify_file(sigfile, str(data))
if verif.valid:
self.logger.print_verbose("Content has been signed by {verif.username} ({verif.pubkey_fingerprint})".format(verif=verif))
if expected_key is not None:
if expected_key != verif.pubkey_fingerprint:
raise LeafException("Content is not signed with {key}".format(key=expected_key))
else:
raise LeafException("Signed content could not be verified")
def rename_remote(self, oldalias: str, newalias: str):
with self.open_user_configuration() as usrc:
remotes = usrc.remotes
if oldalias not in remotes:
raise LeafException("Cannot find remote {alias}".format(alias=oldalias))
if newalias in remotes:
raise LeafException("Remote {alias} already exists".format(alias=newalias))
remotes[newalias] = remotes[oldalias]
del remotes[oldalias]
self.__clean_remote_files(oldalias)
self.__clean_remote_files(newalias)
def hash_parse(hashstr: str):
parts = hashstr.split(":")
if len(parts) != 2:
raise LeafException("Invalid hash format {hash}".format(hash=hashstr))
if parts[0] != __HASH_NAME:
raise LeafException("Unsupported hash method, expecting {hash}".format(
hash=__HASH_NAME))
if len(parts[1]) != __HASH_LEN:
raise LeafException(
"Hash value '{hash}' has not the correct length, expecting {len}".
format(hash=parts[1], len=__HASH_LEN))
return parts
def rename_remote(self, oldalias: str, newalias: str):
# Do some checks
if not RemoteManager.__REMOTE_ALIAS_PATTERN.fullmatch(newalias):
raise LeafException("Invalid remote alias: '{0}'".format(newalias))
with self.open_user_configuration() as usrc:
remotes = usrc.remotes
if oldalias not in remotes:
raise LeafException("Cannot find remote {alias}".format(alias=oldalias))
if newalias in remotes:
raise LeafException("Remote {alias} already exists".format(alias=newalias))
remotes[newalias] = remotes[oldalias]
del remotes[oldalias]
self.__clean_remote_files(oldalias)
self.__clean_remote_files(newalias)
def get_profile(self, name: str) -> Profile:
if name is None:
raise LeafException("Cannot find profile")
pfmap = self.list_profiles()
if name not in pfmap:
raise InvalidProfileNameException(name)
return pfmap[name]
def update_remote(self, remote: Remote):
with self.open_user_configuration() as usrc:
remotes = usrc.remotes
if remote.alias not in remotes:
raise LeafException("Cannot find remote {remote.alias}".format(remote=remote))
remotes[remote.alias] = remote.json
self.__clean_remote_files(remote.alias)
def delete_remote(self, alias: str):
with self.open_user_configuration() as usrc:
remotes = usrc.remotes
if alias not in remotes:
raise LeafException("Cannot find remote {alias}".format(alias=alias))
del remotes[alias]
self.__clean_remote_files(alias)
def list_available_packages(self, force_refresh=False) -> dict:
"""
List all available package
"""
out = OrderedDict()
self.fetch_remotes(force_refresh=force_refresh)
for remote in self.list_remotes(only_enabled=True).values():
if remote.is_fetched:
for ap in remote.available_packages:
if ap.identifier not in out:
out[ap.identifier] = ap
else:
ap2 = out[ap.identifier]
ap2.remotes.append(remote)
if ap.hashsum != ap2.hashsum:
self.logger.print_error(
"Package {ap.identifier} is available in several remotes with same version but different content!"
.format(ap=ap))
raise LeafException(
"Package {ap.identifier} has multiple artifacts for the same version"
.format(ap=ap))
# Keep tags
for t in ap.tags:
if t not in ap2.tags:
ap2.tags.append(t)
if len(out) == 0:
raise NoPackagesInCacheException()
return out
def available_packages(self) -> list:
if not self.is_fetched:
raise LeafException("Remote is not fetched")
out = []
for json in JsonObject(self.content).jsonget(
JsonConstants.REMOTE_PACKAGES, []):
ap = AvailablePackage(json, remote=self)
out.append(ap)
return out
def execute(self, args, uargs):
# Check argcomplete
if subprocess.call(["which", CompletionPlugin.__ARGCOMPLETE_BIN],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL) != 0:
raise LeafException(
"Cannot find argcomplete: {bin}".format(
bin=CompletionPlugin.__ARGCOMPLETE_BIN),
hints=[
"You can install argcomplete with 'sudo apt-get install python3-argcomplete'",
"or using 'pip install argcomplete' if you are in a virtualenv",
],
)
# Guess shell if not provided
shell = args.shell
if shell is None and CommonSettings.SHELL.is_set():
shell = CommonSettings.SHELL.as_path().name
# Check supported shell
if shell not in CompletionPlugin.__SUPPORTED_SHELLS:
raise LeafException("Unsupported shell")
# Print commands
logger = TextLogger()
logger.print_default(
"# Evaluate the following lines to load leaf completion for {shell}"
.format(shell=shell))
logger.print_default(
'# e.g. with: eval "$(leaf completion -q -s {shell})"'.format(
shell=shell))
if shell == "bash":
logger.print_quiet('eval "$({bin} -s bash leaf)";'.format(
bin=CompletionPlugin.__ARGCOMPLETE_BIN))
elif shell == "zsh":
logger.print_quiet("autoload bashcompinit;")
logger.print_quiet("bashcompinit;")
logger.print_quiet("autoload compinit;")
logger.print_quiet("compinit;")
logger.print_quiet('eval "$({bin} -s bash leaf)";'.format(
bin=CompletionPlugin.__ARGCOMPLETE_BIN))
elif shell == "tcsh":
logger.print_quiet('eval "$({bin} -s tcsh leaf)";'.format(
bin=CompletionPlugin.__ARGCOMPLETE_BIN))
def set_setting(self, setting_id: str, value: str, scope: Scope = None):
# Retrieve setting
setting = self.get_setting(setting_id)
if value is None:
# If value is None, unset setting
self.__unset_setting(setting)
else:
# In case of enum, resolve value
if not setting.is_valid(value):
raise LeafException(
"Value for setting {id} must match '{validator}'".format(
id=setting_id, validator=setting.is_valid))
# If no scope specified, use setting only scope if it is unique
if scope is None:
if len(setting.scopes) == 1:
scope = setting.scopes[0]
else:
raise LeafException(
"No scope specified to update setting {id}".format(
id=setting_id))
# Check taht the setting can be set in given scope
if scope not in setting.scopes:
raise LeafException(
"Cannot set '{id}' in scope {scope}".format(
id=setting_id, scope=scope.name.lower()))
# Set the setting in expected scope
if scope == Scope.USER:
with self.open_user_configuration() as config:
config.update_environment(set_map={setting.key: value})
elif scope == Scope.WORKSPACE:
with self.open_ws_configuration() as config:
config.update_environment(set_map={setting.key: value})
elif scope == Scope.PROFILE:
profile = self.get_profile(self.current_profile_name)
profile.update_environment(set_map={setting.key: value})
self.update_profile(profile)
else:
raise LeafException(
"Cannot update setting {id} in scope {scope}".format(
id=setting.identifier, scope=scope))
def execute(self, args, uargs):
rm = RemoteManager()
for alias in args.aliases:
remote = rm.list_remotes().get(alias)
if remote is None:
raise LeafException(
"Cannot find remote {alias}".format(alias=alias))
remote.enabled = True
rm.update_remote(remote)
def add_duplicate(self, dupp_ap):
if not isinstance(dupp_ap, AvailablePackage):
raise ValueError()
if dupp_ap.hashsum is not None:
for c in self.candidates:
if c.hashsum is not None and c.hashsum != dupp_ap.hashsum:
raise LeafException(
"Package {ap.identifier} has multiple artifacts for the same version"
.format(ap=self))
self.__duplicates.append(dupp_ap)
def __create_exception(self, cause=None):
if cause is None:
cause = Exception("This is a fake cause exception")
return LeafException(
"Random message for this exception",
cause=cause,
hints=[
"this is a first hint with a 'command'",
"another one with 'a first command' and 'a second one'"
],
)
def gpg_import_keys(self, *keys: str, keyserver: str = None):
if keyserver is None:
keyserver = LeafSettings.GPG_KEYSERVER.value
if len(keys) > 0:
self.logger.print_verbose("Update GPG keys for {keys} from {server}".format(keys=", ".join(keys), server=keyserver))
gpg_import = self.gpg.recv_keys(keyserver, *keys)
for result in gpg_import.results:
if "ok" in result:
self.logger.print_verbose("Received GPG key {fingerprint}".format(**result))
else:
raise LeafException("Error receiving GPG keys: {text}".format(**result))
def leaf_exec(generator, logger, verb, arguments=None):
command = generator.gen_command(verb, arguments=arguments)
logger.print_quiet(" -> Execute:", *command)
rc = subprocess.call(command,
env=os.environ,
stdout=None,
stderr=subprocess.STDOUT)
if rc != 0:
logger.print_error("Command exited with {rc}".format(rc=rc))
raise LeafException(
"Sub command failed: '{cmd}'".format(cmd=" ".join(command)))
def create_remote(self, alias: str, url: str, enabled: bool = True, insecure: bool = False, gpgkey: str = None, priority: int = None):
# Do some checks
if not RemoteManager.__REMOTE_ALIAS_PATTERN.fullmatch(alias):
raise LeafException("Invalid remote alias: '{0}'".format(alias))
if len(url) == 0:
raise LeafException("Invalid remote url")
with self.open_user_configuration() as usrc:
remotes = usrc.remotes
if alias in remotes:
raise LeafException("Remote {alias} already exists".format(alias=alias))
remotes[alias] = {JsonConstants.CONFIG_REMOTE_URL: str(url), JsonConstants.CONFIG_REMOTE_ENABLED: enabled}
# Set the priority
if isinstance(priority, int):
if priority not in PRIORITIES_RANGE:
raise LeafException("Invalid priority, must be between {min} and {max}".format(min=PRIORITIES_RANGE[0], max=PRIORITIES_RANGE[-1]))
remotes[alias][JsonConstants.CONFIG_REMOTE_PRIORITY] = priority
# Set the GPG key if set
if isinstance(gpgkey, str):
remotes[alias][JsonConstants.CONFIG_REMOTE_GPGKEY] = gpgkey
self.__clean_remote_files(alias)
def execute(self, args, uargs):
wm = self.get_workspacemanager(check_parents=True,
check_initialized=False)
cmd_generator = LeafCommandGenerator()
cmd_generator.init_common_args(args)
# Checks
if args.packages is None or len(args.packages) == 0:
raise LeafException(
"You need to add at least one package to your profile")
# Compute PackageIdentifiers
pilist = resolve_latest(args.packages, wm)
# Find or create workspace
if not wm.is_initialized:
wm.print_with_confirm(
"Cannot find workspace, initialize one in {wm.ws_root_folder}?"
.format(wm=wm),
raise_on_decline=True)
leaf_exec(cmd_generator, wm.logger, "init")
# Profile name
pfname = args.profiles
if pfname is None:
pfname = Profile.generate_default_name(pilist)
wm.logger.print_default(
"No profile name given, the new profile will be automatically named {name}"
.format(name=pfname))
# Create profile
leaf_exec(cmd_generator, wm.logger, ("profile", "create"), [pfname])
# Update profile with packages
config_args = [pfname]
for pi in pilist:
config_args += ["-p", str(pi)]
leaf_exec(cmd_generator, wm.logger, ("profile", "config"), config_args)
# Set profile env
if args.env_vars is not None:
config_args = [pfname]
for e in args.env_vars:
config_args += ["--set", e]
leaf_exec(cmd_generator, wm.logger, ("env", "profile"),
config_args)
# Run sync command
leaf_exec(cmd_generator, wm.logger, ("profile", "sync"), [pfname])
def find_topic(self, topics: list, name: str):
# Exact match
for t in topics:
if name == t.full_name:
return t
# Partial match
matching_topics = [t for t in topics if t.name == name]
if len(matching_topics) == 0:
# No partial matches found
# Add default "leaf-" prefix to search name and try again
matching_topics = [t for t in topics if t.name == ("leaf-" + name)]
if len(matching_topics) == 1:
return matching_topics[0]
raise LeafException(
"Cannot find topic: {0}".format(name),
hints="Use 'leaf help' to list all available topics")
if len(matching_topics) > 1:
raise LeafException(
"Ambiguous topic name: {0}".format(name),
hints=[
"You can use 'leaf help {t.full_name}'".format(t=t)
for t in matching_topics
])
return matching_topics[0]
def execute(self, args, uargs):
shell_folder = self.installed_package.folder / ShellPlugin.SHELL_DIRNAME
if shell_folder is None:
raise LeafException("Cannot find leaf shell configuration files")
shell_name = None
# Was the shell name specified by the user directly?
if args.shell is not None:
shell_name = args.shell
elif CommonSettings.SHELL.is_set():
# No, so see if the parent shell advertised it's name.
shell_name = CommonSettings.SHELL.as_path().name
else:
# If nothing else was found, assume Bash.
shell_name = "bash"
# Now run our shell.
self.__run_sub_shell(shell_folder, shell_name, args.command)
def __execute(self, step, label):
command = step[JsonConstants.STEP_EXEC_COMMAND]
command = list(map(self.__vr.resolve, command))
command_text = " ".join(command)
self.__logger.print_verbose("Execute: {command}".format(command=command_text))
env = Environment()
env.append(self.__env)
env.append(Environment(content=step.get(JsonConstants.STEP_EXEC_ENV)))
verbose = step.get(JsonConstants.STEP_EXEC_VERBOSE, False)
rc = execute_command(*command, cwd=self.__target_folder, env=env, print_stdout=verbose or self.__logger.isverbose())
if rc != 0:
self.__logger.print_verbose("Command '{command}' exited with {rc}".format(command=command_text, rc=rc))
if step.get(JsonConstants.STEP_IGNORE_FAIL, False):
self.__logger.print_verbose("Step ignores failure")
else:
raise LeafException("Error during {label} step for {ip.identifier} (command returned {rc})".format(label=label, ip=self.__package, rc=rc))
def execute(self, args, uargs):
rm = RelengManager()
# Guess output file
output_file = Path(LeafFiles.MANIFEST)
if args.output_folder is not None:
if not args.output_folder.is_dir():
raise LeafException("Invalid output folder: {folder}".format(
folder=args.output_folder))
output_file = args.output_folder / LeafFiles.MANIFEST
# Build the info map
info_map = {}
for k, v in vars(args).items():
if k.startswith(JsonConstants.INFO + "_"):
info_map[k[(len(JsonConstants.INFO) + 1):]] = v
rm.generate_manifest(output_file,
fragment_files=args.fragment_files,
info_map=info_map,
resolve_envvars=args.resolve_envvars)
def uninstall_packages(self, pilist: list):
"""
Remove given package
"""
with self.application_lock.acquire():
ipmap = self.list_installed_packages()
iplist_to_remove = DependencyUtils.uninstall(pilist,
ipmap,
logger=self.logger)
if len(iplist_to_remove) == 0:
self.logger.print_default("No package to remove")
else:
# Confirm
text = ", ".join(
[str(ip.identifier) for ip in iplist_to_remove])
self.logger.print_quiet(
"Packages to uninstall: {packages}".format(packages=text))
self.print_with_confirm(raise_on_decline=True)
for ip in iplist_to_remove:
if ip.read_only:
raise LeafException(
"Cannot uninstall system package {ip.identifier}".
format(ip=ip))
self.logger.print_default(
"Removing {ip.identifier}".format(ip=ip))
self.__execute_steps(ip.identifier, ipmap,
StepExecutor.uninstall)
self.logger.print_verbose(
"Remove folder: {ip.folder}".format(ip=ip))
rmtree_force(ip.folder)
del ipmap[ip.identifier]
self.logger.print_default("{count} package(s) removed".format(
count=len(iplist_to_remove)))
def info_node(self):
if not self.is_fetched:
raise LeafException("Remote is not fetched")
return JsonObject(self.content).jsonget(JsonConstants.INFO, default={})
