def create_package(self, input_folder: Path, output_file: Path, store_extenal_info: bool = True, tar_extra_args: list = None, validate_only: bool = False): """ Create a leaf artifact from given folder containing a manifest.json """ mffile = input_folder / LeafFiles.MANIFEST infofile = self.find_external_info_file(output_file) if not mffile.exists(): raise LeafException("Cannot find manifest: {file}".format(file=mffile)) manifest = Manifest.parse(mffile) manifest.validate_model() if not validate_only: if is_latest_package(manifest.identifier): raise LeafException("Invalid version for manifest {mf} ({kw} is a reserved keyword)".format(mf=mffile, kw=LeafConstants.LATEST)) self.logger.print_default("Found package {mf.identifier} in {folder}".format(mf=manifest, folder=input_folder)) # Check if external info file exists if not store_extenal_info and infofile.exists(): raise LeafException( "A previous info file ({file}) exists for your package".format(file=infofile), hints="You should remove it with 'rm {file}'".format(file=infofile), ) self.__exec_tar(output_file, input_folder, extra_args=tar_extra_args) self.logger.print_default("Leaf package created: {file}".format(file=output_file)) if store_extenal_info: self.logger.print_default("Write info to {file}".format(file=infofile)) jwritefile(infofile, self.__build_pkg_node(output_file, manifest=manifest), pp=True)
def __exec_tar(self, output: Path, workdir: Path, extra_args: list = None): tar = "tar" if LeafSettings.CUSTOM_TAR.is_set(): tar = LeafSettings.CUSTOM_TAR.value command = [tar, "-c"] command += ["-f", output] command += ["-C", workdir] if extra_args is not None and len(extra_args) > 0: forbidden_args = set( extra_args) & RelengManager.__TAR_FORBIDDEN_ARGS if len(forbidden_args) > 0: raise LeafException( "You should not use tar extra arguments: {invalid_args}". format(invalid_args=" ".join(forbidden_args))) command += extra_args else: command.append(".") command_text = " ".join(map(str, command)) self.logger.print_default( "Executing command: {cmd}".format(cmd=command_text)) rc = subprocess.call(list(map(str, command)), stdout=None, stderr=subprocess.STDOUT) if rc != 0: raise LeafException( "Error executing: {cmd}".format(cmd=command_text))
def is_profile_sync(self, profile: Profile, raise_if_not_sync=False): """ Check if a profile contains all needed links to all contained packages """ try: linked_pi_list = [ ip.identifier for ip in profile.list_linked_packages() ] needed_pi_list = [ ip.identifier for ip in self.get_profile_dependencies(profile) ] for pi in needed_pi_list: if pi not in linked_pi_list: raise LeafException( "Missing package link for {pi}".format(pi=pi)) for pi in linked_pi_list: if pi not in needed_pi_list: raise LeafException( "Package should not be linked: {pi}".format(pi=pi)) except Exception as e: if raise_if_not_sync: raise ProfileOutOfSyncException(profile, cause=e) self.logger.print_verbose(str(e)) return False return True
def create_remote(self, alias: str, url: str, enabled: bool = True, insecure: bool = False, gpgkey: str = None): with self.open_user_configuration() as usrc: remotes = usrc.remotes if alias in remotes: raise LeafException( "Remote {alias} already exists".format(alias=alias)) if insecure: remotes[alias] = { JsonConstants.CONFIG_REMOTE_URL: str(url), JsonConstants.CONFIG_REMOTE_ENABLED: enabled } elif gpgkey is not None: remotes[alias] = { JsonConstants.CONFIG_REMOTE_URL: str(url), JsonConstants.CONFIG_REMOTE_ENABLED: enabled, JsonConstants.CONFIG_REMOTE_GPGKEY: gpgkey, } else: raise LeafException( "Invalid security for remote {alias}".format(alias=alias)) self.__clean_remote_files(alias)
def __extract_artifact( self, la: LeafArtifact, env: Environment, install_folder: Path, ipmap: dict = None, keep_folder_on_error: bool = False) -> InstalledPackage: """ Install a leaf artifact @return InstalledPackage """ target_folder = install_folder / str(la.identifier) if target_folder.is_dir(): raise LeafException( "Folder already exists: {folder}".format(folder=target_folder)) # Check already installed ipmap = ipmap or self.list_installed_packages() if la.identifier in ipmap: raise LeafException( "Package is already installed: {la.identifier}".format(la=la)) # Check leaf min version min_version = check_leaf_min_version([la]) if min_version: raise LeafOutOfDateException( "You need to upgrade leaf to v{version} to install {la.identifier}" .format(version=min_version, la=la)) # Create folder target_folder.mkdir(parents=True) try: # Extract content self.logger.print_verbose("Extract {la.path} in {dest}".format( la=la, dest=target_folder)) with TarFile.open(str(la.path)) as tf: tf.extractall(str(target_folder)) # Execute post install steps out = InstalledPackage(target_folder / LeafFiles.MANIFEST) ipmap[out.identifier] = out self.__execute_steps(out.identifier, ipmap, StepExecutor.install, env=env) return out except Exception as e: self.logger.print_error("Error during installation:", e) if keep_folder_on_error: target_folder = mark_folder_as_ignored(target_folder) self.logger.print_verbose( "Mark folder as ignored: {folder}".format( folder=target_folder)) else: self.logger.print_verbose( "Remove folder: {folder}".format(folder=target_folder)) rmtree_force(target_folder) raise e
def check_valid_name(name): if not isinstance(name, str): raise LeafException("Profile name must be a string") if name in Profile.__RESERVED_NAMES: raise LeafException( "'{name}' is not a valid profile name".format(name=name)) if " " in name: raise LeafException("Profile cannot contain space") return name
def gpg_verify_file(self, data: Path, sig: Path, expected_key=None): self.logger.print_verbose("Known GPG keys: {count}".format(count=len(self.gpg.list_keys()))) with sig.open("rb") as sigfile: verif = self.gpg.verify_file(sigfile, str(data)) if verif.valid: self.logger.print_verbose("Content has been signed by {verif.username} ({verif.pubkey_fingerprint})".format(verif=verif)) if expected_key is not None: if expected_key != verif.pubkey_fingerprint: raise LeafException("Content is not signed with {key}".format(key=expected_key)) else: raise LeafException("Signed content could not be verified")
def rename_remote(self, oldalias: str, newalias: str): with self.open_user_configuration() as usrc: remotes = usrc.remotes if oldalias not in remotes: raise LeafException("Cannot find remote {alias}".format(alias=oldalias)) if newalias in remotes: raise LeafException("Remote {alias} already exists".format(alias=newalias)) remotes[newalias] = remotes[oldalias] del remotes[oldalias] self.__clean_remote_files(oldalias) self.__clean_remote_files(newalias)
def hash_parse(hashstr: str): parts = hashstr.split(":") if len(parts) != 2: raise LeafException("Invalid hash format {hash}".format(hash=hashstr)) if parts[0] != __HASH_NAME: raise LeafException("Unsupported hash method, expecting {hash}".format( hash=__HASH_NAME)) if len(parts[1]) != __HASH_LEN: raise LeafException( "Hash value '{hash}' has not the correct length, expecting {len}". format(hash=parts[1], len=__HASH_LEN)) return parts
def rename_remote(self, oldalias: str, newalias: str): # Do some checks if not RemoteManager.__REMOTE_ALIAS_PATTERN.fullmatch(newalias): raise LeafException("Invalid remote alias: '{0}'".format(newalias)) with self.open_user_configuration() as usrc: remotes = usrc.remotes if oldalias not in remotes: raise LeafException("Cannot find remote {alias}".format(alias=oldalias)) if newalias in remotes: raise LeafException("Remote {alias} already exists".format(alias=newalias)) remotes[newalias] = remotes[oldalias] del remotes[oldalias] self.__clean_remote_files(oldalias) self.__clean_remote_files(newalias)
def get_profile(self, name: str) -> Profile: if name is None: raise LeafException("Cannot find profile") pfmap = self.list_profiles() if name not in pfmap: raise InvalidProfileNameException(name) return pfmap[name]
def update_remote(self, remote: Remote): with self.open_user_configuration() as usrc: remotes = usrc.remotes if remote.alias not in remotes: raise LeafException("Cannot find remote {remote.alias}".format(remote=remote)) remotes[remote.alias] = remote.json self.__clean_remote_files(remote.alias)
def delete_remote(self, alias: str): with self.open_user_configuration() as usrc: remotes = usrc.remotes if alias not in remotes: raise LeafException("Cannot find remote {alias}".format(alias=alias)) del remotes[alias] self.__clean_remote_files(alias)
def list_available_packages(self, force_refresh=False) -> dict: """ List all available package """ out = OrderedDict() self.fetch_remotes(force_refresh=force_refresh) for remote in self.list_remotes(only_enabled=True).values(): if remote.is_fetched: for ap in remote.available_packages: if ap.identifier not in out: out[ap.identifier] = ap else: ap2 = out[ap.identifier] ap2.remotes.append(remote) if ap.hashsum != ap2.hashsum: self.logger.print_error( "Package {ap.identifier} is available in several remotes with same version but different content!" .format(ap=ap)) raise LeafException( "Package {ap.identifier} has multiple artifacts for the same version" .format(ap=ap)) # Keep tags for t in ap.tags: if t not in ap2.tags: ap2.tags.append(t) if len(out) == 0: raise NoPackagesInCacheException() return out
def available_packages(self) -> list: if not self.is_fetched: raise LeafException("Remote is not fetched") out = [] for json in JsonObject(self.content).jsonget( JsonConstants.REMOTE_PACKAGES, []): ap = AvailablePackage(json, remote=self) out.append(ap) return out
def execute(self, args, uargs): # Check argcomplete if subprocess.call(["which", CompletionPlugin.__ARGCOMPLETE_BIN], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) != 0: raise LeafException( "Cannot find argcomplete: {bin}".format( bin=CompletionPlugin.__ARGCOMPLETE_BIN), hints=[ "You can install argcomplete with 'sudo apt-get install python3-argcomplete'", "or using 'pip install argcomplete' if you are in a virtualenv", ], ) # Guess shell if not provided shell = args.shell if shell is None and CommonSettings.SHELL.is_set(): shell = CommonSettings.SHELL.as_path().name # Check supported shell if shell not in CompletionPlugin.__SUPPORTED_SHELLS: raise LeafException("Unsupported shell") # Print commands logger = TextLogger() logger.print_default( "# Evaluate the following lines to load leaf completion for {shell}" .format(shell=shell)) logger.print_default( '# e.g. with: eval "$(leaf completion -q -s {shell})"'.format( shell=shell)) if shell == "bash": logger.print_quiet('eval "$({bin} -s bash leaf)";'.format( bin=CompletionPlugin.__ARGCOMPLETE_BIN)) elif shell == "zsh": logger.print_quiet("autoload bashcompinit;") logger.print_quiet("bashcompinit;") logger.print_quiet("autoload compinit;") logger.print_quiet("compinit;") logger.print_quiet('eval "$({bin} -s bash leaf)";'.format( bin=CompletionPlugin.__ARGCOMPLETE_BIN)) elif shell == "tcsh": logger.print_quiet('eval "$({bin} -s tcsh leaf)";'.format( bin=CompletionPlugin.__ARGCOMPLETE_BIN))
def set_setting(self, setting_id: str, value: str, scope: Scope = None): # Retrieve setting setting = self.get_setting(setting_id) if value is None: # If value is None, unset setting self.__unset_setting(setting) else: # In case of enum, resolve value if not setting.is_valid(value): raise LeafException( "Value for setting {id} must match '{validator}'".format( id=setting_id, validator=setting.is_valid)) # If no scope specified, use setting only scope if it is unique if scope is None: if len(setting.scopes) == 1: scope = setting.scopes[0] else: raise LeafException( "No scope specified to update setting {id}".format( id=setting_id)) # Check taht the setting can be set in given scope if scope not in setting.scopes: raise LeafException( "Cannot set '{id}' in scope {scope}".format( id=setting_id, scope=scope.name.lower())) # Set the setting in expected scope if scope == Scope.USER: with self.open_user_configuration() as config: config.update_environment(set_map={setting.key: value}) elif scope == Scope.WORKSPACE: with self.open_ws_configuration() as config: config.update_environment(set_map={setting.key: value}) elif scope == Scope.PROFILE: profile = self.get_profile(self.current_profile_name) profile.update_environment(set_map={setting.key: value}) self.update_profile(profile) else: raise LeafException( "Cannot update setting {id} in scope {scope}".format( id=setting.identifier, scope=scope))
def execute(self, args, uargs): rm = RemoteManager() for alias in args.aliases: remote = rm.list_remotes().get(alias) if remote is None: raise LeafException( "Cannot find remote {alias}".format(alias=alias)) remote.enabled = True rm.update_remote(remote)
def add_duplicate(self, dupp_ap): if not isinstance(dupp_ap, AvailablePackage): raise ValueError() if dupp_ap.hashsum is not None: for c in self.candidates: if c.hashsum is not None and c.hashsum != dupp_ap.hashsum: raise LeafException( "Package {ap.identifier} has multiple artifacts for the same version" .format(ap=self)) self.__duplicates.append(dupp_ap)
def __create_exception(self, cause=None): if cause is None: cause = Exception("This is a fake cause exception") return LeafException( "Random message for this exception", cause=cause, hints=[ "this is a first hint with a 'command'", "another one with 'a first command' and 'a second one'" ], )
def gpg_import_keys(self, *keys: str, keyserver: str = None): if keyserver is None: keyserver = LeafSettings.GPG_KEYSERVER.value if len(keys) > 0: self.logger.print_verbose("Update GPG keys for {keys} from {server}".format(keys=", ".join(keys), server=keyserver)) gpg_import = self.gpg.recv_keys(keyserver, *keys) for result in gpg_import.results: if "ok" in result: self.logger.print_verbose("Received GPG key {fingerprint}".format(**result)) else: raise LeafException("Error receiving GPG keys: {text}".format(**result))
def leaf_exec(generator, logger, verb, arguments=None): command = generator.gen_command(verb, arguments=arguments) logger.print_quiet(" -> Execute:", *command) rc = subprocess.call(command, env=os.environ, stdout=None, stderr=subprocess.STDOUT) if rc != 0: logger.print_error("Command exited with {rc}".format(rc=rc)) raise LeafException( "Sub command failed: '{cmd}'".format(cmd=" ".join(command)))
def create_remote(self, alias: str, url: str, enabled: bool = True, insecure: bool = False, gpgkey: str = None, priority: int = None): # Do some checks if not RemoteManager.__REMOTE_ALIAS_PATTERN.fullmatch(alias): raise LeafException("Invalid remote alias: '{0}'".format(alias)) if len(url) == 0: raise LeafException("Invalid remote url") with self.open_user_configuration() as usrc: remotes = usrc.remotes if alias in remotes: raise LeafException("Remote {alias} already exists".format(alias=alias)) remotes[alias] = {JsonConstants.CONFIG_REMOTE_URL: str(url), JsonConstants.CONFIG_REMOTE_ENABLED: enabled} # Set the priority if isinstance(priority, int): if priority not in PRIORITIES_RANGE: raise LeafException("Invalid priority, must be between {min} and {max}".format(min=PRIORITIES_RANGE[0], max=PRIORITIES_RANGE[-1])) remotes[alias][JsonConstants.CONFIG_REMOTE_PRIORITY] = priority # Set the GPG key if set if isinstance(gpgkey, str): remotes[alias][JsonConstants.CONFIG_REMOTE_GPGKEY] = gpgkey self.__clean_remote_files(alias)
def execute(self, args, uargs): wm = self.get_workspacemanager(check_parents=True, check_initialized=False) cmd_generator = LeafCommandGenerator() cmd_generator.init_common_args(args) # Checks if args.packages is None or len(args.packages) == 0: raise LeafException( "You need to add at least one package to your profile") # Compute PackageIdentifiers pilist = resolve_latest(args.packages, wm) # Find or create workspace if not wm.is_initialized: wm.print_with_confirm( "Cannot find workspace, initialize one in {wm.ws_root_folder}?" .format(wm=wm), raise_on_decline=True) leaf_exec(cmd_generator, wm.logger, "init") # Profile name pfname = args.profiles if pfname is None: pfname = Profile.generate_default_name(pilist) wm.logger.print_default( "No profile name given, the new profile will be automatically named {name}" .format(name=pfname)) # Create profile leaf_exec(cmd_generator, wm.logger, ("profile", "create"), [pfname]) # Update profile with packages config_args = [pfname] for pi in pilist: config_args += ["-p", str(pi)] leaf_exec(cmd_generator, wm.logger, ("profile", "config"), config_args) # Set profile env if args.env_vars is not None: config_args = [pfname] for e in args.env_vars: config_args += ["--set", e] leaf_exec(cmd_generator, wm.logger, ("env", "profile"), config_args) # Run sync command leaf_exec(cmd_generator, wm.logger, ("profile", "sync"), [pfname])
def find_topic(self, topics: list, name: str): # Exact match for t in topics: if name == t.full_name: return t # Partial match matching_topics = [t for t in topics if t.name == name] if len(matching_topics) == 0: # No partial matches found # Add default "leaf-" prefix to search name and try again matching_topics = [t for t in topics if t.name == ("leaf-" + name)] if len(matching_topics) == 1: return matching_topics[0] raise LeafException( "Cannot find topic: {0}".format(name), hints="Use 'leaf help' to list all available topics") if len(matching_topics) > 1: raise LeafException( "Ambiguous topic name: {0}".format(name), hints=[ "You can use 'leaf help {t.full_name}'".format(t=t) for t in matching_topics ]) return matching_topics[0]
def execute(self, args, uargs): shell_folder = self.installed_package.folder / ShellPlugin.SHELL_DIRNAME if shell_folder is None: raise LeafException("Cannot find leaf shell configuration files") shell_name = None # Was the shell name specified by the user directly? if args.shell is not None: shell_name = args.shell elif CommonSettings.SHELL.is_set(): # No, so see if the parent shell advertised it's name. shell_name = CommonSettings.SHELL.as_path().name else: # If nothing else was found, assume Bash. shell_name = "bash" # Now run our shell. self.__run_sub_shell(shell_folder, shell_name, args.command)
def __execute(self, step, label): command = step[JsonConstants.STEP_EXEC_COMMAND] command = list(map(self.__vr.resolve, command)) command_text = " ".join(command) self.__logger.print_verbose("Execute: {command}".format(command=command_text)) env = Environment() env.append(self.__env) env.append(Environment(content=step.get(JsonConstants.STEP_EXEC_ENV))) verbose = step.get(JsonConstants.STEP_EXEC_VERBOSE, False) rc = execute_command(*command, cwd=self.__target_folder, env=env, print_stdout=verbose or self.__logger.isverbose()) if rc != 0: self.__logger.print_verbose("Command '{command}' exited with {rc}".format(command=command_text, rc=rc)) if step.get(JsonConstants.STEP_IGNORE_FAIL, False): self.__logger.print_verbose("Step ignores failure") else: raise LeafException("Error during {label} step for {ip.identifier} (command returned {rc})".format(label=label, ip=self.__package, rc=rc))
def execute(self, args, uargs): rm = RelengManager() # Guess output file output_file = Path(LeafFiles.MANIFEST) if args.output_folder is not None: if not args.output_folder.is_dir(): raise LeafException("Invalid output folder: {folder}".format( folder=args.output_folder)) output_file = args.output_folder / LeafFiles.MANIFEST # Build the info map info_map = {} for k, v in vars(args).items(): if k.startswith(JsonConstants.INFO + "_"): info_map[k[(len(JsonConstants.INFO) + 1):]] = v rm.generate_manifest(output_file, fragment_files=args.fragment_files, info_map=info_map, resolve_envvars=args.resolve_envvars)
def uninstall_packages(self, pilist: list): """ Remove given package """ with self.application_lock.acquire(): ipmap = self.list_installed_packages() iplist_to_remove = DependencyUtils.uninstall(pilist, ipmap, logger=self.logger) if len(iplist_to_remove) == 0: self.logger.print_default("No package to remove") else: # Confirm text = ", ".join( [str(ip.identifier) for ip in iplist_to_remove]) self.logger.print_quiet( "Packages to uninstall: {packages}".format(packages=text)) self.print_with_confirm(raise_on_decline=True) for ip in iplist_to_remove: if ip.read_only: raise LeafException( "Cannot uninstall system package {ip.identifier}". format(ip=ip)) self.logger.print_default( "Removing {ip.identifier}".format(ip=ip)) self.__execute_steps(ip.identifier, ipmap, StepExecutor.uninstall) self.logger.print_verbose( "Remove folder: {ip.folder}".format(ip=ip)) rmtree_force(ip.folder) del ipmap[ip.identifier] self.logger.print_default("{count} package(s) removed".format( count=len(iplist_to_remove)))
def info_node(self): if not self.is_fetched: raise LeafException("Remote is not fetched") return JsonObject(self.content).jsonget(JsonConstants.INFO, default={})