def execute(self, args, uargs): wm = self.get_workspacemanager() profile = wm.get_profile(self._find_profile_name(args, wm=wm)) if args.pkg_add_list is not None: valid_pilist = list(wm.list_available_packages().keys()) + list( wm.list_installed_packages().keys()) pilist = [] for motif in args.pkg_add_list: if PackageIdentifier.is_valid_identifier(motif): pilist.append(PackageIdentifier.parse(motif)) else: pilist.append(find_latest_version(motif, valid_pilist)) profile.add_packages(pilist) if args.pkg_rm_list is not None: valid_pilist = profile.packages pilist = [] for motif in args.pkg_rm_list: if PackageIdentifier.is_valid_identifier(motif): pilist.append(PackageIdentifier.parse(motif)) else: pilist.append(find_latest_version(motif, valid_pilist)) profile.remove_packages(pilist) wm.update_profile(profile)
def test_prereq(self): self.assertEqual( APMAP[PackageIdentifier.parse("prereq-A_1.0")].requires_packages, ["prereq-true_1.0"]) self.assertEqual( APMAP[PackageIdentifier.parse("prereq-C_1.0")].requires_packages, ["prereq-A_1.0", "prereq-B_1.0"]) self.assertEqual( APMAP[PackageIdentifier.parse("prereq-D_1.0")].requires_packages, ["prereq-true_1.0", "prereq-false_1.0"])
def resolve_latest(motif_list, pm): out = [] grouped_packages = {} group_package_identifiers_by_name(pm.list_installed_packages().keys(), pkgmap=grouped_packages) group_package_identifiers_by_name(pm.list_available_packages().keys(), pkgmap=grouped_packages) for motif in motif_list: pi = None if PackageIdentifier.is_valid_identifier(motif): pi = PackageIdentifier.parse(motif) if pi.name not in grouped_packages or pi not in grouped_packages[ pi.name]: # Unknwon package pi = None elif motif in grouped_packages: # Get latest of the sorted list pi = grouped_packages[motif][-1] # Check if package identifier has been found if pi is None: raise InvalidPackageNameException(motif) out.append(pi) return out
def test_multiple_volatile_tags(self): def toggle_remote(name, enabled): remote = self.pm.list_remotes()[name] remote.enabled = enabled self.pm.update_remote(remote) pi = PackageIdentifier.parse("multitags_1.0") toggle_remote("other", False) self.assertEqual( ["staticTag1", "staticTag2", "volatileTag1", "volatileTag2"], self.pm.list_available_packages()[pi].tags) toggle_remote("other", True) toggle_remote("default", False) self.assertEqual( ["staticTag1", "staticTag2", "volatileTag3", "volatileTag4"], self.pm.list_available_packages()[pi].tags) toggle_remote("default", True) self.assertEqual([ "staticTag1", "staticTag2", "volatileTag1", "volatileTag2", "volatileTag3", "volatileTag4" ], self.pm.list_available_packages()[pi].tags)
def prereq(pilist: list, apmap: dict, ipmap: dict, env: Environment = None): """ Return the list of prereq packages to install Packages are sorted in alpha order. Returns a list of AvailablePackages """ # All available packages mfmap = reduce(lambda a, b: a.update(b) or a, [apmap or {}, ipmap or {}], {}) # Get the list of prereq prereq_pilist = [] for pi in pilist: mf = find_manifest(pi, mfmap) for pis in mf.requires_packages: prereq_pi = PackageIdentifier.parse(pis) if prereq_pi not in prereq_pilist: prereq_pilist.append(prereq_pi) # Compute prereq dependencies out = [] DependencyUtils.__build_tree(prereq_pilist, mfmap, out, env=env) return out
def test_prereq_order(self): pi = "prereq-D_1.0" prereqs = APMAP[PackageIdentifier.parse(pi)].requires_packages self.assertEqual(["prereq-true_1.0", "prereq-false_1.0"], prereqs) prereqs = DependencyUtils.prereq(PackageIdentifier.parse_list([pi]), APMAP, {}) self.assertEqual(["prereq-false_1.0", "prereq-true_1.0"], list(map(str, map(IDENTIFIER_GETTER, prereqs))))
def get_latest_ap(motif, pilist): if PackageIdentifier.is_valid_identifier(motif): pi = PackageIdentifier.parse(motif) return pi if pi in pilist else None out = None for pi in pilist: if pi.name == motif: if out is None or pi > out: out = pi return out
def execute(self, args, uargs): wm = self.get_workspacemanager() logger = wm.logger pfname = wm.current_profile_name profile = wm.get_profile(pfname) profile_pkg_map = profile.pkg_map grouped_packagesmap = group_package_identifiers_by_name(wm.list_installed_packages()) grouped_packagesmap = group_package_identifiers_by_name(wm.list_available_packages(), pkgmap=grouped_packagesmap) update_pilist = [] motiflist = args.packages if args.packages is not None else profile_pkg_map.keys() for motif in motiflist: pi = None if PackageIdentifier.is_valid_identifier(motif): # User force specific version candidate = PackageIdentifier.parse(motif) if candidate.name in grouped_packagesmap: if candidate in grouped_packagesmap[candidate.name]: pi = candidate elif motif in grouped_packagesmap: # Get latest version pi = grouped_packagesmap[motif][-1] if pi is None: # Unknown package identifier raise InvalidPackageNameException(motif) if pi is not None and pi not in update_pilist: # Get PI in profile previous_pi = PackageIdentifier(pi.name, profile_pkg_map[pi.name]) if pi.name in profile_pkg_map else None if previous_pi is None: # Package not in profile yet, add it if wm.print_with_confirm("Do you want to add package {pi}?".format(pi=pi)): update_pilist.append(pi) elif previous_pi != pi: # Package already in profile with a different version, update it if wm.print_with_confirm("Do you want to update package {pi.name} from {oldpi.version} to {pi.version}?".format(pi=pi, oldpi=previous_pi)): update_pilist.append(pi) else: # Package already in profile with same version, do nothing pass if len(update_pilist) == 0: logger.print_default("Nothing to do") else: profile.add_packages(update_pilist) wm.update_profile(profile) wm.provision_profile(profile)
def get_value(self, var: str, pis: str): pkg = None if pis is None: # No PI specified means current package pkg = self.__current_package else: pkg = find_manifest(PackageIdentifier.parse(pis), self.__all_packages, ignore_unknown=True) if pkg is not None: if var == "NAME": return pkg.name if var == "VERSION": return pkg.version if var == "DIR": return str(pkg.folder)
def test_sort_pi(self): a10 = PackageIdentifier.parse("a_1.0") a11 = PackageIdentifier.parse("a_1.1") a20 = PackageIdentifier.parse("a_2.0") a21 = PackageIdentifier.parse("a_2.1") b10 = PackageIdentifier.parse("b_1.0") b11 = PackageIdentifier.parse("b_1.1") b20 = PackageIdentifier.parse("b_2.0") b21 = PackageIdentifier.parse("b_2.1") inputlist = [a10, a11, a20, a21, b10, b11, b20, b21] for _ in range(100): shuffle(inputlist) latest_pilist = keep_latest( [b10, a20, a10, b11, b21, b20, a21, a11]) self.assertEqual(latest_pilist, [a21, b21])
def test_sync_with_package_not_available(self): self.wm.init_ws() self.assertEqual(len(self.wm.list_remotes(only_enabled=True)), 2) self.assertTrue(PackageIdentifier.parse("container-A_1.0") in self.wm.list_available_packages()) profile = self.wm.create_profile("myprofile") profile.add_packages(PackageIdentifier.parse_list(["container-A_1.0"])) self.wm.update_profile(profile) self.wm.provision_profile(profile) self.assertTrue(self.wm.is_profile_sync(profile)) remote2 = self.wm.list_remotes()["default"] remote2.enabled = False self.wm.update_remote(remote2) self.assertEqual(len(self.wm.list_remotes(only_enabled=True)), 1) self.assertFalse(PackageIdentifier.parse("container-A_1.0") in self.wm.list_available_packages()) profile = self.wm.create_profile("myprofile2") profile.add_packages(PackageIdentifier.parse_list(["container-A_1.0"])) self.wm.update_profile(profile) self.wm.provision_profile(profile) self.assertTrue(self.wm.is_profile_sync(profile))
def test_regex(self): pi = PackageIdentifier.parse("foo_1.2-beta") cpi = ConditionalPackageIdentifier.parse("foo_1.2-beta") self.assertEqual(pi, cpi) self.assertEqual([], cpi.conditions) cpi = ConditionalPackageIdentifier.parse("foo_1.2-beta(FOO=BAR)") self.assertEqual(pi, cpi) self.assertEqual(["FOO=BAR"], cpi.conditions) cpi = ConditionalPackageIdentifier.parse( "foo_1.2-beta(FOO)(!BAR)(FOO=BAR)(FOO!=BAR)(FOO~BaR)(FOO!~BaR)") self.assertEqual(pi, cpi) self.assertEqual( ["FOO", "!BAR", "FOO=BAR", "FOO!=BAR", "FOO~BaR", "FOO!~BaR"], cpi.conditions)
def check_content(self, content, pislist): self.assertEqual(len(content), len(pislist)) for pis in pislist: self.assertTrue(PackageIdentifier.parse(pis) in content)
TestCase.maxDiff = None LEAF_UT_DEBUG = EnvVar("LEAF_UT_DEBUG") LEAF_UT_SKIP = EnvVar("LEAF_UT_SKIP", "") LEAF_UT_CREATE_TEMPLATE = EnvVar("LEAF_UT_CREATE_TEMPLATE") LEAF_PROJECT_ROOT_FOLDER = Path(__file__).parent.parent.parent LEAF_SYSTEM_ROOT = LEAF_PROJECT_ROOT_FOLDER / "resources" / "share" / "leaf" / "packages" TEST_RESOURCES_FOLDER = LEAF_PROJECT_ROOT_FOLDER / "src" / "tests" / "resources" TEST_REMOTE_PACKAGE_SOURCE = TEST_RESOURCES_FOLDER / "packages" TEST_LEAF_SYSTEM_ROOT = TEST_RESOURCES_FOLDER / "system_packages" TEST_GPG_HOMEDIR = TEST_RESOURCES_FOLDER / "gpg" TAR_EXTRA_ARGS = { PackageIdentifier.parse("compress-xz_1.0"): ("-z", "."), PackageIdentifier.parse("compress-bz2_1.0"): ("-j", "."), PackageIdentifier.parse("compress-gz_1.0"): ("-J", "."), } ALT_INDEX_CONTENT = { "multitags_1.0": True, "version_2.0": False, "upgrade_1.2": False, "upgrade_2.0": False } TEST_GPG_FINGERPRINT = "E35D6817397359074160F68952ECE808A2BC372C" class StringIOWrapper(StringIO): def __init__(self, stream, *args, **kwargs): StringIO.__init__(self, *args, **kwargs)
def test_env(self): try: LeafSettings.PROFILE_NORELATIVE.value = 1 self.wm.init_ws() profile = self.wm.create_profile("myenv") profile.add_packages([PackageIdentifier.parse(pis) for pis in ["env-A_1.0", "env-A_1.0"]]) self.wm.update_profile(profile) self.wm.switch_profile(profile) self.wm.provision_profile(profile) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("LEAF_PROFILE", "myenv"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) self.wm.update_user_environment(set_map=OrderedDict((("scope", "user"), ("HELLO", "world")))) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("scope", "user"), ("HELLO", "world"), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("LEAF_PROFILE", "myenv"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) self.wm.update_user_environment(unset_list=["HELLO"]) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("scope", "user"), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("LEAF_PROFILE", "myenv"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) self.wm.update_ws_environment(set_map=OrderedDict((("scope", "workspace"), ("HELLO", "world")))) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("scope", "user"), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("scope", "workspace"), ("HELLO", "world"), ("LEAF_PROFILE", "myenv"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) self.wm.update_ws_environment(unset_list=["HELLO"]) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("scope", "user"), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("scope", "workspace"), ("LEAF_PROFILE", "myenv"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) profile.update_environment(set_map=OrderedDict((("scope", "profile"), ("HELLO", "world")))) self.wm.update_profile(profile) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("scope", "user"), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("scope", "workspace"), ("LEAF_PROFILE", "myenv"), ("scope", "profile"), ("HELLO", "world"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) profile.update_environment(unset_list=["HELLO"]) self.wm.update_profile(profile) self.assertEqual( [ ("LEAF_VERSION", leaf.__version__), ("LEAF_PLATFORM_SYSTEM", platform.system()), ("LEAF_PLATFORM_MACHINE", platform.machine()), ("LEAF_PLATFORM_RELEASE", platform.release()), ("scope", "user"), ("LEAF_WORKSPACE", str(self.workspace_folder)), ("scope", "workspace"), ("LEAF_PROFILE", "myenv"), ("scope", "profile"), ("LEAF_ENV_B", "BAR"), ("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)), ("LEAF_ENV_A", "FOO"), ("LEAF_ENV_A2", "Hello"), ("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)), ], env_tolist(self.wm.build_full_environment(profile)), ) finally: LeafSettings.PROFILE_NORELATIVE.value = None
def generate_manifest(self, output_file: Path, fragment_files: list = None, info_map: dict = None, resolve_envvars: bool = False): """ Used to create a manifest.json file """ model = OrderedDict() # Load fragments if fragment_files is not None: for fragment_file in fragment_files: self.logger.print_default( "Use json fragment: {fragment}".format( fragment=fragment_file)) jlayer_update(model, jloadfile(fragment_file), list_append=True) # Load model manifest = Manifest(model) info = manifest.jsonget(JsonConstants.INFO, default=OrderedDict()) # Set the common info if info_map is not None: for key in ( JsonConstants.INFO_NAME, JsonConstants.INFO_VERSION, JsonConstants.INFO_DESCRIPTION, JsonConstants.INFO_MASTER, JsonConstants.INFO_DATE, JsonConstants.INFO_REQUIRES, JsonConstants.INFO_DEPENDS, JsonConstants.INFO_TAGS, JsonConstants.INFO_LEAF_MINVER, JsonConstants.INFO_AUTOUPGRADE, ): if key in info_map: value = info_map[key] if value is not None: if key in (JsonConstants.INFO_REQUIRES, JsonConstants.INFO_DEPENDS, JsonConstants.INFO_TAGS): # Handle lists model_list = manifest.jsonpath( [JsonConstants.INFO, key], default=[]) for motif in value: if motif not in model_list: if key == JsonConstants.INFO_DEPENDS: # Try to parse as a conditional package # identifier ConditionalPackageIdentifier.parse( motif) elif key == JsonConstants.INFO_REQUIRES: # Try to parse as a package identifier PackageIdentifier.parse(motif) self.logger.print_verbose( "Add '{motif}' to '{key}' list".format( motif=motif, key=key)) model_list.append(motif) else: self.logger.print_verbose( "Set '{key}' = '{value}'".format(key=key, value=value)) info[key] = value # String replacement jsonstr = jtostring(manifest.json, pp=True) if resolve_envvars: for var in set( re.compile(r"#\{([a-zA-Z0-9_]+)\}").findall(jsonstr)): value = os.environ.get(var) if value is None: raise LeafException( "Cannot find '{var}' in env".format(var=var), hints="Set the variable with 'export {var}=xxx'". format(var=var)) self.logger.print_default("Replace {key} --> {value}".format( key=var, value=value)) jsonstr = jsonstr.replace("#{{{var}}}".format(var=var), value) if is_latest_package(manifest.identifier): raise LeafException( "Invalid version ({word} is a reserved keyword)".format( word=LeafConstants.LATEST)) self.logger.print_default( "Save '{mf.identifier}' manifest to {file}".format( mf=manifest, file=output_file)) with output_file.open("w") as fp: fp.write(jsonstr)
def install_packages(self, items: list, env: Environment = None, keep_folder_on_error: bool = False): """ Compute dependency tree, check compatibility, download from remotes and extract needed packages @return: InstalledPackage list """ with self.application_lock.acquire(): ipmap = self.list_installed_packages() apmap = self.list_available_packages() pilist = [] for item in items: if isinstance(item, PackageIdentifier): # Package identifier is given pilist.append(item) elif PackageIdentifier.is_valid_identifier(item): # Package identifier string given pilist.append(PackageIdentifier.parse(item)) else: # If leaf artifacts are given, add/replace identifiers of available packages la = LeafArtifact(Path(item)) pilist.append(la.identifier) apmap[la.identifier] = la out = [] # Build env to resolve dynamic dependencies if env is None: env = Environment.build(self.build_builtin_environment(), self.build_user_environment()) ap_to_install = DependencyUtils.install(pilist, apmap, ipmap, env=env) # Check leaf min version min_version = check_leaf_min_version(ap_to_install) if min_version: raise LeafOutOfDateException( "You need to upgrade leaf to v{version} to install {text}". format(version=min_version, text=", ".join( [str(ap.identifier) for ap in ap_to_install]))) # Check nothing to do if len(ap_to_install) == 0: self.logger.print_default("All packages are installed") else: # Check available size download_totalsize = 0 download_count = 0 for ap in [ ap for ap in ap_to_install if isinstance(ap, AvailablePackage) ]: download_count += 1 if ap.size is not None: download_totalsize += ap.size fs_check_free_space(self.download_cache_folder, download_totalsize) # Confirm text = ", ".join([str(ap.identifier) for ap in ap_to_install]) self.logger.print_quiet( "Packages to install: {packages}".format(packages=text)) if download_totalsize > 0: self.logger.print_default("Total size:", sizeof_fmt(download_totalsize)) self.print_with_confirm(raise_on_decline=True) # Install prereq prereq_to_install = DependencyUtils.prereq( [ap.identifier for ap in ap_to_install], apmap, ipmap, env=env) if len(prereq_to_install) > 0: try: self.__install_prereq( prereq_to_install, ipmap, env=env, keep_folder_on_error=keep_folder_on_error) except BaseException as e: raise PrereqException(e) # Download ap list self.logger.print_default( "Downloading {size} package(s)".format( size=download_count)) la_to_install = [] for mf in ap_to_install: if isinstance(mf, AvailablePackage): la_to_install.append(self.__download_ap(mf)) elif isinstance(mf, LeafArtifact): la_to_install.append(mf) # Check the extracted size extracted_totalsize = 0 for la in la_to_install: if la.final_size is not None: extracted_totalsize += la.final_size else: extracted_totalsize += la.get_total_size() fs_check_free_space(self.install_folder, extracted_totalsize) # Extract la list for la in la_to_install: self.logger.print_default( "[{current}/{total}] Installing {la.identifier}". format(current=(len(out) + 1), total=len(la_to_install), la=la)) ip = self.__extract_artifact( la, env, ipmap, keep_folder_on_error=keep_folder_on_error) out.append(ip) return out