def execute(self, args, uargs):
wm = self.get_workspacemanager()
profile = wm.get_profile(self._find_profile_name(args, wm=wm))
if args.pkg_add_list is not None:
valid_pilist = list(wm.list_available_packages().keys()) + list(
wm.list_installed_packages().keys())
pilist = []
for motif in args.pkg_add_list:
if PackageIdentifier.is_valid_identifier(motif):
pilist.append(PackageIdentifier.parse(motif))
else:
pilist.append(find_latest_version(motif, valid_pilist))
profile.add_packages(pilist)
if args.pkg_rm_list is not None:
valid_pilist = profile.packages
pilist = []
for motif in args.pkg_rm_list:
if PackageIdentifier.is_valid_identifier(motif):
pilist.append(PackageIdentifier.parse(motif))
else:
pilist.append(find_latest_version(motif, valid_pilist))
profile.remove_packages(pilist)
wm.update_profile(profile)
def test_prereq(self):
self.assertEqual(
APMAP[PackageIdentifier.parse("prereq-A_1.0")].requires_packages,
["prereq-true_1.0"])
self.assertEqual(
APMAP[PackageIdentifier.parse("prereq-C_1.0")].requires_packages,
["prereq-A_1.0", "prereq-B_1.0"])
self.assertEqual(
APMAP[PackageIdentifier.parse("prereq-D_1.0")].requires_packages,
["prereq-true_1.0", "prereq-false_1.0"])
def resolve_latest(motif_list, pm):
out = []
grouped_packages = {}
group_package_identifiers_by_name(pm.list_installed_packages().keys(),
pkgmap=grouped_packages)
group_package_identifiers_by_name(pm.list_available_packages().keys(),
pkgmap=grouped_packages)
for motif in motif_list:
pi = None
if PackageIdentifier.is_valid_identifier(motif):
pi = PackageIdentifier.parse(motif)
if pi.name not in grouped_packages or pi not in grouped_packages[
pi.name]:
# Unknwon package
pi = None
elif motif in grouped_packages:
# Get latest of the sorted list
pi = grouped_packages[motif][-1]
# Check if package identifier has been found
if pi is None:
raise InvalidPackageNameException(motif)
out.append(pi)
return out
def test_multiple_volatile_tags(self):
def toggle_remote(name, enabled):
remote = self.pm.list_remotes()[name]
remote.enabled = enabled
self.pm.update_remote(remote)
pi = PackageIdentifier.parse("multitags_1.0")
toggle_remote("other", False)
self.assertEqual(
["staticTag1", "staticTag2", "volatileTag1", "volatileTag2"],
self.pm.list_available_packages()[pi].tags)
toggle_remote("other", True)
toggle_remote("default", False)
self.assertEqual(
["staticTag1", "staticTag2", "volatileTag3", "volatileTag4"],
self.pm.list_available_packages()[pi].tags)
toggle_remote("default", True)
self.assertEqual([
"staticTag1", "staticTag2", "volatileTag1", "volatileTag2",
"volatileTag3", "volatileTag4"
],
self.pm.list_available_packages()[pi].tags)
def prereq(pilist: list,
apmap: dict,
ipmap: dict,
env: Environment = None):
"""
Return the list of prereq packages to install
Packages are sorted in alpha order.
Returns a list of AvailablePackages
"""
# All available packages
mfmap = reduce(lambda a, b: a.update(b) or a,
[apmap or {}, ipmap or {}], {})
# Get the list of prereq
prereq_pilist = []
for pi in pilist:
mf = find_manifest(pi, mfmap)
for pis in mf.requires_packages:
prereq_pi = PackageIdentifier.parse(pis)
if prereq_pi not in prereq_pilist:
prereq_pilist.append(prereq_pi)
# Compute prereq dependencies
out = []
DependencyUtils.__build_tree(prereq_pilist, mfmap, out, env=env)
return out
def test_prereq_order(self):
pi = "prereq-D_1.0"
prereqs = APMAP[PackageIdentifier.parse(pi)].requires_packages
self.assertEqual(["prereq-true_1.0", "prereq-false_1.0"], prereqs)
prereqs = DependencyUtils.prereq(PackageIdentifier.parse_list([pi]),
APMAP, {})
self.assertEqual(["prereq-false_1.0", "prereq-true_1.0"],
list(map(str, map(IDENTIFIER_GETTER, prereqs))))
def get_latest_ap(motif, pilist):
if PackageIdentifier.is_valid_identifier(motif):
pi = PackageIdentifier.parse(motif)
return pi if pi in pilist else None
out = None
for pi in pilist:
if pi.name == motif:
if out is None or pi > out:
out = pi
return out
def execute(self, args, uargs):
wm = self.get_workspacemanager()
logger = wm.logger
pfname = wm.current_profile_name
profile = wm.get_profile(pfname)
profile_pkg_map = profile.pkg_map
grouped_packagesmap = group_package_identifiers_by_name(wm.list_installed_packages())
grouped_packagesmap = group_package_identifiers_by_name(wm.list_available_packages(), pkgmap=grouped_packagesmap)
update_pilist = []
motiflist = args.packages if args.packages is not None else profile_pkg_map.keys()
for motif in motiflist:
pi = None
if PackageIdentifier.is_valid_identifier(motif):
# User force specific version
candidate = PackageIdentifier.parse(motif)
if candidate.name in grouped_packagesmap:
if candidate in grouped_packagesmap[candidate.name]:
pi = candidate
elif motif in grouped_packagesmap:
# Get latest version
pi = grouped_packagesmap[motif][-1]
if pi is None:
# Unknown package identifier
raise InvalidPackageNameException(motif)
if pi is not None and pi not in update_pilist:
# Get PI in profile
previous_pi = PackageIdentifier(pi.name, profile_pkg_map[pi.name]) if pi.name in profile_pkg_map else None
if previous_pi is None:
# Package not in profile yet, add it
if wm.print_with_confirm("Do you want to add package {pi}?".format(pi=pi)):
update_pilist.append(pi)
elif previous_pi != pi:
# Package already in profile with a different version, update it
if wm.print_with_confirm("Do you want to update package {pi.name} from {oldpi.version} to {pi.version}?".format(pi=pi, oldpi=previous_pi)):
update_pilist.append(pi)
else:
# Package already in profile with same version, do nothing
pass
if len(update_pilist) == 0:
logger.print_default("Nothing to do")
else:
profile.add_packages(update_pilist)
wm.update_profile(profile)
wm.provision_profile(profile)
def get_value(self, var: str, pis: str):
pkg = None
if pis is None:
# No PI specified means current package
pkg = self.__current_package
else:
pkg = find_manifest(PackageIdentifier.parse(pis), self.__all_packages, ignore_unknown=True)
if pkg is not None:
if var == "NAME":
return pkg.name
if var == "VERSION":
return pkg.version
if var == "DIR":
return str(pkg.folder)
def test_sort_pi(self):
a10 = PackageIdentifier.parse("a_1.0")
a11 = PackageIdentifier.parse("a_1.1")
a20 = PackageIdentifier.parse("a_2.0")
a21 = PackageIdentifier.parse("a_2.1")
b10 = PackageIdentifier.parse("b_1.0")
b11 = PackageIdentifier.parse("b_1.1")
b20 = PackageIdentifier.parse("b_2.0")
b21 = PackageIdentifier.parse("b_2.1")
inputlist = [a10, a11, a20, a21, b10, b11, b20, b21]
for _ in range(100):
shuffle(inputlist)
latest_pilist = keep_latest(
[b10, a20, a10, b11, b21, b20, a21, a11])
self.assertEqual(latest_pilist, [a21, b21])
def test_sync_with_package_not_available(self):
self.wm.init_ws()
self.assertEqual(len(self.wm.list_remotes(only_enabled=True)), 2)
self.assertTrue(PackageIdentifier.parse("container-A_1.0") in self.wm.list_available_packages())
profile = self.wm.create_profile("myprofile")
profile.add_packages(PackageIdentifier.parse_list(["container-A_1.0"]))
self.wm.update_profile(profile)
self.wm.provision_profile(profile)
self.assertTrue(self.wm.is_profile_sync(profile))
remote2 = self.wm.list_remotes()["default"]
remote2.enabled = False
self.wm.update_remote(remote2)
self.assertEqual(len(self.wm.list_remotes(only_enabled=True)), 1)
self.assertFalse(PackageIdentifier.parse("container-A_1.0") in self.wm.list_available_packages())
profile = self.wm.create_profile("myprofile2")
profile.add_packages(PackageIdentifier.parse_list(["container-A_1.0"]))
self.wm.update_profile(profile)
self.wm.provision_profile(profile)
self.assertTrue(self.wm.is_profile_sync(profile))
def test_regex(self):
pi = PackageIdentifier.parse("foo_1.2-beta")
cpi = ConditionalPackageIdentifier.parse("foo_1.2-beta")
self.assertEqual(pi, cpi)
self.assertEqual([], cpi.conditions)
cpi = ConditionalPackageIdentifier.parse("foo_1.2-beta(FOO=BAR)")
self.assertEqual(pi, cpi)
self.assertEqual(["FOO=BAR"], cpi.conditions)
cpi = ConditionalPackageIdentifier.parse(
"foo_1.2-beta(FOO)(!BAR)(FOO=BAR)(FOO!=BAR)(FOO~BaR)(FOO!~BaR)")
self.assertEqual(pi, cpi)
self.assertEqual(
["FOO", "!BAR", "FOO=BAR", "FOO!=BAR", "FOO~BaR", "FOO!~BaR"],
cpi.conditions)
def check_content(self, content, pislist):
self.assertEqual(len(content), len(pislist))
for pis in pislist:
self.assertTrue(PackageIdentifier.parse(pis) in content)
TestCase.maxDiff = None
LEAF_UT_DEBUG = EnvVar("LEAF_UT_DEBUG")
LEAF_UT_SKIP = EnvVar("LEAF_UT_SKIP", "")
LEAF_UT_CREATE_TEMPLATE = EnvVar("LEAF_UT_CREATE_TEMPLATE")
LEAF_PROJECT_ROOT_FOLDER = Path(__file__).parent.parent.parent
LEAF_SYSTEM_ROOT = LEAF_PROJECT_ROOT_FOLDER / "resources" / "share" / "leaf" / "packages"
TEST_RESOURCES_FOLDER = LEAF_PROJECT_ROOT_FOLDER / "src" / "tests" / "resources"
TEST_REMOTE_PACKAGE_SOURCE = TEST_RESOURCES_FOLDER / "packages"
TEST_LEAF_SYSTEM_ROOT = TEST_RESOURCES_FOLDER / "system_packages"
TEST_GPG_HOMEDIR = TEST_RESOURCES_FOLDER / "gpg"
TAR_EXTRA_ARGS = {
PackageIdentifier.parse("compress-xz_1.0"): ("-z", "."),
PackageIdentifier.parse("compress-bz2_1.0"): ("-j", "."),
PackageIdentifier.parse("compress-gz_1.0"): ("-J", "."),
}
ALT_INDEX_CONTENT = {
"multitags_1.0": True,
"version_2.0": False,
"upgrade_1.2": False,
"upgrade_2.0": False
}
TEST_GPG_FINGERPRINT = "E35D6817397359074160F68952ECE808A2BC372C"
class StringIOWrapper(StringIO):
def __init__(self, stream, *args, **kwargs):
StringIO.__init__(self, *args, **kwargs)
def test_env(self):
try:
LeafSettings.PROFILE_NORELATIVE.value = 1
self.wm.init_ws()
profile = self.wm.create_profile("myenv")
profile.add_packages([PackageIdentifier.parse(pis) for pis in ["env-A_1.0", "env-A_1.0"]])
self.wm.update_profile(profile)
self.wm.switch_profile(profile)
self.wm.provision_profile(profile)
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("LEAF_PROFILE", "myenv"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
self.wm.update_user_environment(set_map=OrderedDict((("scope", "user"), ("HELLO", "world"))))
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("scope", "user"),
("HELLO", "world"),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("LEAF_PROFILE", "myenv"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
self.wm.update_user_environment(unset_list=["HELLO"])
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("scope", "user"),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("LEAF_PROFILE", "myenv"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
self.wm.update_ws_environment(set_map=OrderedDict((("scope", "workspace"), ("HELLO", "world"))))
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("scope", "user"),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("scope", "workspace"),
("HELLO", "world"),
("LEAF_PROFILE", "myenv"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
self.wm.update_ws_environment(unset_list=["HELLO"])
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("scope", "user"),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("scope", "workspace"),
("LEAF_PROFILE", "myenv"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
profile.update_environment(set_map=OrderedDict((("scope", "profile"), ("HELLO", "world"))))
self.wm.update_profile(profile)
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("scope", "user"),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("scope", "workspace"),
("LEAF_PROFILE", "myenv"),
("scope", "profile"),
("HELLO", "world"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
profile.update_environment(unset_list=["HELLO"])
self.wm.update_profile(profile)
self.assertEqual(
[
("LEAF_VERSION", leaf.__version__),
("LEAF_PLATFORM_SYSTEM", platform.system()),
("LEAF_PLATFORM_MACHINE", platform.machine()),
("LEAF_PLATFORM_RELEASE", platform.release()),
("scope", "user"),
("LEAF_WORKSPACE", str(self.workspace_folder)),
("scope", "workspace"),
("LEAF_PROFILE", "myenv"),
("scope", "profile"),
("LEAF_ENV_B", "BAR"),
("LEAF_PATH_B", "$PATH:{folder}/env-B_1.0".format(folder=self.install_folder)),
("LEAF_ENV_A", "FOO"),
("LEAF_ENV_A2", "Hello"),
("LEAF_PATH_A", "$PATH:{folder}/env-A_1.0:{folder}/env-B_1.0".format(folder=self.install_folder)),
],
env_tolist(self.wm.build_full_environment(profile)),
)
finally:
LeafSettings.PROFILE_NORELATIVE.value = None
def generate_manifest(self,
output_file: Path,
fragment_files: list = None,
info_map: dict = None,
resolve_envvars: bool = False):
"""
Used to create a manifest.json file
"""
model = OrderedDict()
# Load fragments
if fragment_files is not None:
for fragment_file in fragment_files:
self.logger.print_default(
"Use json fragment: {fragment}".format(
fragment=fragment_file))
jlayer_update(model,
jloadfile(fragment_file),
list_append=True)
# Load model
manifest = Manifest(model)
info = manifest.jsonget(JsonConstants.INFO, default=OrderedDict())
# Set the common info
if info_map is not None:
for key in (
JsonConstants.INFO_NAME,
JsonConstants.INFO_VERSION,
JsonConstants.INFO_DESCRIPTION,
JsonConstants.INFO_MASTER,
JsonConstants.INFO_DATE,
JsonConstants.INFO_REQUIRES,
JsonConstants.INFO_DEPENDS,
JsonConstants.INFO_TAGS,
JsonConstants.INFO_LEAF_MINVER,
JsonConstants.INFO_AUTOUPGRADE,
):
if key in info_map:
value = info_map[key]
if value is not None:
if key in (JsonConstants.INFO_REQUIRES,
JsonConstants.INFO_DEPENDS,
JsonConstants.INFO_TAGS):
# Handle lists
model_list = manifest.jsonpath(
[JsonConstants.INFO, key], default=[])
for motif in value:
if motif not in model_list:
if key == JsonConstants.INFO_DEPENDS:
# Try to parse as a conditional package
# identifier
ConditionalPackageIdentifier.parse(
motif)
elif key == JsonConstants.INFO_REQUIRES:
# Try to parse as a package identifier
PackageIdentifier.parse(motif)
self.logger.print_verbose(
"Add '{motif}' to '{key}' list".format(
motif=motif, key=key))
model_list.append(motif)
else:
self.logger.print_verbose(
"Set '{key}' = '{value}'".format(key=key,
value=value))
info[key] = value
# String replacement
jsonstr = jtostring(manifest.json, pp=True)
if resolve_envvars:
for var in set(
re.compile(r"#\{([a-zA-Z0-9_]+)\}").findall(jsonstr)):
value = os.environ.get(var)
if value is None:
raise LeafException(
"Cannot find '{var}' in env".format(var=var),
hints="Set the variable with 'export {var}=xxx'".
format(var=var))
self.logger.print_default("Replace {key} --> {value}".format(
key=var, value=value))
jsonstr = jsonstr.replace("#{{{var}}}".format(var=var), value)
if is_latest_package(manifest.identifier):
raise LeafException(
"Invalid version ({word} is a reserved keyword)".format(
word=LeafConstants.LATEST))
self.logger.print_default(
"Save '{mf.identifier}' manifest to {file}".format(
mf=manifest, file=output_file))
with output_file.open("w") as fp:
fp.write(jsonstr)
def install_packages(self,
items: list,
env: Environment = None,
keep_folder_on_error: bool = False):
"""
Compute dependency tree, check compatibility, download from remotes and extract needed packages
@return: InstalledPackage list
"""
with self.application_lock.acquire():
ipmap = self.list_installed_packages()
apmap = self.list_available_packages()
pilist = []
for item in items:
if isinstance(item, PackageIdentifier):
# Package identifier is given
pilist.append(item)
elif PackageIdentifier.is_valid_identifier(item):
# Package identifier string given
pilist.append(PackageIdentifier.parse(item))
else:
# If leaf artifacts are given, add/replace identifiers of available packages
la = LeafArtifact(Path(item))
pilist.append(la.identifier)
apmap[la.identifier] = la
out = []
# Build env to resolve dynamic dependencies
if env is None:
env = Environment.build(self.build_builtin_environment(),
self.build_user_environment())
ap_to_install = DependencyUtils.install(pilist,
apmap,
ipmap,
env=env)
# Check leaf min version
min_version = check_leaf_min_version(ap_to_install)
if min_version:
raise LeafOutOfDateException(
"You need to upgrade leaf to v{version} to install {text}".
format(version=min_version,
text=", ".join(
[str(ap.identifier) for ap in ap_to_install])))
# Check nothing to do
if len(ap_to_install) == 0:
self.logger.print_default("All packages are installed")
else:
# Check available size
download_totalsize = 0
download_count = 0
for ap in [
ap for ap in ap_to_install
if isinstance(ap, AvailablePackage)
]:
download_count += 1
if ap.size is not None:
download_totalsize += ap.size
fs_check_free_space(self.download_cache_folder,
download_totalsize)
# Confirm
text = ", ".join([str(ap.identifier) for ap in ap_to_install])
self.logger.print_quiet(
"Packages to install: {packages}".format(packages=text))
if download_totalsize > 0:
self.logger.print_default("Total size:",
sizeof_fmt(download_totalsize))
self.print_with_confirm(raise_on_decline=True)
# Install prereq
prereq_to_install = DependencyUtils.prereq(
[ap.identifier for ap in ap_to_install],
apmap,
ipmap,
env=env)
if len(prereq_to_install) > 0:
try:
self.__install_prereq(
prereq_to_install,
ipmap,
env=env,
keep_folder_on_error=keep_folder_on_error)
except BaseException as e:
raise PrereqException(e)
# Download ap list
self.logger.print_default(
"Downloading {size} package(s)".format(
size=download_count))
la_to_install = []
for mf in ap_to_install:
if isinstance(mf, AvailablePackage):
la_to_install.append(self.__download_ap(mf))
elif isinstance(mf, LeafArtifact):
la_to_install.append(mf)
# Check the extracted size
extracted_totalsize = 0
for la in la_to_install:
if la.final_size is not None:
extracted_totalsize += la.final_size
else:
extracted_totalsize += la.get_total_size()
fs_check_free_space(self.install_folder, extracted_totalsize)
# Extract la list
for la in la_to_install:
self.logger.print_default(
"[{current}/{total}] Installing {la.identifier}".
format(current=(len(out) + 1),
total=len(la_to_install),
la=la))
ip = self.__extract_artifact(
la,
env,
ipmap,
keep_folder_on_error=keep_folder_on_error)
out.append(ip)
return out