def create_package(self, input_folder: Path, output_file: Path, store_extenal_info: bool = True, tar_extra_args: list = None, validate_only: bool = False):
"""
Create a leaf artifact from given folder containing a manifest.json
"""
mffile = input_folder / LeafFiles.MANIFEST
infofile = self.find_external_info_file(output_file)
if not mffile.exists():
raise LeafException("Cannot find manifest: {file}".format(file=mffile))
manifest = Manifest.parse(mffile)
manifest.validate_model()
if not validate_only:
if is_latest_package(manifest.identifier):
raise LeafException("Invalid version for manifest {mf} ({kw} is a reserved keyword)".format(mf=mffile, kw=LeafConstants.LATEST))
self.logger.print_default("Found package {mf.identifier} in {folder}".format(mf=manifest, folder=input_folder))
# Check if external info file exists
if not store_extenal_info and infofile.exists():
raise LeafException(
"A previous info file ({file}) exists for your package".format(file=infofile),
hints="You should remove it with 'rm {file}'".format(file=infofile),
)
self.__exec_tar(output_file, input_folder, extra_args=tar_extra_args)
self.logger.print_default("Leaf package created: {file}".format(file=output_file))
if store_extenal_info:
self.logger.print_default("Write info to {file}".format(file=infofile))
jwritefile(infofile, self.__build_pkg_node(output_file, manifest=manifest), pp=True)
def setUpClass(cls):
LeafTestCase.setUpClass()
for f in sorted(TEST_REMOTE_PACKAGE_SOURCE.iterdir(),
key=operator.attrgetter("name")):
if "failure" in f.name:
# Skip these packages for tests
continue
mffile = f / LeafFiles.MANIFEST
if mffile.exists():
try:
mf = Manifest.parse(mffile)
ip = InstalledPackage(mffile)
ap = AvailablePackage({"info": mf.info_node},
"https://fake.tld/foo")
APMAP[mf.identifier] = ap
IPMAP[mf.identifier] = ip
except Exception:
pass
print("Found", len(APMAP), LeafFiles.MANIFEST)
def generate_repository(source_folder, output_folder):
mkdirs(output_folder)
artifacts_list1 = []
artifacts_list2 = []
rm = RelengManager()
for package_folder in source_folder.iterdir():
if package_folder.is_dir() and PackageIdentifier.is_valid_identifier(
package_folder.name):
manifest_file = package_folder / LeafFiles.MANIFEST
if manifest_file.is_file():
manifest = Manifest.parse(manifest_file)
if str(manifest.identifier) != package_folder.name:
raise ValueError(
"Naming error: {mf.identifier} != {folder.name}".
format(mf=manifest, folder=package_folder))
filename = str(manifest.identifier) + ".leaf"
output_file = output_folder / filename
tar_extraargs = TAR_EXTRA_ARGS.get(manifest.identifier)
rm.create_package(package_folder,
output_file,
tar_extra_args=tar_extraargs)
# Check that the generated archive is OK
check_archive_format(
output_file,
tar_extraargs[0] if tar_extraargs is not None else None)
# Create multi index.json
if str(manifest.identifier) in ALT_INDEX_CONTENT:
artifacts_list2.append(output_file)
if ALT_INDEX_CONTENT[str(manifest.identifier)]:
artifacts_list1.append(output_file)
else:
artifacts_list1.append(output_file)
# Create a problem with failure-badhash package
if manifest.name == "failure-badhash":
info_node = jloadfile(
rm.find_external_info_file(output_file))
# chosen by fair dice roll.
# garanteed to be random.
info_node[
JsonConstants.
REMOTE_PACKAGE_HASH] = "sha384:d1083143b5c4cf7f1ddaadc391b2d0102fc9fffeb0951ec51020b512ef9548d40cd1af079a1221133faa949fdc304c41"
jwritefile(rm.find_external_info_file(output_file),
info_node,
pp=True)
if len(artifacts_list1) == 0 or len(artifacts_list2) == 0:
raise ValueError("Empty index!")
with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp:
fp.write("volatileTag1\n")
fp.write("volatileTag2")
rm.generate_index(output_folder / "index.json",
artifacts_list1,
name="First repository",
description="First repository description",
prettyprint=True)
with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp:
fp.write("volatileTag3\n")
fp.write("volatileTag4")
rm.generate_index(output_folder / "index2.json",
artifacts_list2,
name="Second repository",
description="Second repository description",
prettyprint=True)
# Alter some values for test purpose
index1json = jloadfile(output_folder / "index.json")
for pkgjson in index1json[JsonConstants.REMOTE_PACKAGES]:
if pkgjson["info"]["name"] == "failure-large-ap":
pkgjson["size"] = 999999999999
jwritefile(output_folder / "index.json", index1json, pp=True)
# Sign with GPG
subprocess.check_call([
"gpg", "--homedir",
str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor",
str(output_folder / "index.json")
])
subprocess.check_call([
"gpg", "--homedir",
str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor",
str(output_folder / "index2.json")
])
def generate_manifest(self,
output_file: Path,
fragment_files: list = None,
info_map: dict = None,
resolve_envvars: bool = False):
"""
Used to create a manifest.json file
"""
model = OrderedDict()
# Load fragments
if fragment_files is not None:
for fragment_file in fragment_files:
self.logger.print_default(
"Use json fragment: {fragment}".format(
fragment=fragment_file))
jlayer_update(model,
jloadfile(fragment_file),
list_append=True)
# Load model
manifest = Manifest(model)
info = manifest.jsonget(JsonConstants.INFO, default=OrderedDict())
# Set the common info
if info_map is not None:
for key in (
JsonConstants.INFO_NAME,
JsonConstants.INFO_VERSION,
JsonConstants.INFO_DESCRIPTION,
JsonConstants.INFO_MASTER,
JsonConstants.INFO_DATE,
JsonConstants.INFO_REQUIRES,
JsonConstants.INFO_DEPENDS,
JsonConstants.INFO_TAGS,
JsonConstants.INFO_LEAF_MINVER,
JsonConstants.INFO_AUTOUPGRADE,
):
if key in info_map:
value = info_map[key]
if value is not None:
if key in (JsonConstants.INFO_REQUIRES,
JsonConstants.INFO_DEPENDS,
JsonConstants.INFO_TAGS):
# Handle lists
model_list = manifest.jsonpath(
[JsonConstants.INFO, key], default=[])
for motif in value:
if motif not in model_list:
if key == JsonConstants.INFO_DEPENDS:
# Try to parse as a conditional package
# identifier
ConditionalPackageIdentifier.parse(
motif)
elif key == JsonConstants.INFO_REQUIRES:
# Try to parse as a package identifier
PackageIdentifier.parse(motif)
self.logger.print_verbose(
"Add '{motif}' to '{key}' list".format(
motif=motif, key=key))
model_list.append(motif)
else:
self.logger.print_verbose(
"Set '{key}' = '{value}'".format(key=key,
value=value))
info[key] = value
# String replacement
jsonstr = jtostring(manifest.json, pp=True)
if resolve_envvars:
for var in set(
re.compile(r"#\{([a-zA-Z0-9_]+)\}").findall(jsonstr)):
value = os.environ.get(var)
if value is None:
raise LeafException(
"Cannot find '{var}' in env".format(var=var),
hints="Set the variable with 'export {var}=xxx'".
format(var=var))
self.logger.print_default("Replace {key} --> {value}".format(
key=var, value=value))
jsonstr = jsonstr.replace("#{{{var}}}".format(var=var), value)
if is_latest_package(manifest.identifier):
raise LeafException(
"Invalid version ({word} is a reserved keyword)".format(
word=LeafConstants.LATEST))
self.logger.print_default(
"Save '{mf.identifier}' manifest to {file}".format(
mf=manifest, file=output_file))
with output_file.open("w") as fp:
fp.write(jsonstr)
class TestRendering(LeafTestCase):
PKG1 = Manifest(
jloads(
'{"info": {"name": "container-A","version": "1.0","description": "Fake description for container A"}}'
))
PKG2 = Manifest(
jloads(
'{"info": {"name": "container-B","version": "1.0","description": "Fake description for container B"}}'
))
PKG3 = Manifest(
jloads(
'{"info": {"name": "container-C","version": "1.0","description": "Fake description for container C"}}'
))
def __init__(self, *args, **kwargs):
LeafTestCase.__init__(self, *args, **kwargs)
self.loggerManager = LoggerManager()
def __print_error(self):
try:
1 / 0 # Ooops !
except Exception as cause:
ex = self.__create_exception(cause)
print("---- Rendered error ----", file=sys.stderr)
self.loggerManager.print_exception(ex)
print("---- Logger error ----", file=sys.stderr)
self.loggerManager.logger.print_error(ex)
def __create_exception(self, cause=None):
if cause is None:
cause = Exception("This is a fake cause exception")
return LeafException(
"Random message for this exception",
cause=cause,
hints=[
"this is a first hint with a 'command'",
"another one with 'a first command' and 'a second one'"
],
)
def __load_manifest(self):
out = []
for folder in TEST_REMOTE_PACKAGE_SOURCE.iterdir():
if folder.is_dir():
mffile = folder / LeafFiles.MANIFEST
if mffile.is_file():
out.append(
AvailablePackage2(jloadfile(mffile),
remote=Remote(
"alias",
{"url": "http://fakeUrl"})))
out.append(InstalledPackage2(mffile))
return sorted(out, key=lambda mf: str(type(mf)) + str(mf))
def test_manifest(self):
mflist = self.__load_manifest()
rend = ManifestListRenderer()
with self.assertStdout(template_out="manifest.out"):
self.loggerManager.print_renderer(rend)
rend.extend(mflist)
self.loggerManager.print_renderer(rend)
def test_remote(self):
rend = RemoteListRenderer()
rend.append(
Remote(
"MyRemote",
{
JsonConstants.CONFIG_REMOTE_ENABLED: True,
JsonConstants.CONFIG_REMOTE_URL: "remote_url1"
},
content={
JsonConstants.INFO: {
JsonConstants.REMOTE_NAME: "remote_name1",
JsonConstants.REMOTE_DESCRIPTION: "remote_desc1",
JsonConstants.REMOTE_DATE: "remote_date1",
}
},
))
rend.append(
Remote(
"AnotherRemoteWithLessInfo", {
JsonConstants.CONFIG_REMOTE_ENABLED: True,
JsonConstants.CONFIG_REMOTE_URL: "remote_url1"
}))
rend.append(
Remote(
"DisabledRemote",
{
JsonConstants.CONFIG_REMOTE_ENABLED: False,
JsonConstants.CONFIG_REMOTE_URL: "remote_url3"
},
content={
JsonConstants.INFO: {
JsonConstants.REMOTE_NAME: "remote_name3",
JsonConstants.REMOTE_DESCRIPTION: "remote_desc3",
JsonConstants.REMOTE_DATE: "remote_date3",
}
},
))
with self.assertStdout(template_out="remotes.out"):
self.loggerManager.print_renderer(RemoteListRenderer())
self.loggerManager.print_renderer(rend)
def test_environment(self):
pm = PackageManager()
env = Environment.build(pm.build_builtin_environment(),
pm.build_user_environment(),
Environment("test", {"FOO": "BAR"}))
rend = EnvironmentRenderer(env)
with self.assertStdout(template_out="env.out"):
self.loggerManager.print_renderer(rend)
def test_status(self):
profile1 = Profile(
"profile1",
"fake/folder",
jloads(
'{"env": {"Foo1": "Bar1", "Foo2": "Bar2", "Foo3": "Bar3"}, "packages": {"container-A": "1.0", "container-B": "1.0"}}'
),
)
profile2 = Profile("profile2", "fake/folder", {})
profile3 = Profile("profile3", "fake/folder",
jloads('{"packages": {"container-C": "1.0"}}'))
with self.assertStdout(template_out="status.out"):
print("####### Test with 2 other profiles, 2 incl, 1 deps #######")
profile1.is_current = True
profile2.is_current = False
profile3.is_current = False
renderer = StatusRenderer(Path("fake/root/folder"),
Environment("test", {"WS_KEY": "VALUE"}))
renderer.append_profile(
profile1, True,
[TestRendering.PKG1, TestRendering.PKG2, TestRendering.PKG3])
renderer.append_profile(profile2, False, [])
renderer.append_profile(profile3, False, [])
self.loggerManager.print_renderer(renderer)
print(
"\n\n\n####### Test with 1 other profile, 0 incl, 0 deps #######"
)
profile1.is_current = False
profile2.is_current = True
profile3.is_current = False
renderer = StatusRenderer(Path("fake/root/folder"),
Environment("test", {"WS_KEY": "VALUE"}))
renderer.append_profile(
profile1, True,
[TestRendering.PKG1, TestRendering.PKG2, TestRendering.PKG3])
renderer.append_profile(profile2, False, [])
self.loggerManager.print_renderer(renderer)
print(
"\n\n\n####### Test with 1 other profile, 1 incl (not fetched), 0 deps #######"
)
profile1.is_current = False
profile2.is_current = False
profile3.is_current = True
renderer = StatusRenderer(Path("fake/root/folder"),
Environment("test", {"WS_KEY": "VALUE"}))
renderer.append_profile(profile2, False, [])
renderer.append_profile(profile3, False, [])
self.loggerManager.print_renderer(renderer)
print(
"\n\n\n####### Test with no other profiles, no included nor deps nor envvars #######"
)
profile1.is_current = False
profile2.is_current = True
profile3.is_current = False
renderer = StatusRenderer(Path("fake/root/folder"),
Environment("test", {"WS_KEY": "VALUE"}))
renderer.append_profile(profile2, False, [])
self.loggerManager.print_renderer(renderer)
def test_profile(self):
profile1 = Profile(
"profile1", "fake/folder",
jloads(
'{"env": {"Foo1": "Bar1", "Foo2": "Bar2", "Foo3": "Bar3"}, "packages": {"container-A": "1.0"}}'
))
profile2 = Profile("profile2", "fake/folder",
jloads('{"packages": {"container-B": "1.0"}}'))
profile2.is_current = True
profile3 = Profile("profile3", "fake/folder",
jloads('{"env": {"Foo2": "Bar2", "Foo3": "Bar3"}}'))
profile4 = Profile("profile4", "fake/folder", {})
with self.assertStdout(template_out="profile.out"):
print("####### Test with no profile #######")
renderer = ProfileListRenderer(
Path("fake/root/folder"),
Environment("test", {"WS_KEY": "VALUE"}))
self.loggerManager.print_renderer(renderer)
print("\n\n\n####### Test with various profiles #######")
renderer.append_profile(
profile1, True,
[TestRendering.PKG1, TestRendering.PKG2, TestRendering.PKG3])
renderer.append_profile(profile2, False, [TestRendering.PKG2])
renderer.append_profile(profile3, True, [TestRendering.PKG2])
renderer.append_profile(profile4, False, [])
self.loggerManager.print_renderer(renderer)
def test_settings(self):
for filter_unset in (True, False):
with self.assertStdout(template_out="filter{filter}.out".format(
filter=filter_unset)):
rend = SettingsListRenderer(Path("/fake/folder"),
{"foo.id1": "Hello world"},
filter_unset=filter_unset)
self.loggerManager.print_renderer(rend)
rend.append(ScopeSetting("foo.id1", "KEY", None, [Scope.USER]))
rend.append(
ScopeSetting("foo.id1", "KEY", "some description",
[Scope.USER]))
rend.append(
ScopeSetting("foo.id2", "KEY", "some description",
[Scope.USER]))
rend.append(
ScopeSetting("foo.id1", "KEY", "some description",
[Scope.USER, Scope.WORKSPACE, Scope.PROFILE]))
rend.append(
ScopeSetting("foo.id1", "KEY", "some description",
[Scope.USER, Scope.WORKSPACE]))
rend.append(
ScopeSetting("foo.id1", "KEY", "some description",
[Scope.USER, Scope.PROFILE]))
rend.append(
ScopeSetting("foo.id1", "KEY", "some description",
[Scope.WORKSPACE, Scope.PROFILE]))
rend.append(
ScopeSetting("foo.id1",
"KEY",
"some description", [Scope.USER],
default="Hello"))
rend.append(
ScopeSetting("foo.id1",
"KEY",
"some description", [Scope.USER],
validator=RegexValidator(".*")))
rend.append(
ScopeSetting("foo.id1",
"KEY",
"some description", [Scope.USER],
validator=RegexValidator("(HELLO|WORLD)")))
self.loggerManager.print_renderer(rend)
def test_hints(self):
with self.assertStdout(template_out="hints.out"):
self.loggerManager.print_hints(
"This is a hints",
"This is another hint with a 'fake command'")
def test_error(self):
with self.assertStdout(template_out="empty.out",
template_err="error.err"):
self.loggerManager.print_exception(self.__create_exception())
def test_trace(self):
with self.assertStdout(template_out="empty.out",
template_err="trace.err"):
LeafSettings.DEBUG_MODE.value = None
print("--------- Production mode ---------", file=sys.stderr)
self.__print_error()
LeafSettings.DEBUG_MODE.value = 1
print("--------- Debug mode ---------", file=sys.stderr)
self.__print_error()
LeafSettings.DEBUG_MODE.value = None