def create_package(self, input_folder: Path, output_file: Path, store_extenal_info: bool = True, tar_extra_args: list = None, validate_only: bool = False):
"""
Create a leaf artifact from given folder containing a manifest.json
"""
mffile = input_folder / LeafFiles.MANIFEST
infofile = self.find_external_info_file(output_file)
if not mffile.exists():
raise LeafException("Cannot find manifest: {file}".format(file=mffile))
manifest = Manifest.parse(mffile)
manifest.validate_model()
if not validate_only:
if is_latest_package(manifest.identifier):
raise LeafException("Invalid version for manifest {mf} ({kw} is a reserved keyword)".format(mf=mffile, kw=LeafConstants.LATEST))
self.logger.print_default("Found package {mf.identifier} in {folder}".format(mf=manifest, folder=input_folder))
# Check if external info file exists
if not store_extenal_info and infofile.exists():
raise LeafException(
"A previous info file ({file}) exists for your package".format(file=infofile),
hints="You should remove it with 'rm {file}'".format(file=infofile),
)
self.__exec_tar(output_file, input_folder, extra_args=tar_extra_args)
self.logger.print_default("Leaf package created: {file}".format(file=output_file))
if store_extenal_info:
self.logger.print_default("Write info to {file}".format(file=infofile))
jwritefile(infofile, self.__build_pkg_node(output_file, manifest=manifest), pp=True)
def test_json(self):
jo = JsonObject({})
self.assertIsNone(jo.jsonpath(["a"]))
self.assertIsNotNone(jo.jsonpath(["a"], {}))
self.assertIsNotNone(jo.jsonpath(["a"]))
self.assertIsNone(jo.jsonpath(["a", "b"]))
self.assertIsNotNone(jo.jsonpath(["a", "b"], {}))
self.assertIsNotNone(jo.jsonpath(["a", "b"]))
self.assertIsNone(jo.jsonpath(["a", "b", "c"]))
self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "hello"))
self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "world"))
self.assertEqual("hello", jo.jsonpath(["a", "b", "c"]))
tmpfile = Path(mktemp(".json", "leaf-ut"))
jwritefile(tmpfile, jo.json, pp=True)
jo = JsonObject(jloadfile(tmpfile))
self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "hello"))
self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "world"))
self.assertEqual("hello", jo.jsonpath(["a", "b", "c"]))
with self.assertRaises(ValueError):
jo.jsonget("z", mandatory=True)
with self.assertRaises(ValueError):
jo.jsonpath(["a", "b", "c", "d"])
with self.assertRaises(ValueError):
jo.jsonpath(["a", "d", "e"])
def write_layer(self,
output: Path,
previous_layer: Path = None,
pp: bool = False):
# Update last leaf version field
self.leaf_min_version = str(CURRENT_LEAF_VERSION)
# Extract layer to write
data = self.json
if previous_layer is not None and previous_layer.exists():
data = jlayer_diff(jloadfile(previous_layer), self.json)
# Write layer
jwritefile(output, data, pp=pp)
def test_update_packages_map(self):
tmpfile = self.test_folder / "ws.json"
jwritefile(tmpfile, {"leafMinVersion": "1.8", "profiles": {"foo": {"packages": ["test_1.0"]}, "bar": {"packages": {"test": "2.0"}}}})
self.force_version("1.8")
ws_config = WorkspaceConfiguration(tmpfile)
self.assertTrue(isinstance(ws_config.json["profiles"]["foo"]["packages"], list))
self.force_version("2.0")
ws_config = WorkspaceConfiguration(tmpfile)
self.assertTrue(isinstance(ws_config.json["profiles"]["foo"]["packages"], OrderedDict))
self.assertEqual("1.0", ws_config.json["profiles"]["foo"]["packages"]["test"])
self.assertTrue(isinstance(ws_config.json["profiles"]["bar"]["packages"], OrderedDict))
self.assertEqual("2.0", ws_config.json["profiles"]["bar"]["packages"]["test"])
def test_update_root_folder(self):
tmpfile = self.test_folder / "user.json"
jwritefile(tmpfile, {"leafMinVersion": "1.8", "rootfolder": str(self.workspace_folder)})
self.force_version("1.8")
user_config = UserConfiguration(tmpfile)
self.assertTrue("rootfolder" in user_config.json)
self.assertEqual(0, len(user_config._getenvmap()))
self.force_version("2.0")
user_config = UserConfiguration(tmpfile)
self.assertFalse("rootfolder" in user_config.json)
self.assertEqual(1, len(user_config._getenvmap()))
self.assertEqual(str(self.workspace_folder), user_config._getenvmap()[LeafSettings.USER_PKG_FOLDER.key])
def test_updaters(self):
def my_updater_foo(model):
model.json["test"].append("foo")
def my_updater_bar(model):
model.json["test"].append("bar")
class MyConfig(ConfigFileWithLayer):
def __init__(self, *layers):
ConfigFileWithLayer.__init__(self, *layers)
def _get_updaters(self):
return super()._get_updaters() + ((None, my_updater_foo), (Version("2.0"), my_updater_bar))
@property
def test_list(self):
return self.json["test"]
tmpfile = self.test_folder / "a.json"
jwritefile(tmpfile, {"test": ["hello"]})
self.force_version("1.0")
model = MyConfig(tmpfile)
self.assertEqual(["hello", "foo", "bar"], model.test_list)
model.write_layer(tmpfile)
jmodel = jloadfile(tmpfile)
self.assertEqual(["hello", "foo", "bar"], jmodel["test"])
self.assertEqual("1.0", jmodel["leafMinVersion"])
model = MyConfig(tmpfile)
self.assertEqual(["hello", "foo", "bar"], model.test_list)
model.write_layer(tmpfile)
self.force_version("1.1")
model = MyConfig(tmpfile)
self.assertEqual(["hello", "foo", "bar", "foo", "bar"], model.test_list)
model.write_layer(tmpfile)
self.force_version("2.0")
model = MyConfig(tmpfile)
self.assertEqual(["hello", "foo", "bar", "foo", "bar", "foo", "bar"], model.test_list)
model = MyConfig(tmpfile)
self.assertEqual(["hello", "foo", "bar", "foo", "bar", "foo", "bar"], model.test_list)
model.write_layer(tmpfile)
self.force_version("2.1")
model = MyConfig(tmpfile)
self.assertEqual(["hello", "foo", "bar", "foo", "bar", "foo", "bar", "foo"], model.test_list)
def test_manifest_with_env(self, resolve_env=True):
mffile = self.workspace_folder / LeafFiles.MANIFEST
try:
os.environ["LEAF_TEST_VARIABLE"] = "hello"
fragment1 = self.workspace_folder / "a.json"
jwritefile(fragment1,
{"a": "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}"})
self.rm.generate_manifest(
mffile,
fragment_files=[fragment1],
info_map={
JsonConstants.INFO_NAME:
"foo",
JsonConstants.INFO_VERSION:
"1.0",
JsonConstants.INFO_DESCRIPTION:
"#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}",
},
resolve_envvars=resolve_env,
)
self.assertTrue(mffile.exists())
with mffile.open() as fp:
motif = "hello hello" if resolve_env else "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}"
self.assertEqual(
{
JsonConstants.INFO: {
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_DESCRIPTION: motif
},
"a": motif,
},
json.load(fp),
)
finally:
del os.environ["LEAF_TEST_VARIABLE"]
def test_validate_model(self):
pkg_folder = self.workspace_folder / "mypackage_1.0"
mffile = pkg_folder / "manifest.json"
artifact = self.workspace_folder / "foo.leaf"
pkg_folder.mkdir(parents=True, exist_ok=True)
# Validate only, invalid model
jwritefile(mffile, {"info": {"name": "my_package", "version": "1.0"}})
self.leaf_exec(("build", "pack"),
"--output",
artifact,
"--input",
pkg_folder,
"--validate-only",
expected_rc=2)
self.assertFalse(artifact.exists())
# Validate only, valid model
jwritefile(mffile, {"info": {"name": "mypackage", "version": "1.0"}})
self.leaf_exec(("build", "pack"), "--output", artifact, "--input",
pkg_folder, "--validate-only")
self.assertFalse(artifact.exists())
# Build package, invalid model
jwritefile(mffile, {"info": {"name": "my_package", "version": "1.0"}})
self.leaf_exec(("build", "pack"),
"--output",
artifact,
"--input",
pkg_folder,
expected_rc=2)
self.assertFalse(artifact.exists())
# Build package, valid model
jwritefile(mffile, {"info": {"name": "mypackage", "version": "1.0"}})
self.leaf_exec(("build", "pack"), "--output", artifact, "--input",
pkg_folder)
self.assertTrue(artifact.exists())
def test_manifest_fragments(self):
mffile = self.workspace_folder / LeafFiles.MANIFEST
fragment1 = self.workspace_folder / "a.json"
jwritefile(fragment1, {"a": 1, "info": {"tags": ["tag1"]}})
fragment2 = self.workspace_folder / "b.json"
jwritefile(fragment2, {"a": 2})
fragment3 = self.workspace_folder / "c.json"
jwritefile(fragment3, {"b": True, "info": {"tags": ["tag2"]}})
self.rm.generate_manifest(
mffile,
fragment_files=[fragment1, fragment2, fragment3],
info_map={
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_TAGS: ["foo", "bar", "foo"],
"ignored_extra_key": "hello",
},
)
self.assertTrue(mffile.exists())
with mffile.open() as fp:
self.assertEqual(
{
JsonConstants.INFO: {
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_TAGS:
["tag1", "tag2", "foo", "bar"],
},
"a": 2,
"b": True,
},
json.load(fp),
)
def generate_repository(source_folder, output_folder):
mkdirs(output_folder)
artifacts_list1 = []
artifacts_list2 = []
rm = RelengManager()
for package_folder in source_folder.iterdir():
if package_folder.is_dir() and PackageIdentifier.is_valid_identifier(
package_folder.name):
manifest_file = package_folder / LeafFiles.MANIFEST
if manifest_file.is_file():
manifest = Manifest.parse(manifest_file)
if str(manifest.identifier) != package_folder.name:
raise ValueError(
"Naming error: {mf.identifier} != {folder.name}".
format(mf=manifest, folder=package_folder))
filename = str(manifest.identifier) + ".leaf"
output_file = output_folder / filename
tar_extraargs = TAR_EXTRA_ARGS.get(manifest.identifier)
rm.create_package(package_folder,
output_file,
tar_extra_args=tar_extraargs)
# Check that the generated archive is OK
check_archive_format(
output_file,
tar_extraargs[0] if tar_extraargs is not None else None)
# Create multi index.json
if str(manifest.identifier) in ALT_INDEX_CONTENT:
artifacts_list2.append(output_file)
if ALT_INDEX_CONTENT[str(manifest.identifier)]:
artifacts_list1.append(output_file)
else:
artifacts_list1.append(output_file)
# Create a problem with failure-badhash package
if manifest.name == "failure-badhash":
info_node = jloadfile(
rm.find_external_info_file(output_file))
# chosen by fair dice roll.
# garanteed to be random.
info_node[
JsonConstants.
REMOTE_PACKAGE_HASH] = "sha384:d1083143b5c4cf7f1ddaadc391b2d0102fc9fffeb0951ec51020b512ef9548d40cd1af079a1221133faa949fdc304c41"
jwritefile(rm.find_external_info_file(output_file),
info_node,
pp=True)
if len(artifacts_list1) == 0 or len(artifacts_list2) == 0:
raise ValueError("Empty index!")
with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp:
fp.write("volatileTag1\n")
fp.write("volatileTag2")
rm.generate_index(output_folder / "index.json",
artifacts_list1,
name="First repository",
description="First repository description",
prettyprint=True)
with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp:
fp.write("volatileTag3\n")
fp.write("volatileTag4")
rm.generate_index(output_folder / "index2.json",
artifacts_list2,
name="Second repository",
description="Second repository description",
prettyprint=True)
# Alter some values for test purpose
index1json = jloadfile(output_folder / "index.json")
for pkgjson in index1json[JsonConstants.REMOTE_PACKAGES]:
if pkgjson["info"]["name"] == "failure-large-ap":
pkgjson["size"] = 999999999999
jwritefile(output_folder / "index.json", index1json, pp=True)
# Sign with GPG
subprocess.check_call([
"gpg", "--homedir",
str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor",
str(output_folder / "index.json")
])
subprocess.check_call([
"gpg", "--homedir",
str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor",
str(output_folder / "index2.json")
])
def generate_index(
self,
index_file: Path,
artifacts: list,
name: str = None,
description: str = None,
use_external_info: bool = True,
use_extra_tags: bool = True,
prettyprint: bool = False,
resolve: bool = True,
):
"""
Create an index.json referencing all given artifacts
"""
if not index_file.exists():
index_file.touch()
if resolve:
index_file = index_file.resolve()
try:
# Create the "info" node
info_node = OrderedDict()
if name is not None:
info_node[JsonConstants.REMOTE_NAME] = name
if description is not None:
info_node[JsonConstants.REMOTE_DESCRIPTION] = description
info_node[JsonConstants.REMOTE_DATE] = self.__get_date_now()
packages_map = OrderedDict()
# Resolve artifacts if needed
if resolve:
artifacts = [a.resolve() for a in artifacts]
for artifact in artifacts:
artifact_node = None
if use_external_info:
infofile = self.find_external_info_file(artifact)
if infofile.exists():
self.logger.print_default(
"Reading info from {file}".format(file=infofile))
artifact_node = jloadfile(infofile)
if artifact_node is None:
self.logger.print_default(
"Compute info for {artifact}".format(
artifact=artifact))
artifact_node = self.__build_pkg_node(artifact)
ap = AvailablePackage(artifact_node)
pi = ap.identifier
if is_latest_package(pi):
raise LeafException(
"Invalid version for package {artifact} ({word} is a reserved keyword)"
.format(artifact=artifact, word=LeafConstants.LATEST))
if pi in packages_map:
self.logger.print_default(
"Artifact already present: {pi}".format(pi=pi))
if ap.hashsum != AvailablePackage(
packages_map[pi]).hashsum:
raise LeafException(
"Artifact {pi} has multiple different artifacts for same version"
.format(pi=pi))
else:
# Read extra tags
extratags_file = artifact.parent / (artifact.name +
".tags")
if use_extra_tags and extratags_file.exists():
with extratags_file.open() as fp:
for tag in filter(
None, map(str.strip,
fp.read().splitlines())):
if tag not in ap.tags:
self.logger.print_default(
"Add extra tag {tag}".format(tag=tag))
ap.tags.append(tag)
self.logger.print_default("Add package {pi}".format(pi=pi))
try:
relative_path = artifact.relative_to(index_file.parent)
artifact_node[JsonConstants.REMOTE_PACKAGE_FILE] = str(
relative_path)
except ValueError:
raise LeafException(
"Artifact {a} must be relative to {i.parent}".
format(a=artifact, i=index_file))
packages_map[pi] = artifact_node
# Create the json structure
root_node = OrderedDict()
root_node[JsonConstants.INFO] = info_node
root_node[JsonConstants.REMOTE_PACKAGES] = list(
packages_map.values())
jwritefile(index_file, root_node, pp=prettyprint)
self.logger.print_default(
"Index created: {index}".format(index=index_file))
except BaseException as e:
# Clean the invalid index file
if index_file.exists():
index_file.unlink()
raise e
def test_manifest_generation(self):
mffile = self.workspace_folder / LeafFiles.MANIFEST
fragment1 = self.workspace_folder / "a.json"
jwritefile(
fragment1, {
"#{LEAF_TEST_VARIABLE}": "#{LEAF_TEST_VARIABLE}",
"a": 1,
"info": {
"tags": ["tag1"]
}
})
fragment2 = self.workspace_folder / "b.json"
jwritefile(fragment2, {"a": 2})
fragment3 = self.workspace_folder / "c.json"
jwritefile(fragment3, {"b": True, "info": {"tags": ["tag2"]}})
try:
os.environ["LEAF_TEST_VARIABLE"] = "hello"
self.leaf_exec(
("build", "manifest"),
"--output",
self.workspace_folder,
"--append",
fragment1,
"--append",
fragment2,
"--append",
fragment3,
"--env",
"--name",
"foo",
"--version",
"1.0",
"--description",
"lorem ipsum #{LEAF_TEST_VARIABLE}",
"--date",
"2012-12-12 12:12:12",
"--master",
"true",
"--minver",
"0.42",
"--requires",
"a_1",
"--requires",
"b_1",
"--requires",
"a_1",
"--depends",
"a_1",
"--depends",
"b_1(FOO=BAR)",
"--depends",
"a_1",
"--tag",
"foo",
"--tag",
"bar",
"--tag",
"foo",
"--upgradable",
"true",
)
finally:
del os.environ["LEAF_TEST_VARIABLE"]
self.assertTrue(mffile.exists())
with mffile.open() as fp:
self.assertEqual(
{
"hello": "hello",
"a": 2,
"b": True,
JsonConstants.INFO: {
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_DESCRIPTION: "lorem ipsum hello",
JsonConstants.INFO_DATE: "2012-12-12 12:12:12",
JsonConstants.INFO_MASTER: True,
JsonConstants.INFO_LEAF_MINVER: "0.42",
JsonConstants.INFO_REQUIRES: ["a_1", "b_1"],
JsonConstants.INFO_DEPENDS: ["a_1", "b_1(FOO=BAR)"],
JsonConstants.INFO_TAGS:
["tag1", "tag2", "foo", "bar"],
JsonConstants.INFO_AUTOUPGRADE: True,
},
},
json.load(fp),
)
def test_invalid_manifest(self):
pkg_folder = self.workspace_folder / "mypackage_1.0"
mffile = pkg_folder / "manifest.json"
artifact = self.workspace_folder / "foo.leaf"
pkg_folder.mkdir(parents=True, exist_ok=True)
# No name/version
jwritefile(mffile, {})
with self.assertRaises(ValidationError):
self.rm.create_package(pkg_folder, artifact)
# Invalid name
jwritefile(mffile, {"info": {"name": "my_package", "version": "1.0"}})
with self.assertRaises(ValidationError):
self.rm.create_package(pkg_folder, artifact)
# Invalid version
jwritefile(mffile, {"info": {"name": "mypackage", "version": "_1.0"}})
with self.assertRaises(ValidationError):
self.rm.create_package(pkg_folder, artifact)
# Invalid dependency
jwritefile(
mffile, {
"info": {
"name": "mypackage",
"version": "1.0",
"depends": ["foo-bar"]
}
})
with self.assertRaises(ValidationError):
self.rm.create_package(pkg_folder, artifact)
jwritefile(
mffile, {
"info": {
"name": "mypackage",
"version": "1.0",
"requires": ["foo_bar(FOO=BAR)(TEDDY!=BEAR)"]
}
})
with self.assertRaises(ValidationError):
self.rm.create_package(pkg_folder, artifact)
jwritefile(
mffile, {
"info": {
"name": "mypackage",
"version": "1.0",
"depends": ["foo_bar(FOO=BAR)(TEDDY!=BEAR)"],
"requires": ["teddy_bear"]
}
})
self.rm.create_package(pkg_folder, artifact)
# Valid model
jwritefile(mffile, {"info": {"name": "mypackage", "version": "1.0"}})
self.rm.create_package(pkg_folder, artifact)
# Empty step
jwritefile(mffile, {
"info": {
"name": "mypackage",
"version": "1.0"
},
"install": [{}]
})
with self.assertRaises(ValidationError):
self.rm.create_package(pkg_folder, artifact)
