def execute(self, args, uargs):
rm = RelengManager()
pkg_folder = None
if args.input_folder is None:
pkg_folder = Path(".")
elif args.input_folder.is_dir():
pkg_folder = args.input_folder
elif args.input_folder.is_file(
) and args.input_folder.name == LeafFiles.MANIFEST:
# Handles legacy, arg is not the folder but the manifest
pkg_folder = args.input_folder.parent
else:
raise ValueError("Invalid input folder")
rm.create_package(pkg_folder,
args.output_file,
store_extenal_info=args.sore_external_info,
tar_extra_args=args.tar_extra_args)
def execute(self, args, uargs):
rm = RelengManager()
# Guess output file
output_file = Path(LeafFiles.MANIFEST)
if args.output_folder is not None:
if not args.output_folder.is_dir():
raise LeafException("Invalid output folder: {folder}".format(
folder=args.output_folder))
output_file = args.output_folder / LeafFiles.MANIFEST
# Build the info map
info_map = {}
for k, v in vars(args).items():
if k.startswith(JsonConstants.INFO + "_"):
info_map[k[(len(JsonConstants.INFO) + 1):]] = v
rm.generate_manifest(output_file,
fragment_files=args.fragment_files,
info_map=info_map,
resolve_envvars=args.resolve_envvars)
def execute(self, args, uargs):
rm = RelengManager()
artifacts = []
artifacts += args.artifacts
if args.input_files:
for input_file in args.input_files:
rm.logger.print_default(
"Using artifacts from {0}".format(input_file))
with input_file.open() as fp:
for line in filter(lambda l: l and not l.startswith("#"),
map(str.strip,
fp.read().splitlines())):
artifacts.append(Path(line))
rm.generate_index(
args.output_file,
artifacts,
name=args.index_name,
description=args.index_description,
use_external_info=args.use_external_info,
use_extra_tags=args.use_extra_tags,
prettyprint=args.prettyprint,
resolve=args.resolve,
)
def generate_repository(source_folder, output_folder):
mkdirs(output_folder)
artifacts_list1 = []
artifacts_list2 = []
rm = RelengManager()
for package_folder in source_folder.iterdir():
if package_folder.is_dir() and PackageIdentifier.is_valid_identifier(
package_folder.name):
manifest_file = package_folder / LeafFiles.MANIFEST
if manifest_file.is_file():
manifest = Manifest.parse(manifest_file)
if str(manifest.identifier) != package_folder.name:
raise ValueError(
"Naming error: {mf.identifier} != {folder.name}".
format(mf=manifest, folder=package_folder))
filename = str(manifest.identifier) + ".leaf"
output_file = output_folder / filename
tar_extraargs = TAR_EXTRA_ARGS.get(manifest.identifier)
rm.create_package(package_folder,
output_file,
tar_extra_args=tar_extraargs)
# Check that the generated archive is OK
check_archive_format(
output_file,
tar_extraargs[0] if tar_extraargs is not None else None)
# Create multi index.json
if str(manifest.identifier) in ALT_INDEX_CONTENT:
artifacts_list2.append(output_file)
if ALT_INDEX_CONTENT[str(manifest.identifier)]:
artifacts_list1.append(output_file)
else:
artifacts_list1.append(output_file)
# Create a problem with failure-badhash package
if manifest.name == "failure-badhash":
info_node = jloadfile(
rm.find_external_info_file(output_file))
# chosen by fair dice roll.
# garanteed to be random.
info_node[
JsonConstants.
REMOTE_PACKAGE_HASH] = "sha384:d1083143b5c4cf7f1ddaadc391b2d0102fc9fffeb0951ec51020b512ef9548d40cd1af079a1221133faa949fdc304c41"
jwritefile(rm.find_external_info_file(output_file),
info_node,
pp=True)
if len(artifacts_list1) == 0 or len(artifacts_list2) == 0:
raise ValueError("Empty index!")
with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp:
fp.write("volatileTag1\n")
fp.write("volatileTag2")
rm.generate_index(output_folder / "index.json",
artifacts_list1,
name="First repository",
description="First repository description",
prettyprint=True)
with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp:
fp.write("volatileTag3\n")
fp.write("volatileTag4")
rm.generate_index(output_folder / "index2.json",
artifacts_list2,
name="Second repository",
description="Second repository description",
prettyprint=True)
# Alter some values for test purpose
index1json = jloadfile(output_folder / "index.json")
for pkgjson in index1json[JsonConstants.REMOTE_PACKAGES]:
if pkgjson["info"]["name"] == "failure-large-ap":
pkgjson["size"] = 999999999999
jwritefile(output_folder / "index.json", index1json, pp=True)
# Sign with GPG
subprocess.check_call([
"gpg", "--homedir",
str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor",
str(output_folder / "index.json")
])
subprocess.check_call([
"gpg", "--homedir",
str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor",
str(output_folder / "index2.json")
])
def setUp(self):
super().setUp()
self.rm = RelengManager()
class TestApiRelengManager(LeafTestCaseWithRepo):
def setUp(self):
super().setUp()
self.rm = RelengManager()
def test_package_compression(self):
folder = TEST_REMOTE_PACKAGE_SOURCE / "install_1.0"
def check_all_compressions(extension, mime):
output_file = self.workspace_folder / ("myPackage" + extension)
for args, mime in ((None, "x-tar"), (("."), "x-tar"),
(("-z", "."), "gzip"), (("-j", "."), "x-bzip2"),
(("-J", "."), "x-xz"), (("-a", "."), mime)):
self.rm.create_package(folder,
output_file,
tar_extra_args=args)
check_mime(output_file, mime)
check_all_compressions(".bin", "x-tar")
check_all_compressions(".tar", "x-tar")
check_all_compressions(".leaf", "x-tar")
check_all_compressions(".tar.gz", "gzip")
check_all_compressions(".tar.bz2", "x-bzip2")
check_all_compressions(".tar.xz", "x-xz")
def test_external_info_file(self):
folder = TEST_REMOTE_PACKAGE_SOURCE / "install_1.0"
artifact = self.workspace_folder / "myPackage.leaf"
info_file = self.workspace_folder / "myPackage.leaf.info"
self.rm.create_package(folder, artifact, store_extenal_info=False)
self.assertTrue(artifact.exists())
self.assertFalse(info_file.exists())
self.rm.create_package(folder, artifact, store_extenal_info=True)
self.assertTrue(artifact.exists())
self.assertTrue(info_file.exists())
self.assertEqual(info_file, self.rm.find_external_info_file(artifact))
self.assertEqual(hash_compute(artifact),
AvailablePackage(jloadfile(info_file), None).hashsum)
with self.assertRaises(LeafException):
self.rm.create_package(folder, artifact, store_extenal_info=False)
def test_manifest_info_map(self):
mffile = self.workspace_folder / LeafFiles.MANIFEST
self.rm.generate_manifest(
mffile,
info_map={
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_DESCRIPTION: "lorem ipsum",
JsonConstants.INFO_DATE: "2012-12-12 12:12:12",
JsonConstants.INFO_MASTER: True,
JsonConstants.INFO_LEAF_MINVER: "0.42",
JsonConstants.INFO_REQUIRES: ["a_1", "b_1", "a_1"],
JsonConstants.INFO_DEPENDS: ["a_1", "b_1(FOO=BAR)", "a_1"],
JsonConstants.INFO_TAGS: ["foo", "bar", "foo"],
"ignored_extra_key": "hello",
},
)
self.assertTrue(mffile.exists())
with mffile.open() as fp:
self.assertEqual(
{
JsonConstants.INFO: {
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_DESCRIPTION: "lorem ipsum",
JsonConstants.INFO_DATE: "2012-12-12 12:12:12",
JsonConstants.INFO_MASTER: True,
JsonConstants.INFO_LEAF_MINVER: "0.42",
JsonConstants.INFO_REQUIRES: ["a_1", "b_1"],
JsonConstants.INFO_DEPENDS: ["a_1", "b_1(FOO=BAR)"],
JsonConstants.INFO_TAGS: ["foo", "bar"],
}
},
json.load(fp),
)
def test_manifest_fragments(self):
mffile = self.workspace_folder / LeafFiles.MANIFEST
fragment1 = self.workspace_folder / "a.json"
jwritefile(fragment1, {"a": 1, "info": {"tags": ["tag1"]}})
fragment2 = self.workspace_folder / "b.json"
jwritefile(fragment2, {"a": 2})
fragment3 = self.workspace_folder / "c.json"
jwritefile(fragment3, {"b": True, "info": {"tags": ["tag2"]}})
self.rm.generate_manifest(
mffile,
fragment_files=[fragment1, fragment2, fragment3],
info_map={
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_TAGS: ["foo", "bar", "foo"],
"ignored_extra_key": "hello",
},
)
self.assertTrue(mffile.exists())
with mffile.open() as fp:
self.assertEqual(
{
JsonConstants.INFO: {
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_TAGS:
["tag1", "tag2", "foo", "bar"],
},
"a": 2,
"b": True,
},
json.load(fp),
)
def test_manifest_with_env(self, resolve_env=True):
mffile = self.workspace_folder / LeafFiles.MANIFEST
try:
os.environ["LEAF_TEST_VARIABLE"] = "hello"
fragment1 = self.workspace_folder / "a.json"
jwritefile(fragment1,
{"a": "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}"})
self.rm.generate_manifest(
mffile,
fragment_files=[fragment1],
info_map={
JsonConstants.INFO_NAME:
"foo",
JsonConstants.INFO_VERSION:
"1.0",
JsonConstants.INFO_DESCRIPTION:
"#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}",
},
resolve_envvars=resolve_env,
)
self.assertTrue(mffile.exists())
with mffile.open() as fp:
motif = "hello hello" if resolve_env else "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}"
self.assertEqual(
{
JsonConstants.INFO: {
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1.0",
JsonConstants.INFO_DESCRIPTION: motif
},
"a": motif,
},
json.load(fp),
)
finally:
del os.environ["LEAF_TEST_VARIABLE"]
def test_manifest_without_env(self):
self.test_manifest_with_env(resolve_env=False)
def test_index(self):
index = self.workspace_folder / "index.json"
# Build some packages
for pis in (
"install_1.0",
"condition_1.0",
"condition-A_1.0",
"condition-A_2.0",
"condition-B_1.0",
"condition-C_1.0",
"condition-D_1.0",
"condition-E_1.0",
"condition-F_1.0",
"condition-G_1.0",
"condition-H_1.0",
):
folder = TEST_REMOTE_PACKAGE_SOURCE / pis
output_file = self.workspace_folder / (pis + ".leaf")
self.rm.create_package(folder, output_file)
self.rm.generate_index(index,
self.workspace_folder.glob("condition*.leaf"),
prettyprint=True)
index_content = jloadfile(index)
self.assertEqual(10, len(index_content[JsonConstants.REMOTE_PACKAGES]))
self.rm.generate_index(index,
self.workspace_folder.glob("*.leaf"),
prettyprint=False)
index_content = jloadfile(index)
self.assertEqual(11, len(index_content[JsonConstants.REMOTE_PACKAGES]))
def test_index_same_artifact_different_hash(self):
mkdirs(self.workspace_folder / "a")
mkdirs(self.workspace_folder / "b")
self.rm.generate_manifest(
self.workspace_folder / "a" / LeafFiles.MANIFEST,
info_map={
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1",
JsonConstants.INFO_DESCRIPTION: "Some description"
},
)
self.rm.generate_manifest(
self.workspace_folder / "b" / LeafFiles.MANIFEST,
info_map={
JsonConstants.INFO_NAME: "foo",
JsonConstants.INFO_VERSION: "1",
JsonConstants.INFO_DESCRIPTION: "Different description"
},
)
self.rm.create_package(self.workspace_folder / "a",
self.workspace_folder / "a.leaf")
self.rm.create_package(self.workspace_folder / "b",
self.workspace_folder / "b.leaf")
self.rm.generate_index(self.workspace_folder / "indexA.json", [
self.workspace_folder / "a.leaf", self.workspace_folder / "a.leaf"
],
prettyprint=True)
self.rm.generate_index(self.workspace_folder / "indexB.json", [
self.workspace_folder / "b.leaf", self.workspace_folder / "b.leaf"
],
prettyprint=True)
with self.assertRaises(LeafException):
self.rm.generate_index(self.workspace_folder / "indexAB.json", [
self.workspace_folder / "a.leaf",
self.workspace_folder / "b.leaf"
],
prettyprint=True)