def create_package(self, input_folder: Path, output_file: Path, store_extenal_info: bool = True, tar_extra_args: list = None, validate_only: bool = False): """ Create a leaf artifact from given folder containing a manifest.json """ mffile = input_folder / LeafFiles.MANIFEST infofile = self.find_external_info_file(output_file) if not mffile.exists(): raise LeafException("Cannot find manifest: {file}".format(file=mffile)) manifest = Manifest.parse(mffile) manifest.validate_model() if not validate_only: if is_latest_package(manifest.identifier): raise LeafException("Invalid version for manifest {mf} ({kw} is a reserved keyword)".format(mf=mffile, kw=LeafConstants.LATEST)) self.logger.print_default("Found package {mf.identifier} in {folder}".format(mf=manifest, folder=input_folder)) # Check if external info file exists if not store_extenal_info and infofile.exists(): raise LeafException( "A previous info file ({file}) exists for your package".format(file=infofile), hints="You should remove it with 'rm {file}'".format(file=infofile), ) self.__exec_tar(output_file, input_folder, extra_args=tar_extra_args) self.logger.print_default("Leaf package created: {file}".format(file=output_file)) if store_extenal_info: self.logger.print_default("Write info to {file}".format(file=infofile)) jwritefile(infofile, self.__build_pkg_node(output_file, manifest=manifest), pp=True)
def test_json(self): jo = JsonObject({}) self.assertIsNone(jo.jsonpath(["a"])) self.assertIsNotNone(jo.jsonpath(["a"], {})) self.assertIsNotNone(jo.jsonpath(["a"])) self.assertIsNone(jo.jsonpath(["a", "b"])) self.assertIsNotNone(jo.jsonpath(["a", "b"], {})) self.assertIsNotNone(jo.jsonpath(["a", "b"])) self.assertIsNone(jo.jsonpath(["a", "b", "c"])) self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "hello")) self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "world")) self.assertEqual("hello", jo.jsonpath(["a", "b", "c"])) tmpfile = Path(mktemp(".json", "leaf-ut")) jwritefile(tmpfile, jo.json, pp=True) jo = JsonObject(jloadfile(tmpfile)) self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "hello")) self.assertEqual("hello", jo.jsonpath(["a", "b", "c"], "world")) self.assertEqual("hello", jo.jsonpath(["a", "b", "c"])) with self.assertRaises(ValueError): jo.jsonget("z", mandatory=True) with self.assertRaises(ValueError): jo.jsonpath(["a", "b", "c", "d"]) with self.assertRaises(ValueError): jo.jsonpath(["a", "d", "e"])
def write_layer(self, output: Path, previous_layer: Path = None, pp: bool = False): # Update last leaf version field self.leaf_min_version = str(CURRENT_LEAF_VERSION) # Extract layer to write data = self.json if previous_layer is not None and previous_layer.exists(): data = jlayer_diff(jloadfile(previous_layer), self.json) # Write layer jwritefile(output, data, pp=pp)
def test_update_packages_map(self): tmpfile = self.test_folder / "ws.json" jwritefile(tmpfile, {"leafMinVersion": "1.8", "profiles": {"foo": {"packages": ["test_1.0"]}, "bar": {"packages": {"test": "2.0"}}}}) self.force_version("1.8") ws_config = WorkspaceConfiguration(tmpfile) self.assertTrue(isinstance(ws_config.json["profiles"]["foo"]["packages"], list)) self.force_version("2.0") ws_config = WorkspaceConfiguration(tmpfile) self.assertTrue(isinstance(ws_config.json["profiles"]["foo"]["packages"], OrderedDict)) self.assertEqual("1.0", ws_config.json["profiles"]["foo"]["packages"]["test"]) self.assertTrue(isinstance(ws_config.json["profiles"]["bar"]["packages"], OrderedDict)) self.assertEqual("2.0", ws_config.json["profiles"]["bar"]["packages"]["test"])
def test_update_root_folder(self): tmpfile = self.test_folder / "user.json" jwritefile(tmpfile, {"leafMinVersion": "1.8", "rootfolder": str(self.workspace_folder)}) self.force_version("1.8") user_config = UserConfiguration(tmpfile) self.assertTrue("rootfolder" in user_config.json) self.assertEqual(0, len(user_config._getenvmap())) self.force_version("2.0") user_config = UserConfiguration(tmpfile) self.assertFalse("rootfolder" in user_config.json) self.assertEqual(1, len(user_config._getenvmap())) self.assertEqual(str(self.workspace_folder), user_config._getenvmap()[LeafSettings.USER_PKG_FOLDER.key])
def test_updaters(self): def my_updater_foo(model): model.json["test"].append("foo") def my_updater_bar(model): model.json["test"].append("bar") class MyConfig(ConfigFileWithLayer): def __init__(self, *layers): ConfigFileWithLayer.__init__(self, *layers) def _get_updaters(self): return super()._get_updaters() + ((None, my_updater_foo), (Version("2.0"), my_updater_bar)) @property def test_list(self): return self.json["test"] tmpfile = self.test_folder / "a.json" jwritefile(tmpfile, {"test": ["hello"]}) self.force_version("1.0") model = MyConfig(tmpfile) self.assertEqual(["hello", "foo", "bar"], model.test_list) model.write_layer(tmpfile) jmodel = jloadfile(tmpfile) self.assertEqual(["hello", "foo", "bar"], jmodel["test"]) self.assertEqual("1.0", jmodel["leafMinVersion"]) model = MyConfig(tmpfile) self.assertEqual(["hello", "foo", "bar"], model.test_list) model.write_layer(tmpfile) self.force_version("1.1") model = MyConfig(tmpfile) self.assertEqual(["hello", "foo", "bar", "foo", "bar"], model.test_list) model.write_layer(tmpfile) self.force_version("2.0") model = MyConfig(tmpfile) self.assertEqual(["hello", "foo", "bar", "foo", "bar", "foo", "bar"], model.test_list) model = MyConfig(tmpfile) self.assertEqual(["hello", "foo", "bar", "foo", "bar", "foo", "bar"], model.test_list) model.write_layer(tmpfile) self.force_version("2.1") model = MyConfig(tmpfile) self.assertEqual(["hello", "foo", "bar", "foo", "bar", "foo", "bar", "foo"], model.test_list)
def test_manifest_with_env(self, resolve_env=True): mffile = self.workspace_folder / LeafFiles.MANIFEST try: os.environ["LEAF_TEST_VARIABLE"] = "hello" fragment1 = self.workspace_folder / "a.json" jwritefile(fragment1, {"a": "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}"}) self.rm.generate_manifest( mffile, fragment_files=[fragment1], info_map={ JsonConstants.INFO_NAME: "foo", JsonConstants.INFO_VERSION: "1.0", JsonConstants.INFO_DESCRIPTION: "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}", }, resolve_envvars=resolve_env, ) self.assertTrue(mffile.exists()) with mffile.open() as fp: motif = "hello hello" if resolve_env else "#{LEAF_TEST_VARIABLE} #{LEAF_TEST_VARIABLE}" self.assertEqual( { JsonConstants.INFO: { JsonConstants.INFO_NAME: "foo", JsonConstants.INFO_VERSION: "1.0", JsonConstants.INFO_DESCRIPTION: motif }, "a": motif, }, json.load(fp), ) finally: del os.environ["LEAF_TEST_VARIABLE"]
def test_validate_model(self): pkg_folder = self.workspace_folder / "mypackage_1.0" mffile = pkg_folder / "manifest.json" artifact = self.workspace_folder / "foo.leaf" pkg_folder.mkdir(parents=True, exist_ok=True) # Validate only, invalid model jwritefile(mffile, {"info": {"name": "my_package", "version": "1.0"}}) self.leaf_exec(("build", "pack"), "--output", artifact, "--input", pkg_folder, "--validate-only", expected_rc=2) self.assertFalse(artifact.exists()) # Validate only, valid model jwritefile(mffile, {"info": {"name": "mypackage", "version": "1.0"}}) self.leaf_exec(("build", "pack"), "--output", artifact, "--input", pkg_folder, "--validate-only") self.assertFalse(artifact.exists()) # Build package, invalid model jwritefile(mffile, {"info": {"name": "my_package", "version": "1.0"}}) self.leaf_exec(("build", "pack"), "--output", artifact, "--input", pkg_folder, expected_rc=2) self.assertFalse(artifact.exists()) # Build package, valid model jwritefile(mffile, {"info": {"name": "mypackage", "version": "1.0"}}) self.leaf_exec(("build", "pack"), "--output", artifact, "--input", pkg_folder) self.assertTrue(artifact.exists())
def test_manifest_fragments(self): mffile = self.workspace_folder / LeafFiles.MANIFEST fragment1 = self.workspace_folder / "a.json" jwritefile(fragment1, {"a": 1, "info": {"tags": ["tag1"]}}) fragment2 = self.workspace_folder / "b.json" jwritefile(fragment2, {"a": 2}) fragment3 = self.workspace_folder / "c.json" jwritefile(fragment3, {"b": True, "info": {"tags": ["tag2"]}}) self.rm.generate_manifest( mffile, fragment_files=[fragment1, fragment2, fragment3], info_map={ JsonConstants.INFO_NAME: "foo", JsonConstants.INFO_VERSION: "1.0", JsonConstants.INFO_TAGS: ["foo", "bar", "foo"], "ignored_extra_key": "hello", }, ) self.assertTrue(mffile.exists()) with mffile.open() as fp: self.assertEqual( { JsonConstants.INFO: { JsonConstants.INFO_NAME: "foo", JsonConstants.INFO_VERSION: "1.0", JsonConstants.INFO_TAGS: ["tag1", "tag2", "foo", "bar"], }, "a": 2, "b": True, }, json.load(fp), )
def generate_repository(source_folder, output_folder): mkdirs(output_folder) artifacts_list1 = [] artifacts_list2 = [] rm = RelengManager() for package_folder in source_folder.iterdir(): if package_folder.is_dir() and PackageIdentifier.is_valid_identifier( package_folder.name): manifest_file = package_folder / LeafFiles.MANIFEST if manifest_file.is_file(): manifest = Manifest.parse(manifest_file) if str(manifest.identifier) != package_folder.name: raise ValueError( "Naming error: {mf.identifier} != {folder.name}". format(mf=manifest, folder=package_folder)) filename = str(manifest.identifier) + ".leaf" output_file = output_folder / filename tar_extraargs = TAR_EXTRA_ARGS.get(manifest.identifier) rm.create_package(package_folder, output_file, tar_extra_args=tar_extraargs) # Check that the generated archive is OK check_archive_format( output_file, tar_extraargs[0] if tar_extraargs is not None else None) # Create multi index.json if str(manifest.identifier) in ALT_INDEX_CONTENT: artifacts_list2.append(output_file) if ALT_INDEX_CONTENT[str(manifest.identifier)]: artifacts_list1.append(output_file) else: artifacts_list1.append(output_file) # Create a problem with failure-badhash package if manifest.name == "failure-badhash": info_node = jloadfile( rm.find_external_info_file(output_file)) # chosen by fair dice roll. # garanteed to be random. info_node[ JsonConstants. REMOTE_PACKAGE_HASH] = "sha384:d1083143b5c4cf7f1ddaadc391b2d0102fc9fffeb0951ec51020b512ef9548d40cd1af079a1221133faa949fdc304c41" jwritefile(rm.find_external_info_file(output_file), info_node, pp=True) if len(artifacts_list1) == 0 or len(artifacts_list2) == 0: raise ValueError("Empty index!") with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp: fp.write("volatileTag1\n") fp.write("volatileTag2") rm.generate_index(output_folder / "index.json", artifacts_list1, name="First repository", description="First repository description", prettyprint=True) with (output_folder / "multitags_1.0.leaf.tags").open("w") as fp: fp.write("volatileTag3\n") fp.write("volatileTag4") rm.generate_index(output_folder / "index2.json", artifacts_list2, name="Second repository", description="Second repository description", prettyprint=True) # Alter some values for test purpose index1json = jloadfile(output_folder / "index.json") for pkgjson in index1json[JsonConstants.REMOTE_PACKAGES]: if pkgjson["info"]["name"] == "failure-large-ap": pkgjson["size"] = 999999999999 jwritefile(output_folder / "index.json", index1json, pp=True) # Sign with GPG subprocess.check_call([ "gpg", "--homedir", str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor", str(output_folder / "index.json") ]) subprocess.check_call([ "gpg", "--homedir", str(TEST_GPG_HOMEDIR), "--detach-sign", "--armor", str(output_folder / "index2.json") ])
def generate_index( self, index_file: Path, artifacts: list, name: str = None, description: str = None, use_external_info: bool = True, use_extra_tags: bool = True, prettyprint: bool = False, resolve: bool = True, ): """ Create an index.json referencing all given artifacts """ if not index_file.exists(): index_file.touch() if resolve: index_file = index_file.resolve() try: # Create the "info" node info_node = OrderedDict() if name is not None: info_node[JsonConstants.REMOTE_NAME] = name if description is not None: info_node[JsonConstants.REMOTE_DESCRIPTION] = description info_node[JsonConstants.REMOTE_DATE] = self.__get_date_now() packages_map = OrderedDict() # Resolve artifacts if needed if resolve: artifacts = [a.resolve() for a in artifacts] for artifact in artifacts: artifact_node = None if use_external_info: infofile = self.find_external_info_file(artifact) if infofile.exists(): self.logger.print_default( "Reading info from {file}".format(file=infofile)) artifact_node = jloadfile(infofile) if artifact_node is None: self.logger.print_default( "Compute info for {artifact}".format( artifact=artifact)) artifact_node = self.__build_pkg_node(artifact) ap = AvailablePackage(artifact_node) pi = ap.identifier if is_latest_package(pi): raise LeafException( "Invalid version for package {artifact} ({word} is a reserved keyword)" .format(artifact=artifact, word=LeafConstants.LATEST)) if pi in packages_map: self.logger.print_default( "Artifact already present: {pi}".format(pi=pi)) if ap.hashsum != AvailablePackage( packages_map[pi]).hashsum: raise LeafException( "Artifact {pi} has multiple different artifacts for same version" .format(pi=pi)) else: # Read extra tags extratags_file = artifact.parent / (artifact.name + ".tags") if use_extra_tags and extratags_file.exists(): with extratags_file.open() as fp: for tag in filter( None, map(str.strip, fp.read().splitlines())): if tag not in ap.tags: self.logger.print_default( "Add extra tag {tag}".format(tag=tag)) ap.tags.append(tag) self.logger.print_default("Add package {pi}".format(pi=pi)) try: relative_path = artifact.relative_to(index_file.parent) artifact_node[JsonConstants.REMOTE_PACKAGE_FILE] = str( relative_path) except ValueError: raise LeafException( "Artifact {a} must be relative to {i.parent}". format(a=artifact, i=index_file)) packages_map[pi] = artifact_node # Create the json structure root_node = OrderedDict() root_node[JsonConstants.INFO] = info_node root_node[JsonConstants.REMOTE_PACKAGES] = list( packages_map.values()) jwritefile(index_file, root_node, pp=prettyprint) self.logger.print_default( "Index created: {index}".format(index=index_file)) except BaseException as e: # Clean the invalid index file if index_file.exists(): index_file.unlink() raise e
def test_manifest_generation(self): mffile = self.workspace_folder / LeafFiles.MANIFEST fragment1 = self.workspace_folder / "a.json" jwritefile( fragment1, { "#{LEAF_TEST_VARIABLE}": "#{LEAF_TEST_VARIABLE}", "a": 1, "info": { "tags": ["tag1"] } }) fragment2 = self.workspace_folder / "b.json" jwritefile(fragment2, {"a": 2}) fragment3 = self.workspace_folder / "c.json" jwritefile(fragment3, {"b": True, "info": {"tags": ["tag2"]}}) try: os.environ["LEAF_TEST_VARIABLE"] = "hello" self.leaf_exec( ("build", "manifest"), "--output", self.workspace_folder, "--append", fragment1, "--append", fragment2, "--append", fragment3, "--env", "--name", "foo", "--version", "1.0", "--description", "lorem ipsum #{LEAF_TEST_VARIABLE}", "--date", "2012-12-12 12:12:12", "--master", "true", "--minver", "0.42", "--requires", "a_1", "--requires", "b_1", "--requires", "a_1", "--depends", "a_1", "--depends", "b_1(FOO=BAR)", "--depends", "a_1", "--tag", "foo", "--tag", "bar", "--tag", "foo", "--upgradable", "true", ) finally: del os.environ["LEAF_TEST_VARIABLE"] self.assertTrue(mffile.exists()) with mffile.open() as fp: self.assertEqual( { "hello": "hello", "a": 2, "b": True, JsonConstants.INFO: { JsonConstants.INFO_NAME: "foo", JsonConstants.INFO_VERSION: "1.0", JsonConstants.INFO_DESCRIPTION: "lorem ipsum hello", JsonConstants.INFO_DATE: "2012-12-12 12:12:12", JsonConstants.INFO_MASTER: True, JsonConstants.INFO_LEAF_MINVER: "0.42", JsonConstants.INFO_REQUIRES: ["a_1", "b_1"], JsonConstants.INFO_DEPENDS: ["a_1", "b_1(FOO=BAR)"], JsonConstants.INFO_TAGS: ["tag1", "tag2", "foo", "bar"], JsonConstants.INFO_AUTOUPGRADE: True, }, }, json.load(fp), )
def test_invalid_manifest(self): pkg_folder = self.workspace_folder / "mypackage_1.0" mffile = pkg_folder / "manifest.json" artifact = self.workspace_folder / "foo.leaf" pkg_folder.mkdir(parents=True, exist_ok=True) # No name/version jwritefile(mffile, {}) with self.assertRaises(ValidationError): self.rm.create_package(pkg_folder, artifact) # Invalid name jwritefile(mffile, {"info": {"name": "my_package", "version": "1.0"}}) with self.assertRaises(ValidationError): self.rm.create_package(pkg_folder, artifact) # Invalid version jwritefile(mffile, {"info": {"name": "mypackage", "version": "_1.0"}}) with self.assertRaises(ValidationError): self.rm.create_package(pkg_folder, artifact) # Invalid dependency jwritefile( mffile, { "info": { "name": "mypackage", "version": "1.0", "depends": ["foo-bar"] } }) with self.assertRaises(ValidationError): self.rm.create_package(pkg_folder, artifact) jwritefile( mffile, { "info": { "name": "mypackage", "version": "1.0", "requires": ["foo_bar(FOO=BAR)(TEDDY!=BEAR)"] } }) with self.assertRaises(ValidationError): self.rm.create_package(pkg_folder, artifact) jwritefile( mffile, { "info": { "name": "mypackage", "version": "1.0", "depends": ["foo_bar(FOO=BAR)(TEDDY!=BEAR)"], "requires": ["teddy_bear"] } }) self.rm.create_package(pkg_folder, artifact) # Valid model jwritefile(mffile, {"info": {"name": "mypackage", "version": "1.0"}}) self.rm.create_package(pkg_folder, artifact) # Empty step jwritefile(mffile, { "info": { "name": "mypackage", "version": "1.0" }, "install": [{}] }) with self.assertRaises(ValidationError): self.rm.create_package(pkg_folder, artifact)