def build_module(copr, form): if not form.validate_on_submit(): # WORKAROUND append those which are not in min_entries for i in range(2, len(form.profile_names)): form.profile_pkgs.append_entry() return render_create_module(copr, form, profiles=len(form.profile_names)) mmd = modulemd.ModuleMetadata() mmd.load(os.path.join(os.path.dirname(__file__), "empty-module.yaml")) mmd.name = copr.name mmd.version = form.version.data mmd.release = form.release.data mmd.summary = "Module from Copr repository: {}".format(copr.full_name) for package in form.filter.data: mmd.components.rpms.add_filter(package) for package in form.api.data: mmd.components.rpms.add_api(package) for i, values in enumerate(zip(form.profile_names.data, form.profile_pkgs.data)): name, packages = values mmd.profiles[name] = modulemd.profile.ModuleProfile() for package in packages: mmd.profiles[name].add_rpm(package) actions_logic.ActionsLogic.send_build_module(copr, mmd.dumps()) db.session.commit() flask.flash("Modulemd yaml file successfully generated and submitted to build") return flask.redirect(url_for_copr_details(copr))
def index_modulemd_files(repo_path): result = [] for fn in sorted(os.listdir(repo_path)): if not fn.endswith(".yaml"): continue yaml_path = os.path.join(repo_path, fn) mmd = modulemd.ModuleMetadata() mmd.load(yaml_path) result.append(mmd) return result
def test_invalid_yaml(self): document = """ document: modulemd version: 0 data """ mmd = modulemd.ModuleMetadata() # Python yaml module raises wrong error message with "found" string # instead of "find". We are testing for both variants here. self.assertRaisesRegexp(ScannerError, r"could not f(?:ou|i)nd expected ':'", mmd.loads, document)
def test_object_value(self, yaml=None, value=""): """ Replaces $VALUE in the the `yaml` input with the value provided in the `value` variable and loads the yaml using modulemd library. """ if not yaml: return yaml = yaml.replace("$VALUE", value) mmd = modulemd.ModuleMetadata() mmd.loads(yaml) mmd.validate()
def test_object_missing(self, yaml=None): """ Removes the line with the $VALUE from the yaml input and loads the yaml using modulemd library. """ if not yaml: return yaml = "\n".join(n for n in yaml.split("\n") if "$VALUE" not in n) mmd = modulemd.ModuleMetadata() mmd.loads(yaml) mmd.validate()
def setUp(self): """ Verify required modulemd file parameter has been specified, exists, and can be loaded. The file name and loaded metadata are saved. """ mdfile = self.params.get('modulemd') if mdfile is None: self.error("modulemd parameter must be supplied") mdfile = str(mdfile) if not os.path.isfile(mdfile): self.error("modulemd file %s must exist" % mdfile) try: mmd = modulemd.ModuleMetadata() mmd.load(mdfile) except Exception as ex: self.error( "There was an error while processing modulemd file %s: %s" % (mdfile, ex)) # Infer the module name from the mdfile name and check that it is sane mdfileModuleName, mdfileExtension = os.path.basename(mdfile).split( '.', 1) if (mdfileExtension != 'yaml') and (mdfileExtension != 'yml'): self.error("modulemd file %s must have a .y[a]ml extension" % mdfile) if mmd.name == '': # The name can be missing from the metadata because the builder # knows how to infer it mmd.name = mdfileModuleName elif mmd.name != mdfileModuleName: self.error( "modulemd file name %s and module name %s do not match" % (mdfileModuleName, mmd.name)) self.mdfile = mdfile self.mmd = mmd try: jargonfile = self.params.get('jargonfile') if jargonfile is not None: jargonfile = str(jargonfile) dict = DictWithPWL("en_US", jargonfile) for w in self.mmd.name.split('-'): dict.add_to_session(w) self.chkr = SpellChecker(dict) else: self.chkr = SpellChecker("en_US") except: self.error( "Could not initialize spell checker with dictionary %s" % dict)
def parse_module_metadata(module_url): """Fetch and parse the YAML describing a module. Raises some kind of exception if something goes wrong. :param module_url: The URL to the repo. :returns: The parsed YAML object :rtype: modulemd.ModuleMetadata """ # The module's yaml file is part of the repo data, so to find it we first # have to fetch and parse yet another metadata file in yet another format. # Important note: both YAML and XML specify the character encoding as part # of the bytestream (YAML via an optional BOM, XML via the <?xml encoding= part), # so the parsers for both should get response.content, which is bytes, and not # response.text, which is str. response = requests.get(module_url + "/repodata/repomd.xml") if response.status_code != requests.codes.ok: raise ValueError("Unable to fetch repomd for " + module_url) # Parse the repomd, and find the piece that describes the module file. # Python does xpath weird: absolute paths are not allowed, and, since the parse # functions return a top-level Element object and not a separate Document type, # "." from the top-level object is the outmost element. So "./data" is what the # rest of the world would call "/repomd/data". # Also important: use response.content, and not response.text, because the parser # does the character conversion stuff itself based on the <?xml encoding= value. tree = ET.fromstring(response.content) namespaces = {"md": "http://linux.duke.edu/metadata/repo"} # If something goes wrong this will throw an AttributeError probably yaml_url = tree.find("./md:data[@type='module']/md:location", namespaces).get("href") # Fetch the YAML response = requests.get(module_url + "/" + yaml_url) if response.status_code != requests.codes.ok: raise ValueError("Unable to fetch YAML for " + module_url) # It's probably compressed. If so, uncompress if yaml_url.endswith('.gz'): yaml_data = gzip.decompress(response.content) else: yaml_data = response.content # Parse the YAML and return the ModuleMetadata object mmd = modulemd.ModuleMetadata() mmd.loads(yaml_data) return mmd
def _load_module_dist_tag(self): module_md_filepath = os.path.join(self.job.destdir, self.job.chroot, "module_md.yaml") try: mmd = modulemd.ModuleMetadata() mmd.load(module_md_filepath) except IOError as e: return None except Exception as e: log.exception(e) return None else: self.log.info("Loaded {}".format(module_md_filepath)) return ("." + mmd.name + '+' + mmd.version + '+' + mmd.release)
def init_on_load(self): data = json.loads(self.data) yaml = base64.b64decode(json.loads(self.data)["modulemd_b64"]) mmd = modulemd.ModuleMetadata() mmd.loads(yaml) attrs = ["name", "version", "release", "summary"] for attr in attrs: setattr(self, attr, getattr(mmd, attr)) self.ownername = data["ownername"] self.projectname = data["projectname"] self.modulemd = mmd self.yaml = mmd.dumps() self.full_name = "-".join([self.name, self.version, self.release])
def _setup_modulemd(self): mdfile = self.params.get('modulemd') if mdfile is None: self.error("modulemd parameter must be supplied") mdfile = str(mdfile) if not os.path.isfile(mdfile): self.error("modulemd file %s must exist" % mdfile) try: mmd = modulemd.ModuleMetadata() mmd.load(mdfile) except: self.error("Could not load modulemd file %s" % mdfile) self.mdfile = mdfile self.mmd = mmd
def create_unreleased_variant(self, pdc, body): """Creates an UnreleasedVariant for a module in PDC. Checks out the module metadata from the supplied SCM repository (currently only anonymous GIT is supported).""" log.debug("create_unreleased_variant(pdc, body=%r)" % body) mmd = modulemd.ModuleMetadata() mmd.loads(body['modulemd']) runtime_deps = [{ 'dependency': dependency, 'stream': stream } for dependency, stream in mmd.requires.items()] build_deps = [{ 'dependency': dependency, 'stream': stream } for dependency, stream in mmd.buildrequires.items()] name = body['name'] # TODO: PDC has to be patched to support stream/version instead of # version/release, but for now we just do the right mapping here... version = body['stream'] release = body['version'] variant_uid = "{n}-{v}-{r}".format(n=name, v=version, r=release) variant_id = name tag_str = '.'.join([name, version, str(release)]) tag_hash = hashlib.sha1(tag_str).hexdigest()[:16] koji_tag = "module-" + tag_hash data = { 'variant_id': variant_id, 'variant_uid': variant_uid, 'variant_name': name, 'variant_version': version, 'variant_release': release, 'variant_type': 'module', 'koji_tag': koji_tag, 'runtime_deps': runtime_deps, 'build_deps': build_deps, 'modulemd': body["modulemd"], } unreleased_variant = pdc['unreleasedvariants']._(data) return unreleased_variant
def setUp(self): """ Verify required modulemd file parameter has been specified, exists, and can be loaded. The file name and loaded metadata are saved. """ mmd = modulemd.ModuleMetadata() mdfile = self.params.get("modulemd") self.tmdfile = None if not mdfile: # try to use module testing farmework if possible # https://pagure.io/modularity-testing-framework try: mtf_backend = module_framework.CommonFunctions() self.tmdfile = tempfile.mkstemp(suffix=".yaml")[1] with open(self.tmdfile, "w+b") as yamlfile: yaml.dump(mtf_backend.getModulemdYamlconfig(), yamlfile, default_flow_style=False) mdfile = self.tmdfile except common.ConfigExc: pass if mdfile is None: self.error("modulemd parameter must be supplied") mdfile = str(mdfile) if not os.path.isfile(mdfile): self.error("modulemd file %s must exist" % mdfile) try: mmd.load(mdfile) except Exception as ex: self.error("There was an error while processing modulemd file %s: %s" % (mdfile, ex)) # Infer the module name from the mdfile name and check that it is sane mdfileModuleName, mdfileExtension = os.path.basename(mdfile).split('.', 1) if (mdfileExtension != "yaml") and (mdfileExtension != "yml"): self.error("modulemd file %s must have a .y[a]ml extension" % mdfile) if mmd.name == '': # The name can be missing from the metadata because the builder # knows how to infer it mmd.name = mdfileModuleName elif mmd.name != mdfileModuleName: self.error("modulemd file name %s and module name %s do not match" % ( mdfileModuleName, mmd.name)) self.mdfile = mdfile self.mmd = mmd
def test_load(self, filename="tests/test.yaml"): mmd = modulemd.ModuleMetadata() mmd.load(filename) self.assertEqual(mmd.mdversion, 0) self.assertEqual(mmd.name, "test") self.assertEqual(mmd.version, "1.23") self.assertEqual(mmd.release, "4") self.assertEqual(mmd.summary, "A test module") self.assertEqual(mmd.description, "This module is a part of the modulemd test suite.") self.assertEqual(mmd.module_licenses, set(["MIT"])) self.assertEqual(mmd.content_licenses, set(["GPL+", "GPLv3"])) self.assertEqual(mmd.buildrequires, {"example": "84-84"}) self.assertEqual(mmd.requires, {"modulemd": "42-42"}) self.assertEqual(mmd.community, "http://www.example.com/community") self.assertEqual(mmd.documentation, "http://www.example.com/documentation") self.assertEqual(mmd.tracker, "http://www.example.com/tracker") self.assertEqual(mmd.xmd, {"userid": "userdata"}) self.assertEqual(sorted(mmd.profiles.keys()), ["default", "minimal"]) self.assertEqual(mmd.profiles["default"].rpms, set(["alfa", "alfa-subpackage"])) self.assertEqual(mmd.profiles["minimal"].description, "Minimal profile installing only the alfa package.") self.assertEqual(mmd.profiles["minimal"].rpms, set(["alfa"])) self.assertTrue(mmd.components.rpms.dependencies) self.assertEqual(mmd.components.rpms.api, set(["alfa", "alfa-extras"])) self.assertEqual( mmd.components.rpms.packages, { "alfa": { "rationale": "alfa rationale" }, "bravo": { "rationale": "bravo rationale", "arches": ["charlie", "delta"], "multilib": ["echo"], "commit": "foxtrot", "repository": "golf", "cache": "hotel" } }) self.assertEqual(mmd.components.rpms.filter, set(["filter_1", "filter_2"]))
def test_dump(self): mmd = modulemd.ModuleMetadata() mmd.mdversion = 0 mmd.name = "test" mmd.version = "1.23" mmd.release = "4" mmd.summary = "A test module" mmd.description = "This module is a part of the modulemd test suite." mmd.add_module_license("MIT") mmd.add_content_license("GPL+") mmd.add_content_license("GPLv3") mmd.add_buildrequires("example", "84-84") mmd.add_requires("modulemd", "42-42") mmd.community = "http://www.example.com/community" mmd.documentation = "http://www.example.com/documentation" mmd.tracker = "http://www.example.com/tracker" mmd.xmd = {"userid": "userdata"} mmd.profiles = { "default": modulemd.ModuleProfile(), "minimal": modulemd.ModuleProfile() } mmd.profiles["default"].rpms = set(["alfa", "alfa-subpackage"]) mmd.profiles["minimal"].rpms = set(["alfa"]) mmd.profiles[ "minimal"].description = "Minimal profile installing only the alfa package." mmd.components = modulemd.ModuleComponents() mmd.components.rpms = modulemd.ModuleRPMs() mmd.components.rpms.dependencies = True mmd.components.rpms.add_api("alfa") mmd.components.rpms.add_api("alfa-extras") mmd.components.rpms.add_package("alfa", rationale="alfa rationale") mmd.components.rpms.add_package("bravo", rationale="bravo rationale", arches=["charlie", "delta"], multilib=["echo"], commit="foxtrot", repository="golf", cache="hotel") mmd.components.rpms.add_filter("filter_1") mmd.components.rpms.add_filter("filter_2") mmd.dump("tests/dump.yaml") self.test_load(filename="tests/dump.yaml")
def setUp(self): """ Verify required modulemd file parameter has been specified, exists, and can be loaded. The file name and loaded metadata are saved. """ mdfile = self.params.get('modulemd') if mdfile is None: self.error("modulemd parameter must be supplied") mdfile = str(mdfile) if not os.path.isfile(mdfile): self.error("modulemd file %s must exist" % mdfile) try: mmd = modulemd.ModuleMetadata() mmd.load(mdfile) except: self.error("Could not load modulemd file %s" % mdfile) self.mdfile = mdfile self.mmd = mmd
def get_unreleased_variant_rpms(self, pdc, variant): """ Returns the list of rpms as defined "rpms" key in "unreleasedvariants" PDC endpoint. The list is obtained from the Koji tag defined by "koji_tag" value of input variant `variant`. """ mmd = modulemd.ModuleMetadata() mmd.loads(variant['modulemd']) koji_rpms = pdcupdater.services.koji_rpms_in_tag( self.koji_url, variant["koji_tag"]) rpms = [] # Flatten into a list and augment the koji dict with tag info. for rpm in koji_rpms: data = dict( name=rpm['name'], version=rpm['version'], release=rpm['release'], epoch=rpm['epoch'] or 0, arch=rpm['arch'], srpm_name=rpm['srpm_name'], ) if 'srpm_nevra' in rpm and rpm['arch'] != 'src': data['srpm_nevra'] = rpm['srpm_nevra'] # For SRPM packages, include the hash and branch from which is # has been built. if (rpm['arch'] == 'src' and rpm['name'] in mmd.components.rpms and 'rpms' in mmd.xmd['mbs'] and rpm['name'] in mmd.xmd['mbs']['rpms']): mmd_rpm = mmd.components.rpms[rpm['name']] xmd_rpm = mmd.xmd['mbs']['rpms'][rpm['name']] data["srpm_commit_hash"] = xmd_rpm['ref'] if xmd_rpm['ref'] != mmd_rpm.ref: data["srpm_commit_branch"] = mmd_rpm.ref rpms.append(data) return rpms
def setUpClass(cls): cls.mmd = modulemd.ModuleMetadata() cls.mmd.name = "test" cls.mmd.version = "42" cls.mmd.release = "1" cls.mmd.summary = "A test module" cls.mmd.description = "It's only used for testing purposes." cls.mmd.module_licenses = set(["MIT"]) cls.mmd.content_licenses = set(["ISC"]) cls.mmd.buildrequires = {"builddepenency": "123-456"} cls.mmd.requires = {"dependency": "1.00-1"} cls.mmd.community = "http://www.example.com/community" cls.mmd.documentation = "http://www.example.com/documentation" cls.mmd.tracker = "http://www.example.com/tracker" cls.mmd.xmd = {"key": "value"} cls.mmd.profiles = {"default": modulemd.ModuleProfile()} cls.mmd.profiles["default"].rpms = set(["prof", "ile"]) cls.mmd.profiles["default"].description = "Default set of packages" cls.mmd.components = modulemd.ModuleComponents() cls.mmd.components.rpms = modulemd.ModuleRPMs() cls.mmd.components.rpms.dependencies = True cls.mmd.components.rpms.api = set(["api"]) cls.mmd.components.rpms.packages = {"rpm": {"rationale": ""}} cls.mmd.components.rpms.filter = set(["filter_1", "filter_2"])
def handle_build_module(self, result): try: data = json.loads(self.data["data"]) ownername = data["ownername"] projectname = data["projectname"] chroots = data["chroots"] modulemd_data = base64.b64decode(data["modulemd_b64"]) project_path = os.path.join(self.opts.destdir, ownername, projectname) try: modules_file_read = open( os.path.join(project_path, "modules", "modules.json"), "r+") modules = json.loads(modules_file_read.read()) modules_file_read.close() except: modules = [] mmd = modulemd.ModuleMetadata() (fh, abspath) = tempfile.mkstemp() f = open(abspath, "w+") f.write(modulemd_data) f.close() mmd.load(abspath) for chroot in chroots: arch = get_chroot_arch(chroot) srcdir = os.path.join(project_path, chroot) module_tag = chroot + '+' + mmd.name + '-' + ( mmd.version or '1') + '-' + (mmd.release or '1') module_relpath = os.path.join(module_tag, "latest", arch) destdir = os.path.join(project_path, "modules", module_relpath) if os.path.exists(destdir): self.log.warning( "Module {0} already exists. Ommitting.".format( destdir)) else: self.log.info("Copy directory: {} as {}".format( srcdir, destdir)) shutil.copytree(srcdir, destdir) mmd.dump(os.path.join(destdir, "module_md.yaml")) modules.append({ "url": module_relpath, "name": mmd.name, "release": (mmd.release or '1'), "version": (mmd.version or '1'), "requires": mmd.requires, "summary": mmd.summary, }) createrepo(path=destdir, front_url=self.front_url, username=ownername, projectname=projectname, override_acr_flag=True) modules_file_write = open( os.path.join(project_path, "modules", "modules.json"), "w+") modules_file_write.write(json.dumps(modules, indent=4)) modules_file_write.close() result.result = ActionResult.SUCCESS os.unlink(abspath) except Exception as e: self.log.error(str(e)) result.result = ActionResult.FAILURE
def setUp(self): self.mmd = modulemd.ModuleMetadata() self.mmd.load("tests/test.yaml")
def test_loads(self, yaml=None): mmd = modulemd.ModuleMetadata() document = """ document: modulemd version: 0 data: name: test version: 1.23 release: 4 summary: A test module description: > This module is a part of the modulemd test suite. license: module: [ MIT ] content: [ GPL+, GPLv3 ] dependencies: buildrequires: { example: 84-84 } requires: { modulemd: 42-42 } references: community: http://www.example.com/community documentation: http://www.example.com/documentation tracker: http://www.example.com/tracker xmd: userid: userdata profiles: default: rpms: - alfa - alfa-subpackage minimal: description: Minimal profile installing only the alfa package. rpms: - alfa components: rpms: dependencies: True api: - alfa - alfa-extras packages: alfa: rationale: alfa rationale bravo: rationale: bravo rationale arches: [ charlie, delta ] multilib: [ echo ] commit: foxtrot repository: golf cache: hotel filter: - filter_1 - filter_2 """ if not yaml: yaml = document mmd.loads(yaml) self.assertEqual(mmd.mdversion, 0) self.assertEqual(mmd.name, "test") self.assertEqual(mmd.version, "1.23") self.assertEqual(mmd.release, "4") self.assertEqual(mmd.summary, "A test module") self.assertEqual(mmd.description, "This module is a part of the modulemd test suite.") self.assertEqual(mmd.module_licenses, set(["MIT"])) self.assertEqual(mmd.content_licenses, set(["GPL+", "GPLv3"])) self.assertEqual(mmd.buildrequires, {"example": "84-84"}) self.assertEqual(mmd.requires, {"modulemd": "42-42"}) self.assertEqual(mmd.community, "http://www.example.com/community") self.assertEqual(mmd.documentation, "http://www.example.com/documentation") self.assertEqual(mmd.tracker, "http://www.example.com/tracker") self.assertEqual(mmd.xmd, {"userid": "userdata"}) self.assertEqual(sorted(mmd.profiles.keys()), ["default", "minimal"]) self.assertEqual(mmd.profiles["default"].rpms, set(["alfa", "alfa-subpackage"])) self.assertEqual(mmd.profiles["minimal"].description, "Minimal profile installing only the alfa package.") self.assertEqual(mmd.profiles["minimal"].rpms, set(["alfa"])) self.assertTrue(mmd.components.rpms.dependencies) self.assertEqual(mmd.components.rpms.api, set(["alfa", "alfa-extras"])) self.assertEqual( mmd.components.rpms.packages, { "alfa": { "rationale": "alfa rationale" }, "bravo": { "rationale": "bravo rationale", "arches": ["charlie", "delta"], "multilib": ["echo"], "commit": "foxtrot", "repository": "golf", "cache": "hotel" } }) self.assertEqual(mmd.components.rpms.filter, set(["filter_1", "filter_2"]))
def setUpClass(cls): cls.mmd = modulemd.ModuleMetadata()
def _populate_modulemd(module_ctx, module_exclusive_packages, api_rpms, include_module_bootstrap): mod_md = modulemd.ModuleMetadata() mod_md.name = str(module_ctx.module) mod_md.description = str(module_ctx.module_desc) for lic in module_ctx.module_license: mod_md.add_module_license(str(lic)) mod_md.community = str(module_ctx.community) mod_md.documentation = str(module_ctx.docs) mod_md.tracker = str(module_ctx.tracker) # All modules require base runtime mod_md.add_requires("base-runtime", "master") for req in module_ctx.module_deps: mod_md.add_requires(str(req), "master") # All modules build-require bootstrap mod_md.add_buildrequires("bootstrap", "master") for req in module_ctx.module_build_deps: mod_md.add_buildrequires(str(req), "master") if include_module_bootstrap: mod_md.add_buildrequires("%s-bootstrap" % str(module_ctx.module), "master") nvr_re = re.compile("\d+\:(.*).src$") nvr_values = list() for srpm_name, srpm_nvr in module_exclusive_packages.items(): match = nvr_re.match(srpm_nvr) if match: nvr_values.append(match.group(1)) else: raise IOError("NVR [%s] didnt parse" % srpm_nvr) ks = koji.ClientSession('https://koji.fedoraproject.org/kojihub') ks.multicall = True for srpm_nvr in nvr_values: ks.getBuild(srpm_nvr) ret = ks.multiCall(strict=True) ks.multicall = True task_nvrs = [] for i in range(len(nvr_values)): if ret[i][0] is not None: if ret[i][0]['task_id'] is not None: ks.getTaskInfo(ret[i][0]['task_id'], request=True) task_nvrs.append(nvr_values[i]) else: print("WARNING: no task ID for %s" % nvrs[i]) else: print("WARNING: no task ID for %s" % nvrs[i]) ret = ks.multiCall(strict=True) deps = dict() commit_re = re.compile("([^\/]+?):([a-f0-9]{40})") for i in range(len(task_nvrs)): match = commit_re.search(koji.taskLabel(ret[i][0])) if match: deps[match.group(1)] = match.group(2) else: raise IOError("Task [%s] didnt parse" % (koji.taskLabel(ret[i][0]))) for name in sorted(deps, key=deps.get): mod_md.components.add_rpm(name, "Automatically generated", ref=deps[name]) for rpm in api_rpms: mod_md.api.add_rpm(rpm) return mod_md
i for i in os.listdir(noarch_module_dir) if i.endswith(".rpm") ] else: noarch_rpms = [] rpms = sorted(set(rpms) | set(noarch_rpms)) # HACK: force epoch to make test data compatible with libmodulemd >= 1.4.0 rpms_with_epoch = [] for i in rpms: n, v, ra = i.rsplit("-", 2) nevra = "%s-0:%s-%s" % (n, v, ra) rpms_with_epoch.append(nevra) rpms = rpms_with_epoch mmd = modulemd.ModuleMetadata() mmd.name = name mmd.stream = stream mmd.version = int(version) mmd.add_module_license("LGPLv2") mmd.summary = "Fake module" mmd.description = mmd.summary for rpm in rpms: mmd.components.add_rpm(rpm.rsplit("-", 2)[0], "") mmd.artifacts.add_rpm(rpm[:-4]) for profile_name in profiles: profile = modulemd.ModuleProfile() profile.rpms.update(profiles[profile_name]["rpms"]) mmd.profiles[profile_name] = profile mmd.dump(os.path.join(module_dir, "%s.%s.yaml" % (module_id, arch)))