def submit_module_build_from_yaml(
db_session, username, handle, params, stream=None, skiptests=False
):
yaml_file = to_text_type(handle.read())
mmd = load_mmd(yaml_file)
dt = datetime.utcfromtimestamp(int(time.time()))
if hasattr(handle, "filename"):
def_name = str(os.path.splitext(os.path.basename(handle.filename))[0])
elif not mmd.get_module_name():
raise ValidationError(
"The module's name was not present in the modulemd file. Please use the "
'"module_name" parameter'
)
def_version = int(dt.strftime("%Y%m%d%H%M%S"))
module_name = mmd.get_module_name() or def_name
module_stream = stream or mmd.get_stream_name() or "master"
if module_name != mmd.get_module_name() or module_stream != mmd.get_stream_name():
# This is how you set the name and stream in the modulemd
mmd = mmd.copy(module_name, module_stream)
mmd.set_version(mmd.get_version() or def_version)
if skiptests:
buildopts = mmd.get_buildopts() or Modulemd.Buildopts()
macros = buildopts.get_rpm_macros() or ""
buildopts.set_rpm_macros(macros + "\n\n%__spec_check_pre exit 0\n")
mmd.set_buildopts(buildopts)
return submit_module_build(db_session, username, mmd, params)
def get_module_srpm_overrides(module):
"""
Make necessary preparations to use any provided custom SRPMs.
:param module: ModuleBuild object representing the module being submitted.
:type module: :class:`models.ModuleBuild`
:return: mapping of package names to SRPM links for all packages which
have custom SRPM overrides specified
:rtype: dict[str, str]
"""
overrides = {}
if not module.srpms:
return overrides
try:
# Make sure we can decode the custom SRPM list
srpms = json.loads(module.srpms)
assert isinstance(srpms, list)
except Exception:
raise ValueError("Invalid srpms list encountered: {}".format(
module.srpms))
for source in srpms:
if source.startswith("cli-build/") and source.endswith(".src.rpm"):
# This is a custom srpm that has been uploaded to koji by rpkg
# using the package name as the basename suffixed with .src.rpm
rpm_name = os.path.basename(source)[:-len(".src.rpm")]
else:
# This should be a local custom srpm path
if not os.path.exists(source):
raise IOError("Provided srpm is missing: {}".format(source))
# Get package name from rpm headers
try:
rpm_hdr = kobo.rpmlib.get_rpm_header(source)
rpm_name = to_text_type(
kobo.rpmlib.get_header_field(rpm_hdr, "name"))
except Exception:
raise ValueError("Provided srpm is invalid: {}".format(source))
if rpm_name in overrides:
log.warning(
'Encountered duplicate custom SRPM "{0}" for package {1}'.
format(source, rpm_name))
continue
log.debug('Using custom SRPM "{0}" for package {1}'.format(
source, rpm_name))
overrides[rpm_name] = source
return overrides
def read_staged_data(yaml_name):
"""Read module YAML content from staged_data directory
:param str yaml_name: name of YAML file which could be with or without
extension ``.yaml``. ``.yaml`` will be added if extension is omitted.
:return: module YAML file's content.
:rtype: str
:raises ValueError: if specified module YAML file does not exist in
staged_data directory.
"""
filename = staged_data_filename(yaml_name if '.' in
yaml_name else "{}.yaml".format(yaml_name))
if not os.path.exists(filename):
raise ValueError(
"Staged data {}.yaml does not exist.".format(yaml_name))
with open(filename, "r") as mmd:
return to_text_type(mmd.read())
def __init__(self, module, config):
"""
:param module: module_build_service.common.models.ModuleBuild instance.
:param config: module_build_service.common.config.Config instance
"""
self.owner = module.owner
self.module = module
self.module_name = module.name
self.mmd = to_text_type(module.modulemd)
self.config = config
self.devel = False
# List of architectures the module is built for.
self.arches = []
# List of RPMs tagged in module.koji_tag as returned by Koji.
self.rpms = []
# Dict constructed from `self.rpms` with NEVRA as a key.
self.rpms_dict = {}
def _get_arch_mmd_output(self, output_path, arch):
"""
Returns the CG "output" dict for architecture specific modulemd file.
:param str output_path: Path where the modulemd files are stored.
:param str arch: Architecture for which to generate the "output" dict.
:param dict rpms_dict: Dictionary with all RPMs built in this module.
The key is NEVRA string, value is RPM dict as obtained from Koji.
This dict is used to generate architecture specific "components"
section in the "output" record.
:rtype: dict
:return: Dictionary with record in "output" list.
"""
ret = {
"buildroot_id": 1,
"arch": arch,
"type": "file",
"extra": {"typeinfo": {"module": {}}},
"checksum_type": "md5",
}
# Noarch architecture represents "generic" modulemd.txt.
if arch == "noarch":
mmd_filename = "modulemd.txt"
else:
mmd_filename = "modulemd.%s.txt" % arch
# Read the modulemd file to get the filesize/checksum and also
# parse it to get the Modulemd instance.
mmd_path = os.path.join(output_path, mmd_filename)
try:
with open(mmd_path, "rb") as mmd_f:
raw_data = mmd_f.read()
data = to_text_type(raw_data)
mmd = load_mmd(data)
ret["filename"] = mmd_filename
ret["filesize"] = len(raw_data)
ret["checksum"] = hashlib.md5(raw_data).hexdigest()
except IOError:
if arch == "src":
# This might happen in case the Module is submitted directly
# using the yaml without SCM URL. This should never happen
# when building production-ready modules using Koji, but in
# theory it is possible.
log.warning("No modulemd.src.txt found.")
return
else:
raise
components = []
if arch in ["noarch", "src"]:
# For generic noarch/src modulemd, include all the RPMs.
for rpm in self.rpms:
components.append(self._koji_rpm_to_component_record(rpm))
else:
# Check the RPM artifacts built for this architecture in modulemd file,
# find the matching RPM in the `rpms_dict` coming from Koji and use it
# to generate list of components for this architecture.
# We cannot simply use the data from MMD here without `rpms_dict`, because
# RPM sigmd5 signature is not stored in MMD.
for rpm in mmd.get_rpm_artifacts():
if rpm not in self.rpms_dict:
raise RuntimeError(
"RPM %s found in the final modulemd but not in Koji tag." % rpm)
tag_rpm = self.rpms_dict[rpm]
components.append(self._koji_rpm_to_component_record(tag_rpm))
ret["components"] = components
return ret
