def setup_method(self, test_method):
self.fn = module_build_service.scheduler.handlers.modules.init
testmodule_yml_path = read_staged_data("testmodule_init")
mmd = load_mmd(testmodule_yml_path)
# Set the name and stream
mmd = mmd.copy("testmodule", "1")
scmurl = "git://pkgs.domain.local/modules/testmodule?#620ec77"
clean_database()
ModuleBuild.create(db_session, conf, "testmodule", "1", 3,
mmd_to_str(mmd), scmurl, "mprahl")
def test_resolve_requires_siblings(self,
require_platform_and_default_arch):
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="db")
mmd = load_mmd(tests.read_staged_data("formatted_testmodule"))
for i in range(3):
build = tests.module_build_from_modulemd(mmd_to_str(mmd))
build.context = "f6e2ae" + str(i)
build.build_context = "f6e2aeec7576196241b9afa0b6b22acf2b6873d" + str(
i)
build.runtime_context = "bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c" + str(
i)
build.state = models.BUILD_STATES["ready"]
db_session.add(build)
db_session.commit()
build_one = ModuleBuild.get_by_id(db_session, 2)
nsvc = ":".join([
build_one.name, build_one.stream, build_one.version,
build_one.context
])
result = resolver.resolve_requires([nsvc])
assert result == {
"testmodule": {
"stream": build_one.stream,
"version": build_one.version,
"context": build_one.context,
"ref": "65a7721ee4eff44d2a63fb8f3a8da6e944ab7f4d",
"koji_tag": None
}
}
db_session.commit()
def upgrade():
connection = op.get_bind()
op.add_column(
"module_builds",
sa.Column("build_context_no_bms", sa.String(), nullable=True))
# Determine what the contexts should be based on the modulemd
for build in connection.execute(modulebuild.select()):
if not build.modulemd:
continue
try:
mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)
mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)
except Exception:
# If the modulemd isn"t parseable then skip this build
continue
# It's possible this module build was built before MBS filled out xmd or before MBS
# filled out the requires in xmd
mbs_xmd_buildrequires = mmd.get_xmd().get("mbs",
{}).get("buildrequires")
if not mbs_xmd_buildrequires:
continue
build_context_no_bms = ModuleBuild.calculate_build_context(
mbs_xmd_buildrequires, True)
connection.execute(
modulebuild.update().where(modulebuild.c.id == build.id).values(
build_context_no_bms=build_context_no_bms))
def test_add_default_modules_request_failed(mock_get_dm):
"""
Test that an exception is raised when the call to _get_default_modules failed.
"""
clean_database()
make_module_in_db("python:3:12345:1")
make_module_in_db("nodejs:11:2345:2")
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]
assert set(xmd_brs.keys()) == {"platform"}
platform = ModuleBuild.get_build_from_nsvc(
db_session,
"platform",
xmd_brs["platform"]["stream"],
xmd_brs["platform"]["version"],
xmd_brs["platform"]["context"],
)
assert platform
platform_mmd = platform.mmd()
platform_xmd = mmd.get_xmd()
platform_xmd["mbs"]["use_default_modules"] = True
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
db_session.commit()
expected_error = "some error"
mock_get_dm.side_effect = ValueError(expected_error)
with pytest.raises(ValueError, match=expected_error):
default_modules.add_default_modules(mmd)
def test_get_buildrequired_modulemds(self):
mmd = load_mmd(tests.read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
import_mmd(db_session, mmd)
platform_f300103 = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
mmd = tests.make_module("testmodule:master:20170109091357:123")
build = ModuleBuild(
name="testmodule",
stream="master",
version=20170109091357,
state=5,
build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
context="7c29193d",
koji_tag="module-testmodule-master-20170109091357-7c29193d",
scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Dr. Pepper",
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
build.buildrequires.append(platform_f300103)
db_session.add(build)
db_session.commit()
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
result = resolver.get_buildrequired_modulemds(
"testmodule", "master", platform_f300103.mmd())
nsvcs = {m.get_nsvc() for m in result}
assert nsvcs == {"testmodule:master:20170109091357:123"}
def upgrade():
with op.batch_alter_table('module_builds', schema=None) as batch_op:
batch_op.add_column(sa.Column('stream_version', sa.Integer()))
op.create_table(
'module_builds_to_module_buildrequires',
sa.Column('module_id', sa.Integer(), nullable=False),
sa.Column('module_buildrequire_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_builds.id']),
sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']),
sa.UniqueConstraint('module_id', 'module_buildrequire_id')
)
connection = op.get_bind()
# Create all the base module buildrequire entries
for build in connection.execute(mb.select()):
if not build.modulemd:
# If the modulemd is empty, skip this build
continue
brs = None
try:
mmd = Modulemd.ModuleStream.read_string(build.modulemd, True)
mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO)
brs = mmd.get_xmd()['mbs']['buildrequires']
except Exception:
# If the modulemd isn't parseable then skip this build
continue
for base_module in conf.base_module_names:
base_module_dict = brs.get(base_module)
if not base_module_dict:
# If this base module isn't a buildrequire, continue to see if the next one is
continue
select = mb.select()\
.where(mb.c.name == base_module)\
.where(mb.c.stream == base_module_dict['stream'])\
.where(mb.c.version == base_module_dict['version'])\
.where(mb.c.context == base_module_dict.get('context'))
br = connection.execute(select).fetchone()
if not br:
# If the buildrequire isn't in the datbase, then skip it
continue
connection.execute(mb_to_mbr.insert().values(
module_id=build.id,
module_buildrequire_id=br.id
))
for base_module in conf.base_module_names:
for build in connection.execute(mb.select().where(mb.c.name == base_module)):
stream_version = ModuleBuild.get_stream_version(build.stream)
if not stream_version:
# If a stream version isn't parseable, then skip it
continue
connection.execute(mb.update().where(mb.c.id == build.id).values(
stream_version=stream_version))
def test_add_default_modules(mock_get_dm, mock_hc,
require_platform_and_default_arch):
"""
Test that default modules present in the database are added, and the others are ignored.
"""
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]
assert set(xmd_brs.keys()) == {"platform"}
platform = ModuleBuild.get_build_from_nsvc(
db_session,
"platform",
xmd_brs["platform"]["stream"],
xmd_brs["platform"]["version"],
xmd_brs["platform"]["context"],
)
assert platform
platform_mmd = platform.mmd()
platform_xmd = mmd.get_xmd()
platform_xmd["mbs"]["use_default_modules"] = True
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
dependencies = [{
"requires": {
"platform": ["f28"]
},
"buildrequires": {
"platform": ["f28"]
}
}]
make_module_in_db("python:3:12345:1",
base_module=platform,
dependencies=dependencies)
make_module_in_db("nodejs:11:2345:2",
base_module=platform,
dependencies=dependencies)
db_session.commit()
mock_get_dm.return_value = {
"nodejs": "11",
"python": "3",
"ruby": "2.6",
}
defaults_added = default_modules.add_default_modules(mmd)
# Make sure that the default modules were added. ruby:2.6 will be ignored since it's not in
# the database
assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {
"nodejs", "platform", "python"
}
mock_get_dm.assert_called_once_with(
"f28",
"https://pagure.io/releng/fedora-module-defaults.git",
)
assert "ursine_rpms" not in mmd.get_xmd()["mbs"]
assert defaults_added is True
def init_basic(self, db_session, get_build_arches, hcwbmr, rscm,
mocked_scm, built_rpms):
FakeSCM(
mocked_scm,
"testmodule",
"testmodule_init.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
)
built_rpms.return_value = [
"foo-0:2.4.48-3.el8+1308+551bfa71",
"foo-debuginfo-0:2.4.48-3.el8+1308+551bfa71",
"bar-0:2.5.48-3.el8+1308+551bfa71",
"bar-debuginfo-0:2.5.48-3.el8+1308+551bfa71",
"x-0:2.5.48-3.el8+1308+551bfa71",
"x-debuginfo-0:2.5.48-3.el8+1308+551bfa71",
]
platform_build = ModuleBuild.get_by_id(db_session, 1)
mmd = platform_build.mmd()
for rpm in mmd.get_rpm_filters():
mmd.remove_rpm_filter(rpm)
mmd.add_rpm_filter("foo")
mmd.add_rpm_filter("bar")
platform_build.modulemd = mmd_to_str(mmd)
db_session.commit()
self.fn(msg_id="msg-id-1",
module_build_id=2,
module_build_state="init")
build = ModuleBuild.get_by_id(db_session, 2)
# Make sure the module entered the wait state
assert build.state == 1, build.state
# Make sure format_mmd was run properly
xmd_mbs = build.mmd().get_xmd()["mbs"]
assert xmd_mbs["buildrequires"]["platform"]["filtered_rpms"] == [
"foo-0:2.4.48-3.el8+1308+551bfa71",
"bar-0:2.5.48-3.el8+1308+551bfa71",
]
hcwbmr.assert_called_once()
return build
def test_init_includedmodule(self, get_build_arches, mocked_scm,
mocked_mod_allow_repo):
FakeSCM(mocked_scm, "includedmodules", ["testmodule_init.yaml"])
includedmodules_yml_path = read_staged_data("includedmodules")
mmd = load_mmd(includedmodules_yml_path)
# Set the name and stream
mmd = mmd.copy("includedmodules", "1")
scmurl = "git://pkgs.domain.local/modules/includedmodule?#da95886"
ModuleBuild.create(db_session, conf, "includemodule", "1", 3,
mmd_to_str(mmd), scmurl, "mprahl")
self.fn(msg_id="msg-id-1",
module_build_id=3,
module_build_state="init")
build = ModuleBuild.get_by_id(db_session, 3)
assert build.state == 1
assert build.name == "includemodule"
batches = {}
for comp_build in build.component_builds:
batches[comp_build.package] = comp_build.batch
assert batches["perl-List-Compare"] == 2
assert batches["perl-Tangerine"] == 2
assert batches["foo"] == 2
assert batches["tangerine"] == 3
assert batches["file"] == 4
# Test that the RPMs are properly merged in xmd
xmd_rpms = {
"perl-List-Compare": {
"ref": "4f26aeafdb"
},
"perl-Tangerine": {
"ref": "4f26aeafdb"
},
"tangerine": {
"ref": "4f26aeafdb"
},
"foo": {
"ref": "93dea37599"
},
"file": {
"ref": "a2740663f8"
},
}
assert build.mmd().get_xmd()["mbs"]["rpms"] == xmd_rpms
def test_add_default_modules_compatible_platforms(mock_get_dm):
"""
Test that default modules built against compatible base module streams are added.
"""
clean_database(add_platform_module=False)
# Create compatible base modules.
mmd = load_mmd(read_staged_data("platform"))
for stream in ["f27", "f28"]:
mmd = mmd.copy("platform", stream)
# Set the virtual stream to "fedora" to make these base modules compatible.
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["fedora"]
xmd["mbs"]["use_default_modules"] = True
mmd.set_xmd(xmd)
import_mmd(db_session, mmd)
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]
assert set(xmd_brs.keys()) == {"platform"}
platform_f27 = ModuleBuild.get_build_from_nsvc(
db_session, "platform", "f27", "3", "00000000")
assert platform_f27
# Create python default module which requires platform:f27 and therefore cannot be used
# as default module for platform:f28.
dependencies = [
{"requires": {"platform": ["f27"]},
"buildrequires": {"platform": ["f27"]}}]
make_module_in_db("python:3:12345:1", base_module=platform_f27, dependencies=dependencies)
# Create nodejs default module which requries any platform stream and therefore can be used
# as default module for platform:f28.
dependencies[0]["requires"]["platform"] = []
make_module_in_db("nodejs:11:2345:2", base_module=platform_f27, dependencies=dependencies)
db_session.commit()
mock_get_dm.return_value = {
"nodejs": "11",
"python": "3",
"ruby": "2.6",
}
defaults_added = default_modules.add_default_modules(mmd)
# Make sure that the default modules were added. ruby:2.6 will be ignored since it's not in
# the database
assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"nodejs", "platform"}
mock_get_dm.assert_called_once_with(
"f28",
"https://pagure.io/releng/fedora-module-defaults.git",
)
assert defaults_added is True
def test_set_cg_build_koji_tag(
self,
cfg,
generic_resolver,
resolver,
create_builder,
dbg,
koji_cg_tag_build,
expected_cg_koji_build_tag,
):
"""
Test that build.cg_build_koji_tag is set.
"""
base_mmd = Modulemd.ModuleStreamV2.new("base-runtime", "f27")
koji_session = mock.MagicMock()
koji_session.newRepo.return_value = 123456
builder = mock.MagicMock()
builder.koji_session = koji_session
builder.module_build_tag = {"name": "module-123-build"}
builder.get_disttag_srpm.return_value = "some srpm disttag"
builder.build.return_value = (
1234,
koji.BUILD_STATES["BUILDING"],
"",
"module-build-macros-1-1",
)
create_builder.return_value = builder
resolver = mock.MagicMock()
resolver.backend = "db"
resolver.get_module_tag.return_value = "module-testmodule-master-20170109091357"
resolver.get_module_build_dependencies.return_value = {
"module-bootstrap-tag": [base_mmd]
}
with patch.object(
module_build_service.scheduler.handlers.modules.conf,
"koji_cg_tag_build",
new=koji_cg_tag_build,
):
generic_resolver.create.return_value = resolver
module_build_service.scheduler.handlers.modules.wait(
msg_id="msg-id-1",
module_build_id=2,
module_build_state="some state")
module_build = ModuleBuild.get_by_id(db_session, 2)
assert module_build.cg_build_koji_tag == expected_cg_koji_build_tag
def test_init_scm_not_available(self, get_build_arches, mocked_scm):
FakeSCM(
mocked_scm,
"testmodule",
"testmodule.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
get_latest_raise=True,
get_latest_error=RuntimeError("Failed in mocked_scm_get_latest"))
self.fn(msg_id="msg-id-1",
module_build_id=2,
module_build_state="init")
build = ModuleBuild.get_by_id(db_session, 2)
# Make sure the module entered the failed state
# since the git server is not available
assert build.state == 4, build.state
def _create_test_modules(self, koji_tag_with_modules="foo-test"):
mmd = load_mmd(tests.read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
import_mmd(db_session, mmd)
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
if koji_tag_with_modules:
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"][
"koji_tag_with_modules"] = koji_tag_with_modules
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
for context in ["7c29193d", "7c29193e"]:
mmd = tests.make_module("testmodule:master:20170109091357:" +
context)
build = ModuleBuild(
name="testmodule",
stream="master",
version=20170109091357,
state=5,
build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
context=context,
koji_tag="module-testmodule-master-20170109091357-" + context,
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Dr. Pepper",
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
build.buildrequires.append(platform)
db_session.add(build)
db_session.commit()
def test_buildroot_add_repos(self, write_config, load_config, patched_open,
base_module_repofiles):
import_fake_base_module("platform:f29:1:000000")
platform = ModuleBuild.get_last_build_in_stream(
db_session, "platform", "f29")
module_deps = [{
"requires": {
"platform": ["f29"]
},
"buildrequires": {
"platform": ["f29"]
},
}]
foo = make_module_in_db("foo:1:1:1", module_deps)
app = make_module_in_db("app:1:1:1", module_deps)
patched_open.side_effect = [
mock.mock_open(read_data="[fake]\nrepofile 1\n").return_value,
mock.mock_open(read_data="[fake]\nrepofile 2\n").return_value,
mock.mock_open(read_data="[fake]\nrepofile 3\n").return_value,
]
builder = MockModuleBuilder(db_session, "user", app, conf,
"module-app", [])
dependencies = {
"repofile://": [platform.mmd()],
"repofile:///etc/yum.repos.d/foo.repo": [foo.mmd(),
app.mmd()],
}
builder.buildroot_add_repos(dependencies)
assert "repofile 1" in builder.yum_conf
assert "repofile 2" in builder.yum_conf
assert "repofile 3" in builder.yum_conf
assert set(builder.enabled_modules) == {"foo:1", "app:1"}
def test_init_when_get_latest_raises(self, get_build_arches, mocked_scm,
mocked_from_module_event):
FakeSCM(
mocked_scm,
"testmodule",
"testmodule.yaml",
"7035bd33614972ac66559ac1fdd019ff6027ad22",
get_latest_raise=True,
)
build = ModuleBuild.get_by_id(db_session, 2)
mocked_from_module_event.return_value = build
self.fn(msg_id="msg-id-1",
module_build_id=2,
module_build_state="init")
# Query the database again to make sure the build object is updated
db_session.refresh(build)
# Make sure the module entered the failed state
assert build.state == 4, build.state
assert "Failed to get the latest commit for" in build.state_reason
def get_corresponding_module_build(nvr):
"""Find corresponding module build from database and return
:param str nvr: module build NVR. This is the subject_identifier included
inside ``greenwave.decision.update`` message.
:return: the corresponding module build object. For whatever the reason,
if the original module build id cannot be found from the Koji build of
``nvr``, None will be returned.
:rtype: :class:`ModuleBuild` or None
"""
koji_session = get_session(conf, login=False)
build_info = koji_session.getBuild(nvr)
if build_info is None:
return None
try:
module_build_id = build_info["extra"]["typeinfo"]["module"][
"module_build_service_id"]
except KeyError:
# If any of the keys is not present, the NVR is not the one for
# handling Greenwave event.
return None
return ModuleBuild.get_by_id(db_session, module_build_id)
def _create_module_with_filters(self, db_session, batch, state):
mmd = load_mmd(read_staged_data("testmodule-with-filters"))
# Set the name and stream
mmd = mmd.copy("mbs-testmodule", "test")
mmd.set_xmd({
"mbs": {
"rpms": {
"ed": {
"ref": "01bf8330812fea798671925cc537f2f29b0bd216"
},
"mksh": {
"ref": "f70fd11ddf96bce0e2c64309706c29156b39141d"
},
},
"buildrequires": {
"host": {
"version": "20171024133034",
"filtered_rpms": [],
"stream": "master",
"ref": "6df253bb3c53e84706c01b8ab2d5cac24f0b6d45",
"context": "00000000",
},
"platform": {
"version": "20171028112959",
"filtered_rpms": [],
"stream": "master",
"ref": "4f7787370a931d57421f9f9555fc41c3e31ff1fa",
"context": "00000000",
},
},
"scmurl": "file:///testdir",
"commit": "5566bc792ec7a03bb0e28edd1b104a96ba342bd8",
"requires": {
"platform": {
"version": "20171028112959",
"filtered_rpms": [],
"stream": "master",
"ref": "4f7787370a931d57421f9f9555fc41c3e31ff1fa",
"context": "00000000",
}
},
}
})
module = ModuleBuild.create(
db_session,
conf,
name="mbs-testmodule",
stream="test",
version="20171027111452",
modulemd=mmd_to_str(mmd),
scmurl="file:///testdir",
username="******",
)
module.koji_tag = "module-mbs-testmodule-test-20171027111452"
module.batch = batch
db_session.add(module)
db_session.commit()
comp_builds = [
{
"module_id":
module.id,
"state":
state,
"package":
"ed",
"format":
"rpms",
"scmurl": ("https://src.fedoraproject.org/rpms/ed"
"?#01bf8330812fea798671925cc537f2f29b0bd216"),
"batch":
2,
"ref":
"01bf8330812fea798671925cc537f2f29b0bd216",
},
{
"module_id":
module.id,
"state":
state,
"package":
"mksh",
"format":
"rpms",
"scmurl": ("https://src.fedoraproject.org/rpms/mksh"
"?#f70fd11ddf96bce0e2c64309706c29156b39141d"),
"batch":
3,
"ref":
"f70fd11ddf96bce0e2c64309706c29156b39141d",
},
]
for build in comp_builds:
db_session.add(ComponentBuild(**build))
db_session.commit()
return module
def module_build_from_modulemd(yaml):
"""
Create a ModuleBuild object and return. It is not written into database.
Please commit by yourself if necessary.
"""
mmd = load_mmd(yaml)
build = ModuleBuild()
build.name = mmd.get_module_name()
build.stream = mmd.get_stream_name()
build.version = mmd.get_version()
build.state = BUILD_STATES["ready"]
build.modulemd = yaml
build.koji_tag = None
build.batch = 0
build.owner = "some_other_user"
build.time_submitted = datetime(2016, 9, 3, 12, 28, 33)
build.time_modified = datetime(2016, 9, 3, 12, 28, 40)
build.time_completed = None
build.rebuild_strategy = "changed-and-after"
return build
def make_module(
nsvc,
dependencies=None,
base_module=None,
filtered_rpms=None,
xmd=None,
store_to_db=False,
virtual_streams=None,
arches=None,
):
"""
Creates new models.ModuleBuild defined by `nsvc` string with requires
and buildrequires set according to ``requires_list`` and ``build_requires_list``.
:param str nsvc: name:stream:version:context of a module.
:param dependencies: list of groups of dependencies (requires and buildrequires).
For example, [
{"requires": {"platform": ["f30"]}, "buildrequires": {"platform": ["f30"]}},
...
]
:type dependencies: list[dict]
:param base_module: a base module build required by the new module created.
:type base_module: :class:`ModuleBuild`
:param filtered_rpms: list of filtered RPMs which are added to filter
section in module metadata.
:type filtered_rpms: list[str]
:param dict xmd: a mapping representing XMD section in module metadata. A
custom xmd could be passed for testing a particular scenario and some
default key/value pairs are added if not present.
:param bool store_to_db: whether to store created module metadata to the
database. If set to True, ``db_session`` is required.
:param virtual_streams: List of virtual streams provided by this module.
If set, This requires ``db_session`` and ``store_to_db`` to be set to a
session object and True.
:type virtual_streams: list[str]
:param arches: List of architectures this module is built against. If set
to None, ``["x86_64"]`` is used as a default. If set, likewise
``virtual_stream``.
:type arches: list[str]
:return: New Module Build if set to store module metadata to database,
otherwise the module metadata is returned.
:rtype: ModuleBuild or Modulemd.Module
"""
if store_to_db:
assert db_session is not None
if base_module:
assert db_session is not None
if virtual_streams:
assert store_to_db
if arches:
assert store_to_db
nsvc_regex = re.compile(r"([^:]+):([^:]+):([^:]+)(?::(.+))?")
match = nsvc_regex.match(nsvc)
if not match:
raise ValueError('Argument nsvc is not in format N:S:V or N:S:V:C')
name, stream, version, context = match.groups()
mmd = Modulemd.ModuleStreamV2.new(name, stream)
mmd.set_version(int(version))
if context:
mmd.set_context(context)
mmd.set_summary("foo")
# Test unicode in mmd.
mmd.set_description(u"foo \u2019s")
mmd.add_module_license("GPL")
if filtered_rpms:
for rpm in filtered_rpms:
mmd.add_rpm_filter(rpm)
def _add_require(mmd_deps, require_type, name, streams):
assert isinstance(mmd_deps, Modulemd.Dependencies)
assert require_type in ("requires", "buildrequires")
assert isinstance(streams, (list, tuple))
if require_type == "requires":
add_stream = mmd_deps.add_runtime_stream
set_empty_deps = mmd_deps.set_empty_runtime_dependencies_for_module
else:
add_stream = mmd_deps.add_buildtime_stream
set_empty_deps = mmd_deps.set_empty_buildtime_dependencies_for_module
for stream in streams:
add_stream(name, stream)
else:
set_empty_deps(name)
for dep_group in dependencies or []:
mmd_deps = Modulemd.Dependencies()
# A deps could be {"platform": ["f30"], "python": []}
for require_type, deps in dep_group.items():
for req_name, req_streams in deps.items():
_add_require(mmd_deps, require_type, req_name, req_streams)
mmd.add_dependencies(mmd_deps)
# Caller could pass whole xmd including mbs, but if something is missing,
# default values are given here.
xmd = xmd or {"mbs": {}}
xmd_mbs = xmd["mbs"]
if "buildrequires" not in xmd_mbs:
xmd_mbs["buildrequires"] = {}
if "requires" not in xmd_mbs:
xmd_mbs["requires"] = {}
if "commit" not in xmd_mbs:
xmd_mbs["commit"] = "ref_%s" % context
if "mse" not in xmd_mbs:
xmd_mbs["mse"] = "true"
if virtual_streams:
xmd_mbs["virtual_streams"] = virtual_streams
mmd.set_xmd(xmd)
if not store_to_db:
return mmd
module_build = ModuleBuild(
name=name,
stream=stream,
stream_version=ModuleBuild.get_stream_version(stream),
version=version,
context=context,
state=BUILD_STATES["ready"],
scmurl="https://src.stg.fedoraproject.org/modules/unused.git?#ff1ea79",
batch=1,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
build_context=context,
runtime_context=context,
modulemd=mmd_to_str(mmd),
koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None,
)
if base_module:
module_build.buildrequires.append(base_module)
db_session.add(module_build)
db_session.commit()
if virtual_streams:
for virtual_stream in virtual_streams:
vs_obj = db_session.query(VirtualStream).filter_by(
name=virtual_stream).first()
if not vs_obj:
vs_obj = VirtualStream(name=virtual_stream)
db_session.add(vs_obj)
db_session.commit()
if vs_obj not in module_build.virtual_streams:
module_build.virtual_streams.append(vs_obj)
db_session.commit()
for arch in arches or ["x86_64"]:
arch_obj = db_session.query(ModuleArch).filter_by(name=arch).first()
if not arch_obj:
arch_obj = ModuleArch(name=arch)
db_session.add(arch_obj)
db_session.commit()
if arch_obj not in module_build.arches:
module_build.arches.append(arch_obj)
db_session.commit()
return module_build
def _populate_data(data_size=10, contexts=False, scratch=False):
# Query arch from passed database session, otherwise there will be an error
# like "Object '<ModuleBuild at 0x7f4ccc805c50>' is already attached to
# session '275' (this is '276')" when add new module build object to passed
# session.
task_id_counter = itertools.count(1)
arch = db_session.query(
module_build_service.common.models.ModuleArch).get(1)
num_contexts = 2 if contexts else 1
for index in range(data_size):
for context in range(num_contexts):
build_one = ModuleBuild(
name="nginx",
stream="1",
version=2 + index,
state=BUILD_STATES["ready"],
scratch=scratch,
modulemd=read_staged_data("nginx_mmd"),
koji_tag="scrmod-nginx-1.2" if scratch else "module-nginx-1.2",
scmurl="git://pkgs.domain.local/modules/nginx"
"?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=2,
# https://www.youtube.com/watch?v=iQGwrK_yDEg,
owner="Moe Szyslak",
time_submitted=datetime(2016, 9, 3, 11, 23, 20) +
timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 11, 25, 32) +
timedelta(minutes=(index * 10)),
time_completed=datetime(2016, 9, 3, 11, 25, 32) +
timedelta(minutes=(index * 10)),
rebuild_strategy="changed-and-after",
)
build_one.arches.append(arch)
if contexts:
build_one.stream = str(index)
nsvc = "{}:{}:{}:{}".format(build_one.name, build_one.stream,
build_one.version, context)
unique_hash = hashlib.sha1(nsvc.encode('utf-8')).hexdigest()
build_one.build_context = unique_hash
build_one.runtime_context = unique_hash
combined_hashes = "{0}:{1}".format(unique_hash, unique_hash)
build_one.context = hashlib.sha1(
combined_hashes.encode("utf-8")).hexdigest()[:8]
db_session.add(build_one)
db_session.commit()
build_one_component_release = get_rpm_release(
db_session, build_one)
db_session.add_all([
ComponentBuild(package="nginx",
scmurl="git://pkgs.domain.local/rpms/nginx?"
"#ga95886c8a443b36a9ce31abda1f9bed22f2f8c3",
format="rpms",
task_id=six.next(task_id_counter),
state=koji.BUILD_STATES["COMPLETE"],
nvr="nginx-1.10.1-2.{0}".format(
build_one_component_release),
batch=1,
module_id=2 + index * 3,
tagged=True,
tagged_in_final=True),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_nginx_1_2.src.rpm",
format="rpms",
task_id=six.next(task_id_counter),
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(
build_one_component_release),
batch=2,
module_id=2 + index * 3,
tagged=True,
tagged_in_final=True)
])
db_session.commit()
build_two = ModuleBuild(
name="postgressql",
stream="1",
version=2 + index,
state=BUILD_STATES["done"],
scratch=scratch,
modulemd=read_staged_data("testmodule"),
koji_tag="scrmod-postgressql-1.2"
if scratch else "module-postgressql-1.2",
scmurl="git://pkgs.domain.local/modules/postgressql"
"?#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=2,
owner="some_user",
time_submitted=datetime(2016, 9, 3, 12, 25, 33) +
timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 12, 27, 19) +
timedelta(minutes=(index * 10)),
time_completed=datetime(2016, 9, 3, 11, 27, 19) +
timedelta(minutes=(index * 10)),
rebuild_strategy="changed-and-after",
)
build_two.arches.append(arch)
db_session.add(build_two)
db_session.commit()
build_two_component_release = get_rpm_release(db_session, build_two)
db_session.add_all([
ComponentBuild(package="postgresql",
scmurl="git://pkgs.domain.local/rpms/postgresql"
"?#dc95586c4a443b26a9ce38abda1f9bed22f2f8c3",
format="rpms",
task_id=2433433 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="postgresql-9.5.3-4.{0}".format(
build_two_component_release),
batch=2,
module_id=3 + index * 3,
tagged=True,
tagged_in_final=True),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_postgresql_1_2.src.rpm",
format="rpms",
task_id=47383993 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(
build_two_component_release),
batch=1,
module_id=3 + index * 3)
])
db_session.commit()
build_three = ModuleBuild(
name="testmodule",
stream="4.3.43",
version=6 + index,
state=BUILD_STATES["wait"],
scratch=scratch,
modulemd=read_staged_data("testmodule"),
koji_tag=None,
scmurl="git://pkgs.domain.local/modules/testmodule"
"?#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=0,
owner="some_other_user",
time_submitted=datetime(2016, 9, 3, 12, 28, 33) +
timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 12, 28, 40) +
timedelta(minutes=(index * 10)),
time_completed=None,
rebuild_strategy="changed-and-after",
)
db_session.add(build_three)
db_session.commit()
build_three_component_release = get_rpm_release(
db_session, build_three)
db_session.add_all([
ComponentBuild(package="rubygem-rails",
scmurl="git://pkgs.domain.local/rpms/rubygem-rails"
"?#dd55886c4a443b26a9ce38abda1f9bed22f2f8c3",
format="rpms",
task_id=2433433 + index,
state=koji.BUILD_STATES["FAILED"],
nvr="postgresql-9.5.3-4.{0}".format(
build_three_component_release),
batch=2,
module_id=4 + index * 3),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_testmodule_1_2.src.rpm",
format="rpms",
task_id=47383993 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(
build_three_component_release),
batch=1,
module_id=4 + index * 3,
tagged=True,
build_time_only=True)
])
db_session.commit()