def test_get_rpm_release_metadata_br_stream_override(mock_admmn):
"""
Test that when a module buildrequires a module in conf.allowed_privileged_module_names,
and that module has the xmd.mbs.disttag_marking field set, it should influence the disttag.
"""
scheduler_init_data(1)
metadata_mmd = load_mmd(read_staged_data("build_metadata_module"))
import_mmd(db_session, metadata_mmd)
build_one = models.ModuleBuild.get_by_id(db_session, 2)
mmd = build_one.mmd()
deps = mmd.get_dependencies()[0]
deps.add_buildtime_stream("build", "product1.2")
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"]["build"] = {
"filtered_rpms": [],
"ref": "virtual",
"stream": "product1.2",
"version": "1",
"context": "00000000",
}
mmd.set_xmd(xmd)
build_one.modulemd = mmd_to_str(mmd)
db_session.add(build_one)
db_session.commit()
release = utils.get_rpm_release(db_session, build_one)
assert release == "module+product12+2+814cfa39"
def test_transform_from_done_to_ready(self, ClientSession, publish):
clean_database()
# This build should be queried and transformed to ready state
module_build = make_module_in_db(
"pkg:0.1:1:c1",
[{
"requires": {
"platform": ["el8"]
},
"buildrequires": {
"platform": ["el8"]
},
}],
)
module_build.transition(db_session, conf, BUILD_STATES["done"],
"Move to done directly for running test.")
db_session.commit()
# Assert this call below
first_publish_call = call(
"module.state.change",
module_build.json(db_session, show_tasks=False),
conf,
"mbs",
)
ClientSession.return_value.getBuild.return_value = {
"extra": {
"typeinfo": {
"module": {
"module_build_service_id": module_build.id
}
}
}
}
msg = {
"msg_id": "msg-id-1",
"topic": "org.fedoraproject.prod.greenwave.decision.update",
"msg": {
"decision_context": "test_dec_context",
"policies_satisfied": True,
"subject_identifier": "pkg-0.1-1.c1",
},
}
hub = Mock(config={"validate_signatures": False})
consumer = MBSConsumer(hub)
consumer.consume(msg)
db_session.add(module_build)
# Load module build again to check its state is moved correctly
db_session.refresh(module_build)
assert BUILD_STATES["ready"] == module_build.state
publish.assert_has_calls([
first_publish_call,
call("module.state.change",
module_build.json(db_session, show_tasks=False), conf, "mbs"),
])
def test_get_buildrequired_modulemds(self):
mmd = load_mmd(tests.read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
import_mmd(db_session, mmd)
platform_f300103 = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one()
mmd = tests.make_module("testmodule:master:20170109091357:123")
build = ModuleBuild(
name="testmodule",
stream="master",
version=20170109091357,
state=5,
build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
context="7c29193d",
koji_tag="module-testmodule-master-20170109091357-7c29193d",
scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Dr. Pepper",
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
build.buildrequires.append(platform_f300103)
db_session.add(build)
db_session.commit()
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
result = resolver.get_buildrequired_modulemds(
"testmodule", "master", platform_f300103.mmd())
nsvcs = {m.get_nsvc() for m in result}
assert nsvcs == {"testmodule:master:20170109091357:123"}
def test_resolve_requires_siblings(self,
require_platform_and_default_arch):
resolver = mbs_resolver.GenericResolver.create(db_session,
conf,
backend="db")
mmd = load_mmd(tests.read_staged_data("formatted_testmodule"))
for i in range(3):
build = tests.module_build_from_modulemd(mmd_to_str(mmd))
build.context = "f6e2ae" + str(i)
build.build_context = "f6e2aeec7576196241b9afa0b6b22acf2b6873d" + str(
i)
build.runtime_context = "bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c" + str(
i)
build.state = models.BUILD_STATES["ready"]
db_session.add(build)
db_session.commit()
build_one = ModuleBuild.get_by_id(db_session, 2)
nsvc = ":".join([
build_one.name, build_one.stream, build_one.version,
build_one.context
])
result = resolver.resolve_requires([nsvc])
assert result == {
"testmodule": {
"stream": build_one.stream,
"version": build_one.version,
"context": build_one.context,
"ref": "65a7721ee4eff44d2a63fb8f3a8da6e944ab7f4d",
"koji_tag": None
}
}
db_session.commit()
def require_platform_and_default_arch_cls(require_empty_database_cls):
"""Provides clean database with platform module and default arch"""
arch_obj = module_build_service.common.models.ModuleArch(name="x86_64")
db_session.add(arch_obj)
db_session.commit()
mmd = load_mmd(read_staged_data("platform"))
import_mmd(db_session, mmd)
def test_record_component_builds_component_exists_already(
self, mocked_scm):
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
mocked_scm.return_value.get_latest.side_effect = [
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
# To simulate that when a module is resubmitted, some ref of
# its components is changed, which will cause MBS stops
# recording component to database and raise an error.
"abcdefg",
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
]
original_mmd = load_mmd(read_staged_data("testmodule"))
# Set the module name and stream
mmd = original_mmd.copy("testmodule", "master")
module_build = module_build_service.common.models.ModuleBuild()
module_build.name = "testmodule"
module_build.stream = "master"
module_build.version = 20170109091357
module_build.state = models.BUILD_STATES["init"]
module_build.scmurl = \
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
module_build.batch = 1
module_build.owner = "Tom Brady"
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = "changed-and-after"
module_build.modulemd = mmd_to_str(mmd)
db_session.add(module_build)
db_session.commit()
format_mmd(mmd, module_build.scmurl)
record_component_builds(mmd, module_build)
db_session.commit()
mmd = original_mmd.copy("testmodule", "master")
from module_build_service.common.errors import ValidationError
with pytest.raises(
ValidationError,
match=
r"Component build .+ of module build .+ already exists in database"
):
format_mmd(mmd, module_build.scmurl)
record_component_builds(mmd, module_build)
def test_get_rpm_release_platform_stream_override():
scheduler_init_data(1)
# Set the disttag_marking override on the platform
platform = (db_session.query(models.ModuleBuild).filter_by(
name="platform", stream="f28").first())
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"]["disttag_marking"] = "fedora28"
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
db_session.add(platform)
db_session.commit()
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release = utils.get_rpm_release(db_session, build_one)
assert release == "module+fedora28+2+814cfa39"
def cleanup_stale_failed_builds():
"""Does various clean up tasks on stale failed module builds"""
if conf.system != "koji":
return
stale_date = datetime.utcnow() - timedelta(days=conf.cleanup_failed_builds_time)
stale_module_builds = db_session.query(models.ModuleBuild).filter(
models.ModuleBuild.state == models.BUILD_STATES["failed"],
models.ModuleBuild.time_modified <= stale_date,
).all()
if stale_module_builds:
log.info(
"%s stale failed module build(s) will be cleaned up",
len(stale_module_builds)
)
for module in stale_module_builds:
log.info("%r is stale and is being cleaned up", module)
# Find completed artifacts in the stale build
artifacts = [c for c in module.component_builds if c.is_completed]
# If there are no completed artifacts, then there is nothing to tag
if artifacts:
# Set buildroot_connect=False so it doesn't recreate the Koji target and etc.
builder = GenericBuilder.create_from_module(
db_session, module, conf, buildroot_connect=False
)
builder.untag_artifacts([c.nvr for c in artifacts])
# Mark the artifacts as untagged in the database
for c in artifacts:
c.tagged = False
c.tagged_in_final = False
db_session.add(c)
state_reason = (
"The module was garbage collected since it has failed over {0}"
" day(s) ago".format(conf.cleanup_failed_builds_time)
)
module.transition(
db_session,
conf,
models.BUILD_STATES["garbage"],
state_reason=state_reason,
failure_type="user",
)
db_session.add(module)
db_session.commit()
def test_record_component_builds_duplicate_components(self, mocked_scm):
# Mock for format_mmd to get components' latest ref
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
mocked_scm.return_value.get_latest.side_effect = [
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
]
mmd = load_mmd(read_staged_data("testmodule"))
mmd = mmd.copy("testmodule-variant", "master")
module_build = module_build_service.common.models.ModuleBuild()
module_build.name = "testmodule-variant"
module_build.stream = "master"
module_build.version = 20170109091357
module_build.state = models.BUILD_STATES["init"]
module_build.scmurl = \
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
module_build.batch = 1
module_build.owner = "Tom Brady"
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = "changed-and-after"
module_build.modulemd = mmd_to_str(mmd)
db_session.add(module_build)
db_session.commit()
# Rename the the modulemd to include
mmd = mmd.copy("testmodule")
# Remove perl-Tangerine and tangerine from the modulemd to include so only one
# component conflicts
mmd.remove_rpm_component("perl-Tangerine")
mmd.remove_rpm_component("tangerine")
error_msg = (
'The included module "testmodule" in "testmodule-variant" have '
"the following conflicting components: perl-List-Compare")
format_mmd(mmd, module_build.scmurl)
with pytest.raises(UnprocessableEntity) as e:
record_component_builds(mmd,
module_build,
main_mmd=module_build.mmd())
assert str(e.value) == error_msg
def _create_test_modules(self, koji_tag_with_modules="foo-test"):
mmd = load_mmd(tests.read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), "f30.1.3")
import_mmd(db_session, mmd)
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
if koji_tag_with_modules:
platform = db_session.query(ModuleBuild).filter_by(
stream="f30.1.3").one()
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"][
"koji_tag_with_modules"] = koji_tag_with_modules
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
for context in ["7c29193d", "7c29193e"]:
mmd = tests.make_module("testmodule:master:20170109091357:" +
context)
build = ModuleBuild(
name="testmodule",
stream="master",
version=20170109091357,
state=5,
build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3",
runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7",
context=context,
koji_tag="module-testmodule-master-20170109091357-" + context,
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Dr. Pepper",
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
build.buildrequires.append(platform)
db_session.add(build)
db_session.commit()
def test_get_reusable_component_different_component(
self, changed_component):
second_module_build = models.ModuleBuild.get_by_id(db_session, 3)
if changed_component:
mmd = second_module_build.mmd()
mmd.get_rpm_component("tangerine").set_ref(
"00ea1da4192a2030f9ae023de3b3143ed647bbab")
second_module_build.modulemd = mmd_to_str(mmd)
second_module_changed_component = models.ComponentBuild.from_component_name(
db_session, changed_component, second_module_build.id)
second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab"
db_session.add(second_module_changed_component)
db_session.commit()
plc_rv = get_reusable_component(second_module_build,
"perl-List-Compare")
pt_rv = get_reusable_component(second_module_build, "perl-Tangerine")
tangerine_rv = get_reusable_component(second_module_build, "tangerine")
if changed_component == "perl-List-Compare":
# perl-Tangerine can be reused even though a component in its batch has changed
assert plc_rv is None
assert pt_rv.package == "perl-Tangerine"
assert tangerine_rv is None
elif changed_component == "perl-Tangerine":
# perl-List-Compare can be reused even though a component in its batch has changed
assert plc_rv.package == "perl-List-Compare"
assert pt_rv is None
assert tangerine_rv is None
elif changed_component == "tangerine":
# perl-List-Compare and perl-Tangerine can be reused since they are in an earlier
# buildorder than tangerine
assert plc_rv.package == "perl-List-Compare"
assert pt_rv.package == "perl-Tangerine"
assert tangerine_rv is None
elif changed_component is None:
# Nothing has changed so everthing can be used
assert plc_rv.package == "perl-List-Compare"
assert pt_rv.package == "perl-Tangerine"
assert tangerine_rv.package == "tangerine"
def model_tests_init_data():
"""Initialize data for model tests
This is refactored from tests/test_models/__init__.py, which was able to be
called directly inside setup_method generally.
The reason to convert it to this fixture is to use fixture ``db_session``
rather than create a new one. That would also benefit the whole test suite
to reduce the number of SQLAlchemy session objects.
"""
clean_database()
model_test_data_dir = os.path.join(os.path.dirname(__file__),
"test_common", "test_models", "data")
for filename in os.listdir(model_test_data_dir):
with open(os.path.join(model_test_data_dir, filename), "r") as f:
yaml = f.read()
build = module_build_from_modulemd(yaml)
db_session.add(build)
db_session.commit()
def test_record_component_builds_set_weight(self, mocked_scm):
# Mock for format_mmd to get components' latest ref
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
mocked_scm.return_value.get_latest.side_effect = [
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
]
mmd = load_mmd(read_staged_data("testmodule"))
# Set the module name and stream
mmd = mmd.copy("testmodule", "master")
module_build = module_build_service.common.models.ModuleBuild()
module_build.name = "testmodule"
module_build.stream = "master"
module_build.version = 20170109091357
module_build.state = models.BUILD_STATES["init"]
module_build.scmurl = \
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
module_build.batch = 1
module_build.owner = "Tom Brady"
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = "changed-and-after"
module_build.modulemd = mmd_to_str(mmd)
db_session.add(module_build)
db_session.commit()
format_mmd(mmd, module_build.scmurl)
record_component_builds(mmd, module_build)
db_session.commit()
assert module_build.state == models.BUILD_STATES["init"]
db_session.refresh(module_build)
for c in module_build.component_builds:
assert c.weight == 1.5
def test__get_base_module_mmds_virtual_streams_only_major_versions(
self, cfg):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1,
multiple_stream_versions=["foo28", "foo29", "foo30"])
# Mark platform:foo28 as garbage to test that it is still considered as compatible.
platform = db_session.query(models.ModuleBuild).filter_by(
name="platform", stream="foo28").first()
platform.state = "garbage"
db_session.add(platform)
db_session.commit()
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "foo29")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
mmds = get_base_module_mmds(db_session, mmd)
expected = {}
expected["ready"] = {"platform:foo29", "platform:foo30"}
expected["garbage"] = {"platform:foo28"}
# Verify no duplicates were returned before doing set operations
assert len(mmds) == len(expected)
for k in expected.keys():
assert len(mmds[k]) == len(expected[k])
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds[k]:
actual.add("{}:{}".format(mmd_.get_module_name(),
mmd_.get_stream_name()))
assert actual == expected[k]
def init(msg_id, module_build_id, module_build_state):
"""Called whenever a module enters the 'init' state.
:param str msg_id: the original id of the message being handled, which is
received from message bus.
:param int module_build_id: the module build id.
:param int module_build_state: the module build state.
"""
build = models.ModuleBuild.get_by_id(db_session, module_build_id)
state_init = models.BUILD_STATES["init"]
if module_build_state == state_init and build.state != state_init:
log.warning("Module build %r has moved to %s state already.", build,
models.INVERSE_BUILD_STATES[build.state])
log.warning(
"Ignore this message %s. Is there something wrong with the frontend"
" that sends duplicate messages?", msg_id)
return
# for MockModuleBuilder, set build logs dir to mock results dir
# before build_logs start
if conf.system == "mock":
build_tag_name = generate_module_build_koji_tag(build)
mock_resultsdir = os.path.join(conf.mock_resultsdir, build_tag_name)
if not os.path.exists(mock_resultsdir):
os.makedirs(mock_resultsdir)
build_logs.build_logs_dir = mock_resultsdir
build_logs.start(db_session, build)
log.info("Start to handle %s which is in init state.",
build.mmd().get_nsvc())
error_msg = ""
failure_reason = "unspec"
try:
mmd = build.mmd()
record_module_build_arches(mmd, build)
arches = [arch.name for arch in build.arches]
defaults_added = add_default_modules(mmd)
# Get map of packages that have SRPM overrides
srpm_overrides = get_module_srpm_overrides(build)
# Format the modulemd by putting in defaults and replacing streams that
# are branches with commit hashes
format_mmd(mmd, build.scmurl, build, db_session, srpm_overrides)
record_component_builds(mmd, build)
# The ursine.handle_stream_collision_modules is Koji specific.
# It is also run only when Ursa Prime is not enabled for the base
# module (`if not defaults_added`).
if conf.system in ["koji", "test"] and not defaults_added:
handle_stream_collision_modules(mmd)
# Sets xmd["mbs"]["ursine_rpms"] with RPMs from the buildrequired base modules which
# conflict with the RPMs from other buildrequired modules. This is done to prefer modular
# RPMs over base module RPMs even if their NVR is lower.
if conf.system in ("koji", "test"):
handle_collisions_with_base_module_rpms(mmd, arches)
else:
log.warning(
"The necessary conflicts could not be generated due to RHBZ#1693683. "
"Some RPMs from the base modules (%s) may end up being used over modular RPMs. "
"This may result in different behavior than a production build.",
", ".join(conf.base_module_names))
mmd = record_filtered_rpms(mmd)
build.modulemd = mmd_to_str(mmd)
build.transition(db_session, conf, models.BUILD_STATES["wait"])
# Catch custom exceptions that we can expose to the user
except (UnprocessableEntity, Forbidden, ValidationError,
RuntimeError) as e:
log.exception(str(e))
error_msg = str(e)
failure_reason = "user"
except (xmlrpclib.ProtocolError, koji.GenericError) as e:
log.exception(str(e))
error_msg = 'Koji communication error: "{0}"'.format(str(e))
failure_reason = "infra"
except Exception as e:
log.exception(str(e))
error_msg = "An unknown error occurred while validating the modulemd"
failure_reason = "user"
else:
db_session.add(build)
db_session.commit()
finally:
if error_msg:
# Rollback changes underway
db_session.rollback()
build.transition(
db_session,
conf,
models.BUILD_STATES["failed"],
state_reason=error_msg,
failure_type=failure_reason,
)
db_session.commit()
def failed(msg_id, module_build_id, module_build_state):
"""Called whenever a module enters the 'failed' state.
We cancel all the remaining component builds of a module
and stop the building.
:param str msg_id: the original id of the message being handled, which is
received from the message bus.
:param int module_build_id: the module build id.
:param int module_build_state: the module build state.
"""
build = models.ModuleBuild.get_by_id(db_session, module_build_id)
if build.state != module_build_state:
log.warning(
"Note that retrieved module state %r doesn't match message module state %r",
build.state,
module_build_state,
)
# This is ok.. it's a race condition we can ignore.
pass
if build.koji_tag:
builder = GenericBuilder.create_from_module(db_session, build, conf)
if build.new_repo_task_id:
builder.cancel_build(build.new_repo_task_id)
for component in (c for c in build.component_builds if c.is_unbuilt):
if component.task_id:
builder.cancel_build(component.task_id)
component.state = koji.BUILD_STATES["FAILED"]
component.state_reason = build.state_reason
db_session.add(component)
# Tell the external buildsystem to wrap up
builder.finalize(succeeded=False)
else:
# Do not overwrite state_reason set by Frontend if any.
if not build.state_reason:
reason = "Missing koji tag. Assuming previously failed module lookup."
log.error(reason)
build.transition(db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=reason,
failure_type="infra")
db_session.commit()
return
# Don't transition it again if it's already been transitioned
if build.state != models.BUILD_STATES["failed"]:
build.transition(db_session,
conf,
state=models.BUILD_STATES["failed"],
failure_type="user")
db_session.commit()
build_logs.stop(build)
GenericBuilder.clear_cache(build)
def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm):
"""
Test that the `get_reusable_module` tries to reuse the latest module in case when
multiple modules can be reused allow_only_compatible_base_modules is True.
"""
cfg.return_value = allow_ocbm
# Set "fedora" virtual stream to platform:f28.
platform_f28 = db_session.query(
models.ModuleBuild).filter_by(name="platform").one()
mmd = platform_f28.mmd()
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["fedora"]
mmd.set_xmd(xmd)
platform_f28.modulemd = mmd_to_str(mmd)
platform_f28.update_virtual_streams(db_session, ["fedora"])
# Create platform:f29 with "fedora" virtual stream.
mmd = load_mmd(read_staged_data("platform"))
mmd = mmd.copy("platform", "f29")
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["fedora"]
mmd.set_xmd(xmd)
platform_f29 = import_mmd(db_session, mmd)[0]
# Create another copy of `testmodule:master` which should be reused, because its
# stream version will be higher than the previous one. Also set its buildrequires
# to platform:f29.
latest_module = db_session.query(models.ModuleBuild).filter_by(
name="testmodule", state=models.BUILD_STATES["ready"]).one()
# This is used to clone the ModuleBuild SQLAlchemy object without recreating it from
# scratch.
db_session.expunge(latest_module)
make_transient(latest_module)
# Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context.
mmd = latest_module.mmd()
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29"
mmd.set_xmd(xmd)
latest_module.modulemd = mmd_to_str(mmd)
contexts = models.ModuleBuild.contexts_from_mmd(latest_module.modulemd)
latest_module.build_context = contexts.build_context
latest_module.context = contexts.context
latest_module.buildrequires = [platform_f29]
# Set the `id` to None, so new one is generated by SQLAlchemy.
latest_module.id = None
db_session.add(latest_module)
db_session.commit()
module = db_session.query(models.ModuleBuild)\
.filter_by(name="testmodule")\
.filter_by(state=models.BUILD_STATES["build"])\
.one()
db_session.commit()
reusable_module = get_reusable_module(module)
if allow_ocbm:
assert reusable_module.id == latest_module.id
else:
# There are two testmodules in ready state, the first one with
# lower id is what we want.
first_module = db_session.query(models.ModuleBuild).filter_by(
name="testmodule",
state=models.BUILD_STATES["ready"]).order_by(
models.ModuleBuild.id).first()
assert reusable_module.id == first_module.id
def scheduler_init_data(tangerine_state=None, scratch=False):
""" Creates a testmodule in the building state with all the components in the same batch
"""
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule"))
mmd.get_rpm_component("tangerine").set_buildorder(0)
module_build = module_build_service.common.models.ModuleBuild(
name="testmodule",
stream="master",
version='20170109091357',
state=BUILD_STATES["build"],
scratch=scratch,
build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
context="7c29193d",
koji_tag="scrmod-testmodule-master-20170109091357-7c29193d"
if scratch else "module-testmodule-master-20170109091357-7c29193d",
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3 if tangerine_state else 2,
# https://www.youtube.com/watch?v=iOKymYVSaJE
owner="Buzz Lightyear",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
db_session.add(module_build)
db_session.commit()
platform_br = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 1)
module_build.buildrequires.append(platform_br)
arch = db_session.query(
module_build_service.common.models.ModuleArch).get(1)
module_build.arches.append(arch)
build_one_component_release = get_rpm_release(db_session, module_build)
db_session.add_all([
module_build_service.common.models.ComponentBuild(
module_id=module_build.id,
package="perl-Tangerine",
scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine"
"?#4ceea43add2366d8b8c5a622a2fb563b625b9abf",
format="rpms",
task_id=90276227,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-Tangerine-0.23-1.{0}".format(
build_one_component_release),
batch=2,
ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=module_build.id,
package="perl-List-Compare",
scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare"
"?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
format="rpms",
task_id=90276228,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-List-Compare-0.53-5.{0}".format(
build_one_component_release),
batch=2,
ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=module_build.id,
package="tangerine",
scmurl="https://src.fedoraproject.org/rpms/tangerine"
"?#fbed359411a1baa08d4a88e0d12d426fbf8f602c",
format="rpms",
batch=3,
ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c",
state=tangerine_state,
task_id=90276315 if tangerine_state else None,
nvr="tangerine-0.22-3.{}".format(build_one_component_release)
if tangerine_state else None,
tagged=tangerine_state == koji.BUILD_STATES["COMPLETE"],
tagged_in_final=tangerine_state == koji.BUILD_STATES["COMPLETE"],
),
module_build_service.common.models.ComponentBuild(
module_id=module_build.id,
package="module-build-macros",
scmurl=
"/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-"
"macros-0.1-1.module_testmodule_master_20170109091357.src.rpm",
format="rpms",
task_id=90276181,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-0.1-1.{}".format(
build_one_component_release),
batch=1,
tagged=True,
build_time_only=True,
),
])
db_session.commit()
def make_module(
nsvc,
dependencies=None,
base_module=None,
filtered_rpms=None,
xmd=None,
store_to_db=False,
virtual_streams=None,
arches=None,
):
"""
Creates new models.ModuleBuild defined by `nsvc` string with requires
and buildrequires set according to ``requires_list`` and ``build_requires_list``.
:param str nsvc: name:stream:version:context of a module.
:param dependencies: list of groups of dependencies (requires and buildrequires).
For example, [
{"requires": {"platform": ["f30"]}, "buildrequires": {"platform": ["f30"]}},
...
]
:type dependencies: list[dict]
:param base_module: a base module build required by the new module created.
:type base_module: :class:`ModuleBuild`
:param filtered_rpms: list of filtered RPMs which are added to filter
section in module metadata.
:type filtered_rpms: list[str]
:param dict xmd: a mapping representing XMD section in module metadata. A
custom xmd could be passed for testing a particular scenario and some
default key/value pairs are added if not present.
:param bool store_to_db: whether to store created module metadata to the
database. If set to True, ``db_session`` is required.
:param virtual_streams: List of virtual streams provided by this module.
If set, This requires ``db_session`` and ``store_to_db`` to be set to a
session object and True.
:type virtual_streams: list[str]
:param arches: List of architectures this module is built against. If set
to None, ``["x86_64"]`` is used as a default. If set, likewise
``virtual_stream``.
:type arches: list[str]
:return: New Module Build if set to store module metadata to database,
otherwise the module metadata is returned.
:rtype: ModuleBuild or Modulemd.Module
"""
if store_to_db:
assert db_session is not None
if base_module:
assert db_session is not None
if virtual_streams:
assert store_to_db
if arches:
assert store_to_db
nsvc_regex = re.compile(r"([^:]+):([^:]+):([^:]+)(?::(.+))?")
match = nsvc_regex.match(nsvc)
if not match:
raise ValueError('Argument nsvc is not in format N:S:V or N:S:V:C')
name, stream, version, context = match.groups()
mmd = Modulemd.ModuleStreamV2.new(name, stream)
mmd.set_version(int(version))
if context:
mmd.set_context(context)
mmd.set_summary("foo")
# Test unicode in mmd.
mmd.set_description(u"foo \u2019s")
mmd.add_module_license("GPL")
if filtered_rpms:
for rpm in filtered_rpms:
mmd.add_rpm_filter(rpm)
def _add_require(mmd_deps, require_type, name, streams):
assert isinstance(mmd_deps, Modulemd.Dependencies)
assert require_type in ("requires", "buildrequires")
assert isinstance(streams, (list, tuple))
if require_type == "requires":
add_stream = mmd_deps.add_runtime_stream
set_empty_deps = mmd_deps.set_empty_runtime_dependencies_for_module
else:
add_stream = mmd_deps.add_buildtime_stream
set_empty_deps = mmd_deps.set_empty_buildtime_dependencies_for_module
for stream in streams:
add_stream(name, stream)
else:
set_empty_deps(name)
for dep_group in dependencies or []:
mmd_deps = Modulemd.Dependencies()
# A deps could be {"platform": ["f30"], "python": []}
for require_type, deps in dep_group.items():
for req_name, req_streams in deps.items():
_add_require(mmd_deps, require_type, req_name, req_streams)
mmd.add_dependencies(mmd_deps)
# Caller could pass whole xmd including mbs, but if something is missing,
# default values are given here.
xmd = xmd or {"mbs": {}}
xmd_mbs = xmd["mbs"]
if "buildrequires" not in xmd_mbs:
xmd_mbs["buildrequires"] = {}
if "requires" not in xmd_mbs:
xmd_mbs["requires"] = {}
if "commit" not in xmd_mbs:
xmd_mbs["commit"] = "ref_%s" % context
if "mse" not in xmd_mbs:
xmd_mbs["mse"] = "true"
if virtual_streams:
xmd_mbs["virtual_streams"] = virtual_streams
mmd.set_xmd(xmd)
if not store_to_db:
return mmd
module_build = ModuleBuild(
name=name,
stream=stream,
stream_version=ModuleBuild.get_stream_version(stream),
version=version,
context=context,
state=BUILD_STATES["ready"],
scmurl="https://src.stg.fedoraproject.org/modules/unused.git?#ff1ea79",
batch=1,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
build_context=context,
runtime_context=context,
modulemd=mmd_to_str(mmd),
koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None,
)
if base_module:
module_build.buildrequires.append(base_module)
db_session.add(module_build)
db_session.commit()
if virtual_streams:
for virtual_stream in virtual_streams:
vs_obj = db_session.query(VirtualStream).filter_by(
name=virtual_stream).first()
if not vs_obj:
vs_obj = VirtualStream(name=virtual_stream)
db_session.add(vs_obj)
db_session.commit()
if vs_obj not in module_build.virtual_streams:
module_build.virtual_streams.append(vs_obj)
db_session.commit()
for arch in arches or ["x86_64"]:
arch_obj = db_session.query(ModuleArch).filter_by(name=arch).first()
if not arch_obj:
arch_obj = ModuleArch(name=arch)
db_session.add(arch_obj)
db_session.commit()
if arch_obj not in module_build.arches:
module_build.arches.append(arch_obj)
db_session.commit()
return module_build
def _create_module_with_filters(self, db_session, batch, state):
mmd = load_mmd(read_staged_data("testmodule-with-filters"))
# Set the name and stream
mmd = mmd.copy("mbs-testmodule", "test")
mmd.set_xmd({
"mbs": {
"rpms": {
"ed": {
"ref": "01bf8330812fea798671925cc537f2f29b0bd216"
},
"mksh": {
"ref": "f70fd11ddf96bce0e2c64309706c29156b39141d"
},
},
"buildrequires": {
"host": {
"version": "20171024133034",
"filtered_rpms": [],
"stream": "master",
"ref": "6df253bb3c53e84706c01b8ab2d5cac24f0b6d45",
"context": "00000000",
},
"platform": {
"version": "20171028112959",
"filtered_rpms": [],
"stream": "master",
"ref": "4f7787370a931d57421f9f9555fc41c3e31ff1fa",
"context": "00000000",
},
},
"scmurl": "file:///testdir",
"commit": "5566bc792ec7a03bb0e28edd1b104a96ba342bd8",
"requires": {
"platform": {
"version": "20171028112959",
"filtered_rpms": [],
"stream": "master",
"ref": "4f7787370a931d57421f9f9555fc41c3e31ff1fa",
"context": "00000000",
}
},
}
})
module = ModuleBuild.create(
db_session,
conf,
name="mbs-testmodule",
stream="test",
version="20171027111452",
modulemd=mmd_to_str(mmd),
scmurl="file:///testdir",
username="******",
)
module.koji_tag = "module-mbs-testmodule-test-20171027111452"
module.batch = batch
db_session.add(module)
db_session.commit()
comp_builds = [
{
"module_id":
module.id,
"state":
state,
"package":
"ed",
"format":
"rpms",
"scmurl": ("https://src.fedoraproject.org/rpms/ed"
"?#01bf8330812fea798671925cc537f2f29b0bd216"),
"batch":
2,
"ref":
"01bf8330812fea798671925cc537f2f29b0bd216",
},
{
"module_id":
module.id,
"state":
state,
"package":
"mksh",
"format":
"rpms",
"scmurl": ("https://src.fedoraproject.org/rpms/mksh"
"?#f70fd11ddf96bce0e2c64309706c29156b39141d"),
"batch":
3,
"ref":
"f70fd11ddf96bce0e2c64309706c29156b39141d",
},
]
for build in comp_builds:
db_session.add(ComponentBuild(**build))
db_session.commit()
return module
def _populate_data(data_size=10, contexts=False, scratch=False):
# Query arch from passed database session, otherwise there will be an error
# like "Object '<ModuleBuild at 0x7f4ccc805c50>' is already attached to
# session '275' (this is '276')" when add new module build object to passed
# session.
task_id_counter = itertools.count(1)
arch = db_session.query(
module_build_service.common.models.ModuleArch).get(1)
num_contexts = 2 if contexts else 1
for index in range(data_size):
for context in range(num_contexts):
build_one = ModuleBuild(
name="nginx",
stream="1",
version=2 + index,
state=BUILD_STATES["ready"],
scratch=scratch,
modulemd=read_staged_data("nginx_mmd"),
koji_tag="scrmod-nginx-1.2" if scratch else "module-nginx-1.2",
scmurl="git://pkgs.domain.local/modules/nginx"
"?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=2,
# https://www.youtube.com/watch?v=iQGwrK_yDEg,
owner="Moe Szyslak",
time_submitted=datetime(2016, 9, 3, 11, 23, 20) +
timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 11, 25, 32) +
timedelta(minutes=(index * 10)),
time_completed=datetime(2016, 9, 3, 11, 25, 32) +
timedelta(minutes=(index * 10)),
rebuild_strategy="changed-and-after",
)
build_one.arches.append(arch)
if contexts:
build_one.stream = str(index)
nsvc = "{}:{}:{}:{}".format(build_one.name, build_one.stream,
build_one.version, context)
unique_hash = hashlib.sha1(nsvc.encode('utf-8')).hexdigest()
build_one.build_context = unique_hash
build_one.runtime_context = unique_hash
combined_hashes = "{0}:{1}".format(unique_hash, unique_hash)
build_one.context = hashlib.sha1(
combined_hashes.encode("utf-8")).hexdigest()[:8]
db_session.add(build_one)
db_session.commit()
build_one_component_release = get_rpm_release(
db_session, build_one)
db_session.add_all([
ComponentBuild(package="nginx",
scmurl="git://pkgs.domain.local/rpms/nginx?"
"#ga95886c8a443b36a9ce31abda1f9bed22f2f8c3",
format="rpms",
task_id=six.next(task_id_counter),
state=koji.BUILD_STATES["COMPLETE"],
nvr="nginx-1.10.1-2.{0}".format(
build_one_component_release),
batch=1,
module_id=2 + index * 3,
tagged=True,
tagged_in_final=True),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_nginx_1_2.src.rpm",
format="rpms",
task_id=six.next(task_id_counter),
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(
build_one_component_release),
batch=2,
module_id=2 + index * 3,
tagged=True,
tagged_in_final=True)
])
db_session.commit()
build_two = ModuleBuild(
name="postgressql",
stream="1",
version=2 + index,
state=BUILD_STATES["done"],
scratch=scratch,
modulemd=read_staged_data("testmodule"),
koji_tag="scrmod-postgressql-1.2"
if scratch else "module-postgressql-1.2",
scmurl="git://pkgs.domain.local/modules/postgressql"
"?#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=2,
owner="some_user",
time_submitted=datetime(2016, 9, 3, 12, 25, 33) +
timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 12, 27, 19) +
timedelta(minutes=(index * 10)),
time_completed=datetime(2016, 9, 3, 11, 27, 19) +
timedelta(minutes=(index * 10)),
rebuild_strategy="changed-and-after",
)
build_two.arches.append(arch)
db_session.add(build_two)
db_session.commit()
build_two_component_release = get_rpm_release(db_session, build_two)
db_session.add_all([
ComponentBuild(package="postgresql",
scmurl="git://pkgs.domain.local/rpms/postgresql"
"?#dc95586c4a443b26a9ce38abda1f9bed22f2f8c3",
format="rpms",
task_id=2433433 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="postgresql-9.5.3-4.{0}".format(
build_two_component_release),
batch=2,
module_id=3 + index * 3,
tagged=True,
tagged_in_final=True),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_postgresql_1_2.src.rpm",
format="rpms",
task_id=47383993 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(
build_two_component_release),
batch=1,
module_id=3 + index * 3)
])
db_session.commit()
build_three = ModuleBuild(
name="testmodule",
stream="4.3.43",
version=6 + index,
state=BUILD_STATES["wait"],
scratch=scratch,
modulemd=read_staged_data("testmodule"),
koji_tag=None,
scmurl="git://pkgs.domain.local/modules/testmodule"
"?#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=0,
owner="some_other_user",
time_submitted=datetime(2016, 9, 3, 12, 28, 33) +
timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 12, 28, 40) +
timedelta(minutes=(index * 10)),
time_completed=None,
rebuild_strategy="changed-and-after",
)
db_session.add(build_three)
db_session.commit()
build_three_component_release = get_rpm_release(
db_session, build_three)
db_session.add_all([
ComponentBuild(package="rubygem-rails",
scmurl="git://pkgs.domain.local/rpms/rubygem-rails"
"?#dd55886c4a443b26a9ce38abda1f9bed22f2f8c3",
format="rpms",
task_id=2433433 + index,
state=koji.BUILD_STATES["FAILED"],
nvr="postgresql-9.5.3-4.{0}".format(
build_three_component_release),
batch=2,
module_id=4 + index * 3),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_testmodule_1_2.src.rpm",
format="rpms",
task_id=47383993 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(
build_three_component_release),
batch=1,
module_id=4 + index * 3,
tagged=True,
build_time_only=True)
])
db_session.commit()
def require_default_arch(require_empty_database):
"""Provides empty database with x86_64 arch"""
arch_obj = module_build_service.common.models.ModuleArch(name="x86_64")
db_session.add(arch_obj)
db_session.commit()
def record_component_builds(mmd,
module,
initial_batch=1,
previous_buildorder=None,
main_mmd=None):
# Imported here to allow import of utils in GenericBuilder.
from module_build_service.builder import GenericBuilder
# When main_mmd is set, merge the metadata from this mmd to main_mmd,
# otherwise our current mmd is main_mmd.
if main_mmd:
# Check for components that are in both MMDs before merging since MBS
# currently can't handle that situation.
main_mmd_rpms = main_mmd.get_rpm_component_names()
mmd_rpms = mmd.get_rpm_component_names()
duplicate_components = [
rpm for rpm in main_mmd_rpms if rpm in mmd_rpms
]
if duplicate_components:
error_msg = (
'The included module "{0}" in "{1}" have the following '
"conflicting components: {2}".format(
mmd.get_module_name(), main_mmd.get_module_name(),
", ".join(duplicate_components)))
raise UnprocessableEntity(error_msg)
merge_included_mmd(main_mmd, mmd)
else:
main_mmd = mmd
# If the modulemd yaml specifies components, then submit them for build
rpm_components = [
mmd.get_rpm_component(name) for name in mmd.get_rpm_component_names()
]
module_components = [
mmd.get_module_component(name)
for name in mmd.get_module_component_names()
]
all_components = list(rpm_components) + list(module_components)
if not all_components:
return
# Get map of packages that have SRPM overrides
srpm_overrides = get_module_srpm_overrides(module)
rpm_weights = GenericBuilder.get_build_weights(
[c.get_name() for c in rpm_components])
all_components.sort(key=lambda x: x.get_buildorder())
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
for component in all_components:
# Increment the batch number when buildorder increases.
if previous_buildorder != component.get_buildorder():
previous_buildorder = component.get_buildorder()
batch += 1
# If the component is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(component, Modulemd.ComponentModule):
full_url = component.get_repository() + "?#" + component.get_ref()
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = fetch_mmd(full_url, whitelist_url=True)[0]
format_mmd(included_mmd, module.scmurl, module, db_session,
srpm_overrides)
batch = record_component_builds(included_mmd, module, batch,
previous_buildorder, main_mmd)
continue
package = component.get_name()
if package in srpm_overrides:
component_ref = None
full_url = srpm_overrides[package]
log.info('Building custom SRPM "{0}"'
" for package {1}".format(full_url, package))
else:
component_ref = mmd.get_xmd()["mbs"]["rpms"][package]["ref"]
full_url = component.get_repository() + "?#" + component_ref
# Skip the ComponentBuild if it already exists in database. This can happen
# in case of module build resubmition.
existing_build = models.ComponentBuild.from_component_name(
db_session, package, module.id)
if existing_build:
# Check that the existing build has the same most important attributes.
# This should never be a problem, but it's good to be defensive here so
# we do not mess things during resubmition.
if (existing_build.batch != batch
or existing_build.scmurl != full_url
or existing_build.ref != component_ref):
raise ValidationError(
"Component build %s of module build %s (id: %d) already "
"exists in database, but its attributes are different from"
" resubmitted one." %
(component.get_name(), module.name, module.id))
continue
build = models.ComponentBuild(module_id=module.id,
package=package,
format="rpms",
scmurl=full_url,
batch=batch,
ref=component_ref,
weight=rpm_weights[package],
buildonly=component.get_buildonly())
db_session.add(build)
return batch
def reuse_shared_userspace_init_data():
# Create shared-userspace-570, state is COMPLETE, all components
# are properly built.
scmurl = "https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453"
mmd = load_mmd(read_staged_data("shared-userspace-570"))
xmd = mmd.get_xmd()
xmd["mbs"]["scmurl"] = scmurl
xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8"
mmd.set_xmd(xmd)
module_build = module_build_service.common.models.ModuleBuild(
name=mmd.get_module_name(),
stream=mmd.get_stream_name(),
version=mmd.get_version(),
build_context=module_build_service.common.models.ModuleBuild.
calculate_build_context(xmd["mbs"]["buildrequires"]),
runtime_context="50dd3eb5dde600d072e45d4120e1548ce66bc94a",
state=BUILD_STATES["ready"],
modulemd=mmd_to_str(mmd),
koji_tag="module-shared-userspace-f26-20170601141014-75f92abb",
scmurl=scmurl,
batch=16,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
time_completed=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
)
components = [
mmd.get_rpm_component(rpm) for rpm in mmd.get_rpm_component_names()
]
components.sort(key=lambda x: x.get_buildorder())
previous_buildorder = None
batch = 1
xmd_mbs_rpms = xmd["mbs"]["rpms"]
for pkg in components:
# Increment the batch number when buildorder increases.
if previous_buildorder != pkg.get_buildorder():
previous_buildorder = pkg.get_buildorder()
batch += 1
pkgref = xmd_mbs_rpms[pkg.get_name()]["ref"]
full_url = pkg.get_repository() + "?#" + pkgref
module_build.component_builds.append(
module_build_service.common.models.ComponentBuild(
package=pkg.get_name(),
format="rpms",
scmurl=full_url,
batch=batch,
ref=pkgref,
state=1,
tagged=True,
tagged_in_final=True,
))
db_session.add(module_build)
db_session.commit()
# Create shared-userspace-577, state is WAIT, no component built
scmurl = "https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453"
mmd2 = load_mmd(read_staged_data("shared-userspace-577"))
xmd = mmd2.get_xmd()
xmd["mbs"]["scmurl"] = scmurl
xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8"
mmd2.set_xmd(xmd)
module_build = module_build_service.common.models.ModuleBuild(
name=mmd2.get_module_name(),
stream=mmd2.get_stream_name(),
version=mmd2.get_version(),
build_context=module_build_service.common.models.ModuleBuild.
calculate_build_context(xmd["mbs"]["buildrequires"]),
runtime_context="50dd3eb5dde600d072e45d4120e1548ce66bc94a",
state=BUILD_STATES["done"],
modulemd=mmd_to_str(mmd2),
koji_tag="module-shared-userspace-f26-20170605091544-75f92abb",
scmurl=scmurl,
batch=0,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
time_completed=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
)
components2 = [
mmd2.get_rpm_component(rpm) for rpm in mmd2.get_rpm_component_names()
]
# Store components to database in different order than for 570 to
# reproduce the reusing issue.
components2.sort(key=lambda x: len(x.get_name()))
components2.sort(key=lambda x: x.get_buildorder())
previous_buildorder = None
batch = 1
xmd_mbs_rpms = mmd2.get_xmd()["mbs"]["rpms"]
for pkg in components2:
# Increment the batch number when buildorder increases.
if previous_buildorder != pkg.get_buildorder():
previous_buildorder = pkg.get_buildorder()
batch += 1
pkgref = xmd_mbs_rpms[pkg.get_name()]["ref"]
full_url = pkg.get_repository() + "?#" + pkgref
module_build.component_builds.append(
module_build_service.common.models.ComponentBuild(
package=pkg.get_name(),
format="rpms",
scmurl=full_url,
batch=batch,
ref=pkgref))
db_session.add(module_build)
db_session.commit()
def reuse_component_init_data():
mmd = load_mmd(read_staged_data("formatted_testmodule"))
build_one = module_build_service.common.models.ModuleBuild(
name="testmodule",
stream="master",
version='20170109091357',
state=BUILD_STATES["ready"],
runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1",
context="78e4a6fd",
koji_tag="module-testmodule-master-20170109091357-78e4a6fd",
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
time_completed=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
)
build_one_component_release = _get_rpm_release_no_db(build_one)
mmd.set_version(int(build_one.version))
xmd = mmd.get_xmd()
xmd["mbs"]["scmurl"] = build_one.scmurl
xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba"
mmd.set_xmd(xmd)
build_one.modulemd = mmd_to_str(mmd)
contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd(
build_one.modulemd)
build_one.build_context = contexts.build_context
build_one.build_context_no_bms = contexts.build_context_no_bms
db_session.add(build_one)
db_session.flush()
platform_br = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 1)
build_one.buildrequires.append(platform_br)
arch = db_session.query(
module_build_service.common.models.ModuleArch).get(1)
build_one.arches.append(arch)
db_session.add_all([
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="perl-Tangerine",
scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine"
"?#4ceea43add2366d8b8c5a622a2fb563b625b9abf",
format="rpms",
task_id=90276227,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-Tangerine-0.23-1.{0}".format(
build_one_component_release),
batch=2,
ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="perl-List-Compare",
scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare"
"?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
format="rpms",
task_id=90276228,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-List-Compare-0.53-5.{0}".format(
build_one_component_release),
batch=2,
ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="tangerine",
scmurl="https://src.fedoraproject.org/rpms/tangerine"
"?#fbed359411a1baa08d4a88e0d12d426fbf8f602c",
format="rpms",
task_id=90276315,
state=koji.BUILD_STATES["COMPLETE"],
nvr="tangerine-0.22-3.{0}".format(build_one_component_release),
batch=3,
ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="module-build-macros",
scmurl=
"/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-"
"macros-0.1-1.module_testmodule_master_20170109091357.src.rpm",
format="rpms",
task_id=90276181,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-0.1-1.{0}".format(
build_one_component_release),
batch=1,
tagged=True,
build_time_only=True,
),
])
build_two = module_build_service.common.models.ModuleBuild(
name="testmodule",
stream="master",
version='20170219191323',
state=BUILD_STATES["build"],
runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1",
context="c40c156c",
koji_tag="module-testmodule-master-20170219191323-c40c156c",
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a",
batch=1,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 19, 16, 8, 18),
time_modified=datetime(2017, 2, 19, 16, 8, 18),
rebuild_strategy="changed-and-after",
)
build_two_component_release = _get_rpm_release_no_db(build_two)
mmd.set_version(int(build_one.version))
xmd = mmd.get_xmd()
xmd["mbs"]["scmurl"] = build_one.scmurl
xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8"
mmd.set_xmd(xmd)
build_two.modulemd = mmd_to_str(mmd)
contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd(
build_two.modulemd)
build_two.build_context = contexts.build_context
build_two.build_context_no_bms = contexts.build_context_no_bms
db_session.add(build_two)
db_session.flush()
build_two.arches.append(arch)
build_two.buildrequires.append(platform_br)
db_session.add_all([
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="perl-Tangerine",
scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine"
"?#4ceea43add2366d8b8c5a622a2fb563b625b9abf",
format="rpms",
batch=2,
ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf",
),
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="perl-List-Compare",
scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare"
"?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
format="rpms",
batch=2,
ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
),
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="tangerine",
scmurl="https://src.fedoraproject.org/rpms/tangerine"
"?#fbed359411a1baa08d4a88e0d12d426fbf8f602c",
format="rpms",
batch=3,
ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c",
),
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="module-build-macros",
scmurl=
"/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-"
"macros-0.1-1.module_testmodule_master_20170219191323.src.rpm",
format="rpms",
task_id=90276186,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-0.1-1.{0}".format(
build_two_component_release),
batch=1,
tagged=True,
build_time_only=True,
),
])
# Commit everything in one go.
db_session.commit()
def wait(msg_id, module_build_id, module_build_state):
""" Called whenever a module enters the 'wait' state.
We transition to this state shortly after a modulebuild is first requested.
All we do here is request preparation of the buildroot.
The kicking off of individual component builds is handled elsewhere,
in module_build_service.schedulers.handlers.repos.
:param str msg_id: the original id of the message being handled which is
received from the message bus.
:param int module_build_id: the module build id.
:param int module_build_state: the module build state.
"""
build = models.ModuleBuild.get_by_id(db_session, module_build_id)
log.info("Found build=%r from message" % build)
log.debug("%r", build.modulemd)
if build.state != module_build_state:
log.warning(
"Note that retrieved module state %r doesn't match message module state %r",
build.state,
module_build_state,
)
# This is ok.. it's a race condition we can ignore.
pass
try:
build_deps = get_module_build_dependencies(build)
except ValueError:
reason = "Failed to get module info from MBS. Max retries reached."
log.exception(reason)
build.transition(db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=reason,
failure_type="infra")
db_session.commit()
raise
tag = generate_module_build_koji_tag(build)
log.debug("Found tag=%s for module %r" % (tag, build))
# Hang on to this information for later. We need to know which build is
# associated with which koji tag, so that when their repos are regenerated
# in koji we can figure out which for which module build that event is
# relevant.
log.debug("Assigning koji tag=%s to module build" % tag)
build.koji_tag = tag
if build.scratch:
log.debug(
"Assigning Content Generator build koji tag is skipped for scratch module build."
)
elif conf.koji_cg_tag_build:
cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps)
log.debug(
"Assigning Content Generator build koji tag=%s to module build",
cg_build_koji_tag)
build.cg_build_koji_tag = cg_build_koji_tag
else:
log.debug(
"It is disabled to tag module build during importing into Koji by Content Generator."
)
log.debug(
"Skip to assign Content Generator build koji tag to module build.")
builder = GenericBuilder.create_from_module(db_session, build, conf)
log.debug(
"Adding dependencies %s into buildroot for module %s:%s:%s",
build_deps.keys(),
build.name,
build.stream,
build.version,
)
builder.buildroot_add_repos(build_deps)
if not build.component_builds:
log.info("There are no components in module %r, skipping build" %
build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
# Return a KojiRepoChange message so that the build can be transitioned to done
# in the repos handler
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
return
# If all components in module build will be reused, we don't have to build
# module-build-macros, because there won't be any build done.
if attempt_to_reuse_all_components(builder, build):
log.info(
"All components have been reused for module %r, skipping build" %
build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
return []
log.debug("Starting build batch 1")
build.batch = 1
db_session.commit()
artifact_name = "module-build-macros"
component_build = models.ComponentBuild.from_component_name(
db_session, artifact_name, build.id)
srpm = builder.get_disttag_srpm(disttag=".%s" %
get_rpm_release(db_session, build),
module_build=build)
if not component_build:
component_build = models.ComponentBuild(
module_id=build.id,
package=artifact_name,
format="rpms",
scmurl=srpm,
batch=1,
build_time_only=True,
)
db_session.add(component_build)
# Commit and refresh so that the SQLAlchemy relationships are available
db_session.commit()
db_session.refresh(component_build)
recovered = builder.recover_orphaned_artifact(component_build)
if recovered:
log.info("Found an existing module-build-macros build")
# There was no existing artifact found, so lets submit the build instead
else:
task_id, state, reason, nvr = builder.build(
artifact_name=artifact_name, source=srpm)
component_build.task_id = task_id
component_build.state = state
component_build.reason = reason
component_build.nvr = nvr
elif not component_build.is_completed:
# It's possible that the build succeeded in the builder but some other step failed which
# caused module-build-macros to be marked as failed in MBS, so check to see if it exists
# first
recovered = builder.recover_orphaned_artifact(component_build)
if recovered:
log.info("Found an existing module-build-macros build")
else:
task_id, state, reason, nvr = builder.build(
artifact_name=artifact_name, source=srpm)
component_build.task_id = task_id
component_build.state = state
component_build.reason = reason
component_build.nvr = nvr
db_session.add(component_build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
# We always have to regenerate the repository.
if conf.system == "koji":
log.info("Regenerating the repository")
task_id = builder.koji_session.newRepo(
builder.module_build_tag["name"])
build.new_repo_task_id = task_id
db_session.commit()
else:
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
