def test_process_waiting_module_build(self, create_builder, dbg, state):
""" Test that processing old waiting module builds works. """
handler = producer.ON_MODULE_CHANGE_HANDLERS[models.BUILD_STATES[state]]
# Change the batch to 2, so the module build is in state where
# it is not building anything, but the state is "build".
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.state = models.BUILD_STATES[state]
original = datetime.utcnow() - timedelta(minutes=11)
module_build.time_modified = original
db_session.commit()
db_session.refresh(module_build)
# Poll :)
producer.process_waiting_module_builds()
handler.delay.assert_called_once_with(
"internal:mbs.module.state.change",
module_build.id,
module_build.state
)
db_session.refresh(module_build)
# ensure the time_modified was changed.
assert module_build.time_modified > original
def test_cant_delete_build_target_if_not_reach_delete_time(
self, ClientSession, create_builder, dbg
):
module_build_2 = models.ModuleBuild.get_by_id(db_session, 2)
# Only module build 1's build target should be deleted.
module_build_2.koji_tag = "module-tag1"
module_build_2.state = models.BUILD_STATES["done"]
# Ensure to exceed the koji_target_delete_time easily later for deletion
module_build_2.time_completed = datetime.utcnow() - timedelta(minutes=5)
db_session.commit()
db_session.refresh(module_build_2)
koji_session = ClientSession.return_value
# No created module build has any of these tags.
koji_session.getBuildTargets.return_value = [
{"id": 1, "dest_tag_name": module_build_2.koji_tag, "name": module_build_2.koji_tag}
]
with patch.object(conf, "koji_tag_prefixes", new=["module"]):
# Use default koji_target_delete_time in config. That time is long
# enough for test.
producer.delete_old_koji_targets()
koji_session.deleteBuildTarget.assert_not_called()
def test_transform_from_done_to_ready(self, ClientSession, publish):
clean_database()
# This build should be queried and transformed to ready state
module_build = make_module_in_db(
"pkg:0.1:1:c1",
[{
"requires": {
"platform": ["el8"]
},
"buildrequires": {
"platform": ["el8"]
},
}],
)
module_build.transition(db_session, conf, BUILD_STATES["done"],
"Move to done directly for running test.")
db_session.commit()
# Assert this call below
first_publish_call = call(
"module.state.change",
module_build.json(db_session, show_tasks=False),
conf,
"mbs",
)
ClientSession.return_value.getBuild.return_value = {
"extra": {
"typeinfo": {
"module": {
"module_build_service_id": module_build.id
}
}
}
}
msg = {
"msg_id": "msg-id-1",
"topic": "org.fedoraproject.prod.greenwave.decision.update",
"msg": {
"decision_context": "test_dec_context",
"policies_satisfied": True,
"subject_identifier": "pkg-0.1-1.c1",
},
}
hub = Mock(config={"validate_signatures": False})
consumer = MBSConsumer(hub)
consumer.consume(msg)
db_session.add(module_build)
# Load module build again to check its state is moved correctly
db_session.refresh(module_build)
assert BUILD_STATES["ready"] == module_build.state
publish.assert_has_calls([
first_publish_call,
call("module.state.change",
module_build.json(db_session, show_tasks=False), conf, "mbs"),
])
def test_cleanup_stale_failed_builds_no_components(self, create_builder, dbg):
""" Test that a module build without any components built gets to the garbage state when
running cleanup_stale_failed_builds.
"""
module_build_one = models.ModuleBuild.get_by_id(db_session, 1)
module_build_one.state = models.BUILD_STATES["failed"]
module_build_one.time_modified = datetime.utcnow()
module_build_two = models.ModuleBuild.get_by_id(db_session, 2)
module_build_two.state = models.BUILD_STATES["failed"]
module_build_two.time_modified = datetime.utcnow() - timedelta(
days=conf.cleanup_failed_builds_time + 1)
module_build_two.koji_tag = None
module_build_two.cg_build_koji_tag = None
for c in module_build_two.component_builds:
c.state = None
db_session.commit()
producer.cleanup_stale_failed_builds()
db_session.refresh(module_build_two)
# Make sure module_build_two was transitioned to garbage
assert module_build_two.state == models.BUILD_STATES["garbage"]
state_reason = (
"The module was garbage collected since it has failed over {0} day(s) ago"
.format(conf.cleanup_failed_builds_time)
)
assert module_build_two.state_reason == state_reason
# Make sure module_build_one stayed the same
assert module_build_one.state == models.BUILD_STATES["failed"]
# Make sure that the builder was never instantiated
create_builder.assert_not_called()
def test_import_mmd_dont_remove_dropped_virtual_streams_associated_with_other_modules(
):
mmd = load_mmd(read_staged_data("formatted_testmodule"))
# Add some virtual streams to this module metadata
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["f28", "f29", "f30"]
mmd.set_xmd(xmd)
import_mmd(db_session, mmd)
# Import another module which has overlapping virtual streams
another_mmd = load_mmd(
read_staged_data("formatted_testmodule-more-components"))
# Add some virtual streams to this module metadata
xmd = another_mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["f29", "f30"]
another_mmd.set_xmd(xmd)
another_module_build, _ = import_mmd(db_session, another_mmd)
# Now, remove f30 from mmd
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["f28", "f29"]
mmd.set_xmd(xmd)
# Reimport formatted_testmodule again
module_build, _ = import_mmd(db_session, mmd)
db_session.refresh(module_build)
assert ["f28", "f29"] == sorted(item.name
for item in module_build.virtual_streams)
# The overlapped f30 should be still there.
db_session.refresh(another_module_build)
assert ["f29",
"f30"] == sorted(item.name
for item in another_module_build.virtual_streams)
def test_only_delete_build_target_with_allowed_koji_tag_prefix(
self, ClientSession, create_builder, dbg
):
module_build_2 = models.ModuleBuild.get_by_id(db_session, 2)
# Only module build 1's build target should be deleted.
module_build_2.koji_tag = "module-tag1"
module_build_2.state = models.BUILD_STATES["done"]
# Ensure to exceed the koji_target_delete_time easily later for deletion
module_build_2.time_completed = datetime.utcnow() - timedelta(hours=24)
module_build_3 = models.ModuleBuild.get_by_id(db_session, 3)
module_build_3.koji_tag = "f28"
db_session.commit()
db_session.refresh(module_build_2)
db_session.refresh(module_build_3)
koji_session = ClientSession.return_value
# No created module build has any of these tags.
koji_session.getBuildTargets.return_value = [
{"id": 1, "dest_tag_name": module_build_2.koji_tag, "name": module_build_2.koji_tag},
{"id": 2, "dest_tag_name": module_build_3.koji_tag, "name": module_build_3.koji_tag},
]
with patch.object(conf, "koji_tag_prefixes", new=["module", "another-prefix"]):
with patch.object(conf, "koji_target_delete_time", new=60):
producer.delete_old_koji_targets()
koji_session.deleteBuildTarget.assert_called_once_with(1)
koji_session.krb_login.assert_called_once()
def process_message(self, event_info):
# Choose a handler for this message
handler, build = self._map_message(db_session, event_info)
if handler is None:
log.debug("No event handler associated with msg %s",
event_info["msg_id"])
return
idx = "%s: %s, %s" % (handler.__name__, event_info["event"],
event_info["msg_id"])
if handler is no_op_handler:
log.debug("Handler is NO_OP: %s", idx)
return
if not build:
log.debug("No module associated with msg %s", event_info["msg_id"])
return
MBSConsumer.current_module_build_id = build.id
log.info("Calling %s", idx)
kwargs = event_info.copy()
kwargs.pop("event")
try:
if conf.celery_broker_url:
# handlers are also Celery tasks, when celery_broker_url is configured,
# call "delay" method to run the handlers as Celery async tasks
func = getattr(handler, "delay")
func(**kwargs)
else:
handler(**kwargs)
except Exception as e:
log.exception("Could not process message handler.")
db_session.rollback()
db_session.refresh(build)
build.transition(
db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=str(e),
failure_type="infra",
)
db_session.commit()
# Allow caller to do something when error is occurred.
raise
finally:
MBSConsumer.current_module_build_id = None
log.debug("Done with %s", idx)
def test_cleanup_stale_failed_builds(self, create_builder, dbg):
""" Test that one of the two module builds gets to the garbage state when running
cleanup_stale_failed_builds.
"""
builder = mock.MagicMock()
create_builder.return_value = builder
module_build_one = models.ModuleBuild.get_by_id(db_session, 2)
module_build_one.state = models.BUILD_STATES["failed"]
module_build_one.time_modified = datetime.utcnow() - timedelta(
days=conf.cleanup_failed_builds_time + 1)
module_build_two = models.ModuleBuild.get_by_id(db_session, 3)
module_build_two.time_modified = datetime.utcnow()
module_build_two.state = models.BUILD_STATES["failed"]
failed_component = db_session.query(models.ComponentBuild).filter_by(
package="tangerine", module_id=3).one()
failed_component.state = koji.BUILD_STATES["FAILED"]
failed_component.tagged = False
failed_component.tagged_in_final = False
db_session.commit()
producer.cleanup_stale_failed_builds()
db_session.refresh(module_build_two)
# Make sure module_build_one was transitioned to garbage
assert module_build_one.state == models.BUILD_STATES["garbage"]
state_reason = (
"The module was garbage collected since it has failed over {0} day(s) ago"
.format(conf.cleanup_failed_builds_time)
)
assert module_build_one.state_reason == state_reason
# Make sure all the components are marked as untagged in the database
for component in module_build_one.component_builds:
assert not component.tagged
assert not component.tagged_in_final
# Make sure module_build_two stayed the same
assert module_build_two.state == models.BUILD_STATES["failed"]
# Make sure the builds were untagged
builder.untag_artifacts.assert_called_once()
args, _ = builder.untag_artifacts.call_args
expected = [
"module-build-macros-0.1-1.module+0+d027b723",
"perl-List-Compare-0.53-5.module+0+d027b723",
"perl-Tangerine-0.23-1.module+0+d027b723",
"tangerine-0.22-3.module+0+d027b723",
]
assert expected == sorted(args[0])
def test_init_called_twice(self):
build = self.init_basic(db_session)
old_component_builds = len(build.component_builds)
old_mmd = load_mmd(build.modulemd)
build.state = 4
db_session.commit()
build = self.init_basic(db_session)
db_session.refresh(build)
assert build.state == 1
assert old_component_builds == len(build.component_builds)
new_mmd = load_mmd(build.modulemd)
# Compare only lengths, because `mmd_to_str` can shuffle the fields randomly.
assert len(mmd_to_str(old_mmd)) == len(mmd_to_str(new_mmd))
def test_sync_koji_build_tags(
self, tagged_handler, ClientSession, create_builder, dbg,
tagged, tagged_in_final, btime
):
module_build_2 = models.ModuleBuild.get_by_id(db_session, 2)
# Only module build 1's build target should be deleted.
module_build_2.koji_tag = "module-tag1"
module_build_2.state = models.BUILD_STATES["build"]
if btime:
module_build_2.time_modified = datetime.utcnow() - timedelta(minutes=12)
c = module_build_2.current_batch()[0]
c.state = koji.BUILD_STATES["COMPLETE"]
c.tagged_in_final = False
c.tagged = False
db_session.commit()
db_session.refresh(module_build_2)
koji_session = ClientSession.return_value
# No created module build has any of these tags.
listtags_return_value = []
expected_tagged_calls = []
if btime:
if tagged:
listtags_return_value.append(
{"id": 1, "name": module_build_2.koji_tag + "-build"})
expected_tagged_calls.append(call(
"internal:sync_koji_build_tags",
module_build_2.koji_tag + "-build", c.nvr
))
if tagged_in_final:
listtags_return_value.append(
{"id": 2, "name": module_build_2.koji_tag})
expected_tagged_calls.append(call(
"internal:sync_koji_build_tags",
module_build_2.koji_tag, c.nvr
))
koji_session.listTags.return_value = listtags_return_value
producer.sync_koji_build_tags()
tagged_handler.delay.assert_has_calls(
expected_tagged_calls, any_order=True)
def test_process_waiting_module_build_not_old_enough(
self, create_builder, dbg, state
):
""" Test that we do not process young waiting builds. """
handler = producer.ON_MODULE_CHANGE_HANDLERS[models.BUILD_STATES[state]]
# Change the batch to build, so the module build is in state where
# it is not building anything, but the state is "build".
module_build = models.ModuleBuild.get_by_id(db_session, 3)
module_build.state = models.BUILD_STATES[state]
original = datetime.utcnow() - timedelta(minutes=9)
module_build.time_modified = original
db_session.commit()
db_session.refresh(module_build)
# Poll :)
producer.process_waiting_module_builds()
handler.assert_not_called()
def test_init_when_get_latest_raises(self, get_build_arches, mocked_scm,
mocked_from_module_event):
FakeSCM(
mocked_scm,
"testmodule",
"testmodule.yaml",
"7035bd33614972ac66559ac1fdd019ff6027ad22",
get_latest_raise=True,
)
build = ModuleBuild.get_by_id(db_session, 2)
mocked_from_module_event.return_value = build
self.fn(msg_id="msg-id-1",
module_build_id=2,
module_build_state="init")
# Query the database again to make sure the build object is updated
db_session.refresh(build)
# Make sure the module entered the failed state
assert build.state == 4, build.state
assert "Failed to get the latest commit for" in build.state_reason
def test_record_component_builds_set_weight(self, mocked_scm):
# Mock for format_mmd to get components' latest ref
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
mocked_scm.return_value.get_latest.side_effect = [
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
"fbed359411a1baa08d4a88e0d12d426fbf8f602c",
"dbed259411a1baa08d4a88e0d12d426fbf8f6037",
]
mmd = load_mmd(read_staged_data("testmodule"))
# Set the module name and stream
mmd = mmd.copy("testmodule", "master")
module_build = module_build_service.common.models.ModuleBuild()
module_build.name = "testmodule"
module_build.stream = "master"
module_build.version = 20170109091357
module_build.state = models.BUILD_STATES["init"]
module_build.scmurl = \
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79"
module_build.batch = 1
module_build.owner = "Tom Brady"
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = "changed-and-after"
module_build.modulemd = mmd_to_str(mmd)
db_session.add(module_build)
db_session.commit()
format_mmd(mmd, module_build.scmurl)
record_component_builds(mmd, module_build)
db_session.commit()
assert module_build.state == models.BUILD_STATES["init"]
db_session.refresh(module_build)
for c in module_build.component_builds:
assert c.weight == 1.5
def test_finalize(self, mock_koji_cg_cls, cg_enabled, cg_devel_enabled,
mock_get_session):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
db_session.refresh(module_build)
module_build.state = 2
db_session.commit()
with patch(
"module_build_service.common.config.Config.koji_enable_content_generator",
new_callable=mock.PropertyMock,
return_value=cg_enabled,
):
with patch(
"module_build_service.common.config.Config.koji_cg_devel_module",
new_callable=mock.PropertyMock,
return_value=cg_devel_enabled,
):
builder = FakeKojiModuleBuilder(
db_session=db_session,
owner=module_build.owner,
module=module_build,
config=conf,
tag_name="module-nginx-1.2",
components=[],
)
builder.finalize()
mock_koji_cg = mock_koji_cg_cls.return_value
if cg_enabled:
if cg_devel_enabled:
assert mock_koji_cg.koji_import.call_count == 2
mock_koji_cg.koji_import.assert_has_calls(
[mock.call(), mock.call(devel=True)])
else:
mock_koji_cg.koji_import.assert_called_once_with()
else:
mock_koji_cg.koji_import.assert_not_called()
def test_buildroot_connect_create_tag(self, blocklist, mock_get_session):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
db_session.refresh(module_build)
if blocklist:
mmd = module_build.mmd()
xmd = mmd.get_xmd()
xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}
mmd.set_xmd(xmd)
module_build.modulemd = mmd_to_str(mmd)
db_session.commit()
builder = FakeKojiModuleBuilder(
db_session=db_session,
owner=module_build.owner,
module=module_build,
config=conf,
tag_name="module-foo",
components=["nginx"],
)
session = builder.koji_session
FakeKojiModuleBuilder.tags = {}
groups = OrderedDict()
groups["build"] = {"unzip"}
groups["srpm-build"] = {"fedora-release"}
builder.buildroot_connect(groups)
if blocklist:
expected_calls = [
mock.call("module-foo-build", "foo"),
mock.call("module-foo-build", "nginx"),
]
else:
expected_calls = []
assert session.packageListBlock.mock_calls == expected_calls
def test_import_mmd_remove_dropped_virtual_streams():
mmd = load_mmd(read_staged_data("formatted_testmodule"))
# Add some virtual streams
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["f28", "f29", "f30"]
mmd.set_xmd(xmd)
# Import mmd into database to simulate the next step to reimport a module
import_mmd(db_session, mmd)
# Now, remove some virtual streams from module metadata
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["f28", "f29"] # Note that, f30 is removed
mmd.set_xmd(xmd)
# Test import modulemd again and the f30 should be removed from database.
module_build, _ = import_mmd(db_session, mmd)
db_session.refresh(module_build)
assert ["f28", "f29"] == sorted(item.name
for item in module_build.virtual_streams)
assert 0 == db_session.query(
models.VirtualStream).filter_by(name="f30").count()
def wait(msg_id, module_build_id, module_build_state):
""" Called whenever a module enters the 'wait' state.
We transition to this state shortly after a modulebuild is first requested.
All we do here is request preparation of the buildroot.
The kicking off of individual component builds is handled elsewhere,
in module_build_service.schedulers.handlers.repos.
:param str msg_id: the original id of the message being handled which is
received from the message bus.
:param int module_build_id: the module build id.
:param int module_build_state: the module build state.
"""
build = models.ModuleBuild.get_by_id(db_session, module_build_id)
log.info("Found build=%r from message" % build)
log.debug("%r", build.modulemd)
if build.state != module_build_state:
log.warning(
"Note that retrieved module state %r doesn't match message module state %r",
build.state,
module_build_state,
)
# This is ok.. it's a race condition we can ignore.
pass
try:
build_deps = get_module_build_dependencies(build)
except ValueError:
reason = "Failed to get module info from MBS. Max retries reached."
log.exception(reason)
build.transition(db_session,
conf,
state=models.BUILD_STATES["failed"],
state_reason=reason,
failure_type="infra")
db_session.commit()
raise
tag = generate_module_build_koji_tag(build)
log.debug("Found tag=%s for module %r" % (tag, build))
# Hang on to this information for later. We need to know which build is
# associated with which koji tag, so that when their repos are regenerated
# in koji we can figure out which for which module build that event is
# relevant.
log.debug("Assigning koji tag=%s to module build" % tag)
build.koji_tag = tag
if build.scratch:
log.debug(
"Assigning Content Generator build koji tag is skipped for scratch module build."
)
elif conf.koji_cg_tag_build:
cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps)
log.debug(
"Assigning Content Generator build koji tag=%s to module build",
cg_build_koji_tag)
build.cg_build_koji_tag = cg_build_koji_tag
else:
log.debug(
"It is disabled to tag module build during importing into Koji by Content Generator."
)
log.debug(
"Skip to assign Content Generator build koji tag to module build.")
builder = GenericBuilder.create_from_module(db_session, build, conf)
log.debug(
"Adding dependencies %s into buildroot for module %s:%s:%s",
build_deps.keys(),
build.name,
build.stream,
build.version,
)
builder.buildroot_add_repos(build_deps)
if not build.component_builds:
log.info("There are no components in module %r, skipping build" %
build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
# Return a KojiRepoChange message so that the build can be transitioned to done
# in the repos handler
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
return
# If all components in module build will be reused, we don't have to build
# module-build-macros, because there won't be any build done.
if attempt_to_reuse_all_components(builder, build):
log.info(
"All components have been reused for module %r, skipping build" %
build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
return []
log.debug("Starting build batch 1")
build.batch = 1
db_session.commit()
artifact_name = "module-build-macros"
component_build = models.ComponentBuild.from_component_name(
db_session, artifact_name, build.id)
srpm = builder.get_disttag_srpm(disttag=".%s" %
get_rpm_release(db_session, build),
module_build=build)
if not component_build:
component_build = models.ComponentBuild(
module_id=build.id,
package=artifact_name,
format="rpms",
scmurl=srpm,
batch=1,
build_time_only=True,
)
db_session.add(component_build)
# Commit and refresh so that the SQLAlchemy relationships are available
db_session.commit()
db_session.refresh(component_build)
recovered = builder.recover_orphaned_artifact(component_build)
if recovered:
log.info("Found an existing module-build-macros build")
# There was no existing artifact found, so lets submit the build instead
else:
task_id, state, reason, nvr = builder.build(
artifact_name=artifact_name, source=srpm)
component_build.task_id = task_id
component_build.state = state
component_build.reason = reason
component_build.nvr = nvr
elif not component_build.is_completed:
# It's possible that the build succeeded in the builder but some other step failed which
# caused module-build-macros to be marked as failed in MBS, so check to see if it exists
# first
recovered = builder.recover_orphaned_artifact(component_build)
if recovered:
log.info("Found an existing module-build-macros build")
else:
task_id, state, reason, nvr = builder.build(
artifact_name=artifact_name, source=srpm)
component_build.task_id = task_id
component_build.state = state
component_build.reason = reason
component_build.nvr = nvr
db_session.add(component_build)
build.transition(db_session, conf, state=models.BUILD_STATES["build"])
db_session.add(build)
db_session.commit()
# We always have to regenerate the repository.
if conf.system == "koji":
log.info("Regenerating the repository")
task_id = builder.koji_session.newRepo(
builder.module_build_tag["name"])
build.new_repo_task_id = task_id
db_session.commit()
else:
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
events.scheduler.add(repos_done_handler,
("fake_msg", builder.module_build_tag["name"]))
def reuse_component_init_data():
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule"))
build_one = module_build_service.common.models.ModuleBuild(
name="testmodule",
stream="master",
version='20170109091357',
state=BUILD_STATES["ready"],
runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1",
context="78e4a6fd",
koji_tag="module-testmodule-master-20170109091357-78e4a6fd",
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
time_completed=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
)
build_one_component_release = get_rpm_release(db_session, build_one)
mmd.set_version(int(build_one.version))
xmd = mmd.get_xmd()
xmd["mbs"]["scmurl"] = build_one.scmurl
xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba"
mmd.set_xmd(xmd)
build_one.modulemd = mmd_to_str(mmd)
contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd(
build_one.modulemd)
build_one.build_context = contexts.build_context
build_one.build_context_no_bms = contexts.build_context_no_bms
db_session.add(build_one)
db_session.commit()
db_session.refresh(build_one)
platform_br = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 1)
build_one.buildrequires.append(platform_br)
arch = db_session.query(
module_build_service.common.models.ModuleArch).get(1)
build_one.arches.append(arch)
db_session.add_all([
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="perl-Tangerine",
scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine"
"?#4ceea43add2366d8b8c5a622a2fb563b625b9abf",
format="rpms",
task_id=90276227,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-Tangerine-0.23-1.{0}".format(
build_one_component_release),
batch=2,
ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="perl-List-Compare",
scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare"
"?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
format="rpms",
task_id=90276228,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-List-Compare-0.53-5.{0}".format(
build_one_component_release),
batch=2,
ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="tangerine",
scmurl="https://src.fedoraproject.org/rpms/tangerine"
"?#fbed359411a1baa08d4a88e0d12d426fbf8f602c",
format="rpms",
task_id=90276315,
state=koji.BUILD_STATES["COMPLETE"],
nvr="tangerine-0.22-3.{0}".format(build_one_component_release),
batch=3,
ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c",
tagged=True,
tagged_in_final=True,
),
module_build_service.common.models.ComponentBuild(
module_id=build_one.id,
package="module-build-macros",
scmurl=
"/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-"
"macros-0.1-1.module_testmodule_master_20170109091357.src.rpm",
format="rpms",
task_id=90276181,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-0.1-1.{0}".format(
build_one_component_release),
batch=1,
tagged=True,
build_time_only=True,
),
])
# Commit component builds added to build_one
db_session.commit()
build_two = module_build_service.common.models.ModuleBuild(
name="testmodule",
stream="master",
version='20170219191323',
state=BUILD_STATES["build"],
runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1",
context="c40c156c",
koji_tag="module-testmodule-master-20170219191323-c40c156c",
scmurl=
"https://src.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a",
batch=1,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 19, 16, 8, 18),
time_modified=datetime(2017, 2, 19, 16, 8, 18),
rebuild_strategy="changed-and-after",
)
build_two_component_release = get_rpm_release(db_session, build_two)
mmd.set_version(int(build_one.version))
xmd = mmd.get_xmd()
xmd["mbs"]["scmurl"] = build_one.scmurl
xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8"
mmd.set_xmd(xmd)
build_two.modulemd = mmd_to_str(mmd)
contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd(
build_two.modulemd)
build_two.build_context = contexts.build_context
build_two.build_context_no_bms = contexts.build_context_no_bms
db_session.add(build_two)
db_session.commit()
db_session.refresh(build_two)
build_two.arches.append(arch)
build_two.buildrequires.append(platform_br)
db_session.add_all([
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="perl-Tangerine",
scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine"
"?#4ceea43add2366d8b8c5a622a2fb563b625b9abf",
format="rpms",
batch=2,
ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf",
),
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="perl-List-Compare",
scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare"
"?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
format="rpms",
batch=2,
ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
),
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="tangerine",
scmurl="https://src.fedoraproject.org/rpms/tangerine"
"?#fbed359411a1baa08d4a88e0d12d426fbf8f602c",
format="rpms",
batch=3,
ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c",
),
module_build_service.common.models.ComponentBuild(
module_id=build_two.id,
package="module-build-macros",
scmurl=
"/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-"
"macros-0.1-1.module_testmodule_master_20170219191323.src.rpm",
format="rpms",
task_id=90276186,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-0.1-1.{0}".format(
build_two_component_release),
batch=1,
tagged=True,
build_time_only=True,
),
])
db_session.commit()
def test_buildroot_connect(self, custom_whitelist, blocklist,
repo_include_all, mock_get_session):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
db_session.refresh(module_build)
if blocklist:
mmd = module_build.mmd()
xmd = mmd.get_xmd()
xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}
mmd.set_xmd(xmd)
module_build.modulemd = mmd_to_str(mmd)
db_session.commit()
if custom_whitelist:
mmd = module_build.mmd()
opts = Modulemd.Buildopts()
opts.add_rpm_to_whitelist("custom1")
opts.add_rpm_to_whitelist("custom2")
mmd.set_buildopts(opts)
module_build.modulemd = mmd_to_str(mmd)
else:
# Set some irrelevant buildopts options to test that KojiModuleBuilder
# is not confused by this.
mmd = module_build.mmd()
opts = Modulemd.Buildopts()
opts.set_rpm_macros("%my_macro 1")
mmd.set_buildopts(opts)
module_build.modulemd = mmd_to_str(mmd)
db_session.commit()
if repo_include_all is False:
mmd = module_build.mmd()
xmd = mmd.get_xmd()
mbs_options = xmd["mbs_options"] if "mbs_options" in xmd.keys(
) else {}
mbs_options["repo_include_all"] = False
xmd["mbs_options"] = mbs_options
mmd.set_xmd(xmd)
module_build.modulemd = mmd_to_str(mmd)
db_session.commit()
module_build.arches.append(
module_build_service.common.models.ModuleArch(name="i686"))
db_session.commit()
builder = FakeKojiModuleBuilder(
db_session=db_session,
owner=module_build.owner,
module=module_build,
config=conf,
tag_name="module-foo",
components=["nginx"],
)
session = builder.koji_session
groups = OrderedDict()
groups["build"] = {"unzip"}
groups["srpm-build"] = {"fedora-release"}
builder.buildroot_connect(groups)
if custom_whitelist:
expected_calls = [
mock.call("module-foo", "custom1", "Moe Szyslak"),
mock.call("module-foo", "custom2", "Moe Szyslak"),
mock.call("module-foo-build", "custom1", "Moe Szyslak"),
mock.call("module-foo-build", "custom2", "Moe Szyslak"),
]
else:
expected_calls = [
mock.call("module-foo", "nginx", "Moe Szyslak"),
mock.call("module-foo-build", "nginx", "Moe Szyslak"),
]
assert session.packageListAdd.mock_calls == expected_calls
expected_calls = [
mock.call("module-foo-build", "build"),
mock.call("module-foo-build", "srpm-build"),
]
assert session.groupListAdd.mock_calls == expected_calls
expected_calls = [
mock.call("module-foo-build", "build", "unzip"),
mock.call("module-foo-build", "srpm-build", "fedora-release"),
]
assert session.groupPackageListAdd.mock_calls == expected_calls
# packageListBlock should not be called, because we set the block list only when creating
# new Koji tag to prevent overriding it on each buildroot_connect.
expected_calls = []
assert session.packageListBlock.mock_calls == expected_calls
expected_arches = "i686 x86_64"
expected_calls = [
mock.call(
"module-foo",
arches=expected_arches,
extra={
"mock.package_manager": "dnf",
"repo_include_all": repo_include_all,
"mock.new_chroot": 0,
'mock.yum.module_hotfixes': 1,
},
),
mock.call(
"module-foo-build",
arches=expected_arches,
extra={
"mock.package_manager": "dnf",
"repo_include_all": repo_include_all,
"mock.new_chroot": 0,
'mock.yum.module_hotfixes': 1,
},
),
]
assert session.editTag2.mock_calls == expected_calls
def test_newrepo_multiple_batches_tagged(self, create_builder,
koji_get_session, dbg):
"""
Test that newRepo is called just once and only when all components
are tagged even if we tag components from the multiple batches in the
same time.
"""
koji_session = mock.MagicMock()
koji_session.getTag = lambda tag_name: {"name": tag_name}
koji_session.getTaskInfo.return_value = {
"state": koji.TASK_STATES["CLOSED"]
}
koji_session.newRepo.return_value = 123456
koji_get_session.return_value = koji_session
builder = mock.MagicMock()
builder.koji_session = koji_session
builder.buildroot_ready.return_value = False
builder.module_build_tag = {
"name": "module-testmodule-master-20170219191323-c40c156c-build"
}
create_builder.return_value = builder
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 3)
module_build.batch = 2
mbm = module_build_service.common.models.ComponentBuild.from_component_name(
db_session, "module-build-macros", 3)
mbm.tagged = False
for c in module_build.current_batch():
if c.package == "perl-Tangerine":
c.nvr = "perl-Tangerine-0.23-1.module+0+d027b723"
elif c.package == "perl-List-Compare":
c.nvr = "perl-List-Compare-0.53-5.module+0+d027b723"
c.state = koji.BUILD_STATES["COMPLETE"]
db_session.commit()
# Tag the first component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
# Tag the first component to the final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
# newRepo should not be called, because there are still components
# to tag.
assert not koji_session.newRepo.called
# Tag the second component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-List-Compare-0.53-5.module+0+d027b723",
)
# Tag the second component to final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="perl-List-Compare-0.53-5.module+0+d027b723",
)
# newRepo should not be called, because there are still components
# to tag.
assert not koji_session.newRepo.called
# Tag the component from first batch to final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="module-build-macros-0.1-1.module+0+b0a1d1f7",
)
# Tag the component from first batch to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="module-build-macros-0.1-1.module+0+b0a1d1f7",
)
# newRepo should be called now - all components have been tagged.
koji_session.newRepo.assert_called_once_with(
"module-testmodule-master-20170219191323-c40c156c-build")
# Refresh our module_build object.
db_session.refresh(module_build)
# newRepo task_id should be stored in database, so we can check its
# status later in poller.
assert module_build.new_repo_task_id == 123456
def test_newrepo_build_time_only(self, create_builder, koji_get_session,
dbg):
"""
Test the component.build_time_only is respected in tag handler.
"""
koji_session = mock.MagicMock()
koji_session.getTag = lambda tag_name: {"name": tag_name}
koji_session.getTaskInfo.return_value = {
"state": koji.TASK_STATES["CLOSED"]
}
koji_session.newRepo.return_value = 123456
koji_get_session.return_value = koji_session
builder = mock.MagicMock()
builder.koji_session = koji_session
builder.buildroot_ready.return_value = False
builder.module_build_tag = {
"name": "module-testmodule-master-20170219191323-c40c156c-build"
}
create_builder.return_value = builder
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 3)
# Set previous components as COMPLETE and tagged.
module_build.batch = 1
for c in module_build.up_to_current_batch():
if c.package == "module-build-macros":
c.nvr = "module-build-macros-0.1-1.module+0+b0a1d1f7"
c.state = koji.BUILD_STATES["COMPLETE"]
c.tagged = True
c.tagged_in_final = True
module_build.batch = 2
component = db_session.query(
module_build_service.common.models.ComponentBuild).filter_by(
package="perl-Tangerine", module_id=module_build.id).one()
component.state = koji.BUILD_STATES["COMPLETE"]
component.build_time_only = True
component.tagged = False
component.tagged_in_final = False
component.nvr = "perl-Tangerine-0.23-1.module+0+d027b723"
component = db_session.query(
module_build_service.common.models.ComponentBuild).filter_by(
package="perl-List-Compare", module_id=module_build.id).one()
component.state = koji.BUILD_STATES["COMPLETE"]
component.nvr = "perl-List-Compare-0.53-5.module+0+d027b723"
db_session.commit()
# Tag the perl-Tangerine component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
assert not koji_session.newRepo.called
# Tag the perl-List-Compare component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-List-Compare-0.53-5.module+0+d027b723",
)
# Tag the perl-List-Compare component to final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="perl-List-Compare-0.53-5.module+0+d027b723",
)
# newRepo should be called now - all successfully built
# components have been tagged.
koji_session.newRepo.assert_called_once_with(
"module-testmodule-master-20170219191323-c40c156c-build")
# Refresh our module_build object.
db_session.refresh(module_build)
# newRepo task_id should be stored in database, so we can check its
# status later in poller.
assert module_build.new_repo_task_id == 123456
def test_newrepo_not_duplicated(self, create_builder, koji_get_session,
dbg, task_state, expect_new_repo):
"""
Test that newRepo is not called if a task is already in progress.
"""
koji_session = mock.MagicMock()
koji_session.getTag = lambda tag_name: {"name": tag_name}
koji_session.getTaskInfo.return_value = {"state": task_state}
koji_session.newRepo.return_value = 123456
koji_get_session.return_value = koji_session
builder = mock.MagicMock()
builder.koji_session = koji_session
builder.buildroot_ready.return_value = False
builder.module_build_tag = {
"name": "module-testmodule-master-20170219191323-c40c156c-build"
}
create_builder.return_value = builder
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 3)
assert module_build
# Set previous components as COMPLETE and tagged.
module_build.batch = 1
for c in module_build.up_to_current_batch():
c.state = koji.BUILD_STATES["COMPLETE"]
c.tagged = True
c.tagged_in_final = True
module_build.batch = 2
for c in module_build.current_batch():
if c.package == "perl-Tangerine":
c.nvr = "perl-Tangerine-0.23-1.module+0+d027b723"
elif c.package == "perl-List-Compare":
c.nvr = "perl-List-Compare-0.53-5.module+0+d027b723"
c.state = koji.BUILD_STATES["COMPLETE"]
if task_state is not None:
module_build.new_repo_task_id = 123456
db_session.commit()
# Tag the first component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
# Tag the first component to the final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="perl-Tangerine-0.23-1.module+0+d027b723",
)
# Tag the second component to the buildroot.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c-build",
build_nvr="perl-List-Compare-0.53-5.module+0+d027b723",
)
# Tag the second component to the final tag.
module_build_service.scheduler.handlers.tags.tagged(
msg_id="id",
tag_name="module-testmodule-master-20170219191323-c40c156c",
build_nvr="perl-List-Compare-0.53-5.module+0+d027b723",
)
# All components are tagged, newRepo should be called if there are no active tasks.
if expect_new_repo:
koji_session.newRepo.assert_called_once_with(
"module-testmodule-master-20170219191323-c40c156c-build")
else:
assert not koji_session.newRepo.called
# Refresh our module_build object.
db_session.refresh(module_build)
# newRepo task_id should be stored in database, so we can check its
# status later in poller.
assert module_build.new_repo_task_id == 123456
koji_session.newRepo.reset_mock()
