def test_process_waiting_module_build(self, create_builder, dbg, state): """ Test that processing old waiting module builds works. """ handler = producer.ON_MODULE_CHANGE_HANDLERS[models.BUILD_STATES[state]] # Change the batch to 2, so the module build is in state where # it is not building anything, but the state is "build". module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.state = models.BUILD_STATES[state] original = datetime.utcnow() - timedelta(minutes=11) module_build.time_modified = original db_session.commit() db_session.refresh(module_build) # Poll :) producer.process_waiting_module_builds() handler.delay.assert_called_once_with( "internal:mbs.module.state.change", module_build.id, module_build.state ) db_session.refresh(module_build) # ensure the time_modified was changed. assert module_build.time_modified > original
def test_cant_delete_build_target_if_not_reach_delete_time( self, ClientSession, create_builder, dbg ): module_build_2 = models.ModuleBuild.get_by_id(db_session, 2) # Only module build 1's build target should be deleted. module_build_2.koji_tag = "module-tag1" module_build_2.state = models.BUILD_STATES["done"] # Ensure to exceed the koji_target_delete_time easily later for deletion module_build_2.time_completed = datetime.utcnow() - timedelta(minutes=5) db_session.commit() db_session.refresh(module_build_2) koji_session = ClientSession.return_value # No created module build has any of these tags. koji_session.getBuildTargets.return_value = [ {"id": 1, "dest_tag_name": module_build_2.koji_tag, "name": module_build_2.koji_tag} ] with patch.object(conf, "koji_tag_prefixes", new=["module"]): # Use default koji_target_delete_time in config. That time is long # enough for test. producer.delete_old_koji_targets() koji_session.deleteBuildTarget.assert_not_called()
def test_transform_from_done_to_ready(self, ClientSession, publish): clean_database() # This build should be queried and transformed to ready state module_build = make_module_in_db( "pkg:0.1:1:c1", [{ "requires": { "platform": ["el8"] }, "buildrequires": { "platform": ["el8"] }, }], ) module_build.transition(db_session, conf, BUILD_STATES["done"], "Move to done directly for running test.") db_session.commit() # Assert this call below first_publish_call = call( "module.state.change", module_build.json(db_session, show_tasks=False), conf, "mbs", ) ClientSession.return_value.getBuild.return_value = { "extra": { "typeinfo": { "module": { "module_build_service_id": module_build.id } } } } msg = { "msg_id": "msg-id-1", "topic": "org.fedoraproject.prod.greenwave.decision.update", "msg": { "decision_context": "test_dec_context", "policies_satisfied": True, "subject_identifier": "pkg-0.1-1.c1", }, } hub = Mock(config={"validate_signatures": False}) consumer = MBSConsumer(hub) consumer.consume(msg) db_session.add(module_build) # Load module build again to check its state is moved correctly db_session.refresh(module_build) assert BUILD_STATES["ready"] == module_build.state publish.assert_has_calls([ first_publish_call, call("module.state.change", module_build.json(db_session, show_tasks=False), conf, "mbs"), ])
def test_cleanup_stale_failed_builds_no_components(self, create_builder, dbg): """ Test that a module build without any components built gets to the garbage state when running cleanup_stale_failed_builds. """ module_build_one = models.ModuleBuild.get_by_id(db_session, 1) module_build_one.state = models.BUILD_STATES["failed"] module_build_one.time_modified = datetime.utcnow() module_build_two = models.ModuleBuild.get_by_id(db_session, 2) module_build_two.state = models.BUILD_STATES["failed"] module_build_two.time_modified = datetime.utcnow() - timedelta( days=conf.cleanup_failed_builds_time + 1) module_build_two.koji_tag = None module_build_two.cg_build_koji_tag = None for c in module_build_two.component_builds: c.state = None db_session.commit() producer.cleanup_stale_failed_builds() db_session.refresh(module_build_two) # Make sure module_build_two was transitioned to garbage assert module_build_two.state == models.BUILD_STATES["garbage"] state_reason = ( "The module was garbage collected since it has failed over {0} day(s) ago" .format(conf.cleanup_failed_builds_time) ) assert module_build_two.state_reason == state_reason # Make sure module_build_one stayed the same assert module_build_one.state == models.BUILD_STATES["failed"] # Make sure that the builder was never instantiated create_builder.assert_not_called()
def test_import_mmd_dont_remove_dropped_virtual_streams_associated_with_other_modules( ): mmd = load_mmd(read_staged_data("formatted_testmodule")) # Add some virtual streams to this module metadata xmd = mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["f28", "f29", "f30"] mmd.set_xmd(xmd) import_mmd(db_session, mmd) # Import another module which has overlapping virtual streams another_mmd = load_mmd( read_staged_data("formatted_testmodule-more-components")) # Add some virtual streams to this module metadata xmd = another_mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["f29", "f30"] another_mmd.set_xmd(xmd) another_module_build, _ = import_mmd(db_session, another_mmd) # Now, remove f30 from mmd xmd = mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["f28", "f29"] mmd.set_xmd(xmd) # Reimport formatted_testmodule again module_build, _ = import_mmd(db_session, mmd) db_session.refresh(module_build) assert ["f28", "f29"] == sorted(item.name for item in module_build.virtual_streams) # The overlapped f30 should be still there. db_session.refresh(another_module_build) assert ["f29", "f30"] == sorted(item.name for item in another_module_build.virtual_streams)
def test_only_delete_build_target_with_allowed_koji_tag_prefix( self, ClientSession, create_builder, dbg ): module_build_2 = models.ModuleBuild.get_by_id(db_session, 2) # Only module build 1's build target should be deleted. module_build_2.koji_tag = "module-tag1" module_build_2.state = models.BUILD_STATES["done"] # Ensure to exceed the koji_target_delete_time easily later for deletion module_build_2.time_completed = datetime.utcnow() - timedelta(hours=24) module_build_3 = models.ModuleBuild.get_by_id(db_session, 3) module_build_3.koji_tag = "f28" db_session.commit() db_session.refresh(module_build_2) db_session.refresh(module_build_3) koji_session = ClientSession.return_value # No created module build has any of these tags. koji_session.getBuildTargets.return_value = [ {"id": 1, "dest_tag_name": module_build_2.koji_tag, "name": module_build_2.koji_tag}, {"id": 2, "dest_tag_name": module_build_3.koji_tag, "name": module_build_3.koji_tag}, ] with patch.object(conf, "koji_tag_prefixes", new=["module", "another-prefix"]): with patch.object(conf, "koji_target_delete_time", new=60): producer.delete_old_koji_targets() koji_session.deleteBuildTarget.assert_called_once_with(1) koji_session.krb_login.assert_called_once()
def process_message(self, event_info): # Choose a handler for this message handler, build = self._map_message(db_session, event_info) if handler is None: log.debug("No event handler associated with msg %s", event_info["msg_id"]) return idx = "%s: %s, %s" % (handler.__name__, event_info["event"], event_info["msg_id"]) if handler is no_op_handler: log.debug("Handler is NO_OP: %s", idx) return if not build: log.debug("No module associated with msg %s", event_info["msg_id"]) return MBSConsumer.current_module_build_id = build.id log.info("Calling %s", idx) kwargs = event_info.copy() kwargs.pop("event") try: if conf.celery_broker_url: # handlers are also Celery tasks, when celery_broker_url is configured, # call "delay" method to run the handlers as Celery async tasks func = getattr(handler, "delay") func(**kwargs) else: handler(**kwargs) except Exception as e: log.exception("Could not process message handler.") db_session.rollback() db_session.refresh(build) build.transition( db_session, conf, state=models.BUILD_STATES["failed"], state_reason=str(e), failure_type="infra", ) db_session.commit() # Allow caller to do something when error is occurred. raise finally: MBSConsumer.current_module_build_id = None log.debug("Done with %s", idx)
def test_cleanup_stale_failed_builds(self, create_builder, dbg): """ Test that one of the two module builds gets to the garbage state when running cleanup_stale_failed_builds. """ builder = mock.MagicMock() create_builder.return_value = builder module_build_one = models.ModuleBuild.get_by_id(db_session, 2) module_build_one.state = models.BUILD_STATES["failed"] module_build_one.time_modified = datetime.utcnow() - timedelta( days=conf.cleanup_failed_builds_time + 1) module_build_two = models.ModuleBuild.get_by_id(db_session, 3) module_build_two.time_modified = datetime.utcnow() module_build_two.state = models.BUILD_STATES["failed"] failed_component = db_session.query(models.ComponentBuild).filter_by( package="tangerine", module_id=3).one() failed_component.state = koji.BUILD_STATES["FAILED"] failed_component.tagged = False failed_component.tagged_in_final = False db_session.commit() producer.cleanup_stale_failed_builds() db_session.refresh(module_build_two) # Make sure module_build_one was transitioned to garbage assert module_build_one.state == models.BUILD_STATES["garbage"] state_reason = ( "The module was garbage collected since it has failed over {0} day(s) ago" .format(conf.cleanup_failed_builds_time) ) assert module_build_one.state_reason == state_reason # Make sure all the components are marked as untagged in the database for component in module_build_one.component_builds: assert not component.tagged assert not component.tagged_in_final # Make sure module_build_two stayed the same assert module_build_two.state == models.BUILD_STATES["failed"] # Make sure the builds were untagged builder.untag_artifacts.assert_called_once() args, _ = builder.untag_artifacts.call_args expected = [ "module-build-macros-0.1-1.module+0+d027b723", "perl-List-Compare-0.53-5.module+0+d027b723", "perl-Tangerine-0.23-1.module+0+d027b723", "tangerine-0.22-3.module+0+d027b723", ] assert expected == sorted(args[0])
def test_init_called_twice(self): build = self.init_basic(db_session) old_component_builds = len(build.component_builds) old_mmd = load_mmd(build.modulemd) build.state = 4 db_session.commit() build = self.init_basic(db_session) db_session.refresh(build) assert build.state == 1 assert old_component_builds == len(build.component_builds) new_mmd = load_mmd(build.modulemd) # Compare only lengths, because `mmd_to_str` can shuffle the fields randomly. assert len(mmd_to_str(old_mmd)) == len(mmd_to_str(new_mmd))
def test_sync_koji_build_tags( self, tagged_handler, ClientSession, create_builder, dbg, tagged, tagged_in_final, btime ): module_build_2 = models.ModuleBuild.get_by_id(db_session, 2) # Only module build 1's build target should be deleted. module_build_2.koji_tag = "module-tag1" module_build_2.state = models.BUILD_STATES["build"] if btime: module_build_2.time_modified = datetime.utcnow() - timedelta(minutes=12) c = module_build_2.current_batch()[0] c.state = koji.BUILD_STATES["COMPLETE"] c.tagged_in_final = False c.tagged = False db_session.commit() db_session.refresh(module_build_2) koji_session = ClientSession.return_value # No created module build has any of these tags. listtags_return_value = [] expected_tagged_calls = [] if btime: if tagged: listtags_return_value.append( {"id": 1, "name": module_build_2.koji_tag + "-build"}) expected_tagged_calls.append(call( "internal:sync_koji_build_tags", module_build_2.koji_tag + "-build", c.nvr )) if tagged_in_final: listtags_return_value.append( {"id": 2, "name": module_build_2.koji_tag}) expected_tagged_calls.append(call( "internal:sync_koji_build_tags", module_build_2.koji_tag, c.nvr )) koji_session.listTags.return_value = listtags_return_value producer.sync_koji_build_tags() tagged_handler.delay.assert_has_calls( expected_tagged_calls, any_order=True)
def test_process_waiting_module_build_not_old_enough( self, create_builder, dbg, state ): """ Test that we do not process young waiting builds. """ handler = producer.ON_MODULE_CHANGE_HANDLERS[models.BUILD_STATES[state]] # Change the batch to build, so the module build is in state where # it is not building anything, but the state is "build". module_build = models.ModuleBuild.get_by_id(db_session, 3) module_build.state = models.BUILD_STATES[state] original = datetime.utcnow() - timedelta(minutes=9) module_build.time_modified = original db_session.commit() db_session.refresh(module_build) # Poll :) producer.process_waiting_module_builds() handler.assert_not_called()
def test_init_when_get_latest_raises(self, get_build_arches, mocked_scm, mocked_from_module_event): FakeSCM( mocked_scm, "testmodule", "testmodule.yaml", "7035bd33614972ac66559ac1fdd019ff6027ad22", get_latest_raise=True, ) build = ModuleBuild.get_by_id(db_session, 2) mocked_from_module_event.return_value = build self.fn(msg_id="msg-id-1", module_build_id=2, module_build_state="init") # Query the database again to make sure the build object is updated db_session.refresh(build) # Make sure the module entered the failed state assert build.state == 4, build.state assert "Failed to get the latest commit for" in build.state_reason
def test_record_component_builds_set_weight(self, mocked_scm): # Mock for format_mmd to get components' latest ref mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4" mocked_scm.return_value.get_latest.side_effect = [ "4ceea43add2366d8b8c5a622a2fb563b625b9abf", "fbed359411a1baa08d4a88e0d12d426fbf8f602c", "dbed259411a1baa08d4a88e0d12d426fbf8f6037", ] mmd = load_mmd(read_staged_data("testmodule")) # Set the module name and stream mmd = mmd.copy("testmodule", "master") module_build = module_build_service.common.models.ModuleBuild() module_build.name = "testmodule" module_build.stream = "master" module_build.version = 20170109091357 module_build.state = models.BUILD_STATES["init"] module_build.scmurl = \ "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79" module_build.batch = 1 module_build.owner = "Tom Brady" module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) module_build.rebuild_strategy = "changed-and-after" module_build.modulemd = mmd_to_str(mmd) db_session.add(module_build) db_session.commit() format_mmd(mmd, module_build.scmurl) record_component_builds(mmd, module_build) db_session.commit() assert module_build.state == models.BUILD_STATES["init"] db_session.refresh(module_build) for c in module_build.component_builds: assert c.weight == 1.5
def test_finalize(self, mock_koji_cg_cls, cg_enabled, cg_devel_enabled, mock_get_session): module_build = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 2) db_session.refresh(module_build) module_build.state = 2 db_session.commit() with patch( "module_build_service.common.config.Config.koji_enable_content_generator", new_callable=mock.PropertyMock, return_value=cg_enabled, ): with patch( "module_build_service.common.config.Config.koji_cg_devel_module", new_callable=mock.PropertyMock, return_value=cg_devel_enabled, ): builder = FakeKojiModuleBuilder( db_session=db_session, owner=module_build.owner, module=module_build, config=conf, tag_name="module-nginx-1.2", components=[], ) builder.finalize() mock_koji_cg = mock_koji_cg_cls.return_value if cg_enabled: if cg_devel_enabled: assert mock_koji_cg.koji_import.call_count == 2 mock_koji_cg.koji_import.assert_has_calls( [mock.call(), mock.call(devel=True)]) else: mock_koji_cg.koji_import.assert_called_once_with() else: mock_koji_cg.koji_import.assert_not_called()
def test_buildroot_connect_create_tag(self, blocklist, mock_get_session): module_build = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 2) db_session.refresh(module_build) if blocklist: mmd = module_build.mmd() xmd = mmd.get_xmd() xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]} mmd.set_xmd(xmd) module_build.modulemd = mmd_to_str(mmd) db_session.commit() builder = FakeKojiModuleBuilder( db_session=db_session, owner=module_build.owner, module=module_build, config=conf, tag_name="module-foo", components=["nginx"], ) session = builder.koji_session FakeKojiModuleBuilder.tags = {} groups = OrderedDict() groups["build"] = {"unzip"} groups["srpm-build"] = {"fedora-release"} builder.buildroot_connect(groups) if blocklist: expected_calls = [ mock.call("module-foo-build", "foo"), mock.call("module-foo-build", "nginx"), ] else: expected_calls = [] assert session.packageListBlock.mock_calls == expected_calls
def test_import_mmd_remove_dropped_virtual_streams(): mmd = load_mmd(read_staged_data("formatted_testmodule")) # Add some virtual streams xmd = mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["f28", "f29", "f30"] mmd.set_xmd(xmd) # Import mmd into database to simulate the next step to reimport a module import_mmd(db_session, mmd) # Now, remove some virtual streams from module metadata xmd = mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["f28", "f29"] # Note that, f30 is removed mmd.set_xmd(xmd) # Test import modulemd again and the f30 should be removed from database. module_build, _ = import_mmd(db_session, mmd) db_session.refresh(module_build) assert ["f28", "f29"] == sorted(item.name for item in module_build.virtual_streams) assert 0 == db_session.query( models.VirtualStream).filter_by(name="f30").count()
def wait(msg_id, module_build_id, module_build_state): """ Called whenever a module enters the 'wait' state. We transition to this state shortly after a modulebuild is first requested. All we do here is request preparation of the buildroot. The kicking off of individual component builds is handled elsewhere, in module_build_service.schedulers.handlers.repos. :param str msg_id: the original id of the message being handled which is received from the message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) log.info("Found build=%r from message" % build) log.debug("%r", build.modulemd) if build.state != module_build_state: log.warning( "Note that retrieved module state %r doesn't match message module state %r", build.state, module_build_state, ) # This is ok.. it's a race condition we can ignore. pass try: build_deps = get_module_build_dependencies(build) except ValueError: reason = "Failed to get module info from MBS. Max retries reached." log.exception(reason) build.transition(db_session, conf, state=models.BUILD_STATES["failed"], state_reason=reason, failure_type="infra") db_session.commit() raise tag = generate_module_build_koji_tag(build) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is # associated with which koji tag, so that when their repos are regenerated # in koji we can figure out which for which module build that event is # relevant. log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag if build.scratch: log.debug( "Assigning Content Generator build koji tag is skipped for scratch module build." ) elif conf.koji_cg_tag_build: cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) log.debug( "Assigning Content Generator build koji tag=%s to module build", cg_build_koji_tag) build.cg_build_koji_tag = cg_build_koji_tag else: log.debug( "It is disabled to tag module build during importing into Koji by Content Generator." ) log.debug( "Skip to assign Content Generator build koji tag to module build.") builder = GenericBuilder.create_from_module(db_session, build, conf) log.debug( "Adding dependencies %s into buildroot for module %s:%s:%s", build_deps.keys(), build.name, build.stream, build.version, ) builder.buildroot_add_repos(build_deps) if not build.component_builds: log.info("There are no components in module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # Return a KojiRepoChange message so that the build can be transitioned to done # in the repos handler from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"])) return # If all components in module build will be reused, we don't have to build # module-build-macros, because there won't be any build done. if attempt_to_reuse_all_components(builder, build): log.info( "All components have been reused for module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() return [] log.debug("Starting build batch 1") build.batch = 1 db_session.commit() artifact_name = "module-build-macros" component_build = models.ComponentBuild.from_component_name( db_session, artifact_name, build.id) srpm = builder.get_disttag_srpm(disttag=".%s" % get_rpm_release(db_session, build), module_build=build) if not component_build: component_build = models.ComponentBuild( module_id=build.id, package=artifact_name, format="rpms", scmurl=srpm, batch=1, build_time_only=True, ) db_session.add(component_build) # Commit and refresh so that the SQLAlchemy relationships are available db_session.commit() db_session.refresh(component_build) recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") # There was no existing artifact found, so lets submit the build instead else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr elif not component_build.is_completed: # It's possible that the build succeeded in the builder but some other step failed which # caused module-build-macros to be marked as failed in MBS, so check to see if it exists # first recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr db_session.add(component_build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # We always have to regenerate the repository. if conf.system == "koji": log.info("Regenerating the repository") task_id = builder.koji_session.newRepo( builder.module_build_tag["name"]) build.new_repo_task_id = task_id db_session.commit() else: from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"]))
def reuse_component_init_data(): clean_database() mmd = load_mmd(read_staged_data("formatted_testmodule")) build_one = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170109091357', state=BUILD_STATES["ready"], runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1", context="78e4a6fd", koji_tag="module-testmodule-master-20170109091357-78e4a6fd", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), time_completed=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", ) build_one_component_release = get_rpm_release(db_session, build_one) mmd.set_version(int(build_one.version)) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = build_one.scmurl xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba" mmd.set_xmd(xmd) build_one.modulemd = mmd_to_str(mmd) contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd( build_one.modulemd) build_one.build_context = contexts.build_context build_one.build_context_no_bms = contexts.build_context_no_bms db_session.add(build_one) db_session.commit() db_session.refresh(build_one) platform_br = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 1) build_one.buildrequires.append(platform_br) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) build_one.arches.append(arch) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", task_id=90276227, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-Tangerine-0.23-1.{0}".format( build_one_component_release), batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", task_id=90276228, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-List-Compare-0.53-5.{0}".format( build_one_component_release), batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", task_id=90276315, state=koji.BUILD_STATES["COMPLETE"], nvr="tangerine-0.22-3.{0}".format(build_one_component_release), batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170109091357.src.rpm", format="rpms", task_id=90276181, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{0}".format( build_one_component_release), batch=1, tagged=True, build_time_only=True, ), ]) # Commit component builds added to build_one db_session.commit() build_two = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170219191323', state=BUILD_STATES["build"], runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1", context="c40c156c", koji_tag="module-testmodule-master-20170219191323-c40c156c", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a", batch=1, owner="Tom Brady", time_submitted=datetime(2017, 2, 19, 16, 8, 18), time_modified=datetime(2017, 2, 19, 16, 8, 18), rebuild_strategy="changed-and-after", ) build_two_component_release = get_rpm_release(db_session, build_two) mmd.set_version(int(build_one.version)) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = build_one.scmurl xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8" mmd.set_xmd(xmd) build_two.modulemd = mmd_to_str(mmd) contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd( build_two.modulemd) build_two.build_context = contexts.build_context build_two.build_context_no_bms = contexts.build_context_no_bms db_session.add(build_two) db_session.commit() db_session.refresh(build_two) build_two.arches.append(arch) build_two.buildrequires.append(platform_br) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170219191323.src.rpm", format="rpms", task_id=90276186, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{0}".format( build_two_component_release), batch=1, tagged=True, build_time_only=True, ), ]) db_session.commit()
def test_buildroot_connect(self, custom_whitelist, blocklist, repo_include_all, mock_get_session): module_build = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 2) db_session.refresh(module_build) if blocklist: mmd = module_build.mmd() xmd = mmd.get_xmd() xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]} mmd.set_xmd(xmd) module_build.modulemd = mmd_to_str(mmd) db_session.commit() if custom_whitelist: mmd = module_build.mmd() opts = Modulemd.Buildopts() opts.add_rpm_to_whitelist("custom1") opts.add_rpm_to_whitelist("custom2") mmd.set_buildopts(opts) module_build.modulemd = mmd_to_str(mmd) else: # Set some irrelevant buildopts options to test that KojiModuleBuilder # is not confused by this. mmd = module_build.mmd() opts = Modulemd.Buildopts() opts.set_rpm_macros("%my_macro 1") mmd.set_buildopts(opts) module_build.modulemd = mmd_to_str(mmd) db_session.commit() if repo_include_all is False: mmd = module_build.mmd() xmd = mmd.get_xmd() mbs_options = xmd["mbs_options"] if "mbs_options" in xmd.keys( ) else {} mbs_options["repo_include_all"] = False xmd["mbs_options"] = mbs_options mmd.set_xmd(xmd) module_build.modulemd = mmd_to_str(mmd) db_session.commit() module_build.arches.append( module_build_service.common.models.ModuleArch(name="i686")) db_session.commit() builder = FakeKojiModuleBuilder( db_session=db_session, owner=module_build.owner, module=module_build, config=conf, tag_name="module-foo", components=["nginx"], ) session = builder.koji_session groups = OrderedDict() groups["build"] = {"unzip"} groups["srpm-build"] = {"fedora-release"} builder.buildroot_connect(groups) if custom_whitelist: expected_calls = [ mock.call("module-foo", "custom1", "Moe Szyslak"), mock.call("module-foo", "custom2", "Moe Szyslak"), mock.call("module-foo-build", "custom1", "Moe Szyslak"), mock.call("module-foo-build", "custom2", "Moe Szyslak"), ] else: expected_calls = [ mock.call("module-foo", "nginx", "Moe Szyslak"), mock.call("module-foo-build", "nginx", "Moe Szyslak"), ] assert session.packageListAdd.mock_calls == expected_calls expected_calls = [ mock.call("module-foo-build", "build"), mock.call("module-foo-build", "srpm-build"), ] assert session.groupListAdd.mock_calls == expected_calls expected_calls = [ mock.call("module-foo-build", "build", "unzip"), mock.call("module-foo-build", "srpm-build", "fedora-release"), ] assert session.groupPackageListAdd.mock_calls == expected_calls # packageListBlock should not be called, because we set the block list only when creating # new Koji tag to prevent overriding it on each buildroot_connect. expected_calls = [] assert session.packageListBlock.mock_calls == expected_calls expected_arches = "i686 x86_64" expected_calls = [ mock.call( "module-foo", arches=expected_arches, extra={ "mock.package_manager": "dnf", "repo_include_all": repo_include_all, "mock.new_chroot": 0, 'mock.yum.module_hotfixes': 1, }, ), mock.call( "module-foo-build", arches=expected_arches, extra={ "mock.package_manager": "dnf", "repo_include_all": repo_include_all, "mock.new_chroot": 0, 'mock.yum.module_hotfixes': 1, }, ), ] assert session.editTag2.mock_calls == expected_calls
def test_newrepo_multiple_batches_tagged(self, create_builder, koji_get_session, dbg): """ Test that newRepo is called just once and only when all components are tagged even if we tag components from the multiple batches in the same time. """ koji_session = mock.MagicMock() koji_session.getTag = lambda tag_name: {"name": tag_name} koji_session.getTaskInfo.return_value = { "state": koji.TASK_STATES["CLOSED"] } koji_session.newRepo.return_value = 123456 koji_get_session.return_value = koji_session builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False builder.module_build_tag = { "name": "module-testmodule-master-20170219191323-c40c156c-build" } create_builder.return_value = builder module_build = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 3) module_build.batch = 2 mbm = module_build_service.common.models.ComponentBuild.from_component_name( db_session, "module-build-macros", 3) mbm.tagged = False for c in module_build.current_batch(): if c.package == "perl-Tangerine": c.nvr = "perl-Tangerine-0.23-1.module+0+d027b723" elif c.package == "perl-List-Compare": c.nvr = "perl-List-Compare-0.53-5.module+0+d027b723" c.state = koji.BUILD_STATES["COMPLETE"] db_session.commit() # Tag the first component to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="perl-Tangerine-0.23-1.module+0+d027b723", ) # Tag the first component to the final tag. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c", build_nvr="perl-Tangerine-0.23-1.module+0+d027b723", ) # newRepo should not be called, because there are still components # to tag. assert not koji_session.newRepo.called # Tag the second component to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="perl-List-Compare-0.53-5.module+0+d027b723", ) # Tag the second component to final tag. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c", build_nvr="perl-List-Compare-0.53-5.module+0+d027b723", ) # newRepo should not be called, because there are still components # to tag. assert not koji_session.newRepo.called # Tag the component from first batch to final tag. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c", build_nvr="module-build-macros-0.1-1.module+0+b0a1d1f7", ) # Tag the component from first batch to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="module-build-macros-0.1-1.module+0+b0a1d1f7", ) # newRepo should be called now - all components have been tagged. koji_session.newRepo.assert_called_once_with( "module-testmodule-master-20170219191323-c40c156c-build") # Refresh our module_build object. db_session.refresh(module_build) # newRepo task_id should be stored in database, so we can check its # status later in poller. assert module_build.new_repo_task_id == 123456
def test_newrepo_build_time_only(self, create_builder, koji_get_session, dbg): """ Test the component.build_time_only is respected in tag handler. """ koji_session = mock.MagicMock() koji_session.getTag = lambda tag_name: {"name": tag_name} koji_session.getTaskInfo.return_value = { "state": koji.TASK_STATES["CLOSED"] } koji_session.newRepo.return_value = 123456 koji_get_session.return_value = koji_session builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False builder.module_build_tag = { "name": "module-testmodule-master-20170219191323-c40c156c-build" } create_builder.return_value = builder module_build = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 3) # Set previous components as COMPLETE and tagged. module_build.batch = 1 for c in module_build.up_to_current_batch(): if c.package == "module-build-macros": c.nvr = "module-build-macros-0.1-1.module+0+b0a1d1f7" c.state = koji.BUILD_STATES["COMPLETE"] c.tagged = True c.tagged_in_final = True module_build.batch = 2 component = db_session.query( module_build_service.common.models.ComponentBuild).filter_by( package="perl-Tangerine", module_id=module_build.id).one() component.state = koji.BUILD_STATES["COMPLETE"] component.build_time_only = True component.tagged = False component.tagged_in_final = False component.nvr = "perl-Tangerine-0.23-1.module+0+d027b723" component = db_session.query( module_build_service.common.models.ComponentBuild).filter_by( package="perl-List-Compare", module_id=module_build.id).one() component.state = koji.BUILD_STATES["COMPLETE"] component.nvr = "perl-List-Compare-0.53-5.module+0+d027b723" db_session.commit() # Tag the perl-Tangerine component to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="perl-Tangerine-0.23-1.module+0+d027b723", ) assert not koji_session.newRepo.called # Tag the perl-List-Compare component to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="perl-List-Compare-0.53-5.module+0+d027b723", ) # Tag the perl-List-Compare component to final tag. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c", build_nvr="perl-List-Compare-0.53-5.module+0+d027b723", ) # newRepo should be called now - all successfully built # components have been tagged. koji_session.newRepo.assert_called_once_with( "module-testmodule-master-20170219191323-c40c156c-build") # Refresh our module_build object. db_session.refresh(module_build) # newRepo task_id should be stored in database, so we can check its # status later in poller. assert module_build.new_repo_task_id == 123456
def test_newrepo_not_duplicated(self, create_builder, koji_get_session, dbg, task_state, expect_new_repo): """ Test that newRepo is not called if a task is already in progress. """ koji_session = mock.MagicMock() koji_session.getTag = lambda tag_name: {"name": tag_name} koji_session.getTaskInfo.return_value = {"state": task_state} koji_session.newRepo.return_value = 123456 koji_get_session.return_value = koji_session builder = mock.MagicMock() builder.koji_session = koji_session builder.buildroot_ready.return_value = False builder.module_build_tag = { "name": "module-testmodule-master-20170219191323-c40c156c-build" } create_builder.return_value = builder module_build = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 3) assert module_build # Set previous components as COMPLETE and tagged. module_build.batch = 1 for c in module_build.up_to_current_batch(): c.state = koji.BUILD_STATES["COMPLETE"] c.tagged = True c.tagged_in_final = True module_build.batch = 2 for c in module_build.current_batch(): if c.package == "perl-Tangerine": c.nvr = "perl-Tangerine-0.23-1.module+0+d027b723" elif c.package == "perl-List-Compare": c.nvr = "perl-List-Compare-0.53-5.module+0+d027b723" c.state = koji.BUILD_STATES["COMPLETE"] if task_state is not None: module_build.new_repo_task_id = 123456 db_session.commit() # Tag the first component to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="perl-Tangerine-0.23-1.module+0+d027b723", ) # Tag the first component to the final tag. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c", build_nvr="perl-Tangerine-0.23-1.module+0+d027b723", ) # Tag the second component to the buildroot. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c-build", build_nvr="perl-List-Compare-0.53-5.module+0+d027b723", ) # Tag the second component to the final tag. module_build_service.scheduler.handlers.tags.tagged( msg_id="id", tag_name="module-testmodule-master-20170219191323-c40c156c", build_nvr="perl-List-Compare-0.53-5.module+0+d027b723", ) # All components are tagged, newRepo should be called if there are no active tasks. if expect_new_repo: koji_session.newRepo.assert_called_once_with( "module-testmodule-master-20170219191323-c40c156c-build") else: assert not koji_session.newRepo.called # Refresh our module_build object. db_session.refresh(module_build) # newRepo task_id should be stored in database, so we can check its # status later in poller. assert module_build.new_repo_task_id == 123456 koji_session.newRepo.reset_mock()