def test_get_rpm_release_metadata_br_stream_override(mock_admmn): """ Test that when a module buildrequires a module in conf.allowed_privileged_module_names, and that module has the xmd.mbs.disttag_marking field set, it should influence the disttag. """ scheduler_init_data(1) metadata_mmd = load_mmd(read_staged_data("build_metadata_module")) import_mmd(db_session, metadata_mmd) build_one = models.ModuleBuild.get_by_id(db_session, 2) mmd = build_one.mmd() deps = mmd.get_dependencies()[0] deps.add_buildtime_stream("build", "product1.2") xmd = mmd.get_xmd() xmd["mbs"]["buildrequires"]["build"] = { "filtered_rpms": [], "ref": "virtual", "stream": "product1.2", "version": "1", "context": "00000000", } mmd.set_xmd(xmd) build_one.modulemd = mmd_to_str(mmd) db_session.add(build_one) db_session.commit() release = utils.get_rpm_release(db_session, build_one) assert release == "module+product12+2+814cfa39"
def test_transform_from_done_to_ready(self, ClientSession, publish): clean_database() # This build should be queried and transformed to ready state module_build = make_module_in_db( "pkg:0.1:1:c1", [{ "requires": { "platform": ["el8"] }, "buildrequires": { "platform": ["el8"] }, }], ) module_build.transition(db_session, conf, BUILD_STATES["done"], "Move to done directly for running test.") db_session.commit() # Assert this call below first_publish_call = call( "module.state.change", module_build.json(db_session, show_tasks=False), conf, "mbs", ) ClientSession.return_value.getBuild.return_value = { "extra": { "typeinfo": { "module": { "module_build_service_id": module_build.id } } } } msg = { "msg_id": "msg-id-1", "topic": "org.fedoraproject.prod.greenwave.decision.update", "msg": { "decision_context": "test_dec_context", "policies_satisfied": True, "subject_identifier": "pkg-0.1-1.c1", }, } hub = Mock(config={"validate_signatures": False}) consumer = MBSConsumer(hub) consumer.consume(msg) db_session.add(module_build) # Load module build again to check its state is moved correctly db_session.refresh(module_build) assert BUILD_STATES["ready"] == module_build.state publish.assert_has_calls([ first_publish_call, call("module.state.change", module_build.json(db_session, show_tasks=False), conf, "mbs"), ])
def test_get_buildrequired_modulemds(self): mmd = load_mmd(tests.read_staged_data("platform")) mmd = mmd.copy(mmd.get_module_name(), "f30.1.3") import_mmd(db_session, mmd) platform_f300103 = db_session.query(ModuleBuild).filter_by(stream="f30.1.3").one() mmd = tests.make_module("testmodule:master:20170109091357:123") build = ModuleBuild( name="testmodule", stream="master", version=20170109091357, state=5, build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3", runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7", context="7c29193d", koji_tag="module-testmodule-master-20170109091357-7c29193d", scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3, owner="Dr. Pepper", time_submitted=datetime(2018, 11, 15, 16, 8, 18), time_modified=datetime(2018, 11, 15, 16, 19, 35), rebuild_strategy="changed-and-after", modulemd=mmd_to_str(mmd), ) build.buildrequires.append(platform_f300103) db_session.add(build) db_session.commit() resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db") result = resolver.get_buildrequired_modulemds( "testmodule", "master", platform_f300103.mmd()) nsvcs = {m.get_nsvc() for m in result} assert nsvcs == {"testmodule:master:20170109091357:123"}
def test_resolve_requires_siblings(self, require_platform_and_default_arch): resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db") mmd = load_mmd(tests.read_staged_data("formatted_testmodule")) for i in range(3): build = tests.module_build_from_modulemd(mmd_to_str(mmd)) build.context = "f6e2ae" + str(i) build.build_context = "f6e2aeec7576196241b9afa0b6b22acf2b6873d" + str( i) build.runtime_context = "bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c" + str( i) build.state = models.BUILD_STATES["ready"] db_session.add(build) db_session.commit() build_one = ModuleBuild.get_by_id(db_session, 2) nsvc = ":".join([ build_one.name, build_one.stream, build_one.version, build_one.context ]) result = resolver.resolve_requires([nsvc]) assert result == { "testmodule": { "stream": build_one.stream, "version": build_one.version, "context": build_one.context, "ref": "65a7721ee4eff44d2a63fb8f3a8da6e944ab7f4d", "koji_tag": None } } db_session.commit()
def require_platform_and_default_arch_cls(require_empty_database_cls): """Provides clean database with platform module and default arch""" arch_obj = module_build_service.common.models.ModuleArch(name="x86_64") db_session.add(arch_obj) db_session.commit() mmd = load_mmd(read_staged_data("platform")) import_mmd(db_session, mmd)
def test_record_component_builds_component_exists_already( self, mocked_scm): mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4" mocked_scm.return_value.get_latest.side_effect = [ "4ceea43add2366d8b8c5a622a2fb563b625b9abf", "fbed359411a1baa08d4a88e0d12d426fbf8f602c", "dbed259411a1baa08d4a88e0d12d426fbf8f6037", "4ceea43add2366d8b8c5a622a2fb563b625b9abf", # To simulate that when a module is resubmitted, some ref of # its components is changed, which will cause MBS stops # recording component to database and raise an error. "abcdefg", "dbed259411a1baa08d4a88e0d12d426fbf8f6037", ] original_mmd = load_mmd(read_staged_data("testmodule")) # Set the module name and stream mmd = original_mmd.copy("testmodule", "master") module_build = module_build_service.common.models.ModuleBuild() module_build.name = "testmodule" module_build.stream = "master" module_build.version = 20170109091357 module_build.state = models.BUILD_STATES["init"] module_build.scmurl = \ "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79" module_build.batch = 1 module_build.owner = "Tom Brady" module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) module_build.rebuild_strategy = "changed-and-after" module_build.modulemd = mmd_to_str(mmd) db_session.add(module_build) db_session.commit() format_mmd(mmd, module_build.scmurl) record_component_builds(mmd, module_build) db_session.commit() mmd = original_mmd.copy("testmodule", "master") from module_build_service.common.errors import ValidationError with pytest.raises( ValidationError, match= r"Component build .+ of module build .+ already exists in database" ): format_mmd(mmd, module_build.scmurl) record_component_builds(mmd, module_build)
def test_get_rpm_release_platform_stream_override(): scheduler_init_data(1) # Set the disttag_marking override on the platform platform = (db_session.query(models.ModuleBuild).filter_by( name="platform", stream="f28").first()) platform_mmd = platform.mmd() platform_xmd = platform_mmd.get_xmd() platform_xmd["mbs"]["disttag_marking"] = "fedora28" platform_mmd.set_xmd(platform_xmd) platform.modulemd = mmd_to_str(platform_mmd) db_session.add(platform) db_session.commit() build_one = models.ModuleBuild.get_by_id(db_session, 2) release = utils.get_rpm_release(db_session, build_one) assert release == "module+fedora28+2+814cfa39"
def cleanup_stale_failed_builds(): """Does various clean up tasks on stale failed module builds""" if conf.system != "koji": return stale_date = datetime.utcnow() - timedelta(days=conf.cleanup_failed_builds_time) stale_module_builds = db_session.query(models.ModuleBuild).filter( models.ModuleBuild.state == models.BUILD_STATES["failed"], models.ModuleBuild.time_modified <= stale_date, ).all() if stale_module_builds: log.info( "%s stale failed module build(s) will be cleaned up", len(stale_module_builds) ) for module in stale_module_builds: log.info("%r is stale and is being cleaned up", module) # Find completed artifacts in the stale build artifacts = [c for c in module.component_builds if c.is_completed] # If there are no completed artifacts, then there is nothing to tag if artifacts: # Set buildroot_connect=False so it doesn't recreate the Koji target and etc. builder = GenericBuilder.create_from_module( db_session, module, conf, buildroot_connect=False ) builder.untag_artifacts([c.nvr for c in artifacts]) # Mark the artifacts as untagged in the database for c in artifacts: c.tagged = False c.tagged_in_final = False db_session.add(c) state_reason = ( "The module was garbage collected since it has failed over {0}" " day(s) ago".format(conf.cleanup_failed_builds_time) ) module.transition( db_session, conf, models.BUILD_STATES["garbage"], state_reason=state_reason, failure_type="user", ) db_session.add(module) db_session.commit()
def test_record_component_builds_duplicate_components(self, mocked_scm): # Mock for format_mmd to get components' latest ref mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4" mocked_scm.return_value.get_latest.side_effect = [ "4ceea43add2366d8b8c5a622a2fb563b625b9abf", "fbed359411a1baa08d4a88e0d12d426fbf8f602c", ] mmd = load_mmd(read_staged_data("testmodule")) mmd = mmd.copy("testmodule-variant", "master") module_build = module_build_service.common.models.ModuleBuild() module_build.name = "testmodule-variant" module_build.stream = "master" module_build.version = 20170109091357 module_build.state = models.BUILD_STATES["init"] module_build.scmurl = \ "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79" module_build.batch = 1 module_build.owner = "Tom Brady" module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) module_build.rebuild_strategy = "changed-and-after" module_build.modulemd = mmd_to_str(mmd) db_session.add(module_build) db_session.commit() # Rename the the modulemd to include mmd = mmd.copy("testmodule") # Remove perl-Tangerine and tangerine from the modulemd to include so only one # component conflicts mmd.remove_rpm_component("perl-Tangerine") mmd.remove_rpm_component("tangerine") error_msg = ( 'The included module "testmodule" in "testmodule-variant" have ' "the following conflicting components: perl-List-Compare") format_mmd(mmd, module_build.scmurl) with pytest.raises(UnprocessableEntity) as e: record_component_builds(mmd, module_build, main_mmd=module_build.mmd()) assert str(e.value) == error_msg
def _create_test_modules(self, koji_tag_with_modules="foo-test"): mmd = load_mmd(tests.read_staged_data("platform")) mmd = mmd.copy(mmd.get_module_name(), "f30.1.3") import_mmd(db_session, mmd) platform = db_session.query(ModuleBuild).filter_by( stream="f30.1.3").one() if koji_tag_with_modules: platform = db_session.query(ModuleBuild).filter_by( stream="f30.1.3").one() platform_mmd = platform.mmd() platform_xmd = platform_mmd.get_xmd() platform_xmd["mbs"][ "koji_tag_with_modules"] = koji_tag_with_modules platform_mmd.set_xmd(platform_xmd) platform.modulemd = mmd_to_str(platform_mmd) for context in ["7c29193d", "7c29193e"]: mmd = tests.make_module("testmodule:master:20170109091357:" + context) build = ModuleBuild( name="testmodule", stream="master", version=20170109091357, state=5, build_context="dd4de1c346dcf09ce77d38cd4e75094ec1c08ec3", runtime_context="ec4de1c346dcf09ce77d38cd4e75094ec1c08ef7", context=context, koji_tag="module-testmodule-master-20170109091357-" + context, scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3, owner="Dr. Pepper", time_submitted=datetime(2018, 11, 15, 16, 8, 18), time_modified=datetime(2018, 11, 15, 16, 19, 35), rebuild_strategy="changed-and-after", modulemd=mmd_to_str(mmd), ) build.buildrequires.append(platform) db_session.add(build) db_session.commit()
def test_get_reusable_component_different_component( self, changed_component): second_module_build = models.ModuleBuild.get_by_id(db_session, 3) if changed_component: mmd = second_module_build.mmd() mmd.get_rpm_component("tangerine").set_ref( "00ea1da4192a2030f9ae023de3b3143ed647bbab") second_module_build.modulemd = mmd_to_str(mmd) second_module_changed_component = models.ComponentBuild.from_component_name( db_session, changed_component, second_module_build.id) second_module_changed_component.ref = "00ea1da4192a2030f9ae023de3b3143ed647bbab" db_session.add(second_module_changed_component) db_session.commit() plc_rv = get_reusable_component(second_module_build, "perl-List-Compare") pt_rv = get_reusable_component(second_module_build, "perl-Tangerine") tangerine_rv = get_reusable_component(second_module_build, "tangerine") if changed_component == "perl-List-Compare": # perl-Tangerine can be reused even though a component in its batch has changed assert plc_rv is None assert pt_rv.package == "perl-Tangerine" assert tangerine_rv is None elif changed_component == "perl-Tangerine": # perl-List-Compare can be reused even though a component in its batch has changed assert plc_rv.package == "perl-List-Compare" assert pt_rv is None assert tangerine_rv is None elif changed_component == "tangerine": # perl-List-Compare and perl-Tangerine can be reused since they are in an earlier # buildorder than tangerine assert plc_rv.package == "perl-List-Compare" assert pt_rv.package == "perl-Tangerine" assert tangerine_rv is None elif changed_component is None: # Nothing has changed so everthing can be used assert plc_rv.package == "perl-List-Compare" assert pt_rv.package == "perl-Tangerine" assert tangerine_rv.package == "tangerine"
def model_tests_init_data(): """Initialize data for model tests This is refactored from tests/test_models/__init__.py, which was able to be called directly inside setup_method generally. The reason to convert it to this fixture is to use fixture ``db_session`` rather than create a new one. That would also benefit the whole test suite to reduce the number of SQLAlchemy session objects. """ clean_database() model_test_data_dir = os.path.join(os.path.dirname(__file__), "test_common", "test_models", "data") for filename in os.listdir(model_test_data_dir): with open(os.path.join(model_test_data_dir, filename), "r") as f: yaml = f.read() build = module_build_from_modulemd(yaml) db_session.add(build) db_session.commit()
def test_record_component_builds_set_weight(self, mocked_scm): # Mock for format_mmd to get components' latest ref mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4" mocked_scm.return_value.get_latest.side_effect = [ "4ceea43add2366d8b8c5a622a2fb563b625b9abf", "fbed359411a1baa08d4a88e0d12d426fbf8f602c", "dbed259411a1baa08d4a88e0d12d426fbf8f6037", ] mmd = load_mmd(read_staged_data("testmodule")) # Set the module name and stream mmd = mmd.copy("testmodule", "master") module_build = module_build_service.common.models.ModuleBuild() module_build.name = "testmodule" module_build.stream = "master" module_build.version = 20170109091357 module_build.state = models.BUILD_STATES["init"] module_build.scmurl = \ "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79" module_build.batch = 1 module_build.owner = "Tom Brady" module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) module_build.rebuild_strategy = "changed-and-after" module_build.modulemd = mmd_to_str(mmd) db_session.add(module_build) db_session.commit() format_mmd(mmd, module_build.scmurl) record_component_builds(mmd, module_build) db_session.commit() assert module_build.state == models.BUILD_STATES["init"] db_session.refresh(module_build) for c in module_build.component_builds: assert c.weight == 1.5
def test__get_base_module_mmds_virtual_streams_only_major_versions( self, cfg): """Ensure the correct results are returned without duplicates.""" init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"]) # Mark platform:foo28 as garbage to test that it is still considered as compatible. platform = db_session.query(models.ModuleBuild).filter_by( name="platform", stream="foo28").first() platform.state = "garbage" db_session.add(platform) db_session.commit() mmd = load_mmd(read_staged_data("testmodule_v2")) deps = mmd.get_dependencies()[0] new_deps = Modulemd.Dependencies() for stream in deps.get_runtime_streams("platform"): new_deps.add_runtime_stream("platform", stream) new_deps.add_buildtime_stream("platform", "foo29") mmd.remove_dependencies(deps) mmd.add_dependencies(new_deps) mmds = get_base_module_mmds(db_session, mmd) expected = {} expected["ready"] = {"platform:foo29", "platform:foo30"} expected["garbage"] = {"platform:foo28"} # Verify no duplicates were returned before doing set operations assert len(mmds) == len(expected) for k in expected.keys(): assert len(mmds[k]) == len(expected[k]) # Verify the expected ones were returned actual = set() for mmd_ in mmds[k]: actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name())) assert actual == expected[k]
def init(msg_id, module_build_id, module_build_state): """Called whenever a module enters the 'init' state. :param str msg_id: the original id of the message being handled, which is received from message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) state_init = models.BUILD_STATES["init"] if module_build_state == state_init and build.state != state_init: log.warning("Module build %r has moved to %s state already.", build, models.INVERSE_BUILD_STATES[build.state]) log.warning( "Ignore this message %s. Is there something wrong with the frontend" " that sends duplicate messages?", msg_id) return # for MockModuleBuilder, set build logs dir to mock results dir # before build_logs start if conf.system == "mock": build_tag_name = generate_module_build_koji_tag(build) mock_resultsdir = os.path.join(conf.mock_resultsdir, build_tag_name) if not os.path.exists(mock_resultsdir): os.makedirs(mock_resultsdir) build_logs.build_logs_dir = mock_resultsdir build_logs.start(db_session, build) log.info("Start to handle %s which is in init state.", build.mmd().get_nsvc()) error_msg = "" failure_reason = "unspec" try: mmd = build.mmd() record_module_build_arches(mmd, build) arches = [arch.name for arch in build.arches] defaults_added = add_default_modules(mmd) # Get map of packages that have SRPM overrides srpm_overrides = get_module_srpm_overrides(build) # Format the modulemd by putting in defaults and replacing streams that # are branches with commit hashes format_mmd(mmd, build.scmurl, build, db_session, srpm_overrides) record_component_builds(mmd, build) # The ursine.handle_stream_collision_modules is Koji specific. # It is also run only when Ursa Prime is not enabled for the base # module (`if not defaults_added`). if conf.system in ["koji", "test"] and not defaults_added: handle_stream_collision_modules(mmd) # Sets xmd["mbs"]["ursine_rpms"] with RPMs from the buildrequired base modules which # conflict with the RPMs from other buildrequired modules. This is done to prefer modular # RPMs over base module RPMs even if their NVR is lower. if conf.system in ("koji", "test"): handle_collisions_with_base_module_rpms(mmd, arches) else: log.warning( "The necessary conflicts could not be generated due to RHBZ#1693683. " "Some RPMs from the base modules (%s) may end up being used over modular RPMs. " "This may result in different behavior than a production build.", ", ".join(conf.base_module_names)) mmd = record_filtered_rpms(mmd) build.modulemd = mmd_to_str(mmd) build.transition(db_session, conf, models.BUILD_STATES["wait"]) # Catch custom exceptions that we can expose to the user except (UnprocessableEntity, Forbidden, ValidationError, RuntimeError) as e: log.exception(str(e)) error_msg = str(e) failure_reason = "user" except (xmlrpclib.ProtocolError, koji.GenericError) as e: log.exception(str(e)) error_msg = 'Koji communication error: "{0}"'.format(str(e)) failure_reason = "infra" except Exception as e: log.exception(str(e)) error_msg = "An unknown error occurred while validating the modulemd" failure_reason = "user" else: db_session.add(build) db_session.commit() finally: if error_msg: # Rollback changes underway db_session.rollback() build.transition( db_session, conf, models.BUILD_STATES["failed"], state_reason=error_msg, failure_type=failure_reason, ) db_session.commit()
def failed(msg_id, module_build_id, module_build_state): """Called whenever a module enters the 'failed' state. We cancel all the remaining component builds of a module and stop the building. :param str msg_id: the original id of the message being handled, which is received from the message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) if build.state != module_build_state: log.warning( "Note that retrieved module state %r doesn't match message module state %r", build.state, module_build_state, ) # This is ok.. it's a race condition we can ignore. pass if build.koji_tag: builder = GenericBuilder.create_from_module(db_session, build, conf) if build.new_repo_task_id: builder.cancel_build(build.new_repo_task_id) for component in (c for c in build.component_builds if c.is_unbuilt): if component.task_id: builder.cancel_build(component.task_id) component.state = koji.BUILD_STATES["FAILED"] component.state_reason = build.state_reason db_session.add(component) # Tell the external buildsystem to wrap up builder.finalize(succeeded=False) else: # Do not overwrite state_reason set by Frontend if any. if not build.state_reason: reason = "Missing koji tag. Assuming previously failed module lookup." log.error(reason) build.transition(db_session, conf, state=models.BUILD_STATES["failed"], state_reason=reason, failure_type="infra") db_session.commit() return # Don't transition it again if it's already been transitioned if build.state != models.BUILD_STATES["failed"]: build.transition(db_session, conf, state=models.BUILD_STATES["failed"], failure_type="user") db_session.commit() build_logs.stop(build) GenericBuilder.clear_cache(build)
def test_get_reusable_module_use_latest_build(self, cfg, allow_ocbm): """ Test that the `get_reusable_module` tries to reuse the latest module in case when multiple modules can be reused allow_only_compatible_base_modules is True. """ cfg.return_value = allow_ocbm # Set "fedora" virtual stream to platform:f28. platform_f28 = db_session.query( models.ModuleBuild).filter_by(name="platform").one() mmd = platform_f28.mmd() xmd = mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["fedora"] mmd.set_xmd(xmd) platform_f28.modulemd = mmd_to_str(mmd) platform_f28.update_virtual_streams(db_session, ["fedora"]) # Create platform:f29 with "fedora" virtual stream. mmd = load_mmd(read_staged_data("platform")) mmd = mmd.copy("platform", "f29") xmd = mmd.get_xmd() xmd["mbs"]["virtual_streams"] = ["fedora"] mmd.set_xmd(xmd) platform_f29 = import_mmd(db_session, mmd)[0] # Create another copy of `testmodule:master` which should be reused, because its # stream version will be higher than the previous one. Also set its buildrequires # to platform:f29. latest_module = db_session.query(models.ModuleBuild).filter_by( name="testmodule", state=models.BUILD_STATES["ready"]).one() # This is used to clone the ModuleBuild SQLAlchemy object without recreating it from # scratch. db_session.expunge(latest_module) make_transient(latest_module) # Change the platform:f28 buildrequirement to platform:f29 and recompute the build_context. mmd = latest_module.mmd() xmd = mmd.get_xmd() xmd["mbs"]["buildrequires"]["platform"]["stream"] = "f29" mmd.set_xmd(xmd) latest_module.modulemd = mmd_to_str(mmd) contexts = models.ModuleBuild.contexts_from_mmd(latest_module.modulemd) latest_module.build_context = contexts.build_context latest_module.context = contexts.context latest_module.buildrequires = [platform_f29] # Set the `id` to None, so new one is generated by SQLAlchemy. latest_module.id = None db_session.add(latest_module) db_session.commit() module = db_session.query(models.ModuleBuild)\ .filter_by(name="testmodule")\ .filter_by(state=models.BUILD_STATES["build"])\ .one() db_session.commit() reusable_module = get_reusable_module(module) if allow_ocbm: assert reusable_module.id == latest_module.id else: # There are two testmodules in ready state, the first one with # lower id is what we want. first_module = db_session.query(models.ModuleBuild).filter_by( name="testmodule", state=models.BUILD_STATES["ready"]).order_by( models.ModuleBuild.id).first() assert reusable_module.id == first_module.id
def scheduler_init_data(tangerine_state=None, scratch=False): """ Creates a testmodule in the building state with all the components in the same batch """ clean_database() mmd = load_mmd(read_staged_data("formatted_testmodule")) mmd.get_rpm_component("tangerine").set_buildorder(0) module_build = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170109091357', state=BUILD_STATES["build"], scratch=scratch, build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", context="7c29193d", koji_tag="scrmod-testmodule-master-20170109091357-7c29193d" if scratch else "module-testmodule-master-20170109091357-7c29193d", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3 if tangerine_state else 2, # https://www.youtube.com/watch?v=iOKymYVSaJE owner="Buzz Lightyear", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", modulemd=mmd_to_str(mmd), ) db_session.add(module_build) db_session.commit() platform_br = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 1) module_build.buildrequires.append(platform_br) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) module_build.arches.append(arch) build_one_component_release = get_rpm_release(db_session, module_build) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", task_id=90276227, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-Tangerine-0.23-1.{0}".format( build_one_component_release), batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", task_id=90276228, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-List-Compare-0.53-5.{0}".format( build_one_component_release), batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", state=tangerine_state, task_id=90276315 if tangerine_state else None, nvr="tangerine-0.22-3.{}".format(build_one_component_release) if tangerine_state else None, tagged=tangerine_state == koji.BUILD_STATES["COMPLETE"], tagged_in_final=tangerine_state == koji.BUILD_STATES["COMPLETE"], ), module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170109091357.src.rpm", format="rpms", task_id=90276181, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{}".format( build_one_component_release), batch=1, tagged=True, build_time_only=True, ), ]) db_session.commit()
def make_module( nsvc, dependencies=None, base_module=None, filtered_rpms=None, xmd=None, store_to_db=False, virtual_streams=None, arches=None, ): """ Creates new models.ModuleBuild defined by `nsvc` string with requires and buildrequires set according to ``requires_list`` and ``build_requires_list``. :param str nsvc: name:stream:version:context of a module. :param dependencies: list of groups of dependencies (requires and buildrequires). For example, [ {"requires": {"platform": ["f30"]}, "buildrequires": {"platform": ["f30"]}}, ... ] :type dependencies: list[dict] :param base_module: a base module build required by the new module created. :type base_module: :class:`ModuleBuild` :param filtered_rpms: list of filtered RPMs which are added to filter section in module metadata. :type filtered_rpms: list[str] :param dict xmd: a mapping representing XMD section in module metadata. A custom xmd could be passed for testing a particular scenario and some default key/value pairs are added if not present. :param bool store_to_db: whether to store created module metadata to the database. If set to True, ``db_session`` is required. :param virtual_streams: List of virtual streams provided by this module. If set, This requires ``db_session`` and ``store_to_db`` to be set to a session object and True. :type virtual_streams: list[str] :param arches: List of architectures this module is built against. If set to None, ``["x86_64"]`` is used as a default. If set, likewise ``virtual_stream``. :type arches: list[str] :return: New Module Build if set to store module metadata to database, otherwise the module metadata is returned. :rtype: ModuleBuild or Modulemd.Module """ if store_to_db: assert db_session is not None if base_module: assert db_session is not None if virtual_streams: assert store_to_db if arches: assert store_to_db nsvc_regex = re.compile(r"([^:]+):([^:]+):([^:]+)(?::(.+))?") match = nsvc_regex.match(nsvc) if not match: raise ValueError('Argument nsvc is not in format N:S:V or N:S:V:C') name, stream, version, context = match.groups() mmd = Modulemd.ModuleStreamV2.new(name, stream) mmd.set_version(int(version)) if context: mmd.set_context(context) mmd.set_summary("foo") # Test unicode in mmd. mmd.set_description(u"foo \u2019s") mmd.add_module_license("GPL") if filtered_rpms: for rpm in filtered_rpms: mmd.add_rpm_filter(rpm) def _add_require(mmd_deps, require_type, name, streams): assert isinstance(mmd_deps, Modulemd.Dependencies) assert require_type in ("requires", "buildrequires") assert isinstance(streams, (list, tuple)) if require_type == "requires": add_stream = mmd_deps.add_runtime_stream set_empty_deps = mmd_deps.set_empty_runtime_dependencies_for_module else: add_stream = mmd_deps.add_buildtime_stream set_empty_deps = mmd_deps.set_empty_buildtime_dependencies_for_module for stream in streams: add_stream(name, stream) else: set_empty_deps(name) for dep_group in dependencies or []: mmd_deps = Modulemd.Dependencies() # A deps could be {"platform": ["f30"], "python": []} for require_type, deps in dep_group.items(): for req_name, req_streams in deps.items(): _add_require(mmd_deps, require_type, req_name, req_streams) mmd.add_dependencies(mmd_deps) # Caller could pass whole xmd including mbs, but if something is missing, # default values are given here. xmd = xmd or {"mbs": {}} xmd_mbs = xmd["mbs"] if "buildrequires" not in xmd_mbs: xmd_mbs["buildrequires"] = {} if "requires" not in xmd_mbs: xmd_mbs["requires"] = {} if "commit" not in xmd_mbs: xmd_mbs["commit"] = "ref_%s" % context if "mse" not in xmd_mbs: xmd_mbs["mse"] = "true" if virtual_streams: xmd_mbs["virtual_streams"] = virtual_streams mmd.set_xmd(xmd) if not store_to_db: return mmd module_build = ModuleBuild( name=name, stream=stream, stream_version=ModuleBuild.get_stream_version(stream), version=version, context=context, state=BUILD_STATES["ready"], scmurl="https://src.stg.fedoraproject.org/modules/unused.git?#ff1ea79", batch=1, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", build_context=context, runtime_context=context, modulemd=mmd_to_str(mmd), koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None, ) if base_module: module_build.buildrequires.append(base_module) db_session.add(module_build) db_session.commit() if virtual_streams: for virtual_stream in virtual_streams: vs_obj = db_session.query(VirtualStream).filter_by( name=virtual_stream).first() if not vs_obj: vs_obj = VirtualStream(name=virtual_stream) db_session.add(vs_obj) db_session.commit() if vs_obj not in module_build.virtual_streams: module_build.virtual_streams.append(vs_obj) db_session.commit() for arch in arches or ["x86_64"]: arch_obj = db_session.query(ModuleArch).filter_by(name=arch).first() if not arch_obj: arch_obj = ModuleArch(name=arch) db_session.add(arch_obj) db_session.commit() if arch_obj not in module_build.arches: module_build.arches.append(arch_obj) db_session.commit() return module_build
def _create_module_with_filters(self, db_session, batch, state): mmd = load_mmd(read_staged_data("testmodule-with-filters")) # Set the name and stream mmd = mmd.copy("mbs-testmodule", "test") mmd.set_xmd({ "mbs": { "rpms": { "ed": { "ref": "01bf8330812fea798671925cc537f2f29b0bd216" }, "mksh": { "ref": "f70fd11ddf96bce0e2c64309706c29156b39141d" }, }, "buildrequires": { "host": { "version": "20171024133034", "filtered_rpms": [], "stream": "master", "ref": "6df253bb3c53e84706c01b8ab2d5cac24f0b6d45", "context": "00000000", }, "platform": { "version": "20171028112959", "filtered_rpms": [], "stream": "master", "ref": "4f7787370a931d57421f9f9555fc41c3e31ff1fa", "context": "00000000", }, }, "scmurl": "file:///testdir", "commit": "5566bc792ec7a03bb0e28edd1b104a96ba342bd8", "requires": { "platform": { "version": "20171028112959", "filtered_rpms": [], "stream": "master", "ref": "4f7787370a931d57421f9f9555fc41c3e31ff1fa", "context": "00000000", } }, } }) module = ModuleBuild.create( db_session, conf, name="mbs-testmodule", stream="test", version="20171027111452", modulemd=mmd_to_str(mmd), scmurl="file:///testdir", username="******", ) module.koji_tag = "module-mbs-testmodule-test-20171027111452" module.batch = batch db_session.add(module) db_session.commit() comp_builds = [ { "module_id": module.id, "state": state, "package": "ed", "format": "rpms", "scmurl": ("https://src.fedoraproject.org/rpms/ed" "?#01bf8330812fea798671925cc537f2f29b0bd216"), "batch": 2, "ref": "01bf8330812fea798671925cc537f2f29b0bd216", }, { "module_id": module.id, "state": state, "package": "mksh", "format": "rpms", "scmurl": ("https://src.fedoraproject.org/rpms/mksh" "?#f70fd11ddf96bce0e2c64309706c29156b39141d"), "batch": 3, "ref": "f70fd11ddf96bce0e2c64309706c29156b39141d", }, ] for build in comp_builds: db_session.add(ComponentBuild(**build)) db_session.commit() return module
def _populate_data(data_size=10, contexts=False, scratch=False): # Query arch from passed database session, otherwise there will be an error # like "Object '<ModuleBuild at 0x7f4ccc805c50>' is already attached to # session '275' (this is '276')" when add new module build object to passed # session. task_id_counter = itertools.count(1) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) num_contexts = 2 if contexts else 1 for index in range(data_size): for context in range(num_contexts): build_one = ModuleBuild( name="nginx", stream="1", version=2 + index, state=BUILD_STATES["ready"], scratch=scratch, modulemd=read_staged_data("nginx_mmd"), koji_tag="scrmod-nginx-1.2" if scratch else "module-nginx-1.2", scmurl="git://pkgs.domain.local/modules/nginx" "?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9", batch=2, # https://www.youtube.com/watch?v=iQGwrK_yDEg, owner="Moe Szyslak", time_submitted=datetime(2016, 9, 3, 11, 23, 20) + timedelta(minutes=(index * 10)), time_modified=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)), time_completed=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)), rebuild_strategy="changed-and-after", ) build_one.arches.append(arch) if contexts: build_one.stream = str(index) nsvc = "{}:{}:{}:{}".format(build_one.name, build_one.stream, build_one.version, context) unique_hash = hashlib.sha1(nsvc.encode('utf-8')).hexdigest() build_one.build_context = unique_hash build_one.runtime_context = unique_hash combined_hashes = "{0}:{1}".format(unique_hash, unique_hash) build_one.context = hashlib.sha1( combined_hashes.encode("utf-8")).hexdigest()[:8] db_session.add(build_one) db_session.commit() build_one_component_release = get_rpm_release( db_session, build_one) db_session.add_all([ ComponentBuild(package="nginx", scmurl="git://pkgs.domain.local/rpms/nginx?" "#ga95886c8a443b36a9ce31abda1f9bed22f2f8c3", format="rpms", task_id=six.next(task_id_counter), state=koji.BUILD_STATES["COMPLETE"], nvr="nginx-1.10.1-2.{0}".format( build_one_component_release), batch=1, module_id=2 + index * 3, tagged=True, tagged_in_final=True), ComponentBuild( package="module-build-macros", scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/" "module-build-macros-0.1-1.module_nginx_1_2.src.rpm", format="rpms", task_id=six.next(task_id_counter), state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-01-1.{0}".format( build_one_component_release), batch=2, module_id=2 + index * 3, tagged=True, tagged_in_final=True) ]) db_session.commit() build_two = ModuleBuild( name="postgressql", stream="1", version=2 + index, state=BUILD_STATES["done"], scratch=scratch, modulemd=read_staged_data("testmodule"), koji_tag="scrmod-postgressql-1.2" if scratch else "module-postgressql-1.2", scmurl="git://pkgs.domain.local/modules/postgressql" "?#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9", batch=2, owner="some_user", time_submitted=datetime(2016, 9, 3, 12, 25, 33) + timedelta(minutes=(index * 10)), time_modified=datetime(2016, 9, 3, 12, 27, 19) + timedelta(minutes=(index * 10)), time_completed=datetime(2016, 9, 3, 11, 27, 19) + timedelta(minutes=(index * 10)), rebuild_strategy="changed-and-after", ) build_two.arches.append(arch) db_session.add(build_two) db_session.commit() build_two_component_release = get_rpm_release(db_session, build_two) db_session.add_all([ ComponentBuild(package="postgresql", scmurl="git://pkgs.domain.local/rpms/postgresql" "?#dc95586c4a443b26a9ce38abda1f9bed22f2f8c3", format="rpms", task_id=2433433 + index, state=koji.BUILD_STATES["COMPLETE"], nvr="postgresql-9.5.3-4.{0}".format( build_two_component_release), batch=2, module_id=3 + index * 3, tagged=True, tagged_in_final=True), ComponentBuild( package="module-build-macros", scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/" "module-build-macros-0.1-1.module_postgresql_1_2.src.rpm", format="rpms", task_id=47383993 + index, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-01-1.{0}".format( build_two_component_release), batch=1, module_id=3 + index * 3) ]) db_session.commit() build_three = ModuleBuild( name="testmodule", stream="4.3.43", version=6 + index, state=BUILD_STATES["wait"], scratch=scratch, modulemd=read_staged_data("testmodule"), koji_tag=None, scmurl="git://pkgs.domain.local/modules/testmodule" "?#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9", batch=0, owner="some_other_user", time_submitted=datetime(2016, 9, 3, 12, 28, 33) + timedelta(minutes=(index * 10)), time_modified=datetime(2016, 9, 3, 12, 28, 40) + timedelta(minutes=(index * 10)), time_completed=None, rebuild_strategy="changed-and-after", ) db_session.add(build_three) db_session.commit() build_three_component_release = get_rpm_release( db_session, build_three) db_session.add_all([ ComponentBuild(package="rubygem-rails", scmurl="git://pkgs.domain.local/rpms/rubygem-rails" "?#dd55886c4a443b26a9ce38abda1f9bed22f2f8c3", format="rpms", task_id=2433433 + index, state=koji.BUILD_STATES["FAILED"], nvr="postgresql-9.5.3-4.{0}".format( build_three_component_release), batch=2, module_id=4 + index * 3), ComponentBuild( package="module-build-macros", scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/" "module-build-macros-0.1-1.module_testmodule_1_2.src.rpm", format="rpms", task_id=47383993 + index, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-01-1.{0}".format( build_three_component_release), batch=1, module_id=4 + index * 3, tagged=True, build_time_only=True) ]) db_session.commit()
def require_default_arch(require_empty_database): """Provides empty database with x86_64 arch""" arch_obj = module_build_service.common.models.ModuleArch(name="x86_64") db_session.add(arch_obj) db_session.commit()
def record_component_builds(mmd, module, initial_batch=1, previous_buildorder=None, main_mmd=None): # Imported here to allow import of utils in GenericBuilder. from module_build_service.builder import GenericBuilder # When main_mmd is set, merge the metadata from this mmd to main_mmd, # otherwise our current mmd is main_mmd. if main_mmd: # Check for components that are in both MMDs before merging since MBS # currently can't handle that situation. main_mmd_rpms = main_mmd.get_rpm_component_names() mmd_rpms = mmd.get_rpm_component_names() duplicate_components = [ rpm for rpm in main_mmd_rpms if rpm in mmd_rpms ] if duplicate_components: error_msg = ( 'The included module "{0}" in "{1}" have the following ' "conflicting components: {2}".format( mmd.get_module_name(), main_mmd.get_module_name(), ", ".join(duplicate_components))) raise UnprocessableEntity(error_msg) merge_included_mmd(main_mmd, mmd) else: main_mmd = mmd # If the modulemd yaml specifies components, then submit them for build rpm_components = [ mmd.get_rpm_component(name) for name in mmd.get_rpm_component_names() ] module_components = [ mmd.get_module_component(name) for name in mmd.get_module_component_names() ] all_components = list(rpm_components) + list(module_components) if not all_components: return # Get map of packages that have SRPM overrides srpm_overrides = get_module_srpm_overrides(module) rpm_weights = GenericBuilder.get_build_weights( [c.get_name() for c in rpm_components]) all_components.sort(key=lambda x: x.get_buildorder()) # We do not start with batch = 0 here, because the first batch is # reserved for module-build-macros. First real components must be # planned for batch 2 and following. batch = initial_batch for component in all_components: # Increment the batch number when buildorder increases. if previous_buildorder != component.get_buildorder(): previous_buildorder = component.get_buildorder() batch += 1 # If the component is another module, we fetch its modulemd file # and record its components recursively with the initial_batch # set to our current batch, so the components of this module # are built in the right global order. if isinstance(component, Modulemd.ComponentModule): full_url = component.get_repository() + "?#" + component.get_ref() # It is OK to whitelist all URLs here, because the validity # of every URL have been already checked in format_mmd(...). included_mmd = fetch_mmd(full_url, whitelist_url=True)[0] format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides) batch = record_component_builds(included_mmd, module, batch, previous_buildorder, main_mmd) continue package = component.get_name() if package in srpm_overrides: component_ref = None full_url = srpm_overrides[package] log.info('Building custom SRPM "{0}"' " for package {1}".format(full_url, package)) else: component_ref = mmd.get_xmd()["mbs"]["rpms"][package]["ref"] full_url = component.get_repository() + "?#" + component_ref # Skip the ComponentBuild if it already exists in database. This can happen # in case of module build resubmition. existing_build = models.ComponentBuild.from_component_name( db_session, package, module.id) if existing_build: # Check that the existing build has the same most important attributes. # This should never be a problem, but it's good to be defensive here so # we do not mess things during resubmition. if (existing_build.batch != batch or existing_build.scmurl != full_url or existing_build.ref != component_ref): raise ValidationError( "Component build %s of module build %s (id: %d) already " "exists in database, but its attributes are different from" " resubmitted one." % (component.get_name(), module.name, module.id)) continue build = models.ComponentBuild(module_id=module.id, package=package, format="rpms", scmurl=full_url, batch=batch, ref=component_ref, weight=rpm_weights[package], buildonly=component.get_buildonly()) db_session.add(build) return batch
def reuse_shared_userspace_init_data(): # Create shared-userspace-570, state is COMPLETE, all components # are properly built. scmurl = "https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453" mmd = load_mmd(read_staged_data("shared-userspace-570")) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = scmurl xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8" mmd.set_xmd(xmd) module_build = module_build_service.common.models.ModuleBuild( name=mmd.get_module_name(), stream=mmd.get_stream_name(), version=mmd.get_version(), build_context=module_build_service.common.models.ModuleBuild. calculate_build_context(xmd["mbs"]["buildrequires"]), runtime_context="50dd3eb5dde600d072e45d4120e1548ce66bc94a", state=BUILD_STATES["ready"], modulemd=mmd_to_str(mmd), koji_tag="module-shared-userspace-f26-20170601141014-75f92abb", scmurl=scmurl, batch=16, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), time_completed=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", ) components = [ mmd.get_rpm_component(rpm) for rpm in mmd.get_rpm_component_names() ] components.sort(key=lambda x: x.get_buildorder()) previous_buildorder = None batch = 1 xmd_mbs_rpms = xmd["mbs"]["rpms"] for pkg in components: # Increment the batch number when buildorder increases. if previous_buildorder != pkg.get_buildorder(): previous_buildorder = pkg.get_buildorder() batch += 1 pkgref = xmd_mbs_rpms[pkg.get_name()]["ref"] full_url = pkg.get_repository() + "?#" + pkgref module_build.component_builds.append( module_build_service.common.models.ComponentBuild( package=pkg.get_name(), format="rpms", scmurl=full_url, batch=batch, ref=pkgref, state=1, tagged=True, tagged_in_final=True, )) db_session.add(module_build) db_session.commit() # Create shared-userspace-577, state is WAIT, no component built scmurl = "https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453" mmd2 = load_mmd(read_staged_data("shared-userspace-577")) xmd = mmd2.get_xmd() xmd["mbs"]["scmurl"] = scmurl xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8" mmd2.set_xmd(xmd) module_build = module_build_service.common.models.ModuleBuild( name=mmd2.get_module_name(), stream=mmd2.get_stream_name(), version=mmd2.get_version(), build_context=module_build_service.common.models.ModuleBuild. calculate_build_context(xmd["mbs"]["buildrequires"]), runtime_context="50dd3eb5dde600d072e45d4120e1548ce66bc94a", state=BUILD_STATES["done"], modulemd=mmd_to_str(mmd2), koji_tag="module-shared-userspace-f26-20170605091544-75f92abb", scmurl=scmurl, batch=0, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), time_completed=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", ) components2 = [ mmd2.get_rpm_component(rpm) for rpm in mmd2.get_rpm_component_names() ] # Store components to database in different order than for 570 to # reproduce the reusing issue. components2.sort(key=lambda x: len(x.get_name())) components2.sort(key=lambda x: x.get_buildorder()) previous_buildorder = None batch = 1 xmd_mbs_rpms = mmd2.get_xmd()["mbs"]["rpms"] for pkg in components2: # Increment the batch number when buildorder increases. if previous_buildorder != pkg.get_buildorder(): previous_buildorder = pkg.get_buildorder() batch += 1 pkgref = xmd_mbs_rpms[pkg.get_name()]["ref"] full_url = pkg.get_repository() + "?#" + pkgref module_build.component_builds.append( module_build_service.common.models.ComponentBuild( package=pkg.get_name(), format="rpms", scmurl=full_url, batch=batch, ref=pkgref)) db_session.add(module_build) db_session.commit()
def reuse_component_init_data(): mmd = load_mmd(read_staged_data("formatted_testmodule")) build_one = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170109091357', state=BUILD_STATES["ready"], runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1", context="78e4a6fd", koji_tag="module-testmodule-master-20170109091357-78e4a6fd", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), time_completed=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", ) build_one_component_release = _get_rpm_release_no_db(build_one) mmd.set_version(int(build_one.version)) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = build_one.scmurl xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba" mmd.set_xmd(xmd) build_one.modulemd = mmd_to_str(mmd) contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd( build_one.modulemd) build_one.build_context = contexts.build_context build_one.build_context_no_bms = contexts.build_context_no_bms db_session.add(build_one) db_session.flush() platform_br = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 1) build_one.buildrequires.append(platform_br) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) build_one.arches.append(arch) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", task_id=90276227, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-Tangerine-0.23-1.{0}".format( build_one_component_release), batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", task_id=90276228, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-List-Compare-0.53-5.{0}".format( build_one_component_release), batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", task_id=90276315, state=koji.BUILD_STATES["COMPLETE"], nvr="tangerine-0.22-3.{0}".format(build_one_component_release), batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170109091357.src.rpm", format="rpms", task_id=90276181, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{0}".format( build_one_component_release), batch=1, tagged=True, build_time_only=True, ), ]) build_two = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170219191323', state=BUILD_STATES["build"], runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1", context="c40c156c", koji_tag="module-testmodule-master-20170219191323-c40c156c", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a", batch=1, owner="Tom Brady", time_submitted=datetime(2017, 2, 19, 16, 8, 18), time_modified=datetime(2017, 2, 19, 16, 8, 18), rebuild_strategy="changed-and-after", ) build_two_component_release = _get_rpm_release_no_db(build_two) mmd.set_version(int(build_one.version)) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = build_one.scmurl xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8" mmd.set_xmd(xmd) build_two.modulemd = mmd_to_str(mmd) contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd( build_two.modulemd) build_two.build_context = contexts.build_context build_two.build_context_no_bms = contexts.build_context_no_bms db_session.add(build_two) db_session.flush() build_two.arches.append(arch) build_two.buildrequires.append(platform_br) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170219191323.src.rpm", format="rpms", task_id=90276186, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{0}".format( build_two_component_release), batch=1, tagged=True, build_time_only=True, ), ]) # Commit everything in one go. db_session.commit()
def wait(msg_id, module_build_id, module_build_state): """ Called whenever a module enters the 'wait' state. We transition to this state shortly after a modulebuild is first requested. All we do here is request preparation of the buildroot. The kicking off of individual component builds is handled elsewhere, in module_build_service.schedulers.handlers.repos. :param str msg_id: the original id of the message being handled which is received from the message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) log.info("Found build=%r from message" % build) log.debug("%r", build.modulemd) if build.state != module_build_state: log.warning( "Note that retrieved module state %r doesn't match message module state %r", build.state, module_build_state, ) # This is ok.. it's a race condition we can ignore. pass try: build_deps = get_module_build_dependencies(build) except ValueError: reason = "Failed to get module info from MBS. Max retries reached." log.exception(reason) build.transition(db_session, conf, state=models.BUILD_STATES["failed"], state_reason=reason, failure_type="infra") db_session.commit() raise tag = generate_module_build_koji_tag(build) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is # associated with which koji tag, so that when their repos are regenerated # in koji we can figure out which for which module build that event is # relevant. log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag if build.scratch: log.debug( "Assigning Content Generator build koji tag is skipped for scratch module build." ) elif conf.koji_cg_tag_build: cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) log.debug( "Assigning Content Generator build koji tag=%s to module build", cg_build_koji_tag) build.cg_build_koji_tag = cg_build_koji_tag else: log.debug( "It is disabled to tag module build during importing into Koji by Content Generator." ) log.debug( "Skip to assign Content Generator build koji tag to module build.") builder = GenericBuilder.create_from_module(db_session, build, conf) log.debug( "Adding dependencies %s into buildroot for module %s:%s:%s", build_deps.keys(), build.name, build.stream, build.version, ) builder.buildroot_add_repos(build_deps) if not build.component_builds: log.info("There are no components in module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # Return a KojiRepoChange message so that the build can be transitioned to done # in the repos handler from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"])) return # If all components in module build will be reused, we don't have to build # module-build-macros, because there won't be any build done. if attempt_to_reuse_all_components(builder, build): log.info( "All components have been reused for module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() return [] log.debug("Starting build batch 1") build.batch = 1 db_session.commit() artifact_name = "module-build-macros" component_build = models.ComponentBuild.from_component_name( db_session, artifact_name, build.id) srpm = builder.get_disttag_srpm(disttag=".%s" % get_rpm_release(db_session, build), module_build=build) if not component_build: component_build = models.ComponentBuild( module_id=build.id, package=artifact_name, format="rpms", scmurl=srpm, batch=1, build_time_only=True, ) db_session.add(component_build) # Commit and refresh so that the SQLAlchemy relationships are available db_session.commit() db_session.refresh(component_build) recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") # There was no existing artifact found, so lets submit the build instead else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr elif not component_build.is_completed: # It's possible that the build succeeded in the builder but some other step failed which # caused module-build-macros to be marked as failed in MBS, so check to see if it exists # first recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr db_session.add(component_build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # We always have to regenerate the repository. if conf.system == "koji": log.info("Regenerating the repository") task_id = builder.koji_session.newRepo( builder.module_build_tag["name"]) build.new_repo_task_id = task_id db_session.commit() else: from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"]))