def test_get_rpm_release_mse_scratch(provide_test_data): build_one = models.ModuleBuild.get_by_id(db_session, 2) release_one = utils.get_rpm_release(db_session, build_one) assert release_one == "scrmod+2+b8645bbb" build_two = models.ModuleBuild.get_by_id(db_session, 3) release_two = utils.get_rpm_release(db_session, build_two) assert release_two == "scrmod+2+17e35784"
def test_get_rpm_release_mse_scratch(): init_data(contexts=True, scratch=True) build_one = models.ModuleBuild.get_by_id(db_session, 2) release_one = utils.get_rpm_release(db_session, build_one) assert release_one == "scrmod+2+b8645bbb" build_two = models.ModuleBuild.get_by_id(db_session, 3) release_two = utils.get_rpm_release(db_session, build_two) assert release_two == "scrmod+2+17e35784"
def test_get_rpm_release_metadata_br_stream_override(mock_admmn): """ Test that when a module buildrequires a module in conf.allowed_privileged_module_names, and that module has the xmd.mbs.disttag_marking field set, it should influence the disttag. """ scheduler_init_data(1) metadata_mmd = load_mmd(read_staged_data("build_metadata_module")) import_mmd(db_session, metadata_mmd) build_one = models.ModuleBuild.get_by_id(db_session, 2) mmd = build_one.mmd() deps = mmd.get_dependencies()[0] deps.add_buildtime_stream("build", "product1.2") xmd = mmd.get_xmd() xmd["mbs"]["buildrequires"]["build"] = { "filtered_rpms": [], "ref": "virtual", "stream": "product1.2", "version": "1", "context": "00000000", } mmd.set_xmd(xmd) build_one.modulemd = mmd_to_str(mmd) db_session.add(build_one) db_session.commit() release = utils.get_rpm_release(db_session, build_one) assert release == "module+product12+2+814cfa39"
def test_get_rpm_release_platform_stream_override(): scheduler_init_data(1) # Set the disttag_marking override on the platform platform = (db_session.query(models.ModuleBuild).filter_by( name="platform", stream="f28").first()) platform_mmd = platform.mmd() platform_xmd = platform_mmd.get_xmd() platform_xmd["mbs"]["disttag_marking"] = "fedora28" platform_mmd.set_xmd(platform_xmd) platform.modulemd = mmd_to_str(platform_mmd) db_session.add(platform) db_session.commit() build_one = models.ModuleBuild.get_by_id(db_session, 2) release = utils.get_rpm_release(db_session, build_one) assert release == "module+fedora28+2+814cfa39"
def scheduler_init_data(tangerine_state=None, scratch=False): """ Creates a testmodule in the building state with all the components in the same batch """ clean_database() mmd = load_mmd(read_staged_data("formatted_testmodule")) mmd.get_rpm_component("tangerine").set_buildorder(0) module_build = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170109091357', state=BUILD_STATES["build"], scratch=scratch, build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", context="7c29193d", koji_tag="scrmod-testmodule-master-20170109091357-7c29193d" if scratch else "module-testmodule-master-20170109091357-7c29193d", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3 if tangerine_state else 2, # https://www.youtube.com/watch?v=iOKymYVSaJE owner="Buzz Lightyear", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", modulemd=mmd_to_str(mmd), ) db_session.add(module_build) db_session.commit() platform_br = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 1) module_build.buildrequires.append(platform_br) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) module_build.arches.append(arch) build_one_component_release = get_rpm_release(db_session, module_build) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", task_id=90276227, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-Tangerine-0.23-1.{0}".format( build_one_component_release), batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", task_id=90276228, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-List-Compare-0.53-5.{0}".format( build_one_component_release), batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", state=tangerine_state, task_id=90276315 if tangerine_state else None, nvr="tangerine-0.22-3.{}".format(build_one_component_release) if tangerine_state else None, tagged=tangerine_state == koji.BUILD_STATES["COMPLETE"], tagged_in_final=tangerine_state == koji.BUILD_STATES["COMPLETE"], ), module_build_service.common.models.ComponentBuild( module_id=module_build.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170109091357.src.rpm", format="rpms", task_id=90276181, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{}".format( build_one_component_release), batch=1, tagged=True, build_time_only=True, ), ]) db_session.commit()
def _populate_data(data_size=10, contexts=False, scratch=False): # Query arch from passed database session, otherwise there will be an error # like "Object '<ModuleBuild at 0x7f4ccc805c50>' is already attached to # session '275' (this is '276')" when add new module build object to passed # session. task_id_counter = itertools.count(1) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) num_contexts = 2 if contexts else 1 for index in range(data_size): for context in range(num_contexts): build_one = ModuleBuild( name="nginx", stream="1", version=2 + index, state=BUILD_STATES["ready"], scratch=scratch, modulemd=read_staged_data("nginx_mmd"), koji_tag="scrmod-nginx-1.2" if scratch else "module-nginx-1.2", scmurl="git://pkgs.domain.local/modules/nginx" "?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9", batch=2, # https://www.youtube.com/watch?v=iQGwrK_yDEg, owner="Moe Szyslak", time_submitted=datetime(2016, 9, 3, 11, 23, 20) + timedelta(minutes=(index * 10)), time_modified=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)), time_completed=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)), rebuild_strategy="changed-and-after", ) build_one.arches.append(arch) if contexts: build_one.stream = str(index) nsvc = "{}:{}:{}:{}".format(build_one.name, build_one.stream, build_one.version, context) unique_hash = hashlib.sha1(nsvc.encode('utf-8')).hexdigest() build_one.build_context = unique_hash build_one.runtime_context = unique_hash combined_hashes = "{0}:{1}".format(unique_hash, unique_hash) build_one.context = hashlib.sha1( combined_hashes.encode("utf-8")).hexdigest()[:8] db_session.add(build_one) db_session.commit() build_one_component_release = get_rpm_release( db_session, build_one) db_session.add_all([ ComponentBuild(package="nginx", scmurl="git://pkgs.domain.local/rpms/nginx?" "#ga95886c8a443b36a9ce31abda1f9bed22f2f8c3", format="rpms", task_id=six.next(task_id_counter), state=koji.BUILD_STATES["COMPLETE"], nvr="nginx-1.10.1-2.{0}".format( build_one_component_release), batch=1, module_id=2 + index * 3, tagged=True, tagged_in_final=True), ComponentBuild( package="module-build-macros", scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/" "module-build-macros-0.1-1.module_nginx_1_2.src.rpm", format="rpms", task_id=six.next(task_id_counter), state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-01-1.{0}".format( build_one_component_release), batch=2, module_id=2 + index * 3, tagged=True, tagged_in_final=True) ]) db_session.commit() build_two = ModuleBuild( name="postgressql", stream="1", version=2 + index, state=BUILD_STATES["done"], scratch=scratch, modulemd=read_staged_data("testmodule"), koji_tag="scrmod-postgressql-1.2" if scratch else "module-postgressql-1.2", scmurl="git://pkgs.domain.local/modules/postgressql" "?#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9", batch=2, owner="some_user", time_submitted=datetime(2016, 9, 3, 12, 25, 33) + timedelta(minutes=(index * 10)), time_modified=datetime(2016, 9, 3, 12, 27, 19) + timedelta(minutes=(index * 10)), time_completed=datetime(2016, 9, 3, 11, 27, 19) + timedelta(minutes=(index * 10)), rebuild_strategy="changed-and-after", ) build_two.arches.append(arch) db_session.add(build_two) db_session.commit() build_two_component_release = get_rpm_release(db_session, build_two) db_session.add_all([ ComponentBuild(package="postgresql", scmurl="git://pkgs.domain.local/rpms/postgresql" "?#dc95586c4a443b26a9ce38abda1f9bed22f2f8c3", format="rpms", task_id=2433433 + index, state=koji.BUILD_STATES["COMPLETE"], nvr="postgresql-9.5.3-4.{0}".format( build_two_component_release), batch=2, module_id=3 + index * 3, tagged=True, tagged_in_final=True), ComponentBuild( package="module-build-macros", scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/" "module-build-macros-0.1-1.module_postgresql_1_2.src.rpm", format="rpms", task_id=47383993 + index, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-01-1.{0}".format( build_two_component_release), batch=1, module_id=3 + index * 3) ]) db_session.commit() build_three = ModuleBuild( name="testmodule", stream="4.3.43", version=6 + index, state=BUILD_STATES["wait"], scratch=scratch, modulemd=read_staged_data("testmodule"), koji_tag=None, scmurl="git://pkgs.domain.local/modules/testmodule" "?#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9", batch=0, owner="some_other_user", time_submitted=datetime(2016, 9, 3, 12, 28, 33) + timedelta(minutes=(index * 10)), time_modified=datetime(2016, 9, 3, 12, 28, 40) + timedelta(minutes=(index * 10)), time_completed=None, rebuild_strategy="changed-and-after", ) db_session.add(build_three) db_session.commit() build_three_component_release = get_rpm_release( db_session, build_three) db_session.add_all([ ComponentBuild(package="rubygem-rails", scmurl="git://pkgs.domain.local/rpms/rubygem-rails" "?#dd55886c4a443b26a9ce38abda1f9bed22f2f8c3", format="rpms", task_id=2433433 + index, state=koji.BUILD_STATES["FAILED"], nvr="postgresql-9.5.3-4.{0}".format( build_three_component_release), batch=2, module_id=4 + index * 3), ComponentBuild( package="module-build-macros", scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/" "module-build-macros-0.1-1.module_testmodule_1_2.src.rpm", format="rpms", task_id=47383993 + index, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-01-1.{0}".format( build_three_component_release), batch=1, module_id=4 + index * 3, tagged=True, build_time_only=True) ]) db_session.commit()
def reuse_component_init_data(): clean_database() mmd = load_mmd(read_staged_data("formatted_testmodule")) build_one = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170109091357', state=BUILD_STATES["ready"], runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1", context="78e4a6fd", koji_tag="module-testmodule-master-20170109091357-78e4a6fd", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79", batch=3, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), time_completed=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", ) build_one_component_release = get_rpm_release(db_session, build_one) mmd.set_version(int(build_one.version)) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = build_one.scmurl xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba" mmd.set_xmd(xmd) build_one.modulemd = mmd_to_str(mmd) contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd( build_one.modulemd) build_one.build_context = contexts.build_context build_one.build_context_no_bms = contexts.build_context_no_bms db_session.add(build_one) db_session.commit() db_session.refresh(build_one) platform_br = module_build_service.common.models.ModuleBuild.get_by_id( db_session, 1) build_one.buildrequires.append(platform_br) arch = db_session.query( module_build_service.common.models.ModuleArch).get(1) build_one.arches.append(arch) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", task_id=90276227, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-Tangerine-0.23-1.{0}".format( build_one_component_release), batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", task_id=90276228, state=koji.BUILD_STATES["COMPLETE"], nvr="perl-List-Compare-0.53-5.{0}".format( build_one_component_release), batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", task_id=90276315, state=koji.BUILD_STATES["COMPLETE"], nvr="tangerine-0.22-3.{0}".format(build_one_component_release), batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", tagged=True, tagged_in_final=True, ), module_build_service.common.models.ComponentBuild( module_id=build_one.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170109091357.src.rpm", format="rpms", task_id=90276181, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{0}".format( build_one_component_release), batch=1, tagged=True, build_time_only=True, ), ]) # Commit component builds added to build_one db_session.commit() build_two = module_build_service.common.models.ModuleBuild( name="testmodule", stream="master", version='20170219191323', state=BUILD_STATES["build"], runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0", build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1", context="c40c156c", koji_tag="module-testmodule-master-20170219191323-c40c156c", scmurl= "https://src.stg.fedoraproject.org/modules/testmodule.git?#55f4a0a", batch=1, owner="Tom Brady", time_submitted=datetime(2017, 2, 19, 16, 8, 18), time_modified=datetime(2017, 2, 19, 16, 8, 18), rebuild_strategy="changed-and-after", ) build_two_component_release = get_rpm_release(db_session, build_two) mmd.set_version(int(build_one.version)) xmd = mmd.get_xmd() xmd["mbs"]["scmurl"] = build_one.scmurl xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8" mmd.set_xmd(xmd) build_two.modulemd = mmd_to_str(mmd) contexts = module_build_service.common.models.ModuleBuild.contexts_from_mmd( build_two.modulemd) build_two.build_context = contexts.build_context build_two.build_context_no_bms = contexts.build_context_no_bms db_session.add(build_two) db_session.commit() db_session.refresh(build_two) build_two.arches.append(arch) build_two.buildrequires.append(platform_br) db_session.add_all([ module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="perl-Tangerine", scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine" "?#4ceea43add2366d8b8c5a622a2fb563b625b9abf", format="rpms", batch=2, ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="perl-List-Compare", scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare" "?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", format="rpms", batch=2, ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="tangerine", scmurl="https://src.fedoraproject.org/rpms/tangerine" "?#fbed359411a1baa08d4a88e0d12d426fbf8f602c", format="rpms", batch=3, ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c", ), module_build_service.common.models.ComponentBuild( module_id=build_two.id, package="module-build-macros", scmurl= "/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-" "macros-0.1-1.module_testmodule_master_20170219191323.src.rpm", format="rpms", task_id=90276186, state=koji.BUILD_STATES["COMPLETE"], nvr="module-build-macros-0.1-1.{0}".format( build_two_component_release), batch=1, tagged=True, build_time_only=True, ), ]) db_session.commit()
def test_get_rpm_release_platform_stream_scratch(): scheduler_init_data(1, scratch=True) build_one = models.ModuleBuild.get_by_id(db_session, 2) release = utils.get_rpm_release(db_session, build_one) assert release == "scrmod+f28+2+814cfa39"
def wait(msg_id, module_build_id, module_build_state): """ Called whenever a module enters the 'wait' state. We transition to this state shortly after a modulebuild is first requested. All we do here is request preparation of the buildroot. The kicking off of individual component builds is handled elsewhere, in module_build_service.schedulers.handlers.repos. :param str msg_id: the original id of the message being handled which is received from the message bus. :param int module_build_id: the module build id. :param int module_build_state: the module build state. """ build = models.ModuleBuild.get_by_id(db_session, module_build_id) log.info("Found build=%r from message" % build) log.debug("%r", build.modulemd) if build.state != module_build_state: log.warning( "Note that retrieved module state %r doesn't match message module state %r", build.state, module_build_state, ) # This is ok.. it's a race condition we can ignore. pass try: build_deps = get_module_build_dependencies(build) except ValueError: reason = "Failed to get module info from MBS. Max retries reached." log.exception(reason) build.transition(db_session, conf, state=models.BUILD_STATES["failed"], state_reason=reason, failure_type="infra") db_session.commit() raise tag = generate_module_build_koji_tag(build) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is # associated with which koji tag, so that when their repos are regenerated # in koji we can figure out which for which module build that event is # relevant. log.debug("Assigning koji tag=%s to module build" % tag) build.koji_tag = tag if build.scratch: log.debug( "Assigning Content Generator build koji tag is skipped for scratch module build." ) elif conf.koji_cg_tag_build: cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps) log.debug( "Assigning Content Generator build koji tag=%s to module build", cg_build_koji_tag) build.cg_build_koji_tag = cg_build_koji_tag else: log.debug( "It is disabled to tag module build during importing into Koji by Content Generator." ) log.debug( "Skip to assign Content Generator build koji tag to module build.") builder = GenericBuilder.create_from_module(db_session, build, conf) log.debug( "Adding dependencies %s into buildroot for module %s:%s:%s", build_deps.keys(), build.name, build.stream, build.version, ) builder.buildroot_add_repos(build_deps) if not build.component_builds: log.info("There are no components in module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # Return a KojiRepoChange message so that the build can be transitioned to done # in the repos handler from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"])) return # If all components in module build will be reused, we don't have to build # module-build-macros, because there won't be any build done. if attempt_to_reuse_all_components(builder, build): log.info( "All components have been reused for module %r, skipping build" % build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() return [] log.debug("Starting build batch 1") build.batch = 1 db_session.commit() artifact_name = "module-build-macros" component_build = models.ComponentBuild.from_component_name( db_session, artifact_name, build.id) srpm = builder.get_disttag_srpm(disttag=".%s" % get_rpm_release(db_session, build), module_build=build) if not component_build: component_build = models.ComponentBuild( module_id=build.id, package=artifact_name, format="rpms", scmurl=srpm, batch=1, build_time_only=True, ) db_session.add(component_build) # Commit and refresh so that the SQLAlchemy relationships are available db_session.commit() db_session.refresh(component_build) recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") # There was no existing artifact found, so lets submit the build instead else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr elif not component_build.is_completed: # It's possible that the build succeeded in the builder but some other step failed which # caused module-build-macros to be marked as failed in MBS, so check to see if it exists # first recovered = builder.recover_orphaned_artifact(component_build) if recovered: log.info("Found an existing module-build-macros build") else: task_id, state, reason, nvr = builder.build( artifact_name=artifact_name, source=srpm) component_build.task_id = task_id component_build.state = state component_build.reason = reason component_build.nvr = nvr db_session.add(component_build) build.transition(db_session, conf, state=models.BUILD_STATES["build"]) db_session.add(build) db_session.commit() # We always have to regenerate the repository. if conf.system == "koji": log.info("Regenerating the repository") task_id = builder.koji_session.newRepo( builder.module_build_tag["name"]) build.new_repo_task_id = task_id db_session.commit() else: from module_build_service.scheduler.handlers.repos import done as repos_done_handler events.scheduler.add(repos_done_handler, ("fake_msg", builder.module_build_tag["name"]))