def upgrade(): with op.batch_alter_table('module_builds', schema=None) as batch_op: batch_op.add_column(sa.Column('stream_version', sa.Integer())) op.create_table( 'module_builds_to_module_buildrequires', sa.Column('module_id', sa.Integer(), nullable=False), sa.Column('module_buildrequire_id', sa.Integer(), nullable=False), sa.ForeignKeyConstraint(['module_buildrequire_id'], ['module_builds.id']), sa.ForeignKeyConstraint(['module_id'], ['module_builds.id']), sa.UniqueConstraint('module_id', 'module_buildrequire_id') ) connection = op.get_bind() # Create all the base module buildrequire entries for build in connection.execute(mb.select()): if not build.modulemd: # If the modulemd is empty, skip this build continue brs = None try: mmd = Modulemd.ModuleStream.read_string(build.modulemd, True) mmd = mmd.upgrade(Modulemd.ModuleStreamVersionEnum.TWO) brs = mmd.get_xmd()['mbs']['buildrequires'] except Exception: # If the modulemd isn't parseable then skip this build continue for base_module in conf.base_module_names: base_module_dict = brs.get(base_module) if not base_module_dict: # If this base module isn't a buildrequire, continue to see if the next one is continue select = mb.select()\ .where(mb.c.name == base_module)\ .where(mb.c.stream == base_module_dict['stream'])\ .where(mb.c.version == base_module_dict['version'])\ .where(mb.c.context == base_module_dict.get('context')) br = connection.execute(select).fetchone() if not br: # If the buildrequire isn't in the datbase, then skip it continue connection.execute(mb_to_mbr.insert().values( module_id=build.id, module_buildrequire_id=br.id )) for base_module in conf.base_module_names: for build in connection.execute(mb.select().where(mb.c.name == base_module)): stream_version = ModuleBuild.get_stream_version(build.stream) if not stream_version: # If a stream version isn't parseable, then skip it continue connection.execute(mb.update().where(mb.c.id == build.id).values( stream_version=stream_version))
def make_module( nsvc, dependencies=None, base_module=None, filtered_rpms=None, xmd=None, store_to_db=False, virtual_streams=None, arches=None, ): """ Creates new models.ModuleBuild defined by `nsvc` string with requires and buildrequires set according to ``requires_list`` and ``build_requires_list``. :param str nsvc: name:stream:version:context of a module. :param dependencies: list of groups of dependencies (requires and buildrequires). For example, [ {"requires": {"platform": ["f30"]}, "buildrequires": {"platform": ["f30"]}}, ... ] :type dependencies: list[dict] :param base_module: a base module build required by the new module created. :type base_module: :class:`ModuleBuild` :param filtered_rpms: list of filtered RPMs which are added to filter section in module metadata. :type filtered_rpms: list[str] :param dict xmd: a mapping representing XMD section in module metadata. A custom xmd could be passed for testing a particular scenario and some default key/value pairs are added if not present. :param bool store_to_db: whether to store created module metadata to the database. If set to True, ``db_session`` is required. :param virtual_streams: List of virtual streams provided by this module. If set, This requires ``db_session`` and ``store_to_db`` to be set to a session object and True. :type virtual_streams: list[str] :param arches: List of architectures this module is built against. If set to None, ``["x86_64"]`` is used as a default. If set, likewise ``virtual_stream``. :type arches: list[str] :return: New Module Build if set to store module metadata to database, otherwise the module metadata is returned. :rtype: ModuleBuild or Modulemd.Module """ if store_to_db: assert db_session is not None if base_module: assert db_session is not None if virtual_streams: assert store_to_db if arches: assert store_to_db nsvc_regex = re.compile(r"([^:]+):([^:]+):([^:]+)(?::(.+))?") match = nsvc_regex.match(nsvc) if not match: raise ValueError('Argument nsvc is not in format N:S:V or N:S:V:C') name, stream, version, context = match.groups() mmd = Modulemd.ModuleStreamV2.new(name, stream) mmd.set_version(int(version)) if context: mmd.set_context(context) mmd.set_summary("foo") # Test unicode in mmd. mmd.set_description(u"foo \u2019s") mmd.add_module_license("GPL") if filtered_rpms: for rpm in filtered_rpms: mmd.add_rpm_filter(rpm) def _add_require(mmd_deps, require_type, name, streams): assert isinstance(mmd_deps, Modulemd.Dependencies) assert require_type in ("requires", "buildrequires") assert isinstance(streams, (list, tuple)) if require_type == "requires": add_stream = mmd_deps.add_runtime_stream set_empty_deps = mmd_deps.set_empty_runtime_dependencies_for_module else: add_stream = mmd_deps.add_buildtime_stream set_empty_deps = mmd_deps.set_empty_buildtime_dependencies_for_module for stream in streams: add_stream(name, stream) else: set_empty_deps(name) for dep_group in dependencies or []: mmd_deps = Modulemd.Dependencies() # A deps could be {"platform": ["f30"], "python": []} for require_type, deps in dep_group.items(): for req_name, req_streams in deps.items(): _add_require(mmd_deps, require_type, req_name, req_streams) mmd.add_dependencies(mmd_deps) # Caller could pass whole xmd including mbs, but if something is missing, # default values are given here. xmd = xmd or {"mbs": {}} xmd_mbs = xmd["mbs"] if "buildrequires" not in xmd_mbs: xmd_mbs["buildrequires"] = {} if "requires" not in xmd_mbs: xmd_mbs["requires"] = {} if "commit" not in xmd_mbs: xmd_mbs["commit"] = "ref_%s" % context if "mse" not in xmd_mbs: xmd_mbs["mse"] = "true" if virtual_streams: xmd_mbs["virtual_streams"] = virtual_streams mmd.set_xmd(xmd) if not store_to_db: return mmd module_build = ModuleBuild( name=name, stream=stream, stream_version=ModuleBuild.get_stream_version(stream), version=version, context=context, state=BUILD_STATES["ready"], scmurl="https://src.stg.fedoraproject.org/modules/unused.git?#ff1ea79", batch=1, owner="Tom Brady", time_submitted=datetime(2017, 2, 15, 16, 8, 18), time_modified=datetime(2017, 2, 15, 16, 19, 35), rebuild_strategy="changed-and-after", build_context=context, runtime_context=context, modulemd=mmd_to_str(mmd), koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None, ) if base_module: module_build.buildrequires.append(base_module) db_session.add(module_build) db_session.commit() if virtual_streams: for virtual_stream in virtual_streams: vs_obj = db_session.query(VirtualStream).filter_by( name=virtual_stream).first() if not vs_obj: vs_obj = VirtualStream(name=virtual_stream) db_session.add(vs_obj) db_session.commit() if vs_obj not in module_build.virtual_streams: module_build.virtual_streams.append(vs_obj) db_session.commit() for arch in arches or ["x86_64"]: arch_obj = db_session.query(ModuleArch).filter_by(name=arch).first() if not arch_obj: arch_obj = ModuleArch(name=arch) db_session.add(arch_obj) db_session.commit() if arch_obj not in module_build.arches: module_build.arches.append(arch_obj) db_session.commit() return module_build