def test_get_module_build_arches_with_unknown_tag(self, ClientSession):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
session = ClientSession.return_value
session.getTag.return_value = None
with pytest.raises(ValueError, match="Unknown Koji tag .*"):
KojiModuleBuilder.get_module_build_arches(module_build)
def test_generate_koji_tag_for_scratch_build():
tag = KojiModuleBuilder.generate_koji_tag("testmodule",
"master",
"20170816080815",
"37c6c57",
scratch=True)
assert tag == "scrmod-testmodule-master-20170816080815-37c6c57+0"
tag = KojiModuleBuilder.generate_koji_tag("testmodule",
"master",
"20170816080815",
"37c6c57",
scratch=True,
scratch_id=1000)
assert tag == "scrmod-testmodule-master-20170816080815-37c6c57+1000"
def test_ursine_rpms_are_added(self):
self._build_srpm()
with open(self.spec_file, "r") as f:
content = f.read()
# Stream collision ursine RPMs
assert "# modulefoo-s-v-c\n" in content
for nevr in ["foo-0:1.0-1.fc28", "bar-0:2.0-1.fc28"]:
assert KojiModuleBuilder.format_conflicts_line(
nevr) + "\n" in content
# Conflicting ursine RPMs
for nevr in ["pizza-0:4.0-1.fc32", "spaghetti-0:3.0-1.fc32"]:
assert KojiModuleBuilder.format_conflicts_line(
nevr) + "\n" in content
def _build_srpm(self, execute_cmd, mkdtemp):
module_build = make_module_in_db(
"{name}:{stream}:{version}:{context}".format(**self.module_nsvc),
xmd=self.xmd)
mkdtemp.return_value = self.tmp_srpm_build_dir
return KojiModuleBuilder.get_disttag_srpm("disttag", module_build)
def test_generate_koji_tag_in_nsvc_format():
name, stream, version, context = ("testmodule", "master", "20170816080815",
"37c6c57")
tag = KojiModuleBuilder.generate_koji_tag(name, stream, version, context)
assert tag == "module-testmodule-master-20170816080815-37c6c57"
def test_get_build_weights_no_build(self, ClientSession):
session = ClientSession.return_value
session.getLoggedInUser.return_value = {"id": 123}
session.multiCall.side_effect = [
# getPackageID response
[[1], [2]],
# listBuilds response
[[[{
"task_id": 456
}]], [[]]],
# getTaskDescendents response
[[{
"1": [],
"2": [],
"3": [{
"weight": 1.0
}, {
"weight": 1.0
}]
}]],
]
session.getAverageBuildDuration.return_value = None
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
assert weights == {"httpd": 2, "apr": 1.5}
expected_calls = [mock.call(456)]
assert session.getTaskDescendents.mock_calls == expected_calls
session.krb_login.assert_called_once()
def test_get_build_weights_listBuilds_failed(self, ClientSession):
session = ClientSession.return_value
session.getLoggedInUser.return_value = {"id": 123}
session.multiCall.side_effect = [[[1], [2]], []]
session.getAverageBuildDuration.return_value = None
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
assert weights == {"httpd": 1.5, "apr": 1.5}
expected_calls = [
mock.call(packageID=1,
userID=123,
state=1,
queryOpts={
"limit": 1,
"order": "-build_id"
}),
mock.call(packageID=2,
userID=123,
state=1,
queryOpts={
"limit": 1,
"order": "-build_id"
}),
]
assert session.listBuilds.mock_calls == expected_calls
session.krb_login.assert_called_once()
def test_get_module_build_arches_with_archless_tag(self, ClientSession):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
session = ClientSession.return_value
session.getTag.return_value = {"arches": ""}
ret = KojiModuleBuilder.get_module_build_arches(module_build)
assert ret == []
def test_filtered_rpms_are_added(self):
self._build_srpm()
with open(self.spec_file, "r") as f:
content = f.read()
for nevr in ["baz-devel-0:0.1-6.fc28", "baz-doc-0:0.1-6.fc28"]:
assert KojiModuleBuilder.format_conflicts_line(
nevr) + "\n" in content
def test_get_module_build_arches(self, ClientSession):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
arches = "x86_64 i686 ppc64le aarch64 s390x"
session = ClientSession.return_value
session.getTag.return_value = {"arches": arches}
ret = KojiModuleBuilder.get_module_build_arches(module_build)
assert " ".join(ret) == arches
def test_generate_koji_tag_in_hash_format():
name, version, context = ("testmodule", "20170816080815", "37c6c57")
stream = "this-is-a-stream-with-very-looooong-name" + "-blah" * 50
nsvc_list = [name, stream, version, context]
tag = KojiModuleBuilder.generate_koji_tag(*nsvc_list)
expected_tag = "module-1cf457d452e54dda"
assert tag == expected_tag
def test_get_module_build_arches_without_tag(self, ClientSession):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
module_build.koji_tag = None
session = ClientSession.return_value
ret = KojiModuleBuilder.get_module_build_arches(module_build)
assert ret == []
session.getTag.assert_not_called()
session.assert_not_called()
def test_get_built_rpms_in_module_build(self, ClientSession):
session = ClientSession.return_value
session.listTaggedRPMS.return_value = (
[
{
"build_id": 735939,
"name": "tar",
"extra": None,
"arch": "ppc64le",
"buildtime": 1533299221,
"id": 6021394,
"epoch": 2,
"version": "1.30",
"metadata_only": False,
"release": "4.el8+1308+551bfa71",
"buildroot_id": 4321122,
"payloadhash": "0621ab2091256d21c47dcac868e7fc2a",
"size": 878684,
},
{
"build_id": 735939,
"name": "bar",
"extra": None,
"arch": "ppc64le",
"buildtime": 1533299221,
"id": 6021394,
"epoch": 2,
"version": "1.30",
"metadata_only": False,
"release": "4.el8+1308+551bfa71",
"buildroot_id": 4321122,
"payloadhash": "0621ab2091256d21c47dcac868e7fc2a",
"size": 878684,
},
],
[],
)
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
# Module builds generated by init_data uses generic modulemd file and
# the module's name/stream/version/context does not have to match it.
# But for this test, we need it to match.
mmd = module_build.mmd()
module_build.name = mmd.get_module_name()
module_build.stream = mmd.get_stream_name()
module_build.version = mmd.get_version()
module_build.context = mmd.get_context()
db_session.commit()
ret = KojiModuleBuilder.get_built_rpms_in_module_build(mmd)
assert set(ret) == {
"bar-2:1.30-4.el8+1308+551bfa71", "tar-2:1.30-4.el8+1308+551bfa71"
}
session.assert_not_called()
def test_get_filtered_rpms_on_self_dep(ClientSession, br_filtered_rpms,
expected):
session = ClientSession.return_value
session.listTaggedRPMS.return_value = (
[
{
"build_id": 12345,
"epoch": None,
"name": "perl-Tangerine",
"release": "1.module+0+d027b723",
"version": "0.23",
},
{
"build_id": 23456,
"epoch": None,
"name": "perl-List-Compare",
"release": "5.module+0+d027b723",
"version": "0.53",
},
{
"build_id": 34567,
"epoch": None,
"name": "tangerine",
"release": "3.module+0+d027b723",
"version": "0.22",
},
],
[
{
"build_id": 12345,
"name": "perl-Tangerine",
"nvr": "perl-Tangerine-0.23-1.module+0+d027b723",
},
{
"build_id": 23456,
"name": "perl-List-Compare",
"nvr": "perl-List-Compare-0.53-5.module+0+d027b723",
},
{
"build_id": 34567,
"name": "tangerine",
"nvr": "tangerine-0.22-3.module+0+d027b723",
},
],
)
current_module = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 3)
with patch.object(module_build_service.common.models.ModuleBuild,
'log_message'):
rv = KojiModuleBuilder._get_filtered_rpms_on_self_dep(
current_module, br_filtered_rpms)
assert set(rv) == set(expected)
session.assert_not_called()
def test_get_build_weights_getPackageID_failed(self, ClientSession):
session = ClientSession.return_value
session.getLoggedInUser.return_value = {"id": 123}
session.multiCall.side_effect = [[], []]
session.getAverageBuildDuration.return_value = None
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
assert weights == {"httpd": 1.5, "apr": 1.5}
expected_calls = [mock.call("httpd"), mock.call("apr")]
assert session.getPackageID.mock_calls == expected_calls
session.krb_login.assert_called_once()
def generate_module_build_koji_tag(build):
"""Used by wait handler to get module build koji tag
:param build: a module build.
:type build: :class:`ModuleBuild`
:return: generated koji tag.
:rtype: str
"""
log.info("Getting tag for %s:%s:%s", build.name, build.stream,
build.version)
if conf.system in ["koji", "test"]:
return KojiModuleBuilder.generate_koji_tag(
build.name,
build.stream,
build.version,
build.context,
scratch=build.scratch,
scratch_id=build.id,
)
else:
return "-".join(["module", build.name, build.stream, build.version])
def test_ensure_builder_use_a_logged_in_koji_session(self, ClientSession):
module_build = module_build_service.common.models.ModuleBuild.get_by_id(
db_session, 2)
builder = KojiModuleBuilder(db_session, "owner", module_build, conf,
"module-tag", [])
builder.koji_session.krb_login.assert_called_once()
def test_get_build_weights_getLoggedInUser_failed(self, ClientSession):
session = ClientSession.return_value
session.getAverageBuildDuration.return_value = None
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
assert weights == {"httpd": 1.5, "apr": 1.5}
session.krb_login.assert_called_once()
def get_disttag_srpm(disttag, module_build):
# @FIXME
return KojiModuleBuilder.get_disttag_srpm(disttag, module_build)
def filter_module_builds(flask_request):
"""
Returns a flask_sqlalchemy.Pagination object based on the request parameters
:param request: Flask request object
:return: flask_sqlalchemy.Pagination
"""
search_query = dict()
special_columns = {
"time_submitted",
"time_modified",
"time_completed",
"state",
"stream_version_lte",
"virtual_stream",
}
columns = models.ModuleBuild.__table__.columns.keys()
for key in set(request.args.keys()) - special_columns:
# Only filter on valid database columns but skip columns that are treated specially or
# ignored
if key in columns:
search_query[key] = flask_request.args[key]
# Multiple states can be supplied => or-ing will take place
states = flask_request.args.getlist("state")
search_states = []
for state in states:
if state.isdigit():
search_states.append(state)
else:
if state in models.BUILD_STATES:
search_states.append(models.BUILD_STATES[state])
else:
raise ValidationError("Invalid state was supplied: %s" % state)
nsvc = flask_request.args.get("nsvc", None)
if nsvc:
nsvc_parts = nsvc.split(":")
query_keys = ["name", "stream", "version", "context"]
for key, part in zip(query_keys, nsvc_parts):
search_query[key] = part
rpm = flask_request.args.get("rpm", None)
koji_tags = []
if rpm:
if conf.system == "koji":
# we are importing the koji builder here so we can search for the rpm metadata
# from koji. If we imported this regulary we would have gotten a circular import error.
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder # noqa
koji_tags = KojiModuleBuilder.get_rpm_module_tag(rpm)
else:
raise ValidationError(
"Configured builder does not allow to search by rpm binary name!"
)
query = models.ModuleBuild.query
if search_query:
query = query.filter_by(**search_query)
if search_states:
query = query.filter(models.ModuleBuild.state.in_(search_states))
if koji_tags:
query = query.filter(
models.ModuleBuild.koji_tag.in_(koji_tags)).filter_by(
**search_query)
# This is used when filtering the date request parameters, but it is here to avoid recompiling
utc_iso_datetime_regex = re.compile(
r"^(?P<datetime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?(?:Z|[-+]00(?::00)?)?$"
)
# Filter the query based on date request parameters
for item in ("submitted", "modified", "completed"):
for context in ("before", "after"):
request_arg = "%s_%s" % (item, context) # i.e. submitted_before
iso_datetime_arg = request.args.get(request_arg, None)
if iso_datetime_arg:
iso_datetime_matches = re.match(utc_iso_datetime_regex,
iso_datetime_arg)
if not iso_datetime_matches or not iso_datetime_matches.group(
"datetime"):
raise ValidationError(
'An invalid Zulu ISO 8601 timestamp was provided for the "%s" parameter'
% request_arg)
# Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter
item_datetime = datetime.strptime(
iso_datetime_matches.group("datetime"),
"%Y-%m-%dT%H:%M:%S")
# Get the database column to filter against
column = getattr(models.ModuleBuild, "time_" + item)
if context == "after":
query = query.filter(column >= item_datetime)
elif context == "before":
query = query.filter(column <= item_datetime)
# Multiple virtual_streams can be supplied for "or" logic filtering
virtual_streams = flask_request.args.getlist("virtual_stream")
query = models.ModuleBuild._add_virtual_streams_filter(
db.session, query, virtual_streams)
stream_version_lte = flask_request.args.get("stream_version_lte")
if stream_version_lte is not None:
invalid_error = (
"An invalid value of stream_version_lte was provided. It must be an "
"integer or float greater than or equal to 10000.")
try:
stream_version_lte = float(stream_version_lte)
except (TypeError, ValueError):
raise ValidationError(invalid_error)
if stream_version_lte < 10000:
raise ValidationError(invalid_error)
query = models.ModuleBuild._add_stream_version_lte_filter(
db.session, query, stream_version_lte)
br_joined = False
module_br_alias = None
for item in (
"base_module_br",
"name",
"stream",
"version",
"context",
"stream_version",
"stream_version_lte",
"stream_version_gte",
):
if item == "base_module_br":
request_arg_name = item
else:
request_arg_name = "base_module_br_{}".format(item)
request_arg = flask_request.args.get(request_arg_name)
if not request_arg:
continue
if not br_joined:
module_br_alias = aliased(models.ModuleBuild, name="module_br")
# Shorten this table name for clarity in the query below
mb_to_br = models.module_builds_to_module_buildrequires
# The following joins get added:
# JOIN module_builds_to_module_buildrequires
# ON module_builds_to_module_buildrequires.module_id = module_builds.id
# JOIN module_builds AS module_br
# ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id
query = query.join(
mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join(
module_br_alias,
mb_to_br.c.module_buildrequire_id == module_br_alias.id)
br_joined = True
if item == "base_module_br":
try:
name, stream, version, context = flask_request.args[
"base_module_br"].split(":")
except ValueError:
raise ValidationError(
'The filter argument for "base_module_br" must be in the format of N:S:V:C'
)
query = query.filter(
module_br_alias.name == name,
module_br_alias.stream == stream,
module_br_alias.version == version,
module_br_alias.context == context,
)
elif item.endswith("_lte"):
column = getattr(module_br_alias, item[:-4])
query = query.filter(column <= request_arg)
elif item.endswith("_gte"):
column = getattr(module_br_alias, item[:-4])
query = query.filter(column >= request_arg)
else:
column = getattr(module_br_alias, item)
query = query.filter(column == request_arg)
query = _add_order_by_clause(flask_request, query, models.ModuleBuild)
page = flask_request.args.get("page", 1, type=int)
per_page = flask_request.args.get("per_page", 10, type=int)
return query.paginate(page, per_page, False)