def test_reset_removes_custom_backend():
"""reset removes a custom registered backend."""
created = []
source_instance = object()
def source_factory(*args, **kwargs):
created.append(True)
return source_instance
Source.register_backend("mytest", source_factory)
# I can get an instance of it now
assert Source.get("mytest:")
assert len(created) == 1
# But if I reset...
Source.reset()
# Then I can't get it any more
with raises(SourceUrlError) as exc_info:
Source.get("mytest:")
assert len(created) == 1
assert "no registered backend 'mytest'" in str(exc_info)
def test_args_from_url():
"""Arguments from URLs are passed through to registered Source as expected."""
calls = []
def gather_args(*args, **kwargs):
calls.append((args, kwargs))
return []
Source.register_backend("abc", gather_args)
Source.get("abc:key=val1&key=val2&threads=10&timeout=60&foo=bar,baz")
# It should have made a call to the registered backend with the given arguments:
assert len(calls) == 1
assert calls[0] == (
(),
{
# Repeating key is passed as a list
"key": ["val1", "val2"],
# Threads/timeout are converted to int
"threads": 10,
"timeout": 60,
# CSV are not automatically split into a list; Source class itself
# must handle this if desired
"foo": "bar,baz",
},
)
def test_reset_restores_overridden_backend():
"""reset restores the original backend for anything which has been
overridden."""
# I can get an instance of a built-in backend, like 'staged'.
# Note: doesn't matter that we're pointing at nonexistent directory
# (reading doesn't happen until we iterate)
Source.get("staged:/notexist")
created = []
source_instance = object()
def source_factory(*args, **kwargs):
created.append(True)
return source_instance
# Can override the built-in 'staged' backend.
Source.register_backend("staged", source_factory)
# Now it will use what I registered
assert Source.get("staged:/notexist")
assert len(created) == 1
# But if I reset...
Source.reset()
# Then I'm not getting what I registered any more, but I'm
# still getting something OK
new_src = Source.get("staged:/notexist")
assert new_src
assert new_src is not source_instance
assert len(created) == 1
def test_bad_url_missing_backend():
with raises(SourceUrlError) as ex_info:
Source.get("notexist:foo=bar&baz=quux")
assert (
"Requested source 'notexist:foo=bar&baz=quux' but "
"there is no registered backend 'notexist'"
) in str(ex_info.value)
def test_source_get_invalid_items():
"""Get registry source with invalid image URIs."""
source = Source.get(
"registry:?image=registry.redhat.io/odf4/mcg-operator-bundle")
with raises(ValueError):
with source:
items = list(source)
source = Source.get(
"registry:?image=registry.redhat.io/odf4/mcg-operator-bundle@latest:latest"
)
with raises(ValueError):
with source:
items = list(source)
def test_koji_modules_filter_filename(fake_koji, koji_dir):
"""Koji source can filter modules by filename"""
source = Source.get(
"koji:https://koji.example.com/?module_build=foo-1.0-1",
basedir=koji_dir,
module_filter_filename="modulemd.x86_64.txt,modulemd.aarch64.txt",
)
fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1")
fake_koji.insert_modules(
["modulemd.x86_64.txt", "modulemd.aarch64.txt", "modulemd.s390x.txt"],
build_nvr="foo-1.0-1",
)
# Eagerly fetch
items = list(source)
# It should have returned push items for the two modules which matched filter
assert len(items) == 2
item_names = sorted([item.name for item in items])
# Should be only those two matching the filter
assert item_names == ["modulemd.aarch64.txt", "modulemd.x86_64.txt"]
def test_staged_empty_no_metadata(tmpdir):
"""Trying to use staged source of empty directory does not work"""
empty = str(tmpdir.mkdir("empty"))
source = Source.get("staged:%s" % empty)
with raises(IOError):
list(source)
def test_staged_notexist(tmpdir):
"""Trying to use staged source of nonexistent directory does not work"""
notexist = str(tmpdir.join("some/notexist/dir"))
source = Source.get("staged:%s" % notexist)
with raises(OSError):
list(source)
def test_koji_multiple_in_stream(fake_koji, koji_dir, caplog):
"""Koji source fails if referenced modulemd file contains multiple modules.
This isn't something expected to happen in practice. The point of this test is to
ensure that, if it *does* happen, we fail in a controlled manner rather than
something odd like silently using the first module in the stream.
"""
source = Source.get(
"koji:https://koji.example.com/?module_build=foo-1.0-1",
basedir=koji_dir)
fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1")
fake_koji.insert_modules(["modulemd.x86_64.txt"], build_nvr="foo-1.0-1")
# Set up a modulemd stream with multiple documents for this file.
modulemd_path = os.path.join(
koji_dir, "packages/foo/1.0/1/files/module/modulemd.x86_64.txt")
os.makedirs(os.path.dirname(modulemd_path))
shutil.copy(os.path.join(DATA_DIR, "modulemd-multiple.yaml"),
modulemd_path)
# Trying to fetch the items should fail
with raises(Exception) as exc_info:
list(source)
# We should find a relevant message from YAML parser
assert "expected a single document in the stream" in str(exc_info)
# It should have told us exactly which file in which build couldn't be parsed
expected_message = (
"In koji build foo-1.0-1, cannot load module metadata from " +
modulemd_path)
assert expected_message in caplog.messages
def test_errata_missing_koji_rpms(fake_errata_tool):
"""Can't obtain errata if referenced RPMs are not in koji"""
class AllMissingKojiSource(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm",
state="NOTFOUND")
yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm",
state="NOTFOUND")
Source.register_backend("missingkoji", AllMissingKojiSource)
source = Source.get(
"errata:https://errata.example.com?errata=RHSA-2020:0509",
koji_source="missingkoji:",
)
with raises(Exception) as exc:
list(source)
# It should raise because an RPM referred by ET was not found in koji
assert (
"Advisory refers to sudo-1.8.25p1-4.el8_0.3.x86_64.rpm but RPM was not found in koji"
in str(exc.value))
def test_yield_timeout_reached_nodupe(mock_path_exists, mock_sleep,
container_push_item, caplog):
"""src polling/timeout logic should only happen once per item even if
multiple layers of source have been created.
"""
class TestKoji(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
yield container_push_item
mock_path_exists.return_value = False
Source.register_backend("test-koji", TestKoji)
Source.register_backend(
"test-koji-outer", Source.get_partial("test-koji:",
whatever="argument"))
source = Source.get("test-koji-outer:")
# Should be able to get the item.
assert len(list(source)) == 1
# It should mention the timeout, only once
assert caplog.text.count("is missing after 900 seconds") == 1
assert mock_path_exists.call_count == 31
assert mock_sleep.call_count == 30
def test_staged_empty_with_metadata(tmpdir):
"""Trying to use staged source of empty directory with valid metadata silently yields nothing"""
empty = tmpdir.mkdir("empty")
empty.join("staged.json").write('{"header": {"version": "0.2"}}')
source = Source.get("staged:%s" % empty)
assert list(source) == []
def test_koji_bad_modulemd(fake_koji, koji_dir, caplog):
"""Koji source logs and raises exception on unparseable modulemd file"""
source = Source.get(
"koji:https://koji.example.com/?module_build=foo-1.0-1",
basedir=koji_dir)
fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1")
fake_koji.insert_modules(["modulemd.x86_64.txt", "modulemd.s390x.txt"],
build_nvr="foo-1.0-1")
# Write invalid modulemd here.
modulemd_path = os.path.join(
koji_dir, "packages/foo/1.0/1/files/module/modulemd.x86_64.txt")
os.makedirs(os.path.dirname(modulemd_path))
with open(modulemd_path, "wt") as f:
f.write("This ain't no valid modulemd")
# Trying to fetch the items should fail
with raises(Exception):
list(source)
# And it should have told us exactly which file in which build couldn't be parsed
expected_message = (
"In koji build foo-1.0-1, cannot load module metadata from " +
modulemd_path)
assert expected_message in caplog.messages
def test_get_registered_partial():
"""Can register a source obtained via get_partial, then get source using registered scheme."""
errata_example = Source.get_partial("errata:https://errata.example.com")
Source.register_backend("errata-example", errata_example)
# We should now be able to request sources using et_example scheme.
# We're just verifying that the call obtains a source, without crashing.
assert Source.get("errata-example:errata=ABC-123")
def test_koji_nobuild(fake_koji):
"""koji source referencing nonexistent build will fail"""
source = Source.get("koji:https://koji.example.com/?module_build=notexist-1.2.3")
with raises(ValueError) as exc_info:
list(source)
assert "Module build not found in koji: notexist-1.2.3" in str(exc_info.value)
def test_staged_bad_metadata(tmpdir):
"""Trying to use staged source with corrupt metadata file raises"""
tmpdir.join("staged.json").write("'oops, not valid json'")
source = Source.get("staged:%s" % tmpdir)
with raises(ValueError):
list(source)
def test_staged_no_header_metadata(tmpdir):
"""Loading a metadata file with no header will fail."""
empty = tmpdir.mkdir("empty")
empty.join("pub-mapfile.json").write("{}")
source = Source.get("staged:%s" % empty)
with raises(ValidationError) as exc_info:
list(source)
def test_basic_iterable_source():
"""Basic push source returning non-class iterable can be used via with statement."""
Source.register_backend("iterable", lambda *_: ITEMS)
# Even though lists don't have enter/exit, it should be automatically wrapped here
# so that it works.
with Source.get("iterable:") as source:
assert list(source) == ITEMS
def test_loads_entrypoints(monkeypatch):
"""Source.get ensures 'pushsource' entry points are loaded."""
# Forcibly set the Source class back to uninitialized state.
monkeypatch.setattr(Source, "_BACKENDS", {})
monkeypatch.setattr(Source, "_BACKENDS_RESET", {})
# Let's set up that some custom backends have registered entry points.
created1 = []
created2 = []
class Backend1(object):
def __init__(self):
created1.append(True)
@classmethod
def resolve(cls):
Source.register_backend("backend1", Backend1)
class Backend2(object):
def __init__(self):
created2.append(True)
@classmethod
def resolve(cls):
Source.register_backend("backend2", Backend2)
with patch("pkg_resources.iter_entry_points") as iter_ep:
iter_ep.return_value = [Backend1, Backend2]
# I should be able to get instances of those two backends
assert Source.get("backend1:")
assert Source.get("backend2:")
# It should have found them via the expected entry point group
iter_ep.assert_called_once_with("pushsource")
# Should have created one instance of each
assert created1 == [True]
assert created2 == [True]
# These should also be retained over a reset()
Source.reset()
assert Source.get("backend1:")
assert Source.get("backend2:")
def test_staged_dupe_metadata():
"""Loading a metadata file with duplicate entries will fail."""
staged_dir = os.path.join(DATADIR, "dupe_meta")
source = Source.get("staged:%s" % staged_dir)
with raises(ValueError) as exc_info:
list(source)
assert "dest1/ISOS/test.txt listed twice in staged.yaml" in str(exc_info.value)
def test_staged_incomplete_channel_dumps():
staged_dir = os.path.join(DATADIR, "incomplete_channel_dumps")
source = Source.get("staged:" + staged_dir)
with raises(ValueError) as exc_info:
list(source)
assert ("missing mandatory attribute 'channel_dump_disc_number' for "
"somedest/CHANNEL_DUMPS/myfile.txt") in str(exc_info.value)
def test_staged_channel_dumps():
staged_dir = os.path.join(DATADIR, "simple_channel_dumps")
source = Source.get("staged:" + staged_dir)
files = list(source)
files.sort(key=lambda item: item.src)
# It should load all the expected files with fields filled in by metadata
assert files == [
ChannelDumpPushItem(
name="myfile.txt",
state="PENDING",
src=os.path.join(staged_dir, "somedest/CHANNEL_DUMPS/myfile.txt"),
dest=["somedest"],
md5sum=None,
sha256sum=
"9481d6638081ff26556e09844ae1fbf680ad83fb98afa2f3f88718537b41f8b9",
origin="staged",
build=None,
signing_key=None,
description="test channel dump 1",
arch="x86_64",
eng_product_ids=[1, 2, 3],
content="some content",
datetime=datetime.datetime(2020, 2, 19, 11, 5, tzinfo=tz.tzutc()),
disk_number=1,
channels=["ch1", "ch2"],
product_name="a product",
product_version="ABCD",
type="base",
),
ChannelDumpPushItem(
name="otherfile.txt",
state="PENDING",
src=os.path.join(staged_dir,
"somedest/CHANNEL_DUMPS/otherfile.txt"),
dest=["somedest"],
md5sum=None,
sha256sum=
"64f31e7083a2bdbdefa86bbe23de536133bee35980353312ad010b3fcc6a13c4",
origin="staged",
build=None,
signing_key=None,
description="test channel dump 2",
arch="i686",
eng_product_ids=[2, 3, 4],
content="other content",
datetime=datetime.datetime(2020, 2, 19, 11, 50, tzinfo=tz.tzutc()),
disk_number=2,
channels=["ch3", "ch4"],
product_name="other product",
product_version="DEF",
type="incremental",
),
]
def test_errata_files_needs_koji_url(fake_errata_tool):
"""Can't obtain errata referring to files if koji source URL is missing"""
source = Source.get(
"errata:https://errata.example.com?errata=RHSA-2020:0509")
with raises(Exception) as exc:
list(source)
assert "A Koji source is required but none is specified" in str(exc.value)
def test_staged_no_header_metadata(tmpdir):
"""Loading a metadata file with no header will fail."""
empty = tmpdir.mkdir("empty")
empty.join("pub-mapfile.json").write("{}")
source = Source.get("staged:%s" % empty)
with raises(ValueError) as exc_info:
list(source)
assert "pub-mapfile.json has unsupported version" in str(exc_info.value)
def test_koji_connect_error():
"""Source raises a reasonable error if server can't be contacted"""
# Note: fake_koji fixture not used here, so this will really try to connect
source = Source.get("koji:https://localhost:1234/this-aint-koji")
with raises(RuntimeError) as exc_info:
list(source)
assert (
"Communication error with koji at https://localhost:1234/this-aint-koji"
in str(exc_info.value)
)
def test_partial_with_url_unbound():
Source.register_backend("returns-url", ReturnsUrlSource)
# Let's say that I now overwrite this source with an argument (not url) bound
partial = Source.get_partial("returns-url:", a=123)
Source.register_backend("returns-url", partial)
# Then I should be able to obtain an instance of this source, and it
# should still stuff the path part of the below string into the 'url' arg
items = [i for i in Source.get("returns-url:/foo/bar?b=88")]
assert items == [PushItem(name="/foo/bar 123 88")]
def test_koji_rpms(fake_koji, koji_dir):
"""Koji source yields requested RPMs"""
source = Source.get(
"koji:https://koji.example.com/?rpm=foo-1.0-1.x86_64.rpm,notfound-2.0-1.noarch.rpm",
basedir=koji_dir,
)
# It should not have done anything yet (lazy loading)
assert not fake_koji.last_url
# Insert some data
fake_koji.rpm_data["foo-1.0-1.x86_64.rpm"] = {
"arch": "x86_64",
"name": "foo",
"version": "1.0",
"release": "1",
"build_id": 1234,
}
fake_koji.build_data[1234] = {
"id": 1234,
"name": "foobuild",
"version": "1.0",
"release": "1.el8",
"nvr": "foobuild-1.0-1.el8",
"volume_name": "somevol",
}
# Eagerly fetch
items = list(source)
# It should have constructed a session for the given URL
assert fake_koji.last_url == "https://koji.example.com/"
# It should have returned push items for the two RPMs
assert len(items) == 2
items = sorted(items, key=lambda pi: pi.name)
# For present RPM, a push item should be yielded using the koji metadata.
assert items[0] == RpmPushItem(
name="foo-1.0-1.x86_64.rpm",
state="PENDING",
src=
"%s/vol/somevol/packages/foobuild/1.0/1.el8/x86_64/foo-1.0-1.x86_64.rpm"
% koji_dir,
build="foobuild-1.0-1.el8",
)
# For missing RPMs, a push item should be yielded with NOTFOUND state.
assert items[1] == RpmPushItem(name="notfound-2.0-1.noarch.rpm",
state="NOTFOUND")
def test_koji_exceptions(fake_koji):
"""Exceptions raised during calls to koji are propagated"""
source = Source.get("koji:https://koji.example.com/?module_build=error-1.2.3")
error = RuntimeError("simulated error")
fake_koji.build_data["error-1.2.3"] = error
with raises(Exception) as exc_info:
list(source)
# It should have propagated *exactly* the exception from koji
assert exc_info.value is error
def test_staged_nometa_channel_dumps(tmpdir):
staged_dir = tmpdir.mkdir("staged")
staged_dir.mkdir("dest").mkdir("CHANNEL_DUMPS").join("testfile").write(
"test")
staged_dir.join("staged.json").write('{"header": {"version": "0.2"}}')
source = Source.get("staged:%s" % staged_dir)
with raises(ValueError) as exc_info:
list(source)
assert "staged.json doesn't contain data for dest/CHANNEL_DUMPS/testfile" in str(
exc_info.value)
def test_staged_file_with_no_metadata(tmpdir):
"""Trying to use staged source where a staged file is missing metadata will raise"""
staged = tmpdir.mkdir("staged")
staged.join("staged.json").write('{"header": {"version": "0.2"}}')
staged.mkdir("dest").mkdir("FILES").join("some-file").write("some-content")
source = Source.get("staged:%s" % staged)
with raises(ValueError) as exc_info:
list(source)
assert "No metadata available for dest/FILES/some-file in staged.json" in str(
exc_info.value)
