def test_args_from_url():
"""Arguments from URLs are passed through to registered Source as expected."""
calls = []
def gather_args(*args, **kwargs):
calls.append((args, kwargs))
return []
Source.register_backend("abc", gather_args)
Source.get("abc:key=val1&key=val2&threads=10&timeout=60&foo=bar,baz")
# It should have made a call to the registered backend with the given arguments:
assert len(calls) == 1
assert calls[0] == (
(),
{
# Repeating key is passed as a list
"key": ["val1", "val2"],
# Threads/timeout are converted to int
"threads": 10,
"timeout": 60,
# CSV are not automatically split into a list; Source class itself
# must handle this if desired
"foo": "bar,baz",
},
)
def test_reset_removes_custom_backend():
"""reset removes a custom registered backend."""
created = []
source_instance = object()
def source_factory(*args, **kwargs):
created.append(True)
return source_instance
Source.register_backend("mytest", source_factory)
# I can get an instance of it now
assert Source.get("mytest:")
assert len(created) == 1
# But if I reset...
Source.reset()
# Then I can't get it any more
with raises(SourceUrlError) as exc_info:
Source.get("mytest:")
assert len(created) == 1
assert "no registered backend 'mytest'" in str(exc_info)
def test_args_via_partial():
"""Arguments are passed into source via get_partial as expected."""
calls = []
def gather_args(*args, **kwargs):
calls.append((args, kwargs))
return []
Source.register_backend("gather", gather_args)
# Let's make a partial bound to 'a' & 'b' by default
gather = Source.get_partial("gather:a=1", b=2)
# Call it a few different ways:
gather()
gather(c=3)
gather(d=4)
# It should have been called with expected args:
# - every call had 'a' & 'b' since they were bound in get_partial
# - 'b' and 'c' only appear when explicitly passed
# - also note 'a' is a string since it came via URL
assert len(calls) == 3
assert calls[0] == ((), {"a": "1", "b": 2})
assert calls[1] == ((), {"a": "1", "b": 2, "c": 3})
assert calls[2] == ((), {"a": "1", "b": 2, "d": 4})
def test_reset_restores_overridden_backend():
"""reset restores the original backend for anything which has been
overridden."""
# I can get an instance of a built-in backend, like 'staged'.
# Note: doesn't matter that we're pointing at nonexistent directory
# (reading doesn't happen until we iterate)
Source.get("staged:/notexist")
created = []
source_instance = object()
def source_factory(*args, **kwargs):
created.append(True)
return source_instance
# Can override the built-in 'staged' backend.
Source.register_backend("staged", source_factory)
# Now it will use what I registered
assert Source.get("staged:/notexist")
assert len(created) == 1
# But if I reset...
Source.reset()
# Then I'm not getting what I registered any more, but I'm
# still getting something OK
new_src = Source.get("staged:/notexist")
assert new_src
assert new_src is not source_instance
assert len(created) == 1
def test_errata_missing_koji_rpms(fake_errata_tool):
"""Can't obtain errata if referenced RPMs are not in koji"""
class AllMissingKojiSource(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm",
state="NOTFOUND")
yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm",
state="NOTFOUND")
Source.register_backend("missingkoji", AllMissingKojiSource)
source = Source.get(
"errata:https://errata.example.com?errata=RHSA-2020:0509",
koji_source="missingkoji:",
)
with raises(Exception) as exc:
list(source)
# It should raise because an RPM referred by ET was not found in koji
assert (
"Advisory refers to sudo-1.8.25p1-4.el8_0.3.x86_64.rpm but RPM was not found in koji"
in str(exc.value))
def test_yield_timeout_reached_nodupe(mock_path_exists, mock_sleep,
container_push_item, caplog):
"""src polling/timeout logic should only happen once per item even if
multiple layers of source have been created.
"""
class TestKoji(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
yield container_push_item
mock_path_exists.return_value = False
Source.register_backend("test-koji", TestKoji)
Source.register_backend(
"test-koji-outer", Source.get_partial("test-koji:",
whatever="argument"))
source = Source.get("test-koji-outer:")
# Should be able to get the item.
assert len(list(source)) == 1
# It should mention the timeout, only once
assert caplog.text.count("is missing after 900 seconds") == 1
assert mock_path_exists.call_count == 31
assert mock_sleep.call_count == 30
def test_basic_iterable_source():
"""Basic push source returning non-class iterable can be used via with statement."""
Source.register_backend("iterable", lambda *_: ITEMS)
# Even though lists don't have enter/exit, it should be automatically wrapped here
# so that it works.
with Source.get("iterable:") as source:
assert list(source) == ITEMS
def test_get_registered_partial():
"""Can register a source obtained via get_partial, then get source using registered scheme."""
errata_example = Source.get_partial("errata:https://errata.example.com")
Source.register_backend("errata-example", errata_example)
# We should now be able to request sources using et_example scheme.
# We're just verifying that the call obtains a source, without crashing.
assert Source.get("errata-example:errata=ABC-123")
def errata_test_backend(fake_errata_tool, koji_test_backend):
# erratatest backend is errata backend pointing at kojitest and our errata testdata
bound = Source.get_partial("errata:https://errata.example.com/",
koji_source="kojitest:")
Source.register_backend("erratatest", bound)
yield
Source.reset()
def test_pre_push_no_dest(fake_controller, data_path, fake_push,
fake_state_path, command_tester):
"""Test usage of --pre-push with an RPM having no dest."""
# Sanity check that the Pulp server is, initially, empty.
client = fake_controller.client
assert list(client.search_content()) == []
# We're going to push just this one RPM.
rpm_src = os.path.join(data_path,
"staged-mixed/dest1/RPMS/walrus-5.21-1.noarch.rpm")
rpm_item = RpmPushItem(name=os.path.basename(rpm_src),
src=rpm_src,
signing_key="a1b2c3")
# Set up a pushsource backend to return just that item.
Source.register_backend("fake", lambda: [rpm_item])
compare_extra = {
"pulp.yaml": {
"filename": fake_state_path,
"normalize": hide_unit_ids,
}
}
args = [
"",
# This option enables pre-push which should avoid making content
# visible to end-users
"--pre-push",
"--source",
"fake:",
"--pulp-url",
"https://pulp.example.com/",
]
run = functools.partial(entry_point, cls=lambda: fake_push)
# It should be able to run without crashing.
command_tester.test(
run,
args,
compare_plaintext=False,
compare_jsonl=False,
compare_extra=compare_extra,
)
# command_tester will have already compared pulp state against baseline,
# but just to be explicit about it we will check here too...
units = list(client.search_content())
# It should have uploaded the one RPM
assert len(units) == 1
assert isinstance(units[0], RpmUnit)
# Only to this repo
assert units[0].repository_memberships == ["all-rpm-content"]
def koji_test_backend(fake_koji, koji_dir):
# kojitest backend is koji backend pointing at our koji testdata
bound = Source.get_partial("koji:https://koji.example.com/",
basedir=koji_dir)
Source.register_backend("kojitest", bound)
yield
Source.reset()
def load_conf(filename):
with open(filename, "rt") as f:
conf = yaml.safe_load(f)
try:
for source in conf.get("sources") or []:
name = source["name"]
url = source["url"]
Source.register_backend(name, Source.get_partial(url))
except Exception: # pylint: disable=broad-except
LOG.exception("Error loading config from %s", filename)
sys.exit(52)
def test_partial_with_url_unbound():
Source.register_backend("returns-url", ReturnsUrlSource)
# Let's say that I now overwrite this source with an argument (not url) bound
partial = Source.get_partial("returns-url:", a=123)
Source.register_backend("returns-url", partial)
# Then I should be able to obtain an instance of this source, and it
# should still stuff the path part of the below string into the 'url' arg
items = [i for i in Source.get("returns-url:/foo/bar?b=88")]
assert items == [PushItem(name="/foo/bar 123 88")]
def test_partial_with_url_bound_overwrite():
Source.register_backend("returns-url", ReturnsUrlSource)
# Let's say that I now overwrite this source and I pre-fill a URL
partial = Source.get_partial("returns-url:", url="/tmp")
Source.register_backend("returns-url", partial)
# Then I should be able to obtain an instance of this source, and
# I can still override the bound URL by passing a new one in the
# normal manner.
items = [i for i in Source.get("returns-url:/other/url?a=1&b=2")]
assert items == [PushItem(name="/other/url 1 2")]
def test_partial_with_url_bound():
Source.register_backend("returns-url", ReturnsUrlSource)
# Let's say that I now overwrite this source and I pre-fill a URL
partial = Source.get_partial("returns-url:", url="/tmp")
Source.register_backend("returns-url", partial)
# Then I should be able to obtain an instance of this source, with
# the URL coming from the value previously bound and other arguments
# still able to be overridden normally.
items = [i for i in Source.get("returns-url:b=123")]
assert items == [PushItem(name="/tmp a 123")]
def test_yield_once_file_present(mock_path_exists, mock_sleep, koji_dir,
container_push_item):
class TestKoji(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
yield container_push_item
mock_path_exists.side_effect = [False, False, False, True]
Source.register_backend("test-koji", TestKoji)
source = Source.get("test-koji:")
items = list(source)
assert len(items) == 1
assert mock_path_exists.call_count == 4
assert mock_sleep.call_count == 3
def test_load_filters():
"""Push items are filtered to supported Pulp destinations."""
ctx = Context()
phase = LoadPushItems(
ctx,
["fake:"],
allow_unsigned=True,
pre_push=False,
)
# Set up these items to be generated by pushsource.
# It simulates the ET case where some files are generated having
# both pulp repo IDs and FTP paths.
fake_items = [
FilePushItem(name="file1",
dest=["some-repo", "other-repo", "/some/path"]),
FilePushItem(name="file2", dest=["/some/path", "/other/path"]),
FilePushItem(name="file3", dest=["final-repo"]),
]
Source.register_backend("fake", lambda: fake_items)
# Let it run to completion...
with phase:
pass
# It should have succeeded
assert not ctx.has_error
# Now let's get everything from the output queue.
all_outputs = []
while True:
item = phase.out_queue.get()
if item is Phase.FINISHED:
break
all_outputs.append(item.pushsource_item)
# We should have got this:
assert all_outputs == [
# we get file1, but only repo IDs have been kept.
FilePushItem(name="file1", dest=["some-repo", "other-repo"]),
# we don't get file2 at all, since dest was filtered down to nothing.
# we get file3 exactly as it was, since no changes were needed.
FilePushItem(name="file3", dest=["final-repo"]),
]
def test_yield_timeout_reached(mock_path_exists, mock_sleep, koji_dir,
container_push_item):
class TestKoji(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
yield container_push_item
mock_path_exists.return_value = False
Source.register_backend("test-koji", TestKoji)
source = Source.get("test-koji:")
items = list(source)
assert len(items) == 1
assert mock_path_exists.call_count == 31
assert mock_sleep.call_count == 30
def test_custom_source():
"""A push source with args, custom enter/exit and used with get_partial works."""
Source.register_backend("custom-base", CustomSource)
spy = []
Source.register_backend("custom-spy",
Source.get_partial("custom-base:", spy=spy))
Source.register_backend("custom1", Source.get_partial("custom-spy:",
a=123))
Source.register_backend("custom2", Source.get_partial("custom1:", b=234))
Source.register_backend("custom3", Source.get_partial("custom2:", c=456))
# Now use the source while going through multiple layers.
with Source.get("custom3:") as source:
assert list(source) == ITEMS
# the enter/exit should propagate all the way through, just once.
assert spy == ["enter [123, 234, 456]", "exit [123, 234, 456]"]
def test_yield_no_source(mock_path_exists, mock_sleep, koji_dir):
class TestKoji(object):
def __init__(self, **kwargs):
pass
def __iter__(self):
class Object:
pass
item1 = Object()
item1.src = None
yield item1
yield Object()
Source.register_backend("test-koji", TestKoji)
source = Source.get("test-koji:")
items = list(source)
assert len(items) == 2
mock_path_exists.assert_not_called()
mock_sleep.assert_not_called()
def test_keep_prepush_no_dest_items():
"""Push item filtering keeps items with no dest if pre-pushable."""
ctx = Context()
phase = LoadPushItems(
ctx,
["fake:"],
allow_unsigned=True,
pre_push=True,
)
fake_items = [
FilePushItem(name="file", dest=["some-repo"]),
RpmPushItem(name="rpm", dest=[]),
]
Source.register_backend("fake", lambda: fake_items)
# Let it run to completion...
with phase:
pass
# It should have succeeded
assert not ctx.has_error
# Now let's get everything from the output queue.
all_outputs = []
while True:
item = phase.out_queue.get()
if item is Phase.FINISHED:
break
all_outputs.append(item.pushsource_item)
# We should have got this:
assert all_outputs == [
# get file as usual
FilePushItem(name="file", dest=["some-repo"]),
# even though this item has no destination, we still get it since rpms
# support pre-push and pre_push was enabled.
RpmPushItem(name="rpm", dest=[]),
]
def test_unsigned_failure(
fake_push,
command_tester,
caplog,
):
"""Test that a failure occurs if an unsigned RPM is encountered without
the --allow-unsigned option.
"""
Source.register_backend(
"unsigned",
lambda:
[RpmPushItem(name="quux", src="/some/unsigned.rpm", dest=["repo1"])],
)
args = [
"",
"--source",
"unsigned:",
"--pulp-url",
"https://pulp.example.com/",
]
run = functools.partial(entry_point, cls=lambda: fake_push)
# It should exit...
with pytest.raises(SystemExit) as excinfo:
command_tester.test(
run,
args,
compare_plaintext=False,
compare_jsonl=False,
)
# ...unsuccessfully
assert excinfo.value.code != 0
# And it should tell us what went wrong
assert "Unsigned content is not permitted: /some/unsigned.rpm" in caplog.text
def test_empty_push(fake_controller, fake_push, fake_state_path,
command_tester, stub_collector):
"""Test a push with no content."""
# Sanity check that the Pulp server is, initially, empty.
client = fake_controller.client
assert list(client.search_content()) == []
# Set up a pushsource backend which returns no supported items
Source.register_backend("null", lambda: [PushItem(name="quux")])
compare_extra = {
"pulp.yaml": {
"filename": fake_state_path,
"normalize": hide_unit_ids,
}
}
args = [
"",
"--source",
"null:",
"--pulp-url",
"https://pulp.example.com/",
]
run = functools.partial(entry_point, cls=lambda: fake_push)
# It should be able to run without crashing.
command_tester.test(
run,
args,
compare_plaintext=False,
compare_jsonl=False,
compare_extra=compare_extra,
)
# It should not record any push items at all.
assert not stub_collector
def test_errata_ignores_unknown_koji_types(mock_path_exists, source_factory,
koji_dir):
"""Errata source, when requesting containers, will skip unknown push item types
yielded by koji source."""
# This is a very niche case, but to get that 100% coverage...
# It's possible that koji_source might produce something other than ContainerImagePushItem
# or OperatorManifestPushItem, and we want to be forwards-compatible with that.
# This is our hacked source which returns whatever koji returns, but also some
# arbitrary objects
class WeirdKoji(object):
def __init__(self, **kwargs):
# Get a normal koji source...
self.koji = Source.get(
"koji:https://koji.example.com?basedir=%s" % koji_dir,
**kwargs)
def __iter__(self):
# We'll yield whatever koji yields but surround it with
# unexpected junk
yield object()
for item in self.koji:
yield item
yield object()
mock_path_exists.return_value = True
Source.register_backend("weird-koji", WeirdKoji)
source = source_factory(errata="RHBA-2020:2807", koji_source="weird-koji:")
# It should still work as normal
items = list(source)
# Sanity check we got the right number of items
assert len(items) == 45
def test_push_copy_fails(fake_controller, fake_nocopy_push, fake_state_path,
command_tester, caplog):
"""Test that push detects and fails in the case where a Pulp content copy
claims to succeed, but doesn't put expected content in the target repo.
While not expected to happen under normal conditions, there have historically
been a handful of Pulp bugs or operational issues which can trigger this.
"""
client = fake_controller.client
iso_dest1 = client.get_repository("iso-dest1").result()
iso_dest2 = client.get_repository("iso-dest2").result()
# Make this file exist but not in all the desired repos.
existing_file = FileUnit(
path="some-file",
sha256sum=
"db68c8a70f8383de71c107dca5fcfe53b1132186d1a6681d9ee3f4eea724fabb",
size=46,
)
fake_controller.insert_units(iso_dest1, [existing_file])
# Unit is now in iso-dest1.
# Set up a pushsource backend which requests push of the same content
# to both (iso-dest1, iso-dest2).
Source.register_backend(
"test",
lambda: [
FilePushItem(
# Note: a real push item would have to have 'src' pointing at an
# existing file here. It's OK to omit that if the checksum exactly
# matches something already in Pulp.
name="some-file",
sha256sum=
"db68c8a70f8383de71c107dca5fcfe53b1132186d1a6681d9ee3f4eea724fabb",
dest=["iso-dest1", "iso-dest2"],
)
],
)
args = [
"",
"--source",
"test:",
"--pulp-url",
"https://pulp.example.com/",
]
run = functools.partial(entry_point, cls=lambda: fake_nocopy_push)
# Ask it to push.
with pytest.raises(SystemExit) as excinfo:
command_tester.test(
run,
args,
# Can't guarantee a stable log order.
compare_plaintext=False,
compare_jsonl=False,
)
# It should have failed.
assert excinfo.value.code == 59
# It should tell us why it failed.
msg = ("Fatal error: Pulp unit not present in repo(s) iso-dest2 "
"after copy: FileUnit(path='some-file'")
assert msg in caplog.text
def test_basic_inherited_source():
"""Basic inherited push source can be used via with statement."""
Source.register_backend("basic", BasicInheritedSource)
with Source.get("basic:") as source:
assert list(source) == ITEMS
def resolve(cls):
Source.register_backend("backend2", Backend2)
def test_register_invalid():
"""Registering non-callable Source fails."""
with raises(TypeError):
Source.register_backend("foobar", "some wrong value")
