def test_args_from_url(): """Arguments from URLs are passed through to registered Source as expected.""" calls = [] def gather_args(*args, **kwargs): calls.append((args, kwargs)) return [] Source.register_backend("abc", gather_args) Source.get("abc:key=val1&key=val2&threads=10&timeout=60&foo=bar,baz") # It should have made a call to the registered backend with the given arguments: assert len(calls) == 1 assert calls[0] == ( (), { # Repeating key is passed as a list "key": ["val1", "val2"], # Threads/timeout are converted to int "threads": 10, "timeout": 60, # CSV are not automatically split into a list; Source class itself # must handle this if desired "foo": "bar,baz", }, )
def test_reset_removes_custom_backend(): """reset removes a custom registered backend.""" created = [] source_instance = object() def source_factory(*args, **kwargs): created.append(True) return source_instance Source.register_backend("mytest", source_factory) # I can get an instance of it now assert Source.get("mytest:") assert len(created) == 1 # But if I reset... Source.reset() # Then I can't get it any more with raises(SourceUrlError) as exc_info: Source.get("mytest:") assert len(created) == 1 assert "no registered backend 'mytest'" in str(exc_info)
def test_args_via_partial(): """Arguments are passed into source via get_partial as expected.""" calls = [] def gather_args(*args, **kwargs): calls.append((args, kwargs)) return [] Source.register_backend("gather", gather_args) # Let's make a partial bound to 'a' & 'b' by default gather = Source.get_partial("gather:a=1", b=2) # Call it a few different ways: gather() gather(c=3) gather(d=4) # It should have been called with expected args: # - every call had 'a' & 'b' since they were bound in get_partial # - 'b' and 'c' only appear when explicitly passed # - also note 'a' is a string since it came via URL assert len(calls) == 3 assert calls[0] == ((), {"a": "1", "b": 2}) assert calls[1] == ((), {"a": "1", "b": 2, "c": 3}) assert calls[2] == ((), {"a": "1", "b": 2, "d": 4})
def test_reset_restores_overridden_backend(): """reset restores the original backend for anything which has been overridden.""" # I can get an instance of a built-in backend, like 'staged'. # Note: doesn't matter that we're pointing at nonexistent directory # (reading doesn't happen until we iterate) Source.get("staged:/notexist") created = [] source_instance = object() def source_factory(*args, **kwargs): created.append(True) return source_instance # Can override the built-in 'staged' backend. Source.register_backend("staged", source_factory) # Now it will use what I registered assert Source.get("staged:/notexist") assert len(created) == 1 # But if I reset... Source.reset() # Then I'm not getting what I registered any more, but I'm # still getting something OK new_src = Source.get("staged:/notexist") assert new_src assert new_src is not source_instance assert len(created) == 1
def test_errata_missing_koji_rpms(fake_errata_tool): """Can't obtain errata if referenced RPMs are not in koji""" class AllMissingKojiSource(object): def __init__(self, **kwargs): pass def __iter__(self): yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm", state="NOTFOUND") yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm", state="NOTFOUND") Source.register_backend("missingkoji", AllMissingKojiSource) source = Source.get( "errata:https://errata.example.com?errata=RHSA-2020:0509", koji_source="missingkoji:", ) with raises(Exception) as exc: list(source) # It should raise because an RPM referred by ET was not found in koji assert ( "Advisory refers to sudo-1.8.25p1-4.el8_0.3.x86_64.rpm but RPM was not found in koji" in str(exc.value))
def test_yield_timeout_reached_nodupe(mock_path_exists, mock_sleep, container_push_item, caplog): """src polling/timeout logic should only happen once per item even if multiple layers of source have been created. """ class TestKoji(object): def __init__(self, **kwargs): pass def __iter__(self): yield container_push_item mock_path_exists.return_value = False Source.register_backend("test-koji", TestKoji) Source.register_backend( "test-koji-outer", Source.get_partial("test-koji:", whatever="argument")) source = Source.get("test-koji-outer:") # Should be able to get the item. assert len(list(source)) == 1 # It should mention the timeout, only once assert caplog.text.count("is missing after 900 seconds") == 1 assert mock_path_exists.call_count == 31 assert mock_sleep.call_count == 30
def test_basic_iterable_source(): """Basic push source returning non-class iterable can be used via with statement.""" Source.register_backend("iterable", lambda *_: ITEMS) # Even though lists don't have enter/exit, it should be automatically wrapped here # so that it works. with Source.get("iterable:") as source: assert list(source) == ITEMS
def test_get_registered_partial(): """Can register a source obtained via get_partial, then get source using registered scheme.""" errata_example = Source.get_partial("errata:https://errata.example.com") Source.register_backend("errata-example", errata_example) # We should now be able to request sources using et_example scheme. # We're just verifying that the call obtains a source, without crashing. assert Source.get("errata-example:errata=ABC-123")
def errata_test_backend(fake_errata_tool, koji_test_backend): # erratatest backend is errata backend pointing at kojitest and our errata testdata bound = Source.get_partial("errata:https://errata.example.com/", koji_source="kojitest:") Source.register_backend("erratatest", bound) yield Source.reset()
def test_pre_push_no_dest(fake_controller, data_path, fake_push, fake_state_path, command_tester): """Test usage of --pre-push with an RPM having no dest.""" # Sanity check that the Pulp server is, initially, empty. client = fake_controller.client assert list(client.search_content()) == [] # We're going to push just this one RPM. rpm_src = os.path.join(data_path, "staged-mixed/dest1/RPMS/walrus-5.21-1.noarch.rpm") rpm_item = RpmPushItem(name=os.path.basename(rpm_src), src=rpm_src, signing_key="a1b2c3") # Set up a pushsource backend to return just that item. Source.register_backend("fake", lambda: [rpm_item]) compare_extra = { "pulp.yaml": { "filename": fake_state_path, "normalize": hide_unit_ids, } } args = [ "", # This option enables pre-push which should avoid making content # visible to end-users "--pre-push", "--source", "fake:", "--pulp-url", "https://pulp.example.com/", ] run = functools.partial(entry_point, cls=lambda: fake_push) # It should be able to run without crashing. command_tester.test( run, args, compare_plaintext=False, compare_jsonl=False, compare_extra=compare_extra, ) # command_tester will have already compared pulp state against baseline, # but just to be explicit about it we will check here too... units = list(client.search_content()) # It should have uploaded the one RPM assert len(units) == 1 assert isinstance(units[0], RpmUnit) # Only to this repo assert units[0].repository_memberships == ["all-rpm-content"]
def koji_test_backend(fake_koji, koji_dir): # kojitest backend is koji backend pointing at our koji testdata bound = Source.get_partial("koji:https://koji.example.com/", basedir=koji_dir) Source.register_backend("kojitest", bound) yield Source.reset()
def load_conf(filename): with open(filename, "rt") as f: conf = yaml.safe_load(f) try: for source in conf.get("sources") or []: name = source["name"] url = source["url"] Source.register_backend(name, Source.get_partial(url)) except Exception: # pylint: disable=broad-except LOG.exception("Error loading config from %s", filename) sys.exit(52)
def test_partial_with_url_unbound(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source with an argument (not url) bound partial = Source.get_partial("returns-url:", a=123) Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, and it # should still stuff the path part of the below string into the 'url' arg items = [i for i in Source.get("returns-url:/foo/bar?b=88")] assert items == [PushItem(name="/foo/bar 123 88")]
def test_partial_with_url_bound_overwrite(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source and I pre-fill a URL partial = Source.get_partial("returns-url:", url="/tmp") Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, and # I can still override the bound URL by passing a new one in the # normal manner. items = [i for i in Source.get("returns-url:/other/url?a=1&b=2")] assert items == [PushItem(name="/other/url 1 2")]
def test_partial_with_url_bound(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source and I pre-fill a URL partial = Source.get_partial("returns-url:", url="/tmp") Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, with # the URL coming from the value previously bound and other arguments # still able to be overridden normally. items = [i for i in Source.get("returns-url:b=123")] assert items == [PushItem(name="/tmp a 123")]
def test_yield_once_file_present(mock_path_exists, mock_sleep, koji_dir, container_push_item): class TestKoji(object): def __init__(self, **kwargs): pass def __iter__(self): yield container_push_item mock_path_exists.side_effect = [False, False, False, True] Source.register_backend("test-koji", TestKoji) source = Source.get("test-koji:") items = list(source) assert len(items) == 1 assert mock_path_exists.call_count == 4 assert mock_sleep.call_count == 3
def test_load_filters(): """Push items are filtered to supported Pulp destinations.""" ctx = Context() phase = LoadPushItems( ctx, ["fake:"], allow_unsigned=True, pre_push=False, ) # Set up these items to be generated by pushsource. # It simulates the ET case where some files are generated having # both pulp repo IDs and FTP paths. fake_items = [ FilePushItem(name="file1", dest=["some-repo", "other-repo", "/some/path"]), FilePushItem(name="file2", dest=["/some/path", "/other/path"]), FilePushItem(name="file3", dest=["final-repo"]), ] Source.register_backend("fake", lambda: fake_items) # Let it run to completion... with phase: pass # It should have succeeded assert not ctx.has_error # Now let's get everything from the output queue. all_outputs = [] while True: item = phase.out_queue.get() if item is Phase.FINISHED: break all_outputs.append(item.pushsource_item) # We should have got this: assert all_outputs == [ # we get file1, but only repo IDs have been kept. FilePushItem(name="file1", dest=["some-repo", "other-repo"]), # we don't get file2 at all, since dest was filtered down to nothing. # we get file3 exactly as it was, since no changes were needed. FilePushItem(name="file3", dest=["final-repo"]), ]
def test_yield_timeout_reached(mock_path_exists, mock_sleep, koji_dir, container_push_item): class TestKoji(object): def __init__(self, **kwargs): pass def __iter__(self): yield container_push_item mock_path_exists.return_value = False Source.register_backend("test-koji", TestKoji) source = Source.get("test-koji:") items = list(source) assert len(items) == 1 assert mock_path_exists.call_count == 31 assert mock_sleep.call_count == 30
def test_custom_source(): """A push source with args, custom enter/exit and used with get_partial works.""" Source.register_backend("custom-base", CustomSource) spy = [] Source.register_backend("custom-spy", Source.get_partial("custom-base:", spy=spy)) Source.register_backend("custom1", Source.get_partial("custom-spy:", a=123)) Source.register_backend("custom2", Source.get_partial("custom1:", b=234)) Source.register_backend("custom3", Source.get_partial("custom2:", c=456)) # Now use the source while going through multiple layers. with Source.get("custom3:") as source: assert list(source) == ITEMS # the enter/exit should propagate all the way through, just once. assert spy == ["enter [123, 234, 456]", "exit [123, 234, 456]"]
def test_yield_no_source(mock_path_exists, mock_sleep, koji_dir): class TestKoji(object): def __init__(self, **kwargs): pass def __iter__(self): class Object: pass item1 = Object() item1.src = None yield item1 yield Object() Source.register_backend("test-koji", TestKoji) source = Source.get("test-koji:") items = list(source) assert len(items) == 2 mock_path_exists.assert_not_called() mock_sleep.assert_not_called()
def test_keep_prepush_no_dest_items(): """Push item filtering keeps items with no dest if pre-pushable.""" ctx = Context() phase = LoadPushItems( ctx, ["fake:"], allow_unsigned=True, pre_push=True, ) fake_items = [ FilePushItem(name="file", dest=["some-repo"]), RpmPushItem(name="rpm", dest=[]), ] Source.register_backend("fake", lambda: fake_items) # Let it run to completion... with phase: pass # It should have succeeded assert not ctx.has_error # Now let's get everything from the output queue. all_outputs = [] while True: item = phase.out_queue.get() if item is Phase.FINISHED: break all_outputs.append(item.pushsource_item) # We should have got this: assert all_outputs == [ # get file as usual FilePushItem(name="file", dest=["some-repo"]), # even though this item has no destination, we still get it since rpms # support pre-push and pre_push was enabled. RpmPushItem(name="rpm", dest=[]), ]
def test_unsigned_failure( fake_push, command_tester, caplog, ): """Test that a failure occurs if an unsigned RPM is encountered without the --allow-unsigned option. """ Source.register_backend( "unsigned", lambda: [RpmPushItem(name="quux", src="/some/unsigned.rpm", dest=["repo1"])], ) args = [ "", "--source", "unsigned:", "--pulp-url", "https://pulp.example.com/", ] run = functools.partial(entry_point, cls=lambda: fake_push) # It should exit... with pytest.raises(SystemExit) as excinfo: command_tester.test( run, args, compare_plaintext=False, compare_jsonl=False, ) # ...unsuccessfully assert excinfo.value.code != 0 # And it should tell us what went wrong assert "Unsigned content is not permitted: /some/unsigned.rpm" in caplog.text
def test_empty_push(fake_controller, fake_push, fake_state_path, command_tester, stub_collector): """Test a push with no content.""" # Sanity check that the Pulp server is, initially, empty. client = fake_controller.client assert list(client.search_content()) == [] # Set up a pushsource backend which returns no supported items Source.register_backend("null", lambda: [PushItem(name="quux")]) compare_extra = { "pulp.yaml": { "filename": fake_state_path, "normalize": hide_unit_ids, } } args = [ "", "--source", "null:", "--pulp-url", "https://pulp.example.com/", ] run = functools.partial(entry_point, cls=lambda: fake_push) # It should be able to run without crashing. command_tester.test( run, args, compare_plaintext=False, compare_jsonl=False, compare_extra=compare_extra, ) # It should not record any push items at all. assert not stub_collector
def test_errata_ignores_unknown_koji_types(mock_path_exists, source_factory, koji_dir): """Errata source, when requesting containers, will skip unknown push item types yielded by koji source.""" # This is a very niche case, but to get that 100% coverage... # It's possible that koji_source might produce something other than ContainerImagePushItem # or OperatorManifestPushItem, and we want to be forwards-compatible with that. # This is our hacked source which returns whatever koji returns, but also some # arbitrary objects class WeirdKoji(object): def __init__(self, **kwargs): # Get a normal koji source... self.koji = Source.get( "koji:https://koji.example.com?basedir=%s" % koji_dir, **kwargs) def __iter__(self): # We'll yield whatever koji yields but surround it with # unexpected junk yield object() for item in self.koji: yield item yield object() mock_path_exists.return_value = True Source.register_backend("weird-koji", WeirdKoji) source = source_factory(errata="RHBA-2020:2807", koji_source="weird-koji:") # It should still work as normal items = list(source) # Sanity check we got the right number of items assert len(items) == 45
def test_push_copy_fails(fake_controller, fake_nocopy_push, fake_state_path, command_tester, caplog): """Test that push detects and fails in the case where a Pulp content copy claims to succeed, but doesn't put expected content in the target repo. While not expected to happen under normal conditions, there have historically been a handful of Pulp bugs or operational issues which can trigger this. """ client = fake_controller.client iso_dest1 = client.get_repository("iso-dest1").result() iso_dest2 = client.get_repository("iso-dest2").result() # Make this file exist but not in all the desired repos. existing_file = FileUnit( path="some-file", sha256sum= "db68c8a70f8383de71c107dca5fcfe53b1132186d1a6681d9ee3f4eea724fabb", size=46, ) fake_controller.insert_units(iso_dest1, [existing_file]) # Unit is now in iso-dest1. # Set up a pushsource backend which requests push of the same content # to both (iso-dest1, iso-dest2). Source.register_backend( "test", lambda: [ FilePushItem( # Note: a real push item would have to have 'src' pointing at an # existing file here. It's OK to omit that if the checksum exactly # matches something already in Pulp. name="some-file", sha256sum= "db68c8a70f8383de71c107dca5fcfe53b1132186d1a6681d9ee3f4eea724fabb", dest=["iso-dest1", "iso-dest2"], ) ], ) args = [ "", "--source", "test:", "--pulp-url", "https://pulp.example.com/", ] run = functools.partial(entry_point, cls=lambda: fake_nocopy_push) # Ask it to push. with pytest.raises(SystemExit) as excinfo: command_tester.test( run, args, # Can't guarantee a stable log order. compare_plaintext=False, compare_jsonl=False, ) # It should have failed. assert excinfo.value.code == 59 # It should tell us why it failed. msg = ("Fatal error: Pulp unit not present in repo(s) iso-dest2 " "after copy: FileUnit(path='some-file'") assert msg in caplog.text
def test_basic_inherited_source(): """Basic inherited push source can be used via with statement.""" Source.register_backend("basic", BasicInheritedSource) with Source.get("basic:") as source: assert list(source) == ITEMS
def resolve(cls): Source.register_backend("backend2", Backend2)
def test_register_invalid(): """Registering non-callable Source fails.""" with raises(TypeError): Source.register_backend("foobar", "some wrong value")