def test_koji_cache(fake_koji, koji_dir): """Koji source can reuse a cache to avoid repeated calls.""" cache = {} source_ctor = Source.get_partial( "koji:https://koji.example.com/?rpm=foo-1.0-1.x86_64.rpm&module_build=foo-1.0-1", basedir=koji_dir, cache=cache, ) fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1") fake_koji.insert_modules( ["modulemd.x86_64.txt", "modulemd.s390x.txt"], build_nvr="foo-1.0-1" ) # Get push items from one source items1 = sorted(list(source_ctor()), key=repr) # It should have three items (1 RPM, two modulemds) assert len(items1) == 3 # Now wipe out all the fake_koji data fake_koji.reset() # Get push items from a new source. # It should succeed even though no test data is defined items2 = sorted(list(source_ctor()), key=repr) # They should give identical results assert items1 == items2 # It should have stored something in the cache assert cache
def test_args_via_partial(): """Arguments are passed into source via get_partial as expected.""" calls = [] def gather_args(*args, **kwargs): calls.append((args, kwargs)) return [] Source.register_backend("gather", gather_args) # Let's make a partial bound to 'a' & 'b' by default gather = Source.get_partial("gather:a=1", b=2) # Call it a few different ways: gather() gather(c=3) gather(d=4) # It should have been called with expected args: # - every call had 'a' & 'b' since they were bound in get_partial # - 'b' and 'c' only appear when explicitly passed # - also note 'a' is a string since it came via URL assert len(calls) == 3 assert calls[0] == ((), {"a": "1", "b": 2}) assert calls[1] == ((), {"a": "1", "b": 2, "c": 3}) assert calls[2] == ((), {"a": "1", "b": 2, "d": 4})
def test_yield_timeout_reached_nodupe(mock_path_exists, mock_sleep, container_push_item, caplog): """src polling/timeout logic should only happen once per item even if multiple layers of source have been created. """ class TestKoji(object): def __init__(self, **kwargs): pass def __iter__(self): yield container_push_item mock_path_exists.return_value = False Source.register_backend("test-koji", TestKoji) Source.register_backend( "test-koji-outer", Source.get_partial("test-koji:", whatever="argument")) source = Source.get("test-koji-outer:") # Should be able to get the item. assert len(list(source)) == 1 # It should mention the timeout, only once assert caplog.text.count("is missing after 900 seconds") == 1 assert mock_path_exists.call_count == 31 assert mock_sleep.call_count == 30
def test_get_registered_partial(): """Can register a source obtained via get_partial, then get source using registered scheme.""" errata_example = Source.get_partial("errata:https://errata.example.com") Source.register_backend("errata-example", errata_example) # We should now be able to request sources using et_example scheme. # We're just verifying that the call obtains a source, without crashing. assert Source.get("errata-example:errata=ABC-123")
def source_factory(fake_errata_tool, fake_koji, koji_dir): ctor = Source.get_partial( "errata:https://errata.example.com", koji_source="koji:https://koji.example.com?basedir=%s" % koji_dir, ) fake_koji.load_all_builds() yield ctor
def errata_test_backend(fake_errata_tool, koji_test_backend): # erratatest backend is errata backend pointing at kojitest and our errata testdata bound = Source.get_partial("errata:https://errata.example.com/", koji_source="kojitest:") Source.register_backend("erratatest", bound) yield Source.reset()
def test_custom_source(): """A push source with args, custom enter/exit and used with get_partial works.""" Source.register_backend("custom-base", CustomSource) spy = [] Source.register_backend("custom-spy", Source.get_partial("custom-base:", spy=spy)) Source.register_backend("custom1", Source.get_partial("custom-spy:", a=123)) Source.register_backend("custom2", Source.get_partial("custom1:", b=234)) Source.register_backend("custom3", Source.get_partial("custom2:", c=456)) # Now use the source while going through multiple layers. with Source.get("custom3:") as source: assert list(source) == ITEMS # the enter/exit should propagate all the way through, just once. assert spy == ["enter [123, 234, 456]", "exit [123, 234, 456]"]
def koji_test_backend(fake_koji, koji_dir): # kojitest backend is koji backend pointing at our koji testdata bound = Source.get_partial("koji:https://koji.example.com/", basedir=koji_dir) Source.register_backend("kojitest", bound) yield Source.reset()
def test_partial_with_url_unbound(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source with an argument (not url) bound partial = Source.get_partial("returns-url:", a=123) Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, and it # should still stuff the path part of the below string into the 'url' arg items = [i for i in Source.get("returns-url:/foo/bar?b=88")] assert items == [PushItem(name="/foo/bar 123 88")]
def load_conf(filename): with open(filename, "rt") as f: conf = yaml.safe_load(f) try: for source in conf.get("sources") or []: name = source["name"] url = source["url"] Source.register_backend(name, Source.get_partial(url)) except Exception: # pylint: disable=broad-except LOG.exception("Error loading config from %s", filename) sys.exit(52)
def test_partial_with_url_bound_overwrite(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source and I pre-fill a URL partial = Source.get_partial("returns-url:", url="/tmp") Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, and # I can still override the bound URL by passing a new one in the # normal manner. items = [i for i in Source.get("returns-url:/other/url?a=1&b=2")] assert items == [PushItem(name="/other/url 1 2")]
def test_partial_with_url_bound(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source and I pre-fill a URL partial = Source.get_partial("returns-url:", url="/tmp") Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, with # the URL coming from the value previously bound and other arguments # still able to be overridden normally. items = [i for i in Source.get("returns-url:b=123")] assert items == [PushItem(name="/tmp a 123")]
def source_factory(fake_errata_tool, fake_koji, koji_dir): ctor = Source.get_partial( "errata:https://errata.example.com", koji_source="koji:https://koji.example.com?basedir=%s" % koji_dir, ) rpm_filenames = [ "pgaudit-1.4.0-4.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "pgaudit-1.4.0-4.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "pgaudit-1.4.0-4.module+el8.1.1+4794+c82b6e09.s390x.rpm", "pgaudit-1.4.0-4.module+el8.1.1+4794+c82b6e09.src.rpm", "pgaudit-1.4.0-4.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "pgaudit-debuginfo-1.4.0-4.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "pgaudit-debuginfo-1.4.0-4.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "pgaudit-debuginfo-1.4.0-4.module+el8.1.1+4794+c82b6e09.s390x.rpm", "pgaudit-debuginfo-1.4.0-4.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "pgaudit-debugsource-1.4.0-4.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "pgaudit-debugsource-1.4.0-4.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "pgaudit-debugsource-1.4.0-4.module+el8.1.1+4794+c82b6e09.s390x.rpm", "pgaudit-debugsource-1.4.0-4.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgres-decoderbufs-0.10.0-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgres-decoderbufs-0.10.0-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgres-decoderbufs-0.10.0-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgres-decoderbufs-0.10.0-2.module+el8.1.1+4794+c82b6e09.src.rpm", "postgres-decoderbufs-0.10.0-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgres-decoderbufs-debuginfo-0.10.0-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgres-decoderbufs-debuginfo-0.10.0-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgres-decoderbufs-debuginfo-0.10.0-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgres-decoderbufs-debuginfo-0.10.0-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgres-decoderbufs-debugsource-0.10.0-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgres-decoderbufs-debugsource-0.10.0-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgres-decoderbufs-debugsource-0.10.0-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgres-decoderbufs-debugsource-0.10.0-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-12.1-2.module+el8.1.1+4794+c82b6e09.src.rpm", "postgresql-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-contrib-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-contrib-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-contrib-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-contrib-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-contrib-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-contrib-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-contrib-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-contrib-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-debugsource-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-debugsource-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-debugsource-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-debugsource-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-docs-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-docs-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-docs-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-docs-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-docs-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-docs-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-docs-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-docs-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-plperl-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-plperl-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-plperl-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-plperl-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-plperl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-plperl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-plperl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-plperl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-plpython3-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-plpython3-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-plpython3-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-plpython3-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-plpython3-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-plpython3-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-plpython3-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-plpython3-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-pltcl-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-pltcl-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-pltcl-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-pltcl-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-pltcl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-pltcl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-pltcl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-pltcl-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-server-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-server-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-server-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-server-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-server-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-server-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-server-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-server-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-server-devel-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-server-devel-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-server-devel-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-server-devel-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-server-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-server-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-server-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-server-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-static-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-static-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-static-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-static-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-test-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-test-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-test-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-test-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-test-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-test-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-test-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-test-rpm-macros-12.1-2.module+el8.1.1+4794+c82b6e09.noarch.rpm", "postgresql-upgrade-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-upgrade-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-upgrade-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-upgrade-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-upgrade-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-upgrade-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-upgrade-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-upgrade-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-upgrade-devel-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-upgrade-devel-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-upgrade-devel-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-upgrade-devel-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-upgrade-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.aarch64.rpm", "postgresql-upgrade-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", "postgresql-upgrade-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.s390x.rpm", "postgresql-upgrade-devel-debuginfo-12.1-2.module+el8.1.1+4794+c82b6e09.x86_64.rpm", "postgresql-test-12.1-2.module+el8.1.1+4794+c82b6e09.ppc64le.rpm", ] # Insert koji RPMs referenced by this advisory fake_koji.insert_rpms( rpm_filenames, koji_dir=koji_dir, signing_key="fd431d51", build_nvr="postgresql-12.1-2.module+el8.1.1+4794+c82b6e09", ) # Insert archives referenced by build fake_koji.insert_modules( [ "modulemd.aarch64.txt", "modulemd.ppc64le.txt", "modulemd.s390x.txt", "modulemd.x86_64.txt", "modulemd.src.txt", ], build_nvr="postgresql-12-8010120191120141335.e4e244f9", ) yield ctor
def source_factory(fake_errata_tool, fake_koji, koji_dir): # Yields constructor for an errata source pointing at a valid advisory with RPMs, # with no arch filter set. Tests can then provide a specific filter and observe # the results. source_ctor = Source.get_partial( "errata:https://errata.example.com", errata="RHSA-2020:0509", koji_source="koji:https://koji.example.com?basedir=%s" % koji_dir, ) # Insert koji RPMs referenced by this advisory fake_koji.rpm_data["sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm"] = { "arch": "ppc64le", "name": "sudo", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.rpm_data["sudo-1.8.25p1-4.el8_0.3.x86_64.rpm"] = { "arch": "x86_64", "name": "sudo", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.rpm_data["sudo-1.8.25p1-4.el8_0.3.src.rpm"] = { "arch": "src", "name": "sudo", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.rpm_data["sudo-debuginfo-1.8.25p1-4.el8_0.3.ppc64le.rpm"] = { "arch": "ppc64le", "name": "sudo-debuginfo", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.rpm_data["sudo-debuginfo-1.8.25p1-4.el8_0.3.x86_64.rpm"] = { "arch": "x86_64", "name": "sudo-debuginfo", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.rpm_data["sudo-debugsource-1.8.25p1-4.el8_0.3.ppc64le.rpm"] = { "arch": "ppc64le", "name": "sudo-debugsource", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.rpm_data["sudo-debugsource-1.8.25p1-4.el8_0.3.x86_64.rpm"] = { "arch": "x86_64", "name": "sudo-debugsource", "version": "1.8.25p1", "release": "4.el8_0.3", "build_id": 1234, } fake_koji.build_data[1234] = { "id": 1234, "name": "sudo", "version": "1.8.25p1", "release": "4.el8_0.3", "nvr": "sudo-1.8.25p1-4.el8_0.3", } fake_koji.build_data["sudo-1.8.25p1-4.el8_0.3"] = fake_koji.build_data[ 1234] signed_rpm_path = os.path.join( koji_dir, "vol/somevol/packages/foobuild/1.0/1.el8", "data/signed/def456/x86_64/foo-1.0-1.x86_64.rpm", ) # Make signed RPMs exist (contents not relevant here) for filename, rpm in fake_koji.rpm_data.items(): signed_rpm_path = os.path.join( koji_dir, "packages/sudo/1.8.25p1/4.el8_0.3/", "data/signed/fd431d51/%s/%s" % (rpm["arch"], filename), ) signed_dir = os.path.dirname(signed_rpm_path) if not os.path.exists(signed_dir): os.makedirs(signed_dir) open(signed_rpm_path, "w") yield source_ctor