def test_reset_removes_custom_backend(): """reset removes a custom registered backend.""" created = [] source_instance = object() def source_factory(*args, **kwargs): created.append(True) return source_instance Source.register_backend("mytest", source_factory) # I can get an instance of it now assert Source.get("mytest:") assert len(created) == 1 # But if I reset... Source.reset() # Then I can't get it any more with raises(SourceUrlError) as exc_info: Source.get("mytest:") assert len(created) == 1 assert "no registered backend 'mytest'" in str(exc_info)
def test_args_from_url(): """Arguments from URLs are passed through to registered Source as expected.""" calls = [] def gather_args(*args, **kwargs): calls.append((args, kwargs)) return [] Source.register_backend("abc", gather_args) Source.get("abc:key=val1&key=val2&threads=10&timeout=60&foo=bar,baz") # It should have made a call to the registered backend with the given arguments: assert len(calls) == 1 assert calls[0] == ( (), { # Repeating key is passed as a list "key": ["val1", "val2"], # Threads/timeout are converted to int "threads": 10, "timeout": 60, # CSV are not automatically split into a list; Source class itself # must handle this if desired "foo": "bar,baz", }, )
def test_reset_restores_overridden_backend(): """reset restores the original backend for anything which has been overridden.""" # I can get an instance of a built-in backend, like 'staged'. # Note: doesn't matter that we're pointing at nonexistent directory # (reading doesn't happen until we iterate) Source.get("staged:/notexist") created = [] source_instance = object() def source_factory(*args, **kwargs): created.append(True) return source_instance # Can override the built-in 'staged' backend. Source.register_backend("staged", source_factory) # Now it will use what I registered assert Source.get("staged:/notexist") assert len(created) == 1 # But if I reset... Source.reset() # Then I'm not getting what I registered any more, but I'm # still getting something OK new_src = Source.get("staged:/notexist") assert new_src assert new_src is not source_instance assert len(created) == 1
def test_bad_url_missing_backend(): with raises(SourceUrlError) as ex_info: Source.get("notexist:foo=bar&baz=quux") assert ( "Requested source 'notexist:foo=bar&baz=quux' but " "there is no registered backend 'notexist'" ) in str(ex_info.value)
def test_source_get_invalid_items(): """Get registry source with invalid image URIs.""" source = Source.get( "registry:?image=registry.redhat.io/odf4/mcg-operator-bundle") with raises(ValueError): with source: items = list(source) source = Source.get( "registry:?image=registry.redhat.io/odf4/mcg-operator-bundle@latest:latest" ) with raises(ValueError): with source: items = list(source)
def test_koji_modules_filter_filename(fake_koji, koji_dir): """Koji source can filter modules by filename""" source = Source.get( "koji:https://koji.example.com/?module_build=foo-1.0-1", basedir=koji_dir, module_filter_filename="modulemd.x86_64.txt,modulemd.aarch64.txt", ) fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1") fake_koji.insert_modules( ["modulemd.x86_64.txt", "modulemd.aarch64.txt", "modulemd.s390x.txt"], build_nvr="foo-1.0-1", ) # Eagerly fetch items = list(source) # It should have returned push items for the two modules which matched filter assert len(items) == 2 item_names = sorted([item.name for item in items]) # Should be only those two matching the filter assert item_names == ["modulemd.aarch64.txt", "modulemd.x86_64.txt"]
def test_staged_empty_no_metadata(tmpdir): """Trying to use staged source of empty directory does not work""" empty = str(tmpdir.mkdir("empty")) source = Source.get("staged:%s" % empty) with raises(IOError): list(source)
def test_staged_notexist(tmpdir): """Trying to use staged source of nonexistent directory does not work""" notexist = str(tmpdir.join("some/notexist/dir")) source = Source.get("staged:%s" % notexist) with raises(OSError): list(source)
def test_koji_multiple_in_stream(fake_koji, koji_dir, caplog): """Koji source fails if referenced modulemd file contains multiple modules. This isn't something expected to happen in practice. The point of this test is to ensure that, if it *does* happen, we fail in a controlled manner rather than something odd like silently using the first module in the stream. """ source = Source.get( "koji:https://koji.example.com/?module_build=foo-1.0-1", basedir=koji_dir) fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1") fake_koji.insert_modules(["modulemd.x86_64.txt"], build_nvr="foo-1.0-1") # Set up a modulemd stream with multiple documents for this file. modulemd_path = os.path.join( koji_dir, "packages/foo/1.0/1/files/module/modulemd.x86_64.txt") os.makedirs(os.path.dirname(modulemd_path)) shutil.copy(os.path.join(DATA_DIR, "modulemd-multiple.yaml"), modulemd_path) # Trying to fetch the items should fail with raises(Exception) as exc_info: list(source) # We should find a relevant message from YAML parser assert "expected a single document in the stream" in str(exc_info) # It should have told us exactly which file in which build couldn't be parsed expected_message = ( "In koji build foo-1.0-1, cannot load module metadata from " + modulemd_path) assert expected_message in caplog.messages
def test_errata_missing_koji_rpms(fake_errata_tool): """Can't obtain errata if referenced RPMs are not in koji""" class AllMissingKojiSource(object): def __init__(self, **kwargs): pass def __iter__(self): yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.x86_64.rpm", state="NOTFOUND") yield RpmPushItem(name="sudo-1.8.25p1-4.el8_0.3.ppc64le.rpm", state="NOTFOUND") Source.register_backend("missingkoji", AllMissingKojiSource) source = Source.get( "errata:https://errata.example.com?errata=RHSA-2020:0509", koji_source="missingkoji:", ) with raises(Exception) as exc: list(source) # It should raise because an RPM referred by ET was not found in koji assert ( "Advisory refers to sudo-1.8.25p1-4.el8_0.3.x86_64.rpm but RPM was not found in koji" in str(exc.value))
def test_yield_timeout_reached_nodupe(mock_path_exists, mock_sleep, container_push_item, caplog): """src polling/timeout logic should only happen once per item even if multiple layers of source have been created. """ class TestKoji(object): def __init__(self, **kwargs): pass def __iter__(self): yield container_push_item mock_path_exists.return_value = False Source.register_backend("test-koji", TestKoji) Source.register_backend( "test-koji-outer", Source.get_partial("test-koji:", whatever="argument")) source = Source.get("test-koji-outer:") # Should be able to get the item. assert len(list(source)) == 1 # It should mention the timeout, only once assert caplog.text.count("is missing after 900 seconds") == 1 assert mock_path_exists.call_count == 31 assert mock_sleep.call_count == 30
def test_staged_empty_with_metadata(tmpdir): """Trying to use staged source of empty directory with valid metadata silently yields nothing""" empty = tmpdir.mkdir("empty") empty.join("staged.json").write('{"header": {"version": "0.2"}}') source = Source.get("staged:%s" % empty) assert list(source) == []
def test_koji_bad_modulemd(fake_koji, koji_dir, caplog): """Koji source logs and raises exception on unparseable modulemd file""" source = Source.get( "koji:https://koji.example.com/?module_build=foo-1.0-1", basedir=koji_dir) fake_koji.insert_rpms(["foo-1.0-1.x86_64.rpm"], build_nvr="foo-1.0-1") fake_koji.insert_modules(["modulemd.x86_64.txt", "modulemd.s390x.txt"], build_nvr="foo-1.0-1") # Write invalid modulemd here. modulemd_path = os.path.join( koji_dir, "packages/foo/1.0/1/files/module/modulemd.x86_64.txt") os.makedirs(os.path.dirname(modulemd_path)) with open(modulemd_path, "wt") as f: f.write("This ain't no valid modulemd") # Trying to fetch the items should fail with raises(Exception): list(source) # And it should have told us exactly which file in which build couldn't be parsed expected_message = ( "In koji build foo-1.0-1, cannot load module metadata from " + modulemd_path) assert expected_message in caplog.messages
def test_get_registered_partial(): """Can register a source obtained via get_partial, then get source using registered scheme.""" errata_example = Source.get_partial("errata:https://errata.example.com") Source.register_backend("errata-example", errata_example) # We should now be able to request sources using et_example scheme. # We're just verifying that the call obtains a source, without crashing. assert Source.get("errata-example:errata=ABC-123")
def test_koji_nobuild(fake_koji): """koji source referencing nonexistent build will fail""" source = Source.get("koji:https://koji.example.com/?module_build=notexist-1.2.3") with raises(ValueError) as exc_info: list(source) assert "Module build not found in koji: notexist-1.2.3" in str(exc_info.value)
def test_staged_bad_metadata(tmpdir): """Trying to use staged source with corrupt metadata file raises""" tmpdir.join("staged.json").write("'oops, not valid json'") source = Source.get("staged:%s" % tmpdir) with raises(ValueError): list(source)
def test_staged_no_header_metadata(tmpdir): """Loading a metadata file with no header will fail.""" empty = tmpdir.mkdir("empty") empty.join("pub-mapfile.json").write("{}") source = Source.get("staged:%s" % empty) with raises(ValidationError) as exc_info: list(source)
def test_basic_iterable_source(): """Basic push source returning non-class iterable can be used via with statement.""" Source.register_backend("iterable", lambda *_: ITEMS) # Even though lists don't have enter/exit, it should be automatically wrapped here # so that it works. with Source.get("iterable:") as source: assert list(source) == ITEMS
def test_loads_entrypoints(monkeypatch): """Source.get ensures 'pushsource' entry points are loaded.""" # Forcibly set the Source class back to uninitialized state. monkeypatch.setattr(Source, "_BACKENDS", {}) monkeypatch.setattr(Source, "_BACKENDS_RESET", {}) # Let's set up that some custom backends have registered entry points. created1 = [] created2 = [] class Backend1(object): def __init__(self): created1.append(True) @classmethod def resolve(cls): Source.register_backend("backend1", Backend1) class Backend2(object): def __init__(self): created2.append(True) @classmethod def resolve(cls): Source.register_backend("backend2", Backend2) with patch("pkg_resources.iter_entry_points") as iter_ep: iter_ep.return_value = [Backend1, Backend2] # I should be able to get instances of those two backends assert Source.get("backend1:") assert Source.get("backend2:") # It should have found them via the expected entry point group iter_ep.assert_called_once_with("pushsource") # Should have created one instance of each assert created1 == [True] assert created2 == [True] # These should also be retained over a reset() Source.reset() assert Source.get("backend1:") assert Source.get("backend2:")
def test_staged_dupe_metadata(): """Loading a metadata file with duplicate entries will fail.""" staged_dir = os.path.join(DATADIR, "dupe_meta") source = Source.get("staged:%s" % staged_dir) with raises(ValueError) as exc_info: list(source) assert "dest1/ISOS/test.txt listed twice in staged.yaml" in str(exc_info.value)
def test_staged_incomplete_channel_dumps(): staged_dir = os.path.join(DATADIR, "incomplete_channel_dumps") source = Source.get("staged:" + staged_dir) with raises(ValueError) as exc_info: list(source) assert ("missing mandatory attribute 'channel_dump_disc_number' for " "somedest/CHANNEL_DUMPS/myfile.txt") in str(exc_info.value)
def test_staged_channel_dumps(): staged_dir = os.path.join(DATADIR, "simple_channel_dumps") source = Source.get("staged:" + staged_dir) files = list(source) files.sort(key=lambda item: item.src) # It should load all the expected files with fields filled in by metadata assert files == [ ChannelDumpPushItem( name="myfile.txt", state="PENDING", src=os.path.join(staged_dir, "somedest/CHANNEL_DUMPS/myfile.txt"), dest=["somedest"], md5sum=None, sha256sum= "9481d6638081ff26556e09844ae1fbf680ad83fb98afa2f3f88718537b41f8b9", origin="staged", build=None, signing_key=None, description="test channel dump 1", arch="x86_64", eng_product_ids=[1, 2, 3], content="some content", datetime=datetime.datetime(2020, 2, 19, 11, 5, tzinfo=tz.tzutc()), disk_number=1, channels=["ch1", "ch2"], product_name="a product", product_version="ABCD", type="base", ), ChannelDumpPushItem( name="otherfile.txt", state="PENDING", src=os.path.join(staged_dir, "somedest/CHANNEL_DUMPS/otherfile.txt"), dest=["somedest"], md5sum=None, sha256sum= "64f31e7083a2bdbdefa86bbe23de536133bee35980353312ad010b3fcc6a13c4", origin="staged", build=None, signing_key=None, description="test channel dump 2", arch="i686", eng_product_ids=[2, 3, 4], content="other content", datetime=datetime.datetime(2020, 2, 19, 11, 50, tzinfo=tz.tzutc()), disk_number=2, channels=["ch3", "ch4"], product_name="other product", product_version="DEF", type="incremental", ), ]
def test_errata_files_needs_koji_url(fake_errata_tool): """Can't obtain errata referring to files if koji source URL is missing""" source = Source.get( "errata:https://errata.example.com?errata=RHSA-2020:0509") with raises(Exception) as exc: list(source) assert "A Koji source is required but none is specified" in str(exc.value)
def test_staged_no_header_metadata(tmpdir): """Loading a metadata file with no header will fail.""" empty = tmpdir.mkdir("empty") empty.join("pub-mapfile.json").write("{}") source = Source.get("staged:%s" % empty) with raises(ValueError) as exc_info: list(source) assert "pub-mapfile.json has unsupported version" in str(exc_info.value)
def test_koji_connect_error(): """Source raises a reasonable error if server can't be contacted""" # Note: fake_koji fixture not used here, so this will really try to connect source = Source.get("koji:https://localhost:1234/this-aint-koji") with raises(RuntimeError) as exc_info: list(source) assert ( "Communication error with koji at https://localhost:1234/this-aint-koji" in str(exc_info.value) )
def test_partial_with_url_unbound(): Source.register_backend("returns-url", ReturnsUrlSource) # Let's say that I now overwrite this source with an argument (not url) bound partial = Source.get_partial("returns-url:", a=123) Source.register_backend("returns-url", partial) # Then I should be able to obtain an instance of this source, and it # should still stuff the path part of the below string into the 'url' arg items = [i for i in Source.get("returns-url:/foo/bar?b=88")] assert items == [PushItem(name="/foo/bar 123 88")]
def test_koji_rpms(fake_koji, koji_dir): """Koji source yields requested RPMs""" source = Source.get( "koji:https://koji.example.com/?rpm=foo-1.0-1.x86_64.rpm,notfound-2.0-1.noarch.rpm", basedir=koji_dir, ) # It should not have done anything yet (lazy loading) assert not fake_koji.last_url # Insert some data fake_koji.rpm_data["foo-1.0-1.x86_64.rpm"] = { "arch": "x86_64", "name": "foo", "version": "1.0", "release": "1", "build_id": 1234, } fake_koji.build_data[1234] = { "id": 1234, "name": "foobuild", "version": "1.0", "release": "1.el8", "nvr": "foobuild-1.0-1.el8", "volume_name": "somevol", } # Eagerly fetch items = list(source) # It should have constructed a session for the given URL assert fake_koji.last_url == "https://koji.example.com/" # It should have returned push items for the two RPMs assert len(items) == 2 items = sorted(items, key=lambda pi: pi.name) # For present RPM, a push item should be yielded using the koji metadata. assert items[0] == RpmPushItem( name="foo-1.0-1.x86_64.rpm", state="PENDING", src= "%s/vol/somevol/packages/foobuild/1.0/1.el8/x86_64/foo-1.0-1.x86_64.rpm" % koji_dir, build="foobuild-1.0-1.el8", ) # For missing RPMs, a push item should be yielded with NOTFOUND state. assert items[1] == RpmPushItem(name="notfound-2.0-1.noarch.rpm", state="NOTFOUND")
def test_koji_exceptions(fake_koji): """Exceptions raised during calls to koji are propagated""" source = Source.get("koji:https://koji.example.com/?module_build=error-1.2.3") error = RuntimeError("simulated error") fake_koji.build_data["error-1.2.3"] = error with raises(Exception) as exc_info: list(source) # It should have propagated *exactly* the exception from koji assert exc_info.value is error
def test_staged_nometa_channel_dumps(tmpdir): staged_dir = tmpdir.mkdir("staged") staged_dir.mkdir("dest").mkdir("CHANNEL_DUMPS").join("testfile").write( "test") staged_dir.join("staged.json").write('{"header": {"version": "0.2"}}') source = Source.get("staged:%s" % staged_dir) with raises(ValueError) as exc_info: list(source) assert "staged.json doesn't contain data for dest/CHANNEL_DUMPS/testfile" in str( exc_info.value)
def test_staged_file_with_no_metadata(tmpdir): """Trying to use staged source where a staged file is missing metadata will raise""" staged = tmpdir.mkdir("staged") staged.join("staged.json").write('{"header": {"version": "0.2"}}') staged.mkdir("dest").mkdir("FILES").join("some-file").write("some-content") source = Source.get("staged:%s" % staged) with raises(ValueError) as exc_info: list(source) assert "No metadata available for dest/FILES/some-file in staged.json" in str( exc_info.value)