def test_get_missing():
"""Can't get a collector using unregistered backend."""
with pytest.raises(ValueError) as excinfo:
Collector.get("not-registered")
value = excinfo.value
assert "No registered pushcollector backend: 'not-registered'" in str(
value)
def _load_metadata(self, topdir):
# Load the top-level metadata file in the staging directory, if any.
for candidate in METADATA_FILES:
metadata_file = os.path.join(topdir, candidate)
if os.path.exists(metadata_file):
break
else:
# no metadata file
return StagingMetadata()
basename = os.path.basename(metadata_file)
with open(metadata_file, "rt") as f:
content = f.read()
# Save a copy of the file for later reference
Collector.get().attach_file(basename, content).result()
if metadata_file.endswith(".json"):
metadata = json.loads(content)
else:
metadata = yaml.safe_load(content)
return StagingMetadata.from_data(metadata,
os.path.basename(metadata_file))
def test_context_manager_backend():
"""Test a backend's context manager protocol methods are called properly."""
collector = MagicMock(spec=Collector)
# use my-collector name because the autouse fixture `reset_backend`
# will clean it up for us
Collector.register_backend("my-collector", lambda: collector)
Collector.set_default_backend("my-collector")
with Collector.get():
pass
collector.__enter__.assert_called_once()
collector.__exit__.assert_called_once_with(*(None, ) * 3)
def test_local_saves_to_artifacts(caplog, tmpdir, monkeypatch):
"""local collector can be obtained and used successfully, and writes
data under 'artifacts' dir as expected."""
monkeypatch.chdir(tmpdir)
caplog.set_level(logging.INFO)
collector = Collector.get("local")
collector.update_push_items(
[
{"filename": "file1", "state": "PUSHED"},
{"filename": "somedir/file2", "state": "INVALIDFILE"},
]
).result()
collector.update_push_items([{"filename": "file3", "state": "MISSING"}]).result()
collector.attach_file("some-file.txt", "Hello, world\n").result()
collector.attach_file("some-file.txt", "Hello again\n").result()
collector.attach_file("some-file.bin", b"\x00\x01\x02").result()
collector.append_file("appended-file.txt", "chunk 1").result()
collector.append_file("appended-file.txt", b"\nchunk 2\n").result()
# It should have created an "artifacts/latest" directory/symlink
artifactsdir = tmpdir.join("artifacts", "latest")
assert artifactsdir.check(dir=True, link=True)
# Resolve symlink for later comparison with log messages
artifactsdir = artifactsdir.realpath()
# It should have saved push item data as JSONL
assert (
artifactsdir.join("pushitems.jsonl").open().read()
== textwrap.dedent(
"""
{"filename": "file1", "state": "PUSHED"}
{"filename": "somedir/file2", "state": "INVALIDFILE"}
{"filename": "file3", "state": "MISSING"}
"""
).lstrip()
)
# It should have saved the text file with requested content
assert artifactsdir.join("some-file.txt").open().read() == "Hello again\n"
# It should have saved the binary file with requested content
assert artifactsdir.join("some-file.bin").open("rb").read() == b"\x00\x01\x02"
# It should have saved the appended-text file with requested content
assert artifactsdir.join("appended-file.txt").open().read() == "chunk 1\nchunk 2\n"
# It should have logged about the created files
assert caplog.messages == [
"Logging to %s" % artifactsdir.join("pushitems.jsonl"),
"Logging to %s" % artifactsdir.join("some-file.txt"),
"Logging to %s" % artifactsdir.join("some-file.bin"),
"Logging to %s" % artifactsdir.join("appended-file.txt"),
]
def test_bad_push_items():
"""Passing push items with incorrect data to update_push_items raises a
validation error."""
coll = Collector.get("dummy")
with pytest.raises(jsonschema.ValidationError):
coll.update_push_items([{"foo": "bar"}])
def test_bad_obj_push():
"""Any invalid object type push item raises an error."""
collector = Collector.get("dummy")
pushitem = object()
with pytest.raises(AttributeError):
collector.update_push_items([pushitem])
def test_pushitem_obj_push():
"""PushItem object passed to the update_push_items works fine."""
collector = Collector.get("dummy")
pushitem = PushItem(name="test_pushitem")
ret_val = collector.update_push_items([pushitem])
assert ret_val.result() is None
def test_unimplemented():
"""Collector base class raises NotImplementedError for all public API."""
collector = Collector()
with pytest.raises(NotImplementedError):
collector.update_push_items([])
with pytest.raises(NotImplementedError):
collector.attach_file("somefile.txt", "foobar")
with pytest.raises(NotImplementedError):
collector.append_file("somefile.txt", "foobar")
def test_local_dir_sequence(tmpdir, monkeypatch):
"""local collector creates timestamped directories per run, with
'latest' symlink pointing to latest"""
monkeypatch.chdir(tmpdir)
# Use first collector
monkeypatch.setattr(LocalCollector, "timestamp", lambda cls: "time1")
Collector.get("local").update_push_items(
[{"filename": "file1", "state": "PUSHED"}]
).result()
# Use another collector a few seconds later
monkeypatch.setattr(LocalCollector, "timestamp", lambda cls: "time2")
Collector.get("local").update_push_items(
[{"filename": "file1", "state": "PUSHED"}]
).result()
# And another even later
monkeypatch.setattr(LocalCollector, "timestamp", lambda cls: "time3")
Collector.get("local").update_push_items(
[{"filename": "file1", "state": "PUSHED"}]
).result()
# It should have created these paths:
artifactsdir = tmpdir.join("artifacts")
assert artifactsdir.check(dir=True)
assert artifactsdir.join("time1").check(dir=True)
assert artifactsdir.join("time2").check(dir=True)
assert artifactsdir.join("time3").check(dir=True)
assert artifactsdir.join("latest").check(dir=True, link=True)
# and latest should be a symlink to the last created timestamp dir
assert artifactsdir.join("latest").readlink() == "time3"
def test_set_default():
"""Can set and use a default collector."""
class MyCollector(object):
INSTANCES = []
def __init__(self):
MyCollector.INSTANCES.append(self)
self.pushed = []
def update_push_items(self, items):
self.pushed.extend(items)
Collector.register_backend("my-collector", MyCollector)
Collector.set_default_backend("my-collector")
items = [
{
"filename": "file1",
"state": "PENDING"
},
{
"filename": "file2",
"state": "PENDING"
},
]
# Updating push items through default collector should succeed
Collector.get().update_push_items(items).result()
# It should have used the class we installed as the default
assert len(MyCollector.INSTANCES) == 1
assert MyCollector.INSTANCES[0].pushed == items
def test_always_returns_future():
"""Collector interface returns futures regardless of backend return type."""
return_value = None
class TestCollector(object):
def update_push_items(self, items):
return return_value
def attach_file(self, filename, content):
return return_value
def append_file(self, filename, content):
return return_value
Collector.register_backend("test", TestCollector)
collector = Collector.get("test")
# If backend returns a successful future (of any value),
# interface returns an empty future
return_value = f_return("abc")
assert collector.update_push_items([]).result() is None
assert collector.attach_file("somefile", "").result() is None
assert collector.append_file("somefile", "").result() is None
# If backend returns a failed future,
# interface returns a failed future with error propagated
error = RuntimeError("oops")
return_value = f_return_error(error)
assert collector.update_push_items([]).exception() is error
assert collector.attach_file("somefile", "").exception() is error
assert collector.append_file("somefile", "").exception() is error
# If backend returns a non-future,
# interface returns an empty future
return_value = "abc"
assert collector.update_push_items([]).result() is None
assert collector.attach_file("somefile", "").result() is None
assert collector.append_file("somefile", "").result() is None
def reset_backend():
"""Resets the default backend both before and after each test,
and clears my-collector backend.
"""
Collector.set_default_backend(None)
yield
Collector.set_default_backend(None)
Collector.register_backend("my-collector", None)
def stub_collector():
"""Installs a custom pushcollector backend which records all push items
onto a plain old list.
Yields the list; it can be inspected to see which push items were recorded."""
itemlist = []
Collector.register_backend("pubtools-pulp-test",
lambda: StubCollector(itemlist))
Collector.set_default_backend("pubtools-pulp-test")
yield itemlist
Collector.set_default_backend(None)
Collector.register_backend("pubtools-pulp-test", None)
def test_can_use_dummy():
"""Dummy collector can be obtained and used successfully."""
coll = Collector.get("dummy")
coll.update_push_items(
[
{"filename": "file1", "state": "PUSHED"},
{"filename": "file2", "state": "UNKNOWN"},
]
).result()
coll.attach_file("somefile.txt", "hello, world").result()
coll.append_file("otherfile.txt", "line of text\n").result()
def test_minimal_pushitem_obj():
"""PushItem object with minimal attributes is translated to
pushitem dict with destination and checksums as None along
with other attributes as in the object"""
mock = Mock()
Collector.register_backend("mock", lambda: mock)
collector = Collector.get("mock")
pushitem = PushItem(name="test_push")
collector.update_push_items([pushitem])
update_push_item_args = mock.update_push_items.call_args[0][0]
assert len(update_push_item_args) == 1
push_args = update_push_item_args[0]
assert push_args["filename"] == pushitem.name
assert push_args["state"] == pushitem.state
assert push_args["src"] == pushitem.src
assert push_args["dest"] is None
assert push_args["checksums"] is None
assert push_args["origin"] == pushitem.origin
assert push_args["build"] == pushitem.build
assert push_args["signing_key"] == pushitem.signing_key
def test_pushitem_obj_attributes():
"""PushItem object attributes are translated and available as expected in
pushitem dict passed to update_push_items. A pushitem is generated for each
destination"""
mock = Mock()
Collector.register_backend("mock", lambda: mock)
collector = Collector.get("mock")
pushitem = PushItem(
name="test_push",
origin="some_origin",
src="source",
dest=["dest1", "dest2"],
md5sum="bb1b0d528129f47798006e73307ba7a7",
sha256sum=
"4fd23ae44f3366f12f769f82398e96dce72adab8e45dea4d721ddf43fdce31e2",
build="test_build-1.0.0-1",
signing_key="FD431D51",
)
collector.update_push_items([pushitem])
update_push_item_args = mock.update_push_items.call_args[0][0]
assert len(update_push_item_args) == 2
for i in range(len(update_push_item_args) - 1):
push_args = update_push_item_args[i]
assert push_args["filename"] == pushitem.name
assert push_args["state"] == pushitem.state
assert push_args["src"] == pushitem.src
assert push_args["dest"] == pushitem.dest[i]
assert push_args["checksums"] == {
"md5": pushitem.md5sum,
"sha256": pushitem.sha256sum,
}
assert push_args["origin"] == pushitem.origin
assert push_args["build"] == pushitem.build
assert push_args["signing_key"] == pushitem.signing_key
def test_base_class_context_manager():
"""Exercise the __enter__ and __exit__ methods of Collector."""
collector = Collector()
# use my-collector name because the autouse fixture `reset_backend`
# will clean it up for us
Collector.register_backend("my-collector", lambda: collector)
Collector.set_default_backend("my-collector")
# empty with just to exercise __enter__ and __exit__
with Collector.get():
pass
def fake_collector():
"""Install fake in-memory backend for pushcollector library.
Recorded push items can be tested via this instance.
This is an autouse fixture so that all tests will automatically
use the fake backend.
"""
collector = FakeCollector()
Collector.register_backend("pubtools-pulp-test", lambda: collector)
Collector.set_default_backend("pubtools-pulp-test")
yield collector
Collector.set_default_backend(None)
def test_unregister_resets_default():
"""Unregistering a backend unsets it as the default backend (if it was set)."""
class CountingCollector(object):
CONSTRUCTED_COUNT = 0
def __init__(self):
CountingCollector.CONSTRUCTED_COUNT += 1
Collector.register_backend("counter", CountingCollector)
Collector.set_default_backend("counter")
# Calling get() should construct custom backend once.
Collector.get()
assert CountingCollector.CONSTRUCTED_COUNT == 1
# Calling get() should construct custom backend again.
Collector.get()
assert CountingCollector.CONSTRUCTED_COUNT == 2
Collector.register_backend("counter", None)
# Calling get() after unregister should succeed
assert Collector.get()
# And it should not have constructed the custom backend
assert CountingCollector.CONSTRUCTED_COUNT == 2
def test_register_backend_wrong_type():
"""register_backend raises TypeError if passed incorrect type"""
with pytest.raises(TypeError):
Collector.register_backend("my-colletor", Collector())
def test_get_default():
"""Can get a default collector."""
collector = Collector.get()
assert collector