def ns(request):
if request.param == "bigtable-real":
pytest.skip("Bigtable is not available in CI")
ns = {
"bigtable-mocked": lambda: MockedBigtableNodeStorage(project="test"),
"bigtable-real": lambda: BigtableNodeStorage(project="test"),
"django": lambda: DjangoNodeStorage(),
}[request.param]()
ns.bootstrap()
return ns
def ns(request):
# backends are returned from context managers to support teardown when required
backends = {
"bigtable-mocked": lambda: nullcontext(MockedBigtableNodeStorage(project="test")),
"bigtable-real": lambda: get_temporary_bigtable_nodestorage(),
"django": lambda: nullcontext(DjangoNodeStorage()),
}
ctx = backends[request.param]()
with ctx as ns:
ns.bootstrap()
yield ns
def setUp(self):
self.ns = DjangoNodeStorage()
class DjangoNodeStorageTest(TestCase):
def setUp(self):
self.ns = DjangoNodeStorage()
def test_get(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"})
result = self.ns.get(node.id)
assert result == node.data
def test_get_multi(self):
nodes = [
Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"}),
Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc",
data={"foo": "baz"}),
]
result = self.ns.get_multi([
"d2502ebbd7df41ceba8d3275595cac33",
"5394aa025b8e401ca6bc3ddee3130edc"
])
assert result == dict((n.id, n.data) for n in nodes)
def test_set(self):
self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
assert Node.objects.get(
id="d2502ebbd7df41ceba8d3275595cac33").data == {
"foo": "bar"
}
def test_set_multi(self):
self.ns.set_multi({
"d2502ebbd7df41ceba8d3275595cac33": {
"foo": "bar"
},
"5394aa025b8e401ca6bc3ddee3130edc": {
"foo": "baz"
},
})
assert Node.objects.get(
id="d2502ebbd7df41ceba8d3275595cac33").data == {
"foo": "bar"
}
assert Node.objects.get(
id="5394aa025b8e401ca6bc3ddee3130edc").data == {
"foo": "baz"
}
def test_create(self):
node_id = self.ns.create({"foo": "bar"})
assert Node.objects.get(id=node_id).data == {"foo": "bar"}
def test_delete(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"})
self.ns.delete(node.id)
assert not Node.objects.filter(id=node.id).exists()
def test_delete_multi(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"})
self.ns.delete_multi([node.id])
assert not Node.objects.filter(id=node.id).exists()
def test_cleanup(self):
now = timezone.now()
cutoff = now - timedelta(days=1)
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
timestamp=now,
data={"foo": "bar"})
node2 = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac34",
timestamp=cutoff,
data={"foo": "bar"})
self.ns.cleanup(cutoff)
assert Node.objects.filter(id=node.id).exists()
assert not Node.objects.filter(id=node2.id).exists()
def test_cache(self):
node_1 = ("a" * 32, {"foo": "a"})
node_2 = ("b" * 32, {"foo": "b"})
node_3 = ("c" * 32, {"foo": "c"})
for node_id, data in [node_1, node_2, node_3]:
Node.objects.create(id=node_id, data=data)
# Get / get multi populates cache
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_2[0], node_3[0]]) == {
node_2[0]: node_2[1],
node_3[0]: node_3[1],
}
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get(node_2[0]) == node_2[1]
assert self.ns.get(node_3[0]) == node_3[1]
assert mock_get.call_count == 0
with mock.patch.object(Node.objects, "filter") as mock_filter:
assert self.ns.get_multi([node_1[0], node_2[0], node_3[0]])
assert mock_filter.call_count == 0
# Manually deleted item should still retreivable from cache
Node.objects.get(id=node_1[0]).delete()
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_1[0]: node_1[1],
node_2[0]: node_2[1],
}
# Deletion clars cache
self.ns.delete(node_1[0])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_2[0]: node_2[1]
}
self.ns.delete_multi([node_1[0], node_2[0]])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {}
# Setting the item updates cache
new_value = {"event_id": "d" * 32}
self.ns.set(node_1[0], new_value)
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == new_value
assert mock_get.call_count == 0
# Missing rows are never cached
assert self.ns.get("node_4") is None
with mock.patch.object(Node.objects, "get") as mock_get:
mock_get.side_effect = Node.DoesNotExist
self.ns.get("node_4")
self.ns.get("node_4")
assert mock_get.call_count == 2
def setup_method(self):
self.ns = DjangoNodeStorage()
class TestDjangoNodeStorage:
def setup_method(self):
self.ns = DjangoNodeStorage()
@pytest.mark.parametrize(
"node_data",
[
compress(b'{"foo": "bar"}'),
compress(pickle.dumps({"foo": "bar"})),
# hardcoded pickle value from python 3.6
compress(
b"\x80\x03}q\x00X\x03\x00\x00\x00fooq\x01X\x03\x00\x00\x00barq\x02s."
),
# hardcoded pickle value from python 2.7
compress(b"(dp0\nS'foo'\np1\nS'bar'\np2\ns."),
],
)
def test_get(self, node_data):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=node_data)
result = self.ns.get(node.id)
assert result == {"foo": "bar"}
def test_get_multi(self):
Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=compress(b'{"foo": "bar"}'))
Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc",
data=compress(b'{"foo": "baz"}'))
result = self.ns.get_multi([
"d2502ebbd7df41ceba8d3275595cac33",
"5394aa025b8e401ca6bc3ddee3130edc"
])
assert result == {
"d2502ebbd7df41ceba8d3275595cac33": {
"foo": "bar"
},
"5394aa025b8e401ca6bc3ddee3130edc": {
"foo": "baz"
},
}
def test_set(self):
self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
assert Node.objects.get(id="d2502ebbd7df41ceba8d3275595cac33"
).data == compress(b'{"foo":"bar"}')
def test_delete(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=b'{"foo": "bar"}')
self.ns.delete(node.id)
assert not Node.objects.filter(id=node.id).exists()
def test_delete_multi(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=b'{"foo": "bar"}')
self.ns.delete_multi([node.id])
assert not Node.objects.filter(id=node.id).exists()
def test_cleanup(self):
now = timezone.now()
cutoff = now - timedelta(days=1)
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
timestamp=now,
data=b'{"foo": "bar"}')
node2 = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac34",
timestamp=cutoff,
data=b'{"foo": "bar"}')
self.ns.cleanup(cutoff)
assert Node.objects.filter(id=node.id).exists()
assert not Node.objects.filter(id=node2.id).exists()
def test_cache(self):
node_1 = ("a" * 32, {"foo": "a"})
node_2 = ("b" * 32, {"foo": "b"})
node_3 = ("c" * 32, {"foo": "c"})
for node_id, data in [node_1, node_2, node_3]:
Node.objects.create(id=node_id,
data=compress(json_dumps(data).encode("utf8")))
# Get / get multi populates cache
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_2[0], node_3[0]]) == {
node_2[0]: node_2[1],
node_3[0]: node_3[1],
}
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get(node_2[0]) == node_2[1]
assert self.ns.get(node_3[0]) == node_3[1]
assert mock_get.call_count == 0
with mock.patch.object(Node.objects, "filter") as mock_filter:
assert self.ns.get_multi([node_1[0], node_2[0], node_3[0]])
assert mock_filter.call_count == 0
# Manually deleted item should still retreivable from cache
Node.objects.get(id=node_1[0]).delete()
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_1[0]: node_1[1],
node_2[0]: node_2[1],
}
# Deletion clars cache
self.ns.delete(node_1[0])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_2[0]: node_2[1]
}
self.ns.delete_multi([node_1[0], node_2[0]])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {}
# Setting the item updates cache
new_value = {"event_id": "d" * 32}
self.ns.set(node_1[0], new_value)
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == new_value
assert mock_get.call_count == 0
# Missing rows are never cached
assert self.ns.get("node_4") is None
with mock.patch.object(Node.objects, "get") as mock_get:
mock_get.side_effect = Node.DoesNotExist
self.ns.get("node_4")
self.ns.get("node_4")
assert mock_get.call_count == 2
class DjangoNodeStorageTest(TestCase):
def setUp(self):
self.ns = DjangoNodeStorage()
def test_get(self):
node = Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
)
result = self.ns.get(node.id)
assert result == node.data
def test_get_multi(self):
nodes = [
Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
),
Node.objects.create(
id='5394aa025b8e401ca6bc3ddee3130edc',
data={
'foo': 'baz',
}
),
]
result = self.ns.get_multi([
'd2502ebbd7df41ceba8d3275595cac33', '5394aa025b8e401ca6bc3ddee3130edc'
])
assert result == dict((n.id, n.data) for n in nodes)
def test_set(self):
self.ns.set('d2502ebbd7df41ceba8d3275595cac33', {
'foo': 'bar',
})
assert Node.objects.get(id='d2502ebbd7df41ceba8d3275595cac33').data == {
'foo': 'bar',
}
def test_set_multi(self):
self.ns.set_multi({
'd2502ebbd7df41ceba8d3275595cac33': {
'foo': 'bar',
},
'5394aa025b8e401ca6bc3ddee3130edc': {
'foo': 'baz',
},
})
assert Node.objects.get(id='d2502ebbd7df41ceba8d3275595cac33').data == {
'foo': 'bar',
}
assert Node.objects.get(id='5394aa025b8e401ca6bc3ddee3130edc').data == {
'foo': 'baz',
}
def test_create(self):
node_id = self.ns.create({
'foo': 'bar',
})
assert Node.objects.get(id=node_id).data == {
'foo': 'bar',
}
def test_delete(self):
node = Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
)
self.ns.delete(node.id)
assert not Node.objects.filter(id=node.id).exists()
def setUp(self):
self.ns = DjangoNodeStorage()
class DjangoNodeStorageTest(TestCase):
def setUp(self):
self.ns = DjangoNodeStorage()
def test_get(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"})
result = self.ns.get(node.id)
assert result == node.data
def test_get_multi(self):
nodes = [
Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"}),
Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc",
data={"foo": "baz"}),
]
result = self.ns.get_multi([
"d2502ebbd7df41ceba8d3275595cac33",
"5394aa025b8e401ca6bc3ddee3130edc"
])
assert result == dict((n.id, n.data) for n in nodes)
def test_set(self):
self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
assert Node.objects.get(
id="d2502ebbd7df41ceba8d3275595cac33").data == {
"foo": "bar"
}
def test_set_multi(self):
self.ns.set_multi({
"d2502ebbd7df41ceba8d3275595cac33": {
"foo": "bar"
},
"5394aa025b8e401ca6bc3ddee3130edc": {
"foo": "baz"
},
})
assert Node.objects.get(
id="d2502ebbd7df41ceba8d3275595cac33").data == {
"foo": "bar"
}
assert Node.objects.get(
id="5394aa025b8e401ca6bc3ddee3130edc").data == {
"foo": "baz"
}
def test_create(self):
node_id = self.ns.create({"foo": "bar"})
assert Node.objects.get(id=node_id).data == {"foo": "bar"}
def test_delete(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"})
self.ns.delete(node.id)
assert not Node.objects.filter(id=node.id).exists()
def test_delete_multi(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data={"foo": "bar"})
self.ns.delete_multi([node.id])
assert not Node.objects.filter(id=node.id).exists()
def test_cleanup(self):
now = timezone.now()
cutoff = now - timedelta(days=1)
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
timestamp=now,
data={"foo": "bar"})
node2 = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac34",
timestamp=cutoff,
data={"foo": "bar"})
self.ns.cleanup(cutoff)
assert Node.objects.filter(id=node.id).exists()
assert not Node.objects.filter(id=node2.id).exists()
class DjangoNodeStorageTest(TestCase):
def setUp(self):
self.ns = DjangoNodeStorage()
def test_get(self):
node = Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
)
result = self.ns.get(node.id)
assert result == node.data
def test_get_multi(self):
nodes = [
Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
),
Node.objects.create(
id='5394aa025b8e401ca6bc3ddee3130edc',
data={
'foo': 'baz',
}
),
]
result = self.ns.get_multi([
'd2502ebbd7df41ceba8d3275595cac33', '5394aa025b8e401ca6bc3ddee3130edc'
])
assert result == dict((n.id, n.data) for n in nodes)
def test_set(self):
self.ns.set('d2502ebbd7df41ceba8d3275595cac33', {
'foo': 'bar',
})
assert Node.objects.get(id='d2502ebbd7df41ceba8d3275595cac33').data == {
'foo': 'bar',
}
def test_set_multi(self):
self.ns.set_multi({
'd2502ebbd7df41ceba8d3275595cac33': {
'foo': 'bar',
},
'5394aa025b8e401ca6bc3ddee3130edc': {
'foo': 'baz',
},
})
assert Node.objects.get(id='d2502ebbd7df41ceba8d3275595cac33').data == {
'foo': 'bar',
}
assert Node.objects.get(id='5394aa025b8e401ca6bc3ddee3130edc').data == {
'foo': 'baz',
}
def test_create(self):
node_id = self.ns.create({
'foo': 'bar',
})
assert Node.objects.get(id=node_id).data == {
'foo': 'bar',
}
def test_delete(self):
node = Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
)
self.ns.delete(node.id)
assert not Node.objects.filter(id=node.id).exists()
def test_delete_multi(self):
node = Node.objects.create(
id='d2502ebbd7df41ceba8d3275595cac33',
data={
'foo': 'bar',
}
)
self.ns.delete_multi([node.id])
assert not Node.objects.filter(id=node.id).exists()