def test_get_multi(self):
Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", data=compress(b'{"foo": "bar"}'))
Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc", data=compress(b'{"foo": "baz"}'))
result = self.ns.get_multi(
["d2502ebbd7df41ceba8d3275595cac33", "5394aa025b8e401ca6bc3ddee3130edc"]
)
assert result == {
"d2502ebbd7df41ceba8d3275595cac33": {"foo": "bar"},
"5394aa025b8e401ca6bc3ddee3130edc": {"foo": "baz"},
}
def _set_bytes(self, id, data, ttl=None):
create_or_update(Node,
id=id,
values={
"data": compress(data),
"timestamp": timezone.now()
})
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# enforce unicode strings to guarantee consistency
if isinstance(value, str):
value = six.text_type(value)
return compress(pickle.dumps(value))
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# enforce unicode strings to guarantee consistency
if isinstance(value, str):
value = unicode(value)
return compress(pickle.dumps(value))
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# enforce strings to guarantee consistency
if isinstance(value, bytes):
value = str(value)
# db values need to be in unicode
return compress(pickle.dumps(value))
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# enforce six.text_type strings to guarantee consistency
if isinstance(value, six.binary_type):
value = six.text_type(value)
# db values need to be in unicode
return compress(pickle.dumps(value))
def test_cache(self):
node_1 = ("a" * 32, {"foo": "a"})
node_2 = ("b" * 32, {"foo": "b"})
node_3 = ("c" * 32, {"foo": "c"})
for node_id, data in [node_1, node_2, node_3]:
Node.objects.create(id=node_id,
data=compress(json_dumps(data).encode("utf8")))
# Get / get multi populates cache
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_2[0], node_3[0]]) == {
node_2[0]: node_2[1],
node_3[0]: node_3[1],
}
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get(node_2[0]) == node_2[1]
assert self.ns.get(node_3[0]) == node_3[1]
assert mock_get.call_count == 0
with mock.patch.object(Node.objects, "filter") as mock_filter:
assert self.ns.get_multi([node_1[0], node_2[0], node_3[0]])
assert mock_filter.call_count == 0
# Manually deleted item should still retreivable from cache
Node.objects.get(id=node_1[0]).delete()
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_1[0]: node_1[1],
node_2[0]: node_2[1],
}
# Deletion clars cache
self.ns.delete(node_1[0])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_2[0]: node_2[1]
}
self.ns.delete_multi([node_1[0], node_2[0]])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {}
# Setting the item updates cache
new_value = {"event_id": "d" * 32}
self.ns.set(node_1[0], new_value)
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == new_value
assert mock_get.call_count == 0
# Missing rows are never cached
assert self.ns.get("node_4") is None
with mock.patch.object(Node.objects, "get") as mock_get:
mock_get.side_effect = Node.DoesNotExist
self.ns.get("node_4")
self.ns.get("node_4")
assert mock_get.call_count == 2
def test_event_node_id(self):
# Create an event without specifying node_id. A node_id should be generated
e1 = Event(project_id=1, event_id='abc', data={'foo': 'bar'})
e1.save()
e1_node_id = e1.data.id
assert e1.data.id is not None, "We should have generated a node_id for this event"
e1_body = nodestore.get(e1_node_id)
assert e1_body == {'foo': 'bar'}, "The event body should be in nodestore"
e1 = Event.objects.get(project_id=1, event_id='abc')
assert e1.data.data == {'foo': 'bar'}, "The event body should be loaded from nodestore"
assert e1.data.id == e1_node_id, "The event's node_id should be the same after load"
# Create another event that references the same nodestore object as the first event.
e2 = Event(project_id=1, event_id='def', data={'node_id': e1_node_id})
assert e2.data.id == e1_node_id, "The event should use the provided node_id"
e2_body = nodestore.get(e1_node_id)
assert e2_body == {'foo': 'bar'}, "The event body should be in nodestore already"
e2.save()
e2_body = nodestore.get(e1_node_id)
assert e2_body == {'foo': 'bar'}, "The event body should not be overwritten by save"
e2 = Event.objects.get(project_id=1, event_id='def')
assert e2.data.data == {'foo': 'bar'}, "The event body should be loaded from nodestore"
assert e2.data.id == e1_node_id, "The event's node_id should be the same after load"
# Create an event with a new event body that specifies the node_id to use.
e3 = Event(project_id=1, event_id='ghi', data={'baz': 'quux', 'node_id': '1:ghi'})
assert e3.data.id == '1:ghi', "Event should have the specified node_id"
assert e3.data.data == {'baz': 'quux'}, "Event body should be the one provided (sans node_id)"
e3.save()
e3_body = nodestore.get('1:ghi')
assert e3_body == {'baz': 'quux'}, "Event body should be saved to nodestore"
e3 = Event.objects.get(project_id=1, event_id='ghi')
assert e3.data.data == {'baz': 'quux'}, "Event body should be loaded from nodestore"
assert e3.data.id == '1:ghi', "Loaded event should have the correct node_id"
# Try load it again, but using the pickled/compressed string we would expect to find
# in the column
e3_pickled_id = compress(pickle.dumps({'node_id': '1:ghi'}))
e3 = Event(project_id=1, event_id='jkl', data=e3_pickled_id)
assert e3.data.data == {'baz': 'quux'}, "Event body should be loaded from nodestore"
# Event with no data should not be saved (or loaded) from nodestore
e4 = Event(project_id=1, event_id='mno', data=None)
e4.save()
assert nodestore.get('1:mno') is None, "We should not have saved anything to nodestore"
e4 = Event.objects.get(project_id=1, event_id='mno')
assert e4.data.id is None
assert e4.data.data == {} # NodeData returns {} by default
Event.objects.bind_nodes([e4], 'data')
assert e4.data.id is None
assert e4.data.data == {}
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# TODO(dcramer): we should probably do this more intelligently
# and manually
if not value.id:
value.id = nodestore.create(value.data)
else:
nodestore.set(value.id, value.data)
return compress(pickle.dumps({'node_id': value.id}))
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# TODO(dcramer): we should probably do this more intelligently
# and manually
if not value.id:
value.id = nodestore.create(value.data)
else:
nodestore.set(value.id, value.data)
return compress(pickle.dumps({'node_id': value.id}))
def test_does_transition_data_to_node(self):
group = self.group
data = {'key': 'value'}
query_bits = [
"INSERT INTO sentry_message (group_id, project_id, data, message, datetime)",
"VALUES(%s, %s, %s, %s, %s)",
]
params = [
group.id, group.project_id,
compress(pickle.dumps(data)), 'test',
timezone.now()
]
# This is pulled from SQLInsertCompiler
if connection.features.can_return_id_from_insert:
r_fmt, r_params = connection.ops.return_insert_id()
if r_fmt:
query_bits.append(r_fmt % Event._meta.pk.column)
params += r_params
cursor = connection.cursor()
cursor.execute(' '.join(query_bits), params)
if connection.features.can_return_id_from_insert:
event_id = connection.ops.fetch_returned_insert_id(cursor)
else:
event_id = connection.ops.last_insert_id(cursor,
Event._meta.db_table,
Event._meta.pk.column)
event = Event.objects.get(id=event_id)
assert type(event.data) == NodeData
assert event.data == data
assert event.data.id is None
event.save()
assert event.data == data
assert event.data.id is not None
node_id = event.data.id
event = Event.objects.get(id=event_id)
Event.objects.bind_nodes([event], 'data')
assert event.data == data
assert event.data.id == node_id
def get_prep_value(self, value):
"""
Prepares the NodeData to be written in a Model.save() call.
Makes sure the event body is written to nodestore and
returns the node_id reference to be written to rowstore.
"""
if not value and self.null:
# save ourselves some storage
return None
if value.id is None:
value.id = self.id_func()
value.save()
return compress(pickle.dumps({'node_id': value.id}))
def get_prep_value(self, value):
"""
Prepares the NodeData to be written in a Model.save() call.
Makes sure the event body is written to nodestore and
returns the node_id reference to be written to rowstore.
"""
if not value and self.null:
# save ourselves some storage
return None
if value.id is None:
value.id = self.id_func()
value.save()
return compress(pickle.dumps({'node_id': value.id}))
def test_does_transition_data_to_node(self):
group = self.group
data = {'key': 'value'}
query_bits = [
"INSERT INTO sentry_message (group_id, project_id, data, message, datetime)",
"VALUES(%s, %s, %s, %s, %s)",
]
params = [group.id, group.project_id, compress(pickle.dumps(data)), 'test', timezone.now()]
# This is pulled from SQLInsertCompiler
if connection.features.can_return_id_from_insert:
r_fmt, r_params = connection.ops.return_insert_id()
if r_fmt:
query_bits.append(r_fmt % Event._meta.pk.column)
params += r_params
cursor = connection.cursor()
cursor.execute(' '.join(query_bits), params)
if connection.features.can_return_id_from_insert:
event_id = connection.ops.fetch_returned_insert_id(cursor)
else:
event_id = connection.ops.last_insert_id(
cursor, Event._meta.db_table, Event._meta.pk.column
)
event = Event.objects.get(id=event_id)
assert type(event.data) == NodeData
assert event.data == data
assert event.data.id is None
event.save()
assert event.data == data
assert event.data.id is not None
node_id = event.data.id
event = Event.objects.get(id=event_id)
Event.objects.bind_nodes([event], 'data')
assert event.data == data
assert event.data.id == node_id
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
# We can't put our wrappers into the nodestore, so we need to
# ensure that the data is converted into a plain old dict
data = value.data
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
# TODO(dcramer): we should probably do this more intelligently
# and manually
if not value.id:
value.id = nodestore.create(data)
else:
nodestore.set(value.id, data)
return compress(pickle.dumps({'node_id': value.id}))
def read_encode(path):
fh = open(path, 'rb')
try:
return compress(fh.read())
finally:
fh.close()
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
return compress(pickle.dumps(value))
def get_prep_value(self, value):
if not value and self.null:
# save ourselves some storage
return None
return compress(pickle.dumps(value))
class TestDjangoNodeStorage:
def setup_method(self):
self.ns = DjangoNodeStorage()
@pytest.mark.parametrize(
"node_data",
[
compress(b'{"foo": "bar"}'),
compress(pickle.dumps({"foo": "bar"})),
# hardcoded pickle value from python 3.6
compress(
b"\x80\x03}q\x00X\x03\x00\x00\x00fooq\x01X\x03\x00\x00\x00barq\x02s."
),
# hardcoded pickle value from python 2.7
compress(b"(dp0\nS'foo'\np1\nS'bar'\np2\ns."),
],
)
def test_get(self, node_data):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=node_data)
result = self.ns.get(node.id)
assert result == {"foo": "bar"}
def test_get_multi(self):
Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=compress(b'{"foo": "bar"}'))
Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc",
data=compress(b'{"foo": "baz"}'))
result = self.ns.get_multi([
"d2502ebbd7df41ceba8d3275595cac33",
"5394aa025b8e401ca6bc3ddee3130edc"
])
assert result == {
"d2502ebbd7df41ceba8d3275595cac33": {
"foo": "bar"
},
"5394aa025b8e401ca6bc3ddee3130edc": {
"foo": "baz"
},
}
def test_set(self):
self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
assert Node.objects.get(id="d2502ebbd7df41ceba8d3275595cac33"
).data == compress(b'{"foo":"bar"}')
def test_delete(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=b'{"foo": "bar"}')
self.ns.delete(node.id)
assert not Node.objects.filter(id=node.id).exists()
def test_delete_multi(self):
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
data=b'{"foo": "bar"}')
self.ns.delete_multi([node.id])
assert not Node.objects.filter(id=node.id).exists()
def test_cleanup(self):
now = timezone.now()
cutoff = now - timedelta(days=1)
node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33",
timestamp=now,
data=b'{"foo": "bar"}')
node2 = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac34",
timestamp=cutoff,
data=b'{"foo": "bar"}')
self.ns.cleanup(cutoff)
assert Node.objects.filter(id=node.id).exists()
assert not Node.objects.filter(id=node2.id).exists()
def test_cache(self):
node_1 = ("a" * 32, {"foo": "a"})
node_2 = ("b" * 32, {"foo": "b"})
node_3 = ("c" * 32, {"foo": "c"})
for node_id, data in [node_1, node_2, node_3]:
Node.objects.create(id=node_id,
data=compress(json_dumps(data).encode("utf8")))
# Get / get multi populates cache
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_2[0], node_3[0]]) == {
node_2[0]: node_2[1],
node_3[0]: node_3[1],
}
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get(node_2[0]) == node_2[1]
assert self.ns.get(node_3[0]) == node_3[1]
assert mock_get.call_count == 0
with mock.patch.object(Node.objects, "filter") as mock_filter:
assert self.ns.get_multi([node_1[0], node_2[0], node_3[0]])
assert mock_filter.call_count == 0
# Manually deleted item should still retreivable from cache
Node.objects.get(id=node_1[0]).delete()
assert self.ns.get(node_1[0]) == node_1[1]
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_1[0]: node_1[1],
node_2[0]: node_2[1],
}
# Deletion clars cache
self.ns.delete(node_1[0])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {
node_2[0]: node_2[1]
}
self.ns.delete_multi([node_1[0], node_2[0]])
assert self.ns.get_multi([node_1[0], node_2[0]]) == {}
# Setting the item updates cache
new_value = {"event_id": "d" * 32}
self.ns.set(node_1[0], new_value)
with mock.patch.object(Node.objects, "get") as mock_get:
assert self.ns.get(node_1[0]) == new_value
assert mock_get.call_count == 0
# Missing rows are never cached
assert self.ns.get("node_4") is None
with mock.patch.object(Node.objects, "get") as mock_get:
mock_get.side_effect = Node.DoesNotExist
self.ns.get("node_4")
self.ns.get("node_4")
assert mock_get.call_count == 2
def test_set(self):
self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
assert Node.objects.get(id="d2502ebbd7df41ceba8d3275595cac33"
).data == compress(b'{"foo":"bar"}')
def test_event_node_id(self):
# Create an event without specifying node_id. A node_id should be generated
e1 = Event(project_id=1, event_id='abc', data={'foo': 'bar'})
e1.save()
e1_node_id = e1.data.id
assert e1.data.id is not None, "We should have generated a node_id for this event"
e1_body = nodestore.get(e1_node_id)
assert e1_body == {
'foo': 'bar'
}, "The event body should be in nodestore"
e1 = Event.objects.get(project_id=1, event_id='abc')
assert e1.data.data == {
'foo': 'bar'
}, "The event body should be loaded from nodestore"
assert e1.data.id == e1_node_id, "The event's node_id should be the same after load"
# Create another event that references the same nodestore object as the first event.
e2 = Event(project_id=1, event_id='def', data={'node_id': e1_node_id})
assert e2.data.id == e1_node_id, "The event should use the provided node_id"
e2_body = nodestore.get(e1_node_id)
assert e2_body == {
'foo': 'bar'
}, "The event body should be in nodestore already"
e2.save()
e2_body = nodestore.get(e1_node_id)
assert e2_body == {
'foo': 'bar'
}, "The event body should not be overwritten by save"
e2 = Event.objects.get(project_id=1, event_id='def')
assert e2.data.data == {
'foo': 'bar'
}, "The event body should be loaded from nodestore"
assert e2.data.id == e1_node_id, "The event's node_id should be the same after load"
# Create an event with a new event body that specifies the node_id to use.
e3 = Event(project_id=1,
event_id='ghi',
data={
'baz': 'quux',
'node_id': '1:ghi'
})
assert e3.data.id == '1:ghi', "Event should have the specified node_id"
assert e3.data.data == {
'baz': 'quux'
}, "Event body should be the one provided (sans node_id)"
e3.save()
e3_body = nodestore.get('1:ghi')
assert e3_body == {
'baz': 'quux'
}, "Event body should be saved to nodestore"
e3 = Event.objects.get(project_id=1, event_id='ghi')
assert e3.data.data == {
'baz': 'quux'
}, "Event body should be loaded from nodestore"
assert e3.data.id == '1:ghi', "Loaded event should have the correct node_id"
# Try load it again, but using the pickled/compressed string we would expect to find
# in the column
e3_pickled_id = compress(pickle.dumps({'node_id': '1:ghi'}))
e3 = Event(project_id=1, event_id='jkl', data=e3_pickled_id)
assert e3.data.data == {
'baz': 'quux'
}, "Event body should be loaded from nodestore"
# Event with no data should not be saved (or loaded) from nodestore
e4 = Event(project_id=1, event_id='mno', data=None)
e4.save()
assert nodestore.get(
'1:mno') is None, "We should not have saved anything to nodestore"
e4 = Event.objects.get(project_id=1, event_id='mno')
assert e4.data.id is None
assert e4.data.data == {} # NodeData returns {} by default
Event.objects.bind_nodes([e4], 'data')
assert e4.data.id is None
assert e4.data.data == {}
def test_event_node_id(self):
# Create an event without specifying node_id. A node_id should be generated
e1 = Event(project_id=1, event_id="abc", data={"foo": "bar"})
e1.save()
e1_node_id = e1.data.id
assert e1.data.id is not None, "We should have generated a node_id for this event"
e1_body = nodestore.get(e1_node_id)
e1.data.save()
e1_body = nodestore.get(e1_node_id)
assert e1_body == {
"foo": "bar"
}, "The event body should be in nodestore"
e1 = Event.objects.get(project_id=1, event_id="abc")
assert e1.data.data == {
"foo": "bar"
}, "The event body should be loaded from nodestore"
assert e1.data.id == e1_node_id, "The event's node_id should be the same after load"
# Create another event that references the same nodestore object as the first event.
e2 = Event(project_id=1, event_id="def", data={"node_id": e1_node_id})
assert e2.data.id == e1_node_id, "The event should use the provided node_id"
e2_body = nodestore.get(e1_node_id)
assert e2_body == {
"foo": "bar"
}, "The event body should be in nodestore already"
e2.save()
e2_body = nodestore.get(e1_node_id)
assert e2_body == {
"foo": "bar"
}, "The event body should not be overwritten by save"
e2 = Event.objects.get(project_id=1, event_id="def")
assert e2.data.data == {
"foo": "bar"
}, "The event body should be loaded from nodestore"
assert e2.data.id == e1_node_id, "The event's node_id should be the same after load"
# Create an event with a new event body that specifies the node_id to use.
e3 = Event(project_id=1,
event_id="ghi",
data={
"baz": "quux",
"node_id": "1:ghi"
})
assert e3.data.id == "1:ghi", "Event should have the specified node_id"
assert e3.data.data == {
"baz": "quux"
}, "Event body should be the one provided (sans node_id)"
e3.save()
e3_body = nodestore.get("1:ghi")
e3.data.save()
e3_body = nodestore.get("1:ghi")
assert e3_body == {
"baz": "quux"
}, "Event body should be saved to nodestore"
e3 = Event.objects.get(project_id=1, event_id="ghi")
assert e3.data.data == {
"baz": "quux"
}, "Event body should be loaded from nodestore"
assert e3.data.id == "1:ghi", "Loaded event should have the correct node_id"
# Try load it again, but using the pickled/compressed string we would expect to find
# in the column
e3_pickled_id = compress(pickle.dumps({"node_id": "1:ghi"}))
e3 = Event(project_id=1, event_id="jkl", data=e3_pickled_id)
assert e3.data.data == {
"baz": "quux"
}, "Event body should be loaded from nodestore"
# Event with no data should not be saved (or loaded) from nodestore
e4 = Event(project_id=1, event_id="mno", data=None)
e4.save()
e4.data.save()
assert nodestore.get(
"1:mno") is None, "We should not have saved anything to nodestore"
e4 = Event.objects.get(project_id=1, event_id="mno")
assert e4.data.id is None
assert e4.data.data == {} # NodeData returns {} by default
e4.bind_node_data()
assert e4.data.id is None
assert e4.data.data == {}
