def test_get_multi(self): Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", data=compress(b'{"foo": "bar"}')) Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc", data=compress(b'{"foo": "baz"}')) result = self.ns.get_multi( ["d2502ebbd7df41ceba8d3275595cac33", "5394aa025b8e401ca6bc3ddee3130edc"] ) assert result == { "d2502ebbd7df41ceba8d3275595cac33": {"foo": "bar"}, "5394aa025b8e401ca6bc3ddee3130edc": {"foo": "baz"}, }
def _set_bytes(self, id, data, ttl=None): create_or_update(Node, id=id, values={ "data": compress(data), "timestamp": timezone.now() })
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None # enforce unicode strings to guarantee consistency if isinstance(value, str): value = six.text_type(value) return compress(pickle.dumps(value))
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None # enforce unicode strings to guarantee consistency if isinstance(value, str): value = unicode(value) return compress(pickle.dumps(value))
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None # enforce strings to guarantee consistency if isinstance(value, bytes): value = str(value) # db values need to be in unicode return compress(pickle.dumps(value))
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None # enforce six.text_type strings to guarantee consistency if isinstance(value, six.binary_type): value = six.text_type(value) # db values need to be in unicode return compress(pickle.dumps(value))
def test_cache(self): node_1 = ("a" * 32, {"foo": "a"}) node_2 = ("b" * 32, {"foo": "b"}) node_3 = ("c" * 32, {"foo": "c"}) for node_id, data in [node_1, node_2, node_3]: Node.objects.create(id=node_id, data=compress(json_dumps(data).encode("utf8"))) # Get / get multi populates cache assert self.ns.get(node_1[0]) == node_1[1] assert self.ns.get_multi([node_2[0], node_3[0]]) == { node_2[0]: node_2[1], node_3[0]: node_3[1], } with mock.patch.object(Node.objects, "get") as mock_get: assert self.ns.get(node_1[0]) == node_1[1] assert self.ns.get(node_2[0]) == node_2[1] assert self.ns.get(node_3[0]) == node_3[1] assert mock_get.call_count == 0 with mock.patch.object(Node.objects, "filter") as mock_filter: assert self.ns.get_multi([node_1[0], node_2[0], node_3[0]]) assert mock_filter.call_count == 0 # Manually deleted item should still retreivable from cache Node.objects.get(id=node_1[0]).delete() assert self.ns.get(node_1[0]) == node_1[1] assert self.ns.get_multi([node_1[0], node_2[0]]) == { node_1[0]: node_1[1], node_2[0]: node_2[1], } # Deletion clars cache self.ns.delete(node_1[0]) assert self.ns.get_multi([node_1[0], node_2[0]]) == { node_2[0]: node_2[1] } self.ns.delete_multi([node_1[0], node_2[0]]) assert self.ns.get_multi([node_1[0], node_2[0]]) == {} # Setting the item updates cache new_value = {"event_id": "d" * 32} self.ns.set(node_1[0], new_value) with mock.patch.object(Node.objects, "get") as mock_get: assert self.ns.get(node_1[0]) == new_value assert mock_get.call_count == 0 # Missing rows are never cached assert self.ns.get("node_4") is None with mock.patch.object(Node.objects, "get") as mock_get: mock_get.side_effect = Node.DoesNotExist self.ns.get("node_4") self.ns.get("node_4") assert mock_get.call_count == 2
def test_event_node_id(self): # Create an event without specifying node_id. A node_id should be generated e1 = Event(project_id=1, event_id='abc', data={'foo': 'bar'}) e1.save() e1_node_id = e1.data.id assert e1.data.id is not None, "We should have generated a node_id for this event" e1_body = nodestore.get(e1_node_id) assert e1_body == {'foo': 'bar'}, "The event body should be in nodestore" e1 = Event.objects.get(project_id=1, event_id='abc') assert e1.data.data == {'foo': 'bar'}, "The event body should be loaded from nodestore" assert e1.data.id == e1_node_id, "The event's node_id should be the same after load" # Create another event that references the same nodestore object as the first event. e2 = Event(project_id=1, event_id='def', data={'node_id': e1_node_id}) assert e2.data.id == e1_node_id, "The event should use the provided node_id" e2_body = nodestore.get(e1_node_id) assert e2_body == {'foo': 'bar'}, "The event body should be in nodestore already" e2.save() e2_body = nodestore.get(e1_node_id) assert e2_body == {'foo': 'bar'}, "The event body should not be overwritten by save" e2 = Event.objects.get(project_id=1, event_id='def') assert e2.data.data == {'foo': 'bar'}, "The event body should be loaded from nodestore" assert e2.data.id == e1_node_id, "The event's node_id should be the same after load" # Create an event with a new event body that specifies the node_id to use. e3 = Event(project_id=1, event_id='ghi', data={'baz': 'quux', 'node_id': '1:ghi'}) assert e3.data.id == '1:ghi', "Event should have the specified node_id" assert e3.data.data == {'baz': 'quux'}, "Event body should be the one provided (sans node_id)" e3.save() e3_body = nodestore.get('1:ghi') assert e3_body == {'baz': 'quux'}, "Event body should be saved to nodestore" e3 = Event.objects.get(project_id=1, event_id='ghi') assert e3.data.data == {'baz': 'quux'}, "Event body should be loaded from nodestore" assert e3.data.id == '1:ghi', "Loaded event should have the correct node_id" # Try load it again, but using the pickled/compressed string we would expect to find # in the column e3_pickled_id = compress(pickle.dumps({'node_id': '1:ghi'})) e3 = Event(project_id=1, event_id='jkl', data=e3_pickled_id) assert e3.data.data == {'baz': 'quux'}, "Event body should be loaded from nodestore" # Event with no data should not be saved (or loaded) from nodestore e4 = Event(project_id=1, event_id='mno', data=None) e4.save() assert nodestore.get('1:mno') is None, "We should not have saved anything to nodestore" e4 = Event.objects.get(project_id=1, event_id='mno') assert e4.data.id is None assert e4.data.data == {} # NodeData returns {} by default Event.objects.bind_nodes([e4], 'data') assert e4.data.id is None assert e4.data.data == {}
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None # TODO(dcramer): we should probably do this more intelligently # and manually if not value.id: value.id = nodestore.create(value.data) else: nodestore.set(value.id, value.data) return compress(pickle.dumps({'node_id': value.id}))
def test_does_transition_data_to_node(self): group = self.group data = {'key': 'value'} query_bits = [ "INSERT INTO sentry_message (group_id, project_id, data, message, datetime)", "VALUES(%s, %s, %s, %s, %s)", ] params = [ group.id, group.project_id, compress(pickle.dumps(data)), 'test', timezone.now() ] # This is pulled from SQLInsertCompiler if connection.features.can_return_id_from_insert: r_fmt, r_params = connection.ops.return_insert_id() if r_fmt: query_bits.append(r_fmt % Event._meta.pk.column) params += r_params cursor = connection.cursor() cursor.execute(' '.join(query_bits), params) if connection.features.can_return_id_from_insert: event_id = connection.ops.fetch_returned_insert_id(cursor) else: event_id = connection.ops.last_insert_id(cursor, Event._meta.db_table, Event._meta.pk.column) event = Event.objects.get(id=event_id) assert type(event.data) == NodeData assert event.data == data assert event.data.id is None event.save() assert event.data == data assert event.data.id is not None node_id = event.data.id event = Event.objects.get(id=event_id) Event.objects.bind_nodes([event], 'data') assert event.data == data assert event.data.id == node_id
def get_prep_value(self, value): """ Prepares the NodeData to be written in a Model.save() call. Makes sure the event body is written to nodestore and returns the node_id reference to be written to rowstore. """ if not value and self.null: # save ourselves some storage return None if value.id is None: value.id = self.id_func() value.save() return compress(pickle.dumps({'node_id': value.id}))
def test_does_transition_data_to_node(self): group = self.group data = {'key': 'value'} query_bits = [ "INSERT INTO sentry_message (group_id, project_id, data, message, datetime)", "VALUES(%s, %s, %s, %s, %s)", ] params = [group.id, group.project_id, compress(pickle.dumps(data)), 'test', timezone.now()] # This is pulled from SQLInsertCompiler if connection.features.can_return_id_from_insert: r_fmt, r_params = connection.ops.return_insert_id() if r_fmt: query_bits.append(r_fmt % Event._meta.pk.column) params += r_params cursor = connection.cursor() cursor.execute(' '.join(query_bits), params) if connection.features.can_return_id_from_insert: event_id = connection.ops.fetch_returned_insert_id(cursor) else: event_id = connection.ops.last_insert_id( cursor, Event._meta.db_table, Event._meta.pk.column ) event = Event.objects.get(id=event_id) assert type(event.data) == NodeData assert event.data == data assert event.data.id is None event.save() assert event.data == data assert event.data.id is not None node_id = event.data.id event = Event.objects.get(id=event_id) Event.objects.bind_nodes([event], 'data') assert event.data == data assert event.data.id == node_id
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None # We can't put our wrappers into the nodestore, so we need to # ensure that the data is converted into a plain old dict data = value.data if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) # TODO(dcramer): we should probably do this more intelligently # and manually if not value.id: value.id = nodestore.create(data) else: nodestore.set(value.id, data) return compress(pickle.dumps({'node_id': value.id}))
def read_encode(path): fh = open(path, 'rb') try: return compress(fh.read()) finally: fh.close()
def get_prep_value(self, value): if not value and self.null: # save ourselves some storage return None return compress(pickle.dumps(value))
class TestDjangoNodeStorage: def setup_method(self): self.ns = DjangoNodeStorage() @pytest.mark.parametrize( "node_data", [ compress(b'{"foo": "bar"}'), compress(pickle.dumps({"foo": "bar"})), # hardcoded pickle value from python 3.6 compress( b"\x80\x03}q\x00X\x03\x00\x00\x00fooq\x01X\x03\x00\x00\x00barq\x02s." ), # hardcoded pickle value from python 2.7 compress(b"(dp0\nS'foo'\np1\nS'bar'\np2\ns."), ], ) def test_get(self, node_data): node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", data=node_data) result = self.ns.get(node.id) assert result == {"foo": "bar"} def test_get_multi(self): Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", data=compress(b'{"foo": "bar"}')) Node.objects.create(id="5394aa025b8e401ca6bc3ddee3130edc", data=compress(b'{"foo": "baz"}')) result = self.ns.get_multi([ "d2502ebbd7df41ceba8d3275595cac33", "5394aa025b8e401ca6bc3ddee3130edc" ]) assert result == { "d2502ebbd7df41ceba8d3275595cac33": { "foo": "bar" }, "5394aa025b8e401ca6bc3ddee3130edc": { "foo": "baz" }, } def test_set(self): self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"}) assert Node.objects.get(id="d2502ebbd7df41ceba8d3275595cac33" ).data == compress(b'{"foo":"bar"}') def test_delete(self): node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", data=b'{"foo": "bar"}') self.ns.delete(node.id) assert not Node.objects.filter(id=node.id).exists() def test_delete_multi(self): node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", data=b'{"foo": "bar"}') self.ns.delete_multi([node.id]) assert not Node.objects.filter(id=node.id).exists() def test_cleanup(self): now = timezone.now() cutoff = now - timedelta(days=1) node = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac33", timestamp=now, data=b'{"foo": "bar"}') node2 = Node.objects.create(id="d2502ebbd7df41ceba8d3275595cac34", timestamp=cutoff, data=b'{"foo": "bar"}') self.ns.cleanup(cutoff) assert Node.objects.filter(id=node.id).exists() assert not Node.objects.filter(id=node2.id).exists() def test_cache(self): node_1 = ("a" * 32, {"foo": "a"}) node_2 = ("b" * 32, {"foo": "b"}) node_3 = ("c" * 32, {"foo": "c"}) for node_id, data in [node_1, node_2, node_3]: Node.objects.create(id=node_id, data=compress(json_dumps(data).encode("utf8"))) # Get / get multi populates cache assert self.ns.get(node_1[0]) == node_1[1] assert self.ns.get_multi([node_2[0], node_3[0]]) == { node_2[0]: node_2[1], node_3[0]: node_3[1], } with mock.patch.object(Node.objects, "get") as mock_get: assert self.ns.get(node_1[0]) == node_1[1] assert self.ns.get(node_2[0]) == node_2[1] assert self.ns.get(node_3[0]) == node_3[1] assert mock_get.call_count == 0 with mock.patch.object(Node.objects, "filter") as mock_filter: assert self.ns.get_multi([node_1[0], node_2[0], node_3[0]]) assert mock_filter.call_count == 0 # Manually deleted item should still retreivable from cache Node.objects.get(id=node_1[0]).delete() assert self.ns.get(node_1[0]) == node_1[1] assert self.ns.get_multi([node_1[0], node_2[0]]) == { node_1[0]: node_1[1], node_2[0]: node_2[1], } # Deletion clars cache self.ns.delete(node_1[0]) assert self.ns.get_multi([node_1[0], node_2[0]]) == { node_2[0]: node_2[1] } self.ns.delete_multi([node_1[0], node_2[0]]) assert self.ns.get_multi([node_1[0], node_2[0]]) == {} # Setting the item updates cache new_value = {"event_id": "d" * 32} self.ns.set(node_1[0], new_value) with mock.patch.object(Node.objects, "get") as mock_get: assert self.ns.get(node_1[0]) == new_value assert mock_get.call_count == 0 # Missing rows are never cached assert self.ns.get("node_4") is None with mock.patch.object(Node.objects, "get") as mock_get: mock_get.side_effect = Node.DoesNotExist self.ns.get("node_4") self.ns.get("node_4") assert mock_get.call_count == 2
def test_set(self): self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"}) assert Node.objects.get(id="d2502ebbd7df41ceba8d3275595cac33" ).data == compress(b'{"foo":"bar"}')
def test_event_node_id(self): # Create an event without specifying node_id. A node_id should be generated e1 = Event(project_id=1, event_id='abc', data={'foo': 'bar'}) e1.save() e1_node_id = e1.data.id assert e1.data.id is not None, "We should have generated a node_id for this event" e1_body = nodestore.get(e1_node_id) assert e1_body == { 'foo': 'bar' }, "The event body should be in nodestore" e1 = Event.objects.get(project_id=1, event_id='abc') assert e1.data.data == { 'foo': 'bar' }, "The event body should be loaded from nodestore" assert e1.data.id == e1_node_id, "The event's node_id should be the same after load" # Create another event that references the same nodestore object as the first event. e2 = Event(project_id=1, event_id='def', data={'node_id': e1_node_id}) assert e2.data.id == e1_node_id, "The event should use the provided node_id" e2_body = nodestore.get(e1_node_id) assert e2_body == { 'foo': 'bar' }, "The event body should be in nodestore already" e2.save() e2_body = nodestore.get(e1_node_id) assert e2_body == { 'foo': 'bar' }, "The event body should not be overwritten by save" e2 = Event.objects.get(project_id=1, event_id='def') assert e2.data.data == { 'foo': 'bar' }, "The event body should be loaded from nodestore" assert e2.data.id == e1_node_id, "The event's node_id should be the same after load" # Create an event with a new event body that specifies the node_id to use. e3 = Event(project_id=1, event_id='ghi', data={ 'baz': 'quux', 'node_id': '1:ghi' }) assert e3.data.id == '1:ghi', "Event should have the specified node_id" assert e3.data.data == { 'baz': 'quux' }, "Event body should be the one provided (sans node_id)" e3.save() e3_body = nodestore.get('1:ghi') assert e3_body == { 'baz': 'quux' }, "Event body should be saved to nodestore" e3 = Event.objects.get(project_id=1, event_id='ghi') assert e3.data.data == { 'baz': 'quux' }, "Event body should be loaded from nodestore" assert e3.data.id == '1:ghi', "Loaded event should have the correct node_id" # Try load it again, but using the pickled/compressed string we would expect to find # in the column e3_pickled_id = compress(pickle.dumps({'node_id': '1:ghi'})) e3 = Event(project_id=1, event_id='jkl', data=e3_pickled_id) assert e3.data.data == { 'baz': 'quux' }, "Event body should be loaded from nodestore" # Event with no data should not be saved (or loaded) from nodestore e4 = Event(project_id=1, event_id='mno', data=None) e4.save() assert nodestore.get( '1:mno') is None, "We should not have saved anything to nodestore" e4 = Event.objects.get(project_id=1, event_id='mno') assert e4.data.id is None assert e4.data.data == {} # NodeData returns {} by default Event.objects.bind_nodes([e4], 'data') assert e4.data.id is None assert e4.data.data == {}
def test_event_node_id(self): # Create an event without specifying node_id. A node_id should be generated e1 = Event(project_id=1, event_id="abc", data={"foo": "bar"}) e1.save() e1_node_id = e1.data.id assert e1.data.id is not None, "We should have generated a node_id for this event" e1_body = nodestore.get(e1_node_id) e1.data.save() e1_body = nodestore.get(e1_node_id) assert e1_body == { "foo": "bar" }, "The event body should be in nodestore" e1 = Event.objects.get(project_id=1, event_id="abc") assert e1.data.data == { "foo": "bar" }, "The event body should be loaded from nodestore" assert e1.data.id == e1_node_id, "The event's node_id should be the same after load" # Create another event that references the same nodestore object as the first event. e2 = Event(project_id=1, event_id="def", data={"node_id": e1_node_id}) assert e2.data.id == e1_node_id, "The event should use the provided node_id" e2_body = nodestore.get(e1_node_id) assert e2_body == { "foo": "bar" }, "The event body should be in nodestore already" e2.save() e2_body = nodestore.get(e1_node_id) assert e2_body == { "foo": "bar" }, "The event body should not be overwritten by save" e2 = Event.objects.get(project_id=1, event_id="def") assert e2.data.data == { "foo": "bar" }, "The event body should be loaded from nodestore" assert e2.data.id == e1_node_id, "The event's node_id should be the same after load" # Create an event with a new event body that specifies the node_id to use. e3 = Event(project_id=1, event_id="ghi", data={ "baz": "quux", "node_id": "1:ghi" }) assert e3.data.id == "1:ghi", "Event should have the specified node_id" assert e3.data.data == { "baz": "quux" }, "Event body should be the one provided (sans node_id)" e3.save() e3_body = nodestore.get("1:ghi") e3.data.save() e3_body = nodestore.get("1:ghi") assert e3_body == { "baz": "quux" }, "Event body should be saved to nodestore" e3 = Event.objects.get(project_id=1, event_id="ghi") assert e3.data.data == { "baz": "quux" }, "Event body should be loaded from nodestore" assert e3.data.id == "1:ghi", "Loaded event should have the correct node_id" # Try load it again, but using the pickled/compressed string we would expect to find # in the column e3_pickled_id = compress(pickle.dumps({"node_id": "1:ghi"})) e3 = Event(project_id=1, event_id="jkl", data=e3_pickled_id) assert e3.data.data == { "baz": "quux" }, "Event body should be loaded from nodestore" # Event with no data should not be saved (or loaded) from nodestore e4 = Event(project_id=1, event_id="mno", data=None) e4.save() e4.data.save() assert nodestore.get( "1:mno") is None, "We should not have saved anything to nodestore" e4 = Event.objects.get(project_id=1, event_id="mno") assert e4.data.id is None assert e4.data.data == {} # NodeData returns {} by default e4.bind_node_data() assert e4.data.id is None assert e4.data.data == {}