class RedisBufferTest(TestCase): def setUp(self): self.buf = RedisBuffer(hosts={ 0: {'db': 9} }) self.buf.conn.flushdb() def test_default_host_is_local(self): buf = RedisBuffer() self.assertEquals(len(buf.conn.hosts), 1) self.assertEquals(buf.conn.hosts[0].host, 'localhost') def test_map_column_handles_foreignkeys(self): self.assertEquals(self.buf._map_column(Group, 'project', Project(id=1)), 1) def test_make_key_response(self): column = 'times_seen' filters = {'pk': 1} self.assertEquals(self.buf._make_key(Group, filters, column), 'sentry.group:88b48b31b5f100719c64316596b10b0f:times_seen') def test_make_extra_key_response(self): filters = {'pk': 1} self.assertEquals(self.buf._make_extra_key(Group, filters), 'sentry.group:extra:88b48b31b5f100719c64316596b10b0f') @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.maybe_delay') def test_incr_delays_task(self, maybe_delay): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} self.buf.incr(model, columns, filters) maybe_delay.assert_called_once_with(process_incr, model=model, columns=columns, filters=filters, extra=None) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.maybe_delay', mock.Mock()) def test_incr_does_buffer_to_conn(self): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} self.buf.incr(model, columns, filters) self.assertEquals(self.buf.conn.get('foo'), '1') @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.Buffer.process') def test_process_does_not_save_empty_results(self, process): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} self.buf.process(Group, columns, filters) self.assertFalse(process.called) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.Buffer.process') def test_process_does_save_call_with_results(self, process): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} self.buf.conn.set('foo', 2) self.buf.process(Group, columns, filters) process.assert_called_once_with(Group, {'times_seen': 2}, filters, None) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.Buffer.process') def test_process_does_clear_buffer(self, process): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} self.buf.conn.set('foo', 2) self.buf.process(Group, columns, filters) self.assertEquals(self.buf.conn.get('foo'), '0') @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.maybe_delay', mock.Mock()) def test_incr_does_buffer_extra_to_conn(self): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} self.buf.incr(model, columns, filters, extra={'foo': 'bar'}) self.assertEquals(self.buf.conn.hget('extra', 'foo'), pickle.dumps('bar')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) def test_process_saves_extra(self): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} the_date = datetime.now() + timedelta(days=5) self.buf.conn.set('foo', 1) self.buf.conn.hset('extra', 'last_seen', pickle.dumps(the_date)) self.buf.process(Group, columns, filters) group_ = Group.objects.get(pk=group.pk) self.assertEquals(group_.last_seen, the_date)
class RedisBufferTest(TestCase): def setUp(self): self.buf = RedisBuffer(hosts={0: {'db': 9}}) self.buf.conn.flushdb() def test_default_host_is_local(self): buf = RedisBuffer() self.assertEquals(len(buf.conn.hosts), 1) self.assertEquals(buf.conn.hosts[0].host, 'localhost') def test_coerce_val_handles_foreignkeys(self): assert self.buf._coerce_val(Project(id=1)) == '1' def test_coerce_val_handles_unicode(self): assert self.buf._coerce_val(u'\u201d') == '”' def test_make_key_response(self): column = 'times_seen' filters = {'pk': 1} self.assertEquals( self.buf._make_key(Group, filters, column), 'sentry.group:88b48b31b5f100719c64316596b10b0f:times_seen') def test_make_extra_key_response(self): filters = {'pk': 1} self.assertEquals( self.buf._make_extra_key(Group, filters), 'sentry.group:extra:88b48b31b5f100719c64316596b10b0f') @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.process_incr') def test_incr_delays_task(self, process_incr): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} self.buf.incr(model, columns, filters) kwargs = dict(model=model, columns=columns, filters=filters, extra=None) process_incr.apply_async.assert_called_once_with(kwargs=kwargs, countdown=5) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.process_incr', mock.Mock()) def test_incr_does_buffer_to_conn(self): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} self.buf.incr(model, columns, filters) self.assertEquals(self.buf.conn.get('foo'), '1') @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.Buffer.process') def test_process_does_not_save_empty_results(self, process): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} self.buf.process(Group, columns, filters) self.assertFalse(process.called) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.Buffer.process') def test_process_does_save_call_with_results(self, process): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} self.buf.conn.set('foo', 2) self.buf.process(Group, columns, filters) process.assert_called_once_with(Group, {'times_seen': 2}, filters, None) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.Buffer.process') def test_process_does_clear_buffer(self, process): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} self.buf.conn.set('foo', 2) self.buf.process(Group, columns, filters) self.assertEquals(self.buf.conn.get('foo'), '0') @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.base.process_incr', mock.Mock()) def test_incr_does_buffer_extra_to_conn(self): model = mock.Mock() columns = {'times_seen': 1} filters = {'pk': 1} self.buf.incr(model, columns, filters, extra={'foo': 'bar'}) self.assertEquals(self.buf.conn.hget('extra', 'foo'), pickle.dumps('bar')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_key', mock.Mock(return_value='foo')) @mock.patch('sentry.buffer.redis.RedisBuffer._make_extra_key', mock.Mock(return_value='extra')) def test_process_saves_extra(self): group = Group.objects.create(project=Project(id=1)) columns = {'times_seen': 1} filters = {'pk': group.pk} the_date = (timezone.now() + timedelta(days=5)).replace(microsecond=0) self.buf.conn.set('foo', 1) self.buf.conn.hset('extra', 'last_seen', pickle.dumps(the_date)) self.buf.process(Group, columns, filters) group_ = Group.objects.get(pk=group.pk) self.assertEquals(group_.last_seen.replace(microsecond=0), the_date)
class RedisBufferTest(TestCase): def setUp(self): self.buf = RedisBuffer() def test_coerce_val_handles_foreignkeys(self): assert self.buf._coerce_val(Project(id=1)) == b"1" def test_coerce_val_handles_unicode(self): assert self.buf._coerce_val("\u201d") == "”".encode() @mock.patch("sentry.buffer.redis.RedisBuffer._make_key", mock.Mock(return_value="foo")) @mock.patch("sentry.buffer.redis.process_incr") def test_process_pending_one_batch(self, process_incr): self.buf.incr_batch_size = 5 with self.buf.cluster.map() as client: client.zadd("b:p", {"foo": 1, "bar": 2}) self.buf.process_pending() assert len(process_incr.apply_async.mock_calls) == 1 process_incr.apply_async.assert_any_call(kwargs={"batch_keys": ["foo", "bar"]}) client = self.buf.cluster.get_routing_client() assert client.zrange("b:p", 0, -1) == [] @mock.patch("sentry.buffer.redis.RedisBuffer._make_key", mock.Mock(return_value="foo")) @mock.patch("sentry.buffer.redis.process_incr") def test_process_pending_multiple_batches(self, process_incr): self.buf.incr_batch_size = 2 with self.buf.cluster.map() as client: client.zadd("b:p", {"foo": 1, "bar": 2, "baz": 3}) self.buf.process_pending() assert len(process_incr.apply_async.mock_calls) == 2 process_incr.apply_async.assert_any_call(kwargs={"batch_keys": ["foo", "bar"]}) process_incr.apply_async.assert_any_call(kwargs={"batch_keys": ["baz"]}) client = self.buf.cluster.get_routing_client() assert client.zrange("b:p", 0, -1) == [] @mock.patch("sentry.buffer.redis.RedisBuffer._make_key", mock.Mock(return_value="foo")) @mock.patch("sentry.buffer.base.Buffer.process") def test_process_does_bubble_up_json(self, process): client = self.buf.cluster.get_routing_client() client.hmset( "foo", { "e+foo": '["s","bar"]', "e+datetime": '["d","1493791566.000000"]', "f": '{"pk": ["i","1"]}', "i+times_seen": "2", "m": "sentry.models.Group", }, ) columns = {"times_seen": 2} filters = {"pk": 1} extra = {"foo": "bar", "datetime": datetime(2017, 5, 3, 6, 6, 6, tzinfo=timezone.utc)} signal_only = None self.buf.process("foo") process.assert_called_once_with(Group, columns, filters, extra, signal_only) @mock.patch("sentry.buffer.redis.RedisBuffer._make_key", mock.Mock(return_value="foo")) @mock.patch("sentry.buffer.base.Buffer.process") def test_process_does_bubble_up_pickle(self, process): client = self.buf.cluster.get_routing_client() client.hmset( "foo", { "e+foo": "S'bar'\np1\n.", "f": "(dp1\nS'pk'\np2\nI1\ns.", "i+times_seen": "2", "m": "sentry.models.Group", }, ) columns = {"times_seen": 2} filters = {"pk": 1} extra = {"foo": "bar"} signal_only = None self.buf.process("foo") process.assert_called_once_with(Group, columns, filters, extra, signal_only) def test_get(self): model = mock.Mock() model.__name__ = "Mock" columns = ["times_seen"] filters = {"pk": 1} # If the value doesn't exist we just assume 0 assert self.buf.get(model, columns, filters=filters) == {"times_seen": 0} self.buf.incr(model, {"times_seen": 1}, filters) assert self.buf.get(model, columns, filters=filters) == {"times_seen": 1} self.buf.incr(model, {"times_seen": 5}, filters) assert self.buf.get(model, columns, filters=filters) == {"times_seen": 6} def test_incr_saves_to_redis(self): now = datetime(2017, 5, 3, 6, 6, 6, tzinfo=timezone.utc) client = self.buf.cluster.get_routing_client() model = mock.Mock() model.__name__ = "Mock" columns = {"times_seen": 1} filters = {"pk": 1, "datetime": now} key = self.buf._make_key(model, filters=filters) self.buf.incr(model, columns, filters, extra={"foo": "bar", "datetime": now}) result = client.hgetall(key) # Force keys to strings result = {force_text(k): v for k, v in result.items()} f = result.pop("f") assert pickle.loads(f) == {"pk": 1, "datetime": now} assert pickle.loads(result.pop("e+datetime")) == now assert pickle.loads(result.pop("e+foo")) == "bar" assert result == {"i+times_seen": b"1", "m": b"unittest.mock.Mock"} pending = client.zrange("b:p", 0, -1) assert pending == [key.encode("utf-8")] self.buf.incr(model, columns, filters, extra={"foo": "baz", "datetime": now}) result = client.hgetall(key) # Force keys to strings result = {force_text(k): v for k, v in result.items()} f = result.pop("f") assert pickle.loads(f) == {"pk": 1, "datetime": now} assert pickle.loads(result.pop("e+datetime")) == now assert pickle.loads(result.pop("e+foo")) == "baz" assert result == {"i+times_seen": b"2", "m": b"unittest.mock.Mock"} pending = client.zrange("b:p", 0, -1) assert pending == [key.encode("utf-8")] @mock.patch("sentry.buffer.redis.RedisBuffer._make_key", mock.Mock(return_value="foo")) @mock.patch("sentry.buffer.redis.process_incr") @mock.patch("sentry.buffer.redis.process_pending") def test_process_pending_partitions_none(self, process_pending, process_incr): self.buf.pending_partitions = 2 with self.buf.cluster.map() as client: client.zadd("b:p:0", {"foo": 1}) client.zadd("b:p:1", {"bar": 1}) client.zadd("b:p", {"baz": 1}) # On first pass, we are expecting to do: # * process the buffer that doesn't have a partition (b:p) # * queue up 2 jobs, one for each partition to process. self.buf.process_pending() assert len(process_incr.apply_async.mock_calls) == 1 process_incr.apply_async.assert_any_call(kwargs={"batch_keys": ["baz"]}) assert len(process_pending.apply_async.mock_calls) == 2 assert process_pending.apply_async.mock_calls == [ mock.call(kwargs={"partition": 0}), mock.call(kwargs={"partition": 1}), ] # Confirm that we've only processed the unpartitioned buffer client = self.buf.cluster.get_routing_client() assert client.zrange("b:p", 0, -1) == [] assert client.zrange("b:p:0", 0, -1) != [] assert client.zrange("b:p:1", 0, -1) != [] # partition 0 self.buf.process_pending(partition=0) assert len(process_incr.apply_async.mock_calls) == 2 process_incr.apply_async.assert_any_call(kwargs={"batch_keys": ["foo"]}) assert client.zrange("b:p:0", 0, -1) == [] # Make sure we didn't queue up more assert len(process_pending.apply_async.mock_calls) == 2 # partition 1 self.buf.process_pending(partition=1) assert len(process_incr.apply_async.mock_calls) == 3 process_incr.apply_async.assert_any_call(kwargs={"batch_keys": ["bar"]}) assert client.zrange("b:p:1", 0, -1) == [] # Make sure we didn't queue up more assert len(process_pending.apply_async.mock_calls) == 2 @mock.patch("sentry.buffer.redis.RedisBuffer._make_key", mock.Mock(return_value="foo")) @mock.patch("sentry.buffer.base.Buffer.process") def test_process_uses_signal_only(self, process): client = self.buf.cluster.get_routing_client() client.hmset( "foo", { "f": '{"pk": ["i","1"]}', "i+times_seen": "1", "m": "unittest.mock.Mock", "s": "1", }, ) self.buf.process("foo") process.assert_called_once_with(mock.Mock, {"times_seen": 1}, {"pk": 1}, {}, True)