def test_register(self):
callback = object()
other_callback = object()
register_subscriber("hello")(callback)
assert subscriber_registry["hello"] == callback
register_subscriber("goodbye")(other_callback)
assert subscriber_registry["goodbye"] == other_callback
def test_normal(self):
cluster_name = settings.KAFKA_TOPICS[self.topic]["cluster"]
conf = {
"bootstrap.servers":
settings.KAFKA_CLUSTERS[cluster_name]["bootstrap.servers"],
"session.timeout.ms":
6000,
}
producer = Producer(conf)
producer.produce(self.topic, json.dumps(self.valid_wrapper))
producer.flush()
mock_callback = Mock()
mock_callback.side_effect = KeyboardInterrupt()
register_subscriber(self.registration_key)(mock_callback)
sub = self.create_subscription()
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=1)
consumer.run()
payload = self.valid_payload
payload["values"] = payload["result"]
payload["timestamp"] = parse_date(
payload["timestamp"]).replace(tzinfo=pytz.utc)
mock_callback.assert_called_once_with(payload, sub)
def test_normal(self):
cluster_name = settings.KAFKA_TOPICS[self.topic]["cluster"]
conf = {
"bootstrap.servers":
settings.KAFKA_CLUSTERS[cluster_name]["bootstrap.servers"],
"session.timeout.ms":
6000,
}
producer = Producer(conf)
producer.produce(self.topic, json.dumps(self.valid_wrapper))
producer.flush()
mock_callback = Mock()
mock_callback.side_effect = KeyboardInterrupt()
register_subscriber(self.registration_key)(mock_callback)
sub = QuerySubscription.objects.create(
project=self.project,
type=self.registration_key,
subscription_id=self.subscription_id,
dataset="something",
query="hello",
aggregations=[],
time_window=1,
resolution=1,
)
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=1)
consumer.run()
mock_callback.assert_called_once_with(self.valid_payload, sub)
def test_subscription_registered(self):
registration_key = "registered_test"
mock_callback = Mock()
register_subscriber(registration_key)(mock_callback)
with self.tasks():
snuba_query = create_snuba_query(
QueryDatasets.EVENTS,
"hello",
QueryAggregations.TOTAL,
timedelta(minutes=10),
timedelta(minutes=1),
None,
)
sub = create_snuba_subscription(self.project, registration_key,
snuba_query)
sub.refresh_from_db()
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
data = deepcopy(data)
data["payload"]["values"] = data["payload"]["result"]
data["payload"]["timestamp"] = parse_date(
data["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock_callback.assert_called_once_with(data["payload"], sub)
def test_already_registered(self):
callback = object()
other_callback = object()
register_subscriber("hello")(callback)
assert subscriber_registry["hello"] == callback
with self.assertRaises(Exception) as cm:
register_subscriber("hello")(other_callback)
assert six.text_type(cm.exception) == "Handler already registered for hello"
def test_shutdown(self):
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
valid_wrapper_2 = deepcopy(self.valid_wrapper)
valid_wrapper_2["payload"]["values"]["hello"] = 25
valid_wrapper_3 = deepcopy(valid_wrapper_2)
valid_wrapper_3["payload"]["values"]["hello"] = 5000
self.producer.produce(self.topic, json.dumps(valid_wrapper_2))
self.producer.flush()
counts = [0]
def mock_callback(*args, **kwargs):
counts[0] += 1
if counts[0] > 1:
raise KeyboardInterrupt()
mock = Mock()
mock.side_effect = mock_callback
register_subscriber(self.registration_key)(mock)
sub = QuerySubscription.objects.create(
project=self.project,
type=self.registration_key,
subscription_id=self.subscription_id,
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100)
consumer.run()
valid_payload = self.valid_payload
valid_payload["timestamp"] = parse_date(
valid_payload["timestamp"]).replace(tzinfo=pytz.utc)
valid_wrapper_2["payload"]["timestamp"] = parse_date(
valid_wrapper_2["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls(
[call(valid_payload, sub),
call(valid_wrapper_2["payload"], sub)])
# Offset should be committed for the first message, so second run should process
# the second message again
self.producer.produce(self.topic, json.dumps(valid_wrapper_3))
self.producer.flush()
mock.reset_mock()
counts[0] = 0
consumer.run()
valid_wrapper_3["payload"]["timestamp"] = parse_date(
valid_wrapper_3["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls([
call(valid_wrapper_2["payload"], sub),
call(valid_wrapper_3["payload"], sub)
])
def test_shutdown(self):
valid_wrapper_2 = deepcopy(self.valid_wrapper)
valid_wrapper_2["payload"]["result"]["hello"] = 25
valid_wrapper_3 = deepcopy(self.valid_wrapper)
valid_wrapper_3["payload"]["result"]["hello"] = 5000
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
self.producer.produce(self.topic, json.dumps(valid_wrapper_2))
self.producer.produce(self.topic, json.dumps(valid_wrapper_3))
self.producer.flush()
def normalize_payload(payload):
return {
**payload,
"values":
payload["result"],
"timestamp":
parse_date(payload["timestamp"]).replace(tzinfo=pytz.utc),
}
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100)
def mock_callback(*args, **kwargs):
if mock.call_count >= len(expected_calls):
consumer.shutdown()
mock = Mock(side_effect=mock_callback)
register_subscriber(self.registration_key)(mock)
sub = self.create_subscription()
expected_calls = [
call(normalize_payload(self.valid_payload), sub),
call(normalize_payload(valid_wrapper_2["payload"]), sub),
]
consumer.run()
mock.assert_has_calls(expected_calls)
expected_calls = [
call(normalize_payload(valid_wrapper_3["payload"]), sub)
]
mock.reset_mock()
consumer.run()
mock.assert_has_calls(expected_calls)
def test_subscription_registered(self):
registration_key = "registered_test"
mock_callback = Mock()
register_subscriber(registration_key)(mock_callback)
sub = QuerySubscription.objects.create(
project=self.project,
type=registration_key,
subscription_id="an_id",
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
mock_callback.assert_called_once_with(data["payload"], sub)
def test_batch_timeout(self, commit_offset_mock):
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
self.producer.flush()
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100,
commit_batch_timeout_ms=1)
def mock_callback(*args, **kwargs):
time.sleep(0.1)
consumer.shutdown()
mock = Mock(side_effect=mock_callback)
register_subscriber(self.registration_key)(mock)
self.create_subscription()
consumer.run()
# Once on revoke, once on shutdown, and once due to batch timeout
assert len(commit_offset_mock.call_args_list) == 3