def test_totp_can_enroll(self, validate_otp):
# XXX: Pretend an unbound function exists.
validate_otp.__func__ = None
with mock.patch("sentry.auth.authenticators.base.generate_secret_key",
return_value="Z" * 32):
resp = self.get_success_response("me", "totp")
assert resp.data["secret"] == "Z" * 32
assert (
resp.data["qrcode"] ==
"otpauth://totp/admin%40localhost?issuer=Sentry&secret=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ"
)
assert resp.data["form"]
assert resp.data["secret"]
# try to enroll
with self.tasks():
self.get_success_response("me",
"totp",
method="post",
**{
"secret": "secret12",
"otp": "1234"
})
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("1234")
interface = Authenticator.objects.get_interface(user=self.user,
interface_id="totp")
assert interface
assert interface.secret == "secret12"
assert interface.config == {"secret": "secret12"}
# also enrolls in recovery codes
recovery = Authenticator.objects.get_interface(user=self.user,
interface_id="recovery")
assert recovery.is_enrolled()
assert_security_email_sent("mfa-added")
# can rotate in place
self.get_success_response("me", "totp")
self.get_success_response("me",
"totp",
method="post",
**{
"secret": "secret56",
"otp": "5678"
})
assert validate_otp.call_args == mock.call("5678")
interface = Authenticator.objects.get_interface(user=self.user,
interface_id="totp")
assert interface.secret == "secret56"
assert interface.config == {"secret": "secret56"}
def test_shutdown(self):
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
valid_wrapper_2 = deepcopy(self.valid_wrapper)
valid_wrapper_2["payload"]["values"]["hello"] = 25
valid_wrapper_3 = deepcopy(valid_wrapper_2)
valid_wrapper_3["payload"]["values"]["hello"] = 5000
self.producer.produce(self.topic, json.dumps(valid_wrapper_2))
self.producer.flush()
counts = [0]
def mock_callback(*args, **kwargs):
counts[0] += 1
if counts[0] > 1:
raise KeyboardInterrupt()
mock = Mock()
mock.side_effect = mock_callback
register_subscriber(self.registration_key)(mock)
sub = QuerySubscription.objects.create(
project=self.project,
type=self.registration_key,
subscription_id=self.subscription_id,
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100)
consumer.run()
valid_payload = self.valid_payload
valid_payload["timestamp"] = parse_date(
valid_payload["timestamp"]).replace(tzinfo=pytz.utc)
valid_wrapper_2["payload"]["timestamp"] = parse_date(
valid_wrapper_2["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls(
[call(valid_payload, sub),
call(valid_wrapper_2["payload"], sub)])
# Offset should be committed for the first message, so second run should process
# the second message again
self.producer.produce(self.topic, json.dumps(valid_wrapper_3))
self.producer.flush()
mock.reset_mock()
counts[0] = 0
consumer.run()
valid_wrapper_3["payload"]["timestamp"] = parse_date(
valid_wrapper_3["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls([
call(valid_wrapper_2["payload"], sub),
call(valid_wrapper_3["payload"], sub)
])
def test_prepares_each_component(self, run):
self.client.get(self.url, format="json")
calls = [
call(component=self.component1,
install=self.install1,
project=self.project),
call(component=self.component2,
install=self.install2,
project=self.project),
]
run.assert_has_calls(calls, any_order=True)
def test_shutdown(self):
valid_wrapper_2 = deepcopy(self.valid_wrapper)
valid_wrapper_2["payload"]["result"]["hello"] = 25
valid_wrapper_3 = deepcopy(self.valid_wrapper)
valid_wrapper_3["payload"]["result"]["hello"] = 5000
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
self.producer.produce(self.topic, json.dumps(valid_wrapper_2))
self.producer.produce(self.topic, json.dumps(valid_wrapper_3))
self.producer.flush()
def normalize_payload(payload):
return {
**payload,
"values":
payload["result"],
"timestamp":
parse_date(payload["timestamp"]).replace(tzinfo=pytz.utc),
}
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100)
def mock_callback(*args, **kwargs):
if mock.call_count >= len(expected_calls):
consumer.shutdown()
mock = Mock(side_effect=mock_callback)
register_subscriber(self.registration_key)(mock)
sub = self.create_subscription()
expected_calls = [
call(normalize_payload(self.valid_payload), sub),
call(normalize_payload(valid_wrapper_2["payload"]), sub),
]
consumer.run()
mock.assert_has_calls(expected_calls)
expected_calls = [
call(normalize_payload(valid_wrapper_3["payload"]), sub)
]
mock.reset_mock()
consumer.run()
mock.assert_has_calls(expected_calls)
def test_prepares_each_component(self, run):
self.get_valid_response(self.org.slug,
qs_params={"projectId": self.project.id})
calls = [
call(component=self.component1,
install=self.install1,
project=self.project),
call(component=self.component2,
install=self.install2,
project=self.project),
]
run.assert_has_calls(calls, any_order=True)
def test_prepares_components_requiring_requests(self, run):
self.component.schema = {
"link": {
"required_fields": [{
"type": "select",
"name": "foo",
"label": "Foo",
"uri": "/sentry/foo"
}],
"optional_fields": [{
"type": "select",
"name": "beep",
"label": "Beep",
"uri": "/sentry/beep"
}],
},
"create": {
"required_fields": [{
"type": "select",
"name": "bar",
"label": "Bar",
"uri": "/sentry/bar"
}],
"optional_fields": [{
"type": "select",
"name": "baz",
"label": "Baz",
"uri": "/sentry/baz",
"skip_load_on_open": True,
}],
},
}
self.preparer.call()
assert call(install=self.install,
project=self.project,
uri="/sentry/foo") in run.mock_calls
assert (call(install=self.install,
project=self.project,
uri="/sentry/beep") in run.mock_calls)
assert call(install=self.install,
project=self.project,
uri="/sentry/bar") in run.mock_calls
assert (not call(install=self.install,
project=self.project,
uri="/sentry/baz") in run.mock_calls)
def test_u2f_can_enroll(self, try_enroll):
new_options = settings.SENTRY_OPTIONS.copy()
new_options["system.url-prefix"] = "https://testserver"
with self.settings(SENTRY_OPTIONS=new_options):
resp = self.get_success_response("me", "u2f")
assert resp.data["form"]
assert "secret" not in resp.data
assert "qrcode" not in resp.data
assert resp.data["challenge"]
with self.tasks():
self.get_success_response(
"me",
"u2f",
method="post",
**{
"deviceName": "device name",
"challenge": "challenge",
"response": "response",
},
)
assert try_enroll.call_count == 1
assert try_enroll.call_args == mock.call("challenge", "response",
"device name")
assert_security_email_sent("mfa-added")
def test_accept_pending_invite__sms_enroll(self, send_text, validate_otp):
om = self.get_om_and_init_invite()
# setup sms
new_options = settings.SENTRY_OPTIONS.copy()
new_options["sms.twilio-account"] = "twilio-account"
with self.settings(SENTRY_OPTIONS=new_options):
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={"user_id": "me", "interface_id": "sms"},
)
resp = self.client.post(url, data={"secret": "secret12", "phone": "1231234"})
assert resp.status_code == 204
resp = self.client.post(
url,
data={
"secret": "secret12",
"phone": "1231234",
"otp": "123123",
"memberId": om.id,
"token": om.token,
},
)
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("123123")
interface = Authenticator.objects.get_interface(user=self.user, interface_id="sms")
assert interface.phone_number == "1231234"
self.assert_invite_accepted(resp, om.id)
def test_u2f_can_enroll(self, try_enroll, email_log):
new_options = settings.SENTRY_OPTIONS.copy()
new_options["system.url-prefix"] = "https://testserver"
with self.settings(SENTRY_OPTIONS=new_options):
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={
"user_id": "me",
"interface_id": "u2f"
},
)
resp = self.client.get(url)
assert resp.status_code == 200
assert resp.data["form"]
assert "secret" not in resp.data
assert "qrcode" not in resp.data
assert resp.data["challenge"]
resp = self.client.post(
url,
data={
"deviceName": "device name",
"challenge": "challenge",
"response": "response",
},
)
assert try_enroll.call_count == 1
assert try_enroll.call_args == mock.call("challenge", "response",
"device name")
assert resp.status_code == 204
self._assert_security_email_sent("mfa-added", email_log)
def test_simple(self, mock_record):
provider = "dummy"
request = RequestFactory().post("/auth/sso/")
request.user = AnonymousUser()
auth_provider = AuthProvider.objects.create(
organization=self.organization, provider=provider)
identity = {"id": "1234", "email": "*****@*****.**", "name": "Morty"}
auth_identity = handle_new_user(auth_provider, self.organization,
request, identity)
user = auth_identity.user
assert user.email == identity["email"]
assert OrganizationMember.objects.filter(
organization=self.organization, user=user).exists()
signup_record = [
r for r in mock_record.call_args_list if r[0][0] == "user.signup"
]
assert signup_record == [
mock.call("user.signup",
user_id=user.id,
source="sso",
provider=provider,
referrer="in-app")
]
def test_should_filter_message(self, mock_is_valid_error_message):
TestItem = namedtuple("TestItem", "value formatted result")
items = [
TestItem({"type": "UnfilteredException"}, "UnfilteredException", True),
TestItem(
{"value": "This is an unfiltered exception."},
"This is an unfiltered exception.",
True,
),
TestItem(
{"type": "UnfilteredException", "value": "This is an unfiltered exception."},
"UnfilteredException: This is an unfiltered exception.",
True,
),
TestItem(
{"type": "FilteredException", "value": "This is a filtered exception."},
"FilteredException: This is a filtered exception.",
False,
),
]
data = {"exception": {"values": [item.value for item in items]}}
project_config = get_project_config(self.project)
manager = EventManager(data, project=self.project, project_config=project_config)
mock_is_valid_error_message.side_effect = [item.result for item in items]
assert manager.should_filter() == (True, FilterStatKeys.ERROR_MESSAGE)
assert mock_is_valid_error_message.call_args_list == [
mock.call(project_config, item.formatted) for item in items
]
def test_registration_valid(self, mock_record):
options.set("auth.allow-registration", True)
with self.feature("auth:register"):
resp = self.client.post(
self.path,
{
"username": "******",
"password": "******",
"name": "Foo Bar",
"op": "register",
},
)
assert resp.status_code == 302, (
resp.context["register_form"].errors if resp.status_code == 200 else None
)
user = User.objects.get(username="******")
assert user.email == "*****@*****.**"
assert user.check_password("foobar")
assert user.name == "Foo Bar"
assert not OrganizationMember.objects.filter(user=user).exists()
signup_record = [r for r in mock_record.call_args_list if r[0][0] == "user.signup"]
assert signup_record == [
mock.call(
"user.signup",
user_id=user.id,
source="register-form",
provider=None,
referrer="in-app",
)
]
def test_sms_can_enroll(self, send_text, validate_otp, email_log):
new_options = settings.SENTRY_OPTIONS.copy()
new_options["sms.twilio-account"] = "twilio-account"
with self.settings(SENTRY_OPTIONS=new_options):
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={"user_id": "me", "interface_id": "sms"},
)
resp = self.client.get(url)
assert resp.status_code == 200
assert resp.data["form"]
assert resp.data["secret"]
resp = self.client.post(url, data={"secret": "secret12", "phone": "1231234"})
assert send_text.call_count == 1
assert validate_otp.call_count == 0
assert resp.status_code == 204
resp = self.client.post(
url, data={"secret": "secret12", "phone": "1231234", "otp": "123123"}
)
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("123123")
interface = Authenticator.objects.get_interface(user=self.user, interface_id="sms")
assert interface.phone_number == "1231234"
self._assert_security_email_sent("mfa-added", email_log)
def test_totp_can_enroll(self, validate_otp, email_log):
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={"user_id": "me", "interface_id": "totp"},
)
resp = self.client.get(url)
assert resp.status_code == 200
assert len(resp.data["qrcode"])
assert resp.data["form"]
assert resp.data["secret"]
# try to enroll
resp = self.client.post(url, data={"secret": "secret12", "otp": "1234"})
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("1234")
assert resp.status_code == 204
interface = Authenticator.objects.get_interface(user=self.user, interface_id="totp")
assert interface
# also enrolls in recovery codes
recovery = Authenticator.objects.get_interface(user=self.user, interface_id="recovery")
assert recovery.is_enrolled
self._assert_security_email_sent("mfa-added", email_log)
# can't enroll again because no multi enrollment is allowed
resp = self.client.get(url)
assert resp.status_code == 400
resp = self.client.post(url)
assert resp.status_code == 400
def test_future_set_callback_empty():
future_set = FutureSet([])
callback = mock.Mock()
future_set.add_done_callback(callback)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
def test_process_pending_partitions_none(self, process_pending,
process_incr):
self.buf.pending_partitions = 2
with self.buf.cluster.map() as client:
client.zadd("b:p:0", {"foo": 1})
client.zadd("b:p:1", {"bar": 1})
client.zadd("b:p", {"baz": 1})
# On first pass, we are expecting to do:
# * process the buffer that doesn't have a partition (b:p)
# * queue up 2 jobs, one for each partition to process.
self.buf.process_pending()
assert len(process_incr.apply_async.mock_calls) == 1
process_incr.apply_async.assert_any_call(
kwargs={"batch_keys": ["baz"]})
assert len(process_pending.apply_async.mock_calls) == 2
process_pending.apply_async.mock_calls == [
mock.call(kwargs={"partition": 0}),
mock.call(kwargs={"partition": 1}),
]
# Confirm that we've only processed the unpartitioned buffer
client = self.buf.cluster.get_routing_client()
assert client.zrange("b:p", 0, -1) == []
assert client.zrange("b:p:0", 0, -1) != []
assert client.zrange("b:p:1", 0, -1) != []
# partition 0
self.buf.process_pending(partition=0)
assert len(process_incr.apply_async.mock_calls) == 2
process_incr.apply_async.assert_any_call(
kwargs={"batch_keys": ["foo"]})
assert client.zrange("b:p:0", 0, -1) == []
# Make sure we didn't queue up more
assert len(process_pending.apply_async.mock_calls) == 2
# partition 1
self.buf.process_pending(partition=1)
assert len(process_incr.apply_async.mock_calls) == 3
process_incr.apply_async.assert_any_call(
kwargs={"batch_keys": ["bar"]})
assert client.zrange("b:p:1", 0, -1) == []
# Make sure we didn't queue up more
assert len(process_pending.apply_async.mock_calls) == 2
def test_future_set_callback_error():
future_set = FutureSet([Future() for i in range(3)])
callback = mock.Mock()
future_set.add_done_callback(callback)
for i, future in enumerate(list(future_set)):
assert callback.call_count == 0
future.set_exception(Exception)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
other_callback = mock.Mock()
future_set.add_done_callback(other_callback)
assert other_callback.call_count == 1
assert other_callback.call_args == mock.call(future_set)
def test_future_set_callback_success():
future_set = FutureSet([Future() for i in range(3)])
callback = mock.Mock()
future_set.add_done_callback(callback)
for i, future in enumerate(list(future_set)):
assert callback.call_count == 0
future.set_result(True)
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
other_callback = mock.Mock()
future_set.add_done_callback(other_callback)
assert other_callback.call_count == 1
assert other_callback.call_args == mock.call(future_set)
def test_invalid_otp(self, validate_otp, email_log):
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={"user_id": "me", "interface_id": "totp"},
)
# try to enroll
resp = self.client.post(url, data={"secret": "secret12", "otp": "1234"})
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("1234")
assert resp.status_code == 400
assert email_log.call_count == 0
def test_future_broken_callback():
future_set = FutureSet([])
callback = mock.Mock(side_effect=Exception("Boom!"))
try:
future_set.add_done_callback(callback)
except Exception:
assert False, "should not raise"
assert callback.call_count == 1
assert callback.call_args == mock.call(future_set)
def assert_action_handler_called_with_actions(self, incident, actions, project=None):
project = self.project if project is None else project
if not actions:
if not incident:
assert not self.email_action_handler.called
else:
for call_args in self.email_action_handler.call_args_list:
assert call_args[0][1] != incident
else:
self.email_action_handler.assert_has_calls(
[call(action, incident, project) for action in actions], any_order=True
)
def test_totp_can_enroll(self, validate_otp, email_log):
# XXX: Pretend an unbound function exists.
validate_otp.__func__ = None
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={
"user_id": "me",
"interface_id": "totp"
},
)
with mock.patch("sentry.auth.authenticators.base.generate_secret_key",
return_value="Z" * 32):
resp = self.client.get(url)
assert resp.status_code == 200
assert resp.data["secret"] == "Z" * 32
assert (
resp.data["qrcode"] ==
"otpauth://totp/a%40example.com?issuer=Sentry&secret=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ"
)
assert resp.data["form"]
assert resp.data["secret"]
# try to enroll
resp = self.client.post(url,
data={
"secret": "secret12",
"otp": "1234"
})
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("1234")
assert resp.status_code == 204
interface = Authenticator.objects.get_interface(user=self.user,
interface_id="totp")
assert interface
# also enrolls in recovery codes
recovery = Authenticator.objects.get_interface(user=self.user,
interface_id="recovery")
assert recovery.is_enrolled()
self._assert_security_email_sent("mfa-added", email_log)
# can't enroll again because no multi enrollment is allowed
resp = self.client.get(url)
assert resp.status_code == 400
resp = self.client.post(url)
assert resp.status_code == 400
def test_totp_can_enroll(self, validate_otp, email_log):
# XXX: Pretend an unbound function exists.
validate_otp.__func__ = None
url = reverse(
"sentry-api-0-user-authenticator-enroll",
kwargs={
"user_id": "me",
"interface_id": "totp"
},
)
with mock.patch("sentry.models.authenticator.generate_secret_key",
return_value="Z" * 32):
resp = self.client.get(url)
assert resp.status_code == 200
assert resp.data["secret"] == "Z" * 32
with io.open(get_fixture_path("totp_qrcode.json")) as f:
assert resp.data["qrcode"] == json.loads(f.read())
assert resp.data["form"]
assert resp.data["secret"]
# try to enroll
resp = self.client.post(url,
data={
"secret": "secret12",
"otp": "1234"
})
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("1234")
assert resp.status_code == 204
interface = Authenticator.objects.get_interface(user=self.user,
interface_id="totp")
assert interface
# also enrolls in recovery codes
recovery = Authenticator.objects.get_interface(user=self.user,
interface_id="recovery")
assert recovery.is_enrolled
self._assert_security_email_sent("mfa-added", email_log)
# can't enroll again because no multi enrollment is allowed
resp = self.client.get(url)
assert resp.status_code == 400
resp = self.client.post(url)
assert resp.status_code == 400
def test_simple(self, mock_record):
auth_identity = self.handler.handle_new_user(self.identity)
user = auth_identity.user
assert user.email == self.identity["email"]
assert OrganizationMember.objects.filter(organization=self.organization, user=user).exists()
signup_record = [r for r in mock_record.call_args_list if r[0][0] == "user.signup"]
assert signup_record == [
mock.call(
"user.signup",
user_id=user.id,
source="sso",
provider=self.provider,
referrer="in-app",
)
]
def test_sms_can_enroll(self, send_text, validate_otp):
# XXX: Pretend an unbound function exists.
validate_otp.__func__ = None
new_options = settings.SENTRY_OPTIONS.copy()
new_options["sms.twilio-account"] = "twilio-account"
with self.settings(SENTRY_OPTIONS=new_options):
resp = self.get_success_response("me", "sms")
assert resp.data["form"]
assert resp.data["secret"]
self.get_success_response("me",
"sms",
method="post",
**{
"secret": "secret12",
"phone": "1231234"
})
assert send_text.call_count == 1
assert validate_otp.call_count == 0
with self.tasks():
self.get_success_response(
"me",
"sms",
method="post",
**{
"secret": "secret12",
"phone": "1231234",
"otp": "123123"
},
)
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("123123")
interface = Authenticator.objects.get_interface(user=self.user,
interface_id="sms")
assert interface.phone_number == "1231234"
assert_security_email_sent("mfa-added")
def test_invalid_otp(self, validate_otp):
# XXX: Pretend an unbound function exists.
validate_otp.__func__ = None
# try to enroll
with self.tasks():
self.get_error_response(
"me",
"totp",
method="post",
status_code=400,
**{
"secret": "secret12",
"otp": "1234"
},
)
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("1234")
assert len(mail.outbox) == 0
def test_accept_pending_invite__sms_enroll(self, send_text, validate_otp):
# XXX: Pretend an unbound function exists.
validate_otp.__func__ = None
om = self.get_om_and_init_invite()
# setup sms
new_options = settings.SENTRY_OPTIONS.copy()
new_options["sms.twilio-account"] = "twilio-account"
with self.settings(SENTRY_OPTIONS=new_options):
self.get_success_response("me",
"sms",
method="post",
**{
"secret": "secret12",
"phone": "1231234"
})
resp = self.get_success_response(
"me",
"sms",
method="post",
**{
"secret": "secret12",
"phone": "1231234",
"otp": "123123",
"memberId": om.id,
"token": om.token,
},
)
assert validate_otp.call_count == 1
assert validate_otp.call_args == mock.call("123123")
interface = Authenticator.objects.get_interface(user=self.user,
interface_id="sms")
assert interface.phone_number == "1231234"
self.assert_invite_accepted(resp, om.id)
def test_transitions():
callback = mock.MagicMock()
state = SynchronizedPartitionStateManager(callback)
with pytest.raises(InvalidState):
state.validate_local_message("topic", 0, 0)
state.set_local_offset("topic", 0, 1)
assert callback.mock_calls[-1] == mock.call(
"topic",
0,
(None, Offsets(None, None)),
(SynchronizedPartitionState.UNKNOWN, Offsets(1, None)),
)
with pytest.raises(InvalidState):
state.validate_local_message("topic", 0, 0)
state.set_remote_offset("topic", 0, 1)
assert callback.mock_calls[-1] == mock.call(
"topic",
0,
(SynchronizedPartitionState.UNKNOWN, Offsets(1, None)),
(SynchronizedPartitionState.SYNCHRONIZED, Offsets(1, 1)),
)
with pytest.raises(InvalidStateTransition):
state.set_local_offset("topic", 0, None)
with pytest.raises(InvalidStateTransition):
state.set_remote_offset("topic", 0, None)
state.set_remote_offset("topic", 0, 2)
assert callback.mock_calls[-1] == mock.call(
"topic",
0,
(SynchronizedPartitionState.SYNCHRONIZED, Offsets(1, 1)),
(SynchronizedPartitionState.LOCAL_BEHIND, Offsets(1, 2)),
)
state.validate_local_message("topic", 0, 1)
with pytest.raises(MessageNotReady):
state.validate_local_message("topic", 0, 2)
state.set_local_offset("topic", 0, 2)
assert callback.mock_calls[-1] == mock.call(
"topic",
0,
(SynchronizedPartitionState.LOCAL_BEHIND, Offsets(1, 2)),
(SynchronizedPartitionState.SYNCHRONIZED, Offsets(2, 2)),
)
state.set_remote_offset("topic", 1, 5)
assert callback.mock_calls[-1] == mock.call(
"topic",
1,
(None, Offsets(None, None)),
(SynchronizedPartitionState.UNKNOWN, Offsets(None, 5)),
)
state.set_local_offset("topic", 1, 0)
assert callback.mock_calls[-1] == mock.call(
"topic",
1,
(SynchronizedPartitionState.UNKNOWN, Offsets(None, 5)),
(SynchronizedPartitionState.LOCAL_BEHIND, Offsets(0, 5)),
)
before_calls = len(callback.mock_calls)
state.set_local_offset("topic", 1, 1)
state.set_local_offset("topic", 1, 2)
state.set_local_offset("topic", 1, 3)
state.set_local_offset("topic", 1, 4)
assert len(callback.mock_calls) == before_calls
state.set_local_offset("topic", 1, 5)
assert callback.mock_calls[-1] == mock.call(
"topic",
1,
(SynchronizedPartitionState.LOCAL_BEHIND, Offsets(4, 5)),
(SynchronizedPartitionState.SYNCHRONIZED, Offsets(5, 5)),
)
state.set_local_offset("topic", 1, 6)
assert callback.mock_calls[-1] == mock.call(
"topic",
1,
(SynchronizedPartitionState.SYNCHRONIZED, Offsets(5, 5)),
(SynchronizedPartitionState.REMOTE_BEHIND, Offsets(6, 5)),
)
def test_compression(self, mock_compress_file):
"""
For files larger than max memcached payload size we want to avoid
pointless compression and caching attempt since it fails silently.
Tests scenarios:
- happy path where compressed file is successfully cached
- compressed payload is too large to cache and we will avoid
compression and caching while the metadata cache exists
"""
project = self.project
release = Release.objects.create(
organization_id=project.organization_id, version="abc")
release.add_project(project)
filename = "file.min.js"
file = File.objects.create(
name=filename,
type="release.file",
headers={"Content-Type": "application/json; charset=utf-8"},
)
binary_body = unicode_body.encode("utf-8")
file.putfile(BytesIO(binary_body))
ReleaseFile.objects.create(name="file.min.js",
release=release,
organization_id=project.organization_id,
file=file)
mock_compress_file.return_value = (binary_body, binary_body)
releasefile_ident = ReleaseFile.get_ident(filename, None)
cache_key = get_release_file_cache_key(
release_id=release.id, releasefile_ident=releasefile_ident)
cache_key_meta = get_release_file_cache_key_meta(
release_id=release.id, releasefile_ident=releasefile_ident)
fetch_release_file(filename, release)
# Here the ANY is File() retrieved from cache/db
assert mock_compress_file.mock_calls == [call(ANY)]
assert cache.get(cache_key_meta)["compressed_size"] == len(binary_body)
assert cache.get(cache_key)
# Remove cache and check that calling fetch_release_file will do the
# compression and caching again
cache.set(cache_key, None)
mock_compress_file.reset_mock()
fetch_release_file(filename, release)
assert mock_compress_file.mock_calls == [call(ANY)]
assert cache.get(cache_key_meta)["compressed_size"] == len(binary_body)
assert cache.get(cache_key)
# If the file is bigger than the max cache value threshold, avoid
# compression and caching
cache.set(cache_key, None)
mock_compress_file.reset_mock()
with patch("sentry.lang.javascript.processor.CACHE_MAX_VALUE_SIZE",
len(binary_body) - 1):
result = fetch_release_file(filename, release)
assert result == http.UrlResult(
filename,
{"content-type": "application/json; charset=utf-8"},
binary_body,
200,
"utf-8",
)
assert mock_compress_file.mock_calls == []
assert cache.get(cache_key_meta)["compressed_size"] == len(binary_body)
assert cache.get(cache_key) is None
# If the file is bigger than the max cache value threshold, but the
# metadata cache is empty as well, compress and attempt to cache anyway
cache.set(cache_key, None)
cache.set(cache_key_meta, None)
mock_compress_file.reset_mock()
with patch("sentry.lang.javascript.processor.CACHE_MAX_VALUE_SIZE",
len(binary_body) - 1):
result = fetch_release_file(filename, release)
assert result == http.UrlResult(
filename,
{"content-type": "application/json; charset=utf-8"},
binary_body,
200,
"utf-8",
)
assert mock_compress_file.mock_calls == [call(ANY)]
assert cache.get(cache_key_meta)["compressed_size"] == len(binary_body)
assert cache.get(cache_key)
# If the file is smaller than the max cache value threshold, but the
# cache is empty, compress and cache
cache.set(cache_key, None)
mock_compress_file.reset_mock()
with patch("sentry.lang.javascript.processor.CACHE_MAX_VALUE_SIZE",
len(binary_body) + 1):
result = fetch_release_file(filename, release)
assert result == http.UrlResult(
filename,
{"content-type": "application/json; charset=utf-8"},
binary_body,
200,
"utf-8",
)
assert mock_compress_file.mock_calls == [call(ANY)]
assert cache.get(cache_key_meta)["compressed_size"] == len(binary_body)
assert cache.get(cache_key)
