def metricsmock():
"""Returns MetricsMock that a context to record metrics records
Usage::
def test_something(metricsmock):
with metricsmock as mm:
# do stuff
assert mm.has_record(
stat='some.stat',
kwargs_contains={
'something': 1
}
)
"""
return MetricsMock()
def test_filter_records_value(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics("foobar")
mymetrics.incr("key1", value=1, tags=["env:stage"])
key1_metrics = mm.filter_records(fun_name="incr", stat="foobar.key1")
assert len(key1_metrics) == 1
key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=1
)
assert len(key1_metrics) == 1
key1_metrics = mm.filter_records(
fun_name="incr", stat="foobar.key1", value=5
)
assert len(key1_metrics) == 0
def test_bugzilla_error_creates_error_notification(self):
self.assertEqual(Notification.objects.count(), 0)
self.mock_bugzilla_requests_post.side_effect = RequestException()
with self.assertRaises(bugzilla.BugzillaError):
with MetricsMock() as mm:
tasks.create_experiment_bug_task(self.user.id,
self.experiment.id)
self.assertTrue(
mm.has_record(
markus.INCR,
"experiments.tasks.create_experiment_bug.started",
value=1,
))
self.assertTrue(
mm.has_record(
markus.INCR,
"experiments.tasks.create_experiment_bug.failed",
value=1,
))
# Failures should abort timing metrics.
self.assertFalse(
mm.has_record(
markus.TIMING,
"experiments.tasks.create_experiment_bug.timing",
))
# Completed metric should not be sent.
self.assertFalse(
mm.has_record(
markus.INCR,
"experiments.tasks.create_experiment_bug.completed",
))
self.mock_bugzilla_requests_post.assert_called()
self.assertEqual(Notification.objects.count(), 1)
experiment = Experiment.objects.get(id=self.experiment.id)
self.assertEqual(experiment.bugzilla_id, None)
notification = Notification.objects.get()
self.assertEqual(notification.user, self.user)
self.assertEqual(notification.message,
tasks.NOTIFICATION_MESSAGE_CREATE_BUG_FAILED)
def test_histogram_helpers(self):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics.histogram("key1", value=1)
mymetrics.histogram("keymultiple", value=1)
mymetrics.histogram("keymultiple", value=1)
mm.assert_histogram(stat="foobar.key1")
mm.assert_histogram_once(stat="foobar.key1")
with pytest.raises(AssertionError):
mm.assert_histogram_once(stat="foobar.keymultiple")
mm.assert_not_histogram(stat="foobar.keynot")
mm.assert_not_histogram(stat="foobar.key1", value=5)
with pytest.raises(AssertionError):
mm.assert_not_histogram(stat="foobar.key1")
def metricsmock():
"""Return a MetricsMock for asserting things on metrics.
Usage::
def test_something(metricsmock):
metricsmock.assert_incr_once("some.stat")
metricsmock.assert_incr_once("other.stat", tags=["tag:enum"])
If you ever need to clear the records in the middle of a test, do::
metricsmock.clear_records()
For more, see: https://markus.readthedocs.io/en/latest/testing.html
"""
with MetricsMock() as mm:
yield mm
def metricsmock():
"""Return MetricsMock that a context to record metrics records.
Usage::
def test_something(metricsmock):
with metricsmock as mm:
# do stuff
assert mm.has_record(
'incr',
stat='some.stat',
value=1
)
https://markus.readthedocs.io/en/latest/testing.html
"""
return MetricsMock()
def test_it_sends_metrics(self, settings, mocked_autograph):
# 3 to sign
RecipeFactory.create_batch(3,
approver=UserFactory(),
enabler=UserFactory(),
signed=False)
# and 1 to unsign
RecipeFactory(signed=True, enabled=False)
with MetricsMock() as mm:
call_command("update_recipe_signatures")
mm.print_records()
assert mm.has_record(GAUGE,
stat="normandy.signing.recipes.signed",
value=3)
assert mm.has_record(GAUGE,
stat="normandy.signing.recipes.unsigned",
value=1)
def test_print_on_failure(self, capsys):
with MetricsMock() as mm:
markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
mymetrics = markus.get_metrics("foobar")
mymetrics.histogram("keymultiple", value=1)
mymetrics.histogram("keymultiple", value=1)
with pytest.raises(AssertionError):
mm.assert_histogram_once(stat="foobar.keymultiple")
# On assertion error, the assert_* methods will print the metrics
# records to stdout.
captured = capsys.readouterr()
expected = (
"<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n"
"<MetricsRecord type=histogram key=foobar.keymultiple value=1 tags=[]>\n"
)
assert captured.out == expected
def test_experiment_bug_successfully_created(self):
self.assertEqual(Notification.objects.count(), 0)
with MetricsMock() as mm:
tasks.create_experiment_bug_task(self.user.id, self.experiment.id)
self.assertTrue(
mm.has_record(
markus.INCR,
"experiments.tasks.create_experiment_bug.started",
value=1,
)
)
self.assertTrue(
mm.has_record(
markus.INCR,
"experiments.tasks.create_experiment_bug.completed",
value=1,
)
)
self.assertTrue(
mm.has_record(
markus.TIMING, "experiments.tasks.create_experiment_bug.timing"
)
)
# Failed metric should not be sent.
self.assertFalse(
mm.has_record(
markus.INCR, "experiments.tasks.create_experiment_bug.failed"
)
)
self.mock_bugzilla_requests_post.assert_called()
experiment = Experiment.objects.get(id=self.experiment.id)
self.assertEqual(experiment.bugzilla_id, self.bugzilla_id)
notification = Notification.objects.get()
self.assertEqual(notification.user, self.user)
self.assertEqual(
notification.message,
tasks.NOTIFICATION_MESSAGE_CREATE_BUG.format(bug_url=experiment.bugzilla_url),
)
def test_simple_recommendation(test_ctx):
# Fix the random seed so that we get stable results between test
# runs
np.random.seed(seed=42)
with mock_install_mock_curated_data(test_ctx):
EXPECTED_RESULTS = [("def", 3320.0), ("klm", 409.99999999999994),
("hij", 3100.0), ("ijk", 3200.0), ("ghi", 3430.0),
("lmn", 420.0), ("jkl", 400.0), ("abc", 23.0),
("fgh", 22.0), ("efg", 21.0)]
with MetricsMock() as mm:
manager = RecommendationManager(test_ctx)
recommendation_list = manager.recommend("some_ignored_id", 10)
assert isinstance(recommendation_list, list)
assert recommendation_list == EXPECTED_RESULTS
assert mm.has_record(TIMING, stat="taar.profile_recommendation")
def test_configure_doesnt_affect_override(self):
with MetricsMock() as mm:
markus.configure([{
'class': 'markus.backends.logging.LoggingMetrics'
}])
mymetrics = markus.get_metrics('foobar')
mymetrics.incr('key1', value=1)
assert mm.has_record(
fun_name='incr',
stat='foobar.key1',
value=1,
)
assert not mm.has_record(
fun_name='incr',
stat='foobar.key1',
value=5,
)
def test_crash_size_capture(self):
"""Verify we capture raw/processed crash sizes in ES crashstorage"""
with MetricsMock() as mm:
es_storage = ESCrashStorage(config=self.config, namespace='processor.es')
es_storage._submit_crash_to_elasticsearch = mock.Mock()
es_storage.save_raw_and_processed(
raw_crash=deepcopy(a_raw_crash),
dumps=None,
processed_crash=deepcopy(a_processed_crash),
crash_id=a_processed_crash['uuid']
)
# NOTE(willkg): The sizes of these json documents depend on what's
# in them. If we changed a_processed_crash and a_raw_crash, then
# these numbers will change.
assert mm.has_record('histogram', stat='processor.es.raw_crash_size', value=27)
assert mm.has_record('histogram', stat='processor.es.processed_crash_size', value=1738)
def test_crash_size_capture(self):
"""Verify we capture raw/processed crash sizes in ES crashstorage"""
with MetricsMock() as mm:
es_storage = ESCrashStorage(config=self.config,
namespace="processor.es")
es_storage._submit_crash_to_elasticsearch = mock.Mock()
es_storage.save_processed_crash(
raw_crash=deepcopy(a_raw_crash),
processed_crash=deepcopy(a_processed_crash),
)
# NOTE(willkg): The sizes of these json documents depend on what's
# in them. If we changed a_processed_crash and a_raw_crash, then
# these numbers will change.
mm.print_records()
mm.assert_histogram("processor.es.raw_crash_size", value=27)
mm.assert_histogram("processor.es.processed_crash_size",
value=1721)
def test_index_data_capture(self):
"""Verify we capture index data in ES crashstorage"""
with MetricsMock() as mm:
es_storage = ESCrashStorage(config=self.config, namespace="processor.es")
mock_connection = mock.Mock()
# Do a successful indexing
es_storage._index_crash(
connection=mock_connection,
es_index=None,
es_doctype=None,
crash_document=None,
crash_id=None,
)
# Do a failed indexing
mock_connection.index.side_effect = Exception
with pytest.raises(Exception):
es_storage._index_crash(
connection=mock_connection,
es_index=None,
es_doctype=None,
crash_document=None,
crash_id=None,
)
assert (
len(
mm.filter_records(
stat="processor.es.index", tags=["outcome:successful"]
)
)
== 1
)
assert (
len(
mm.filter_records(
stat="processor.es.index", tags=["outcome:failed"]
)
)
== 1
)
def test_crash_size_capture(self):
"""Verify we capture raw/processed crash sizes in ES crashstorage"""
raw_crash = {"ProductName": "Firefox", "ReleaseChannel": "nightly"}
processed_crash = {
"date_processed": "2012-04-08 10:56:41.558922",
"uuid": "936ce666-ff3b-4c7a-9674-367fe2120408",
}
with MetricsMock() as mm:
es_storage = ESCrashStorage(config=self.config,
namespace="processor.es")
es_storage._submit_crash_to_elasticsearch = mock.Mock()
es_storage.save_processed_crash(
raw_crash=raw_crash,
processed_crash=processed_crash,
)
mm.assert_histogram("processor.es.raw_crash_size", value=55)
mm.assert_histogram("processor.es.processed_crash_size", value=96)
def metricsmock():
"""Return a MetricsMock for asserting things on metrics.
Usage::
def test_something(metricsmock):
assert metricsmock.has_record(
'incr',
stat='some.stat',
value=1
)
If you ever need to clear the records in the middle of a test, do::
metricsmock.clear_records()
For more, see: https://markus.readthedocs.io/en/latest/testing.html
"""
with MetricsMock() as mm:
yield mm
def test_has_record(self):
# NOTE(willkg): .has_record() is implemented using .filter_records() so
# we can test that aggressively and just make sure the .has_record()
# wrapper works fine.
#
# If that ever changes, we should update this test.
with MetricsMock() as mm:
mymetrics = markus.get_metrics('foobar')
mymetrics.incr('key1', value=1)
assert mm.has_record(
fun_name='incr',
stat='foobar.key1',
value=1,
)
assert not mm.has_record(
fun_name='incr',
stat='foobar.key1',
value=5,
)
def test_submit_data_capture(self):
with MetricsMock() as mm:
conn = setup_mocked_s3_storage()
# Do a successful submit
conn.submit('fff13cf0-5671-4496-ab89-47a922141114',
'name_of_thing', thing_as_binary)
# Do a failed submit
conn._connect = mock.Mock()
conn._connect.side_effect = Exception
with pytest.raises(Exception):
conn.submit('fff13cf0-5671-4496-ab89-47a922141114',
'name_of_thing', thing_as_binary)
assert len(
mm.filter_records(
stat='processor.s3.submit',
tags=['kind:name_of_thing', 'outcome:successful'])) == 1
assert len(
mm.filter_records(
stat='processor.s3.submit',
tags=['kind:name_of_thing', 'outcome:failed'])) == 1
def test_everything_we_hoped_for(self, mocked_subprocess_module):
rule = self.build_rule()
raw_crash = copy.copy(canonical_standard_raw_crash)
raw_dumps = {rule.dump_field: 'a_fake_dump.dump'}
processed_crash = DotDict()
processor_meta = get_basic_processor_meta()
mocked_subprocess_handle = mocked_subprocess_module.Popen.return_value
mocked_subprocess_handle.stdout.read.return_value = canonical_stackwalker_output_str
mocked_subprocess_handle.wait.return_value = 0
with MetricsMock() as mm:
rule.act(raw_crash, raw_dumps, processed_crash, processor_meta)
assert processed_crash.json_dump == canonical_stackwalker_output
assert processed_crash.mdsw_return_code == 0
assert processed_crash.mdsw_status_string == "OK"
assert processed_crash.success is True
assert mm.has_record('incr',
stat='processor.breakpadstackwalkerrule.run',
value=1,
tags=['outcome:success', 'exitcode:0'])
def test_filter_records_value(self):
with MetricsMock() as mm:
mymetrics = markus.get_metrics('foobar')
mymetrics.incr('key1', value=1, tags=['env:stage'])
key1_metrics = mm.filter_records(
fun_name='incr',
stat='foobar.key1',
)
assert len(key1_metrics) == 1
key1_metrics = mm.filter_records(
fun_name='incr',
stat='foobar.key1',
value=1,
)
assert len(key1_metrics) == 1
key1_metrics = mm.filter_records(
fun_name='incr',
stat='foobar.key1',
value=5,
)
assert len(key1_metrics) == 0
def test_upload(self):
"""Verify API symbols upload"""
user = User.objects.create(username='******', email='*****@*****.**')
self._add_permission(user, 'upload_symbols')
token = Token.objects.create(
user=user,
)
token.permissions.add(
Permission.objects.get(codename='upload_symbols')
)
url = reverse('symbols:upload')
response = self.client.get(url)
assert response.status_code == 405
with MetricsMock() as metrics_mock:
with self.settings(MEDIA_ROOT=self.tmp_dir):
with open(ZIP_FILE, 'rb') as file_object:
response = self.client.post(
url,
{'file.zip': file_object},
# note! No HTTP_AUTH_TOKEN
)
assert response.status_code == 403
with open(ZIP_FILE, 'rb') as file_object:
response = self.client.post(
url,
{'file.zip': file_object},
HTTP_AUTH_TOKEN=''
)
assert response.status_code == 403
with open(ZIP_FILE, 'rb') as file_object:
response = self.client.post(
url,
{'file.zip': file_object},
HTTP_AUTH_TOKEN='somejunk'
)
assert response.status_code == 403
with open(ZIP_FILE, 'rb') as file_object:
response = self.client.post(
url,
{'file.zip': file_object},
HTTP_AUTH_TOKEN=token.key
)
assert response.status_code == 201
symbol_upload = models.SymbolsUpload.objects.get(user=user)
assert symbol_upload.filename == 'file.zip'
assert symbol_upload.size
assert symbol_upload.content
# This should have made one S3 connection
connection_parameters, = self.connection_parameters
args, kwargs = connection_parameters
region, = args
assert region
assert region == settings.SYMBOLS_BUCKET_DEFAULT_LOCATION
assert kwargs['aws_access_key_id'] == settings.AWS_ACCESS_KEY
assert kwargs['aws_secret_access_key'] == settings.AWS_SECRET_ACCESS_KEY
assert isinstance(kwargs['calling_format'], OrdinaryCallingFormat)
# the ZIP_FILE contains a file called south-africa-flag.jpeg
key = os.path.join(
settings.SYMBOLS_FILE_PREFIX,
'south-africa-flag.jpeg'
)
assert self.uploaded_keys[key]
assert self.uploaded_headers[key] == {'Content-Type': 'image/jpeg'}
# and a file called xpcshell.sym
key = os.path.join(
settings.SYMBOLS_FILE_PREFIX,
'xpcshell.sym'
)
assert self.uploaded_keys[key]
expected = {
'Content-Type': 'text/plain',
'Content-Encoding': 'gzip'
}
assert self.uploaded_headers[key] == expected
# The sample.zip file contains the file xpcshell.sym and it's
# 1156 bytes when un-archived. Just gzip'in the content of the
# file will yield a file that is 476 bytes.
# But if you do it properly there's header information in the
# string which is a couple of extra bytes.
assert len(self.uploaded_keys[key]) == 488
# Verify it generated the metrics we need--since only one POST was
# successful, then there's only one incr record
assert metrics_mock.has_record(
markus.INCR, 'symbols.upload.api_upload', 1, tags=['email:test_example.com']
)
def test_it_sends_metrics(self, settings, mocked_autograph):
ActionFactory.create_batch(3, signed=False)
with MetricsMock() as mm:
call_command("update_action_signatures")
mm.print_records()
assert mm.has_record(GAUGE, stat="normandy.signing.actions.signed", value=3)
def metricsmock():
return MetricsMock()
def test_web_upload(self):
url = reverse('symbols:web_upload')
response = self.client.get(url)
assert response.status_code == 302
self.assertRedirects(
response,
reverse('crashstats:login') + '?next=%s' % url
)
user = self._login()
response = self.client.get(url)
assert response.status_code == 302
self.assertRedirects(
response,
reverse('crashstats:login') + '?next=%s' % url
)
# you need to have the permission
self._add_permission(user, 'upload_symbols')
response = self.client.get(url)
assert response.status_code == 200
with MetricsMock() as metrics_mock:
# now we can post
with self.settings(SYMBOLS_MIME_OVERRIDES={'jpeg': 'text/plain'}):
with open(ZIP_FILE) as file_object:
response = self.client.post(
url,
{'file': file_object}
)
assert response.status_code == 302
symbol_upload = models.SymbolsUpload.objects.get(user=user)
assert symbol_upload.filename == os.path.basename(ZIP_FILE)
assert symbol_upload.size
# We expect the content to be a `+` because it was new,
# followed by the bucket name, followed by a comma, followed
# by the symbols prefix + filename.
line = "+%s,%s/%s\n" % (
settings.SYMBOLS_BUCKET_DEFAULT_NAME,
settings.SYMBOLS_FILE_PREFIX,
'south-africa-flag.jpeg'
)
line += "+%s,%s/%s\n" % (
settings.SYMBOLS_BUCKET_DEFAULT_NAME,
settings.SYMBOLS_FILE_PREFIX,
'xpcshell.sym'
)
assert symbol_upload.content == line
assert symbol_upload.content_type == 'text/plain'
assert self.uploaded_keys
# the mocked key object should have its content_type set too
assert self.created_keys[0].content_type == 'text/plain'
expected = [
(settings.SYMBOLS_BUCKET_DEFAULT_NAME, settings.SYMBOLS_BUCKET_DEFAULT_LOCATION)
]
assert self.created_buckets == expected
# Verify it generated the metrics we need
assert metrics_mock.has_record(
markus.INCR, 'symbols.upload.web_upload', 1, tags=['email:test_example.com']
)
