def test_high_n_buckets(self):
SAMPLE_HISTOGRAM = {
"TEST_HISTOGRAM_WHITELIST_N_BUCKETS": {
"record_in_processes": ["main", "content"],
"alert_emails": ["*****@*****.**"],
"bug_numbers": [1383793],
"expires_in_version": "never",
"kind": "exponential",
"low": 1024,
"high": 16777216,
"n_buckets": 200,
"description": "Test histogram",
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
parse_histograms.load_whitelist()
parse_histograms.Histogram('TEST_HISTOGRAM_WHITELIST_N_BUCKETS',
histograms['TEST_HISTOGRAM_WHITELIST_N_BUCKETS'],
strict_type_checks=True)
self.assertRaises(SystemExit, ParserError.exit_func)
# Set global whitelists for parse_histograms.
parse_histograms.whitelists = {
"alert_emails": [],
"bug_numbers": [],
"n_buckets": [
"TEST_HISTOGRAM_WHITELIST_N_BUCKETS"
],
"expiry_default": [],
"kind": []
}
hist = parse_histograms.Histogram('TEST_HISTOGRAM_WHITELIST_N_BUCKETS',
histograms['TEST_HISTOGRAM_WHITELIST_N_BUCKETS'],
strict_type_checks=True)
ParserError.exit_func()
self.assertEqual(hist.expiration(), 'never')
self.assertEqual(hist.kind(), 'exponential')
self.assertEqual(hist.record_in_processes(), ["main", "content"])
self.assertEqual(hist.keyed(), False)
self.assertEqual(hist.low(), 1024)
self.assertEqual(hist.high(), 16777216)
self.assertEqual(hist.n_buckets(), 200)
parse_histograms.whitelists = None
def test_expiry_default(self):
SAMPLE_HISTOGRAM = {
"TEST_HISTOGRAM_WHITELIST_EXPIRY_DEFAULT": {
"record_in_processes": ["main", "content"],
"expires_in_version": "default",
"alert_emails": ["*****@*****.**"],
"bug_numbers": [1383793],
"kind": "boolean",
"description": "Test histogram",
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
parse_histograms.load_whitelist()
parse_histograms.Histogram('TEST_HISTOGRAM_WHITELIST_EXPIRY_DEFAULT',
histograms['TEST_HISTOGRAM_WHITELIST_EXPIRY_DEFAULT'],
strict_type_checks=True)
self.assertRaises(SystemExit, ParserError.exit_func)
# Set global whitelists for parse_histograms.
parse_histograms.whitelists = {
"alert_emails": [],
"bug_numbers": [],
"n_buckets": [],
"expiry_default": [
"TEST_HISTOGRAM_WHITELIST_EXPIRY_DEFAULT"
],
"kind": []
}
hist = parse_histograms.Histogram('TEST_HISTOGRAM_WHITELIST_EXPIRY_DEFAULT',
histograms['TEST_HISTOGRAM_WHITELIST_EXPIRY_DEFAULT'],
strict_type_checks=True)
ParserError.exit_func()
self.assertEqual(hist.expiration(), 'default')
self.assertEqual(hist.kind(), 'boolean')
self.assertEqual(hist.record_in_processes(), ["main", "content"])
self.assertEqual(hist.keyed(), False)
parse_histograms.whitelists = None
def test_unknown_field(self):
SAMPLE_HISTOGRAM = {
"A11Y_INSTANTIATED_FLAG": {
"record_in_processes": ["main", "content"],
"expires_in_version": "never",
"kind": "flag",
"description": "has accessibility support been instantiated",
"new_field": "Its a new field"
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
hist = parse_histograms.Histogram('A11Y_INSTANTIATED_FLAG',
histograms['A11Y_INSTANTIATED_FLAG'],
strict_type_checks=False)
self.assertEqual(hist.expiration(), 'never')
self.assertEqual(hist.kind(), 'flag')
self.assertEqual(hist.record_in_processes(), ["main", "content"])
def test_non_numeric_expressions(self):
SAMPLE_HISTOGRAM = {
"TEST_NON_NUMERIC_HISTOGRAM": {
"kind": "linear",
"description": "sample",
"n_buckets": "JS::gcreason::NUM_TELEMETRY_REASONS",
"high": "mozilla::StartupTimeline::MAX_EVENT_ID"
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
hist = parse_histograms.Histogram(
'TEST_NON_NUMERIC_HISTOGRAM',
histograms['TEST_NON_NUMERIC_HISTOGRAM'],
strict_type_checks=False)
# expected values come off parse_histograms.py
self.assertEqual(hist.n_buckets(), 101)
self.assertEqual(hist.high(), 12)
def test_unsupported_kind_count(self):
SAMPLE_HISTOGRAM = {
"TEST_HISTOGRAM_WHITELIST_KIND": {
"record_in_processes": ["main", "content"],
"expires_in_version": "never",
"kind": "count",
"releaseChannelCollection": "opt-out",
"alert_emails": ["*****@*****.**"],
"bug_numbers": [1383793],
"description": "Test histogram",
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
parse_histograms.load_whitelist()
self.assertRaises(SystemExit, parse_histograms.Histogram,
'TEST_HISTOGRAM_WHITELIST_KIND',
histograms['TEST_HISTOGRAM_WHITELIST_KIND'],
strict_type_checks=True)
# Set global whitelists for parse_histograms.
parse_histograms.whitelists = {
"alert_emails": [],
"bug_numbers": [],
"n_buckets": [],
"expiry_default": [],
"kind": [
"TEST_HISTOGRAM_WHITELIST_KIND"
]
}
hist = parse_histograms.Histogram('TEST_HISTOGRAM_WHITELIST_KIND',
histograms['TEST_HISTOGRAM_WHITELIST_KIND'],
strict_type_checks=True)
ParserError.exit_func()
self.assertEqual(hist.expiration(), 'never')
self.assertEqual(hist.kind(), 'count')
self.assertEqual(hist.record_in_processes(), ["main", "content"])
self.assertEqual(hist.keyed(), False)
parse_histograms.whitelists = None
def test_valid_histogram(self):
SAMPLE_HISTOGRAM = {
"TEST_VALID_HISTOGRAM": {
"record_in_processes": ["main", "content"],
"alert_emails": ["*****@*****.**"],
"bug_numbers": [1383793],
"expires_in_version": "never",
"kind": "boolean",
"description": "Test histogram"
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
parse_histograms.load_whitelist()
hist = parse_histograms.Histogram('TEST_VALID_HISTOGRAM',
histograms['TEST_VALID_HISTOGRAM'],
strict_type_checks=True)
ParserError.exit_func()
self.assertTrue(hist.expiration(), "never")
self.assertTrue(hist.kind(), "boolean")
self.assertTrue(hist.record_in_processes, ["main", "content"])
def test_high_value(self):
SAMPLE_HISTOGRAM = {
"TEST_HISTOGRAM_WHITELIST_N_BUCKETS": {
"record_in_processes": ["main", "content"],
"alert_emails": ["*****@*****.**"],
"bug_numbers": [1383793],
"expires_in_version": "never",
"kind": "exponential",
"low": 1024,
"high": 2**64,
"n_buckets": 100,
"description": "Test histogram",
}
}
histograms = load_histogram(SAMPLE_HISTOGRAM)
parse_histograms.load_whitelist()
parse_histograms.Histogram(
'TEST_HISTOGRAM_WHITELIST_N_BUCKETS',
histograms['TEST_HISTOGRAM_WHITELIST_N_BUCKETS'],
strict_type_checks=True)
self.assertRaises(SystemExit, ParserError.exit_func)
def __init__(self,
name,
instance,
revision=None,
histograms_url=None,
additional_histograms=None):
"""
Initialize a histogram from its name and a telemetry submission.
Raises a KeyError if a definition for the histogram could not be found.
:param histograms_url: url to a JSON file describing available
histograms. Defaults to the latest
Histograms.json in mozilla-central.
:param additional_histograms: dict describing histograms to use in
addition to those in histograms_url. The
dict should resemble the format of
Histograms.json.
"""
if revision and histograms_url:
raise ValueError("Invalid use of both revision and histograms_url")
# For backwards compatibility.
if not histograms_url:
revision = \
(revision or HISTOGRAMS_JSON_REVISION).replace("/rev/", "/raw-file/")
histograms_url = revision + HISTOGRAMS_JSON_PATH
self.histograms_url = histograms_url
histograms_definition = _fetch_histograms_definition(histograms_url)
if additional_histograms:
histograms_definition.update(additional_histograms)
# TODO: implement centralized revision service which handles all the quirks...
if name.startswith("USE_COUNTER_") or name.startswith("USE_COUNTER2_"):
self.definition = parse_histograms.Histogram(
name, {
"kind": "boolean",
"description": "",
"expires_in_version": "never"
})
else:
proper_name = name
if "/" in name: # key in a keyed histogram, like BLOCKED_ON_PLUGIN_INSTANCE_INIT_MS/'Shockwave Flash14.0.0.145'
proper_name = name.split("/")[
0] # just keep the name of the parent histogram
try:
self.definition = parse_histograms.Histogram(
name, histograms_definition[proper_name])
except KeyError:
# Some histograms are collected twice: during startup and during normal execution.
# In the former case the STARTUP_ prefix prepends the histogram name, even though
# the prefixed histogram name is not part of the histogram definition file.
# Other histograms, like STARTUP_CRASH_DETECTED, are instead collected only once
# and are defined the histogram definition file.
self.definition = parse_histograms.Histogram(
name, histograms_definition[re.sub("^STARTUP_", "",
proper_name)])
self.kind = self.definition.kind()
self.name = name
labels, ranges = self.definition.labels(), self.definition.ranges()
if self.kind == 'categorical':
pd_index = labels + [CATEGORICAL_HISTOGRAM_SPILL_BUCKET_NAME]
if isinstance(instance, dict):
# pandas clobbers values if instance is a dict with string labels
entries = Histogram.values_to_dict(instance, {})
instance = [entries.get(k, 0) for k in ranges]
else:
pd_index = ranges
if isinstance(instance, list) or isinstance(
instance, np.ndarray) or isinstance(instance, pd.Series):
if self.kind == 'categorical':
values = instance[:len(labels) + 1]
elif len(instance) == self.definition.n_buckets():
values = instance
else:
values = instance[:-5]
self.buckets = pd.Series(values, index=pd_index, dtype='int64')
else:
entries = Histogram.values_to_dict(
instance, [0 for _ in enumerate(pd_index)])
self.buckets = pd.Series(entries, index=pd_index,
dtype='int64').fillna(0)
