def _ProcessRowAndHistogram(params, bot_whitelist):
revision = int(params['revision'])
test_path = params['test_path']
benchmark_description = params['benchmark_description']
data_dict = params['data']
logging.info('Processing: %s', test_path)
hist = histogram_module.Histogram.FromDict(data_dict)
if hist.num_values == 0:
return []
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
benchmark_name = test_path_parts[2]
histogram_name = test_path_parts[3]
if len(test_path_parts) > 4:
rest = '/'.join(test_path_parts[4:])
else:
rest = None
full_test_name = '/'.join(test_path_parts[2:])
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = GetUnitArgs(hist.unit)
unescaped_story_name = _GetStoryFromDiagnosticsDict(params.get('diagnostics'))
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
# https://github.com/catapult-project/catapult/issues/4096
parent_test = add_point_queue.GetOrCreateAncestors(
master, bot, full_test_name, internal_only=internal_only,
unescaped_story_name=unescaped_story_name,
benchmark_description=benchmark_description, **extra_args)
test_key = parent_test.key
statistics_scalars = hist.statistics_scalars
legacy_parent_tests = {}
# TODO(#4213): Stop doing this.
if benchmark_name in LEGACY_BENCHMARKS:
statistics_scalars = {}
for stat_name, scalar in statistics_scalars.iteritems():
if _ShouldFilter(histogram_name, benchmark_name, stat_name):
continue
extra_args = GetUnitArgs(scalar.unit)
suffixed_name = '%s/%s_%s' % (
benchmark_name, histogram_name, stat_name)
if rest is not None:
suffixed_name += '/' + rest
legacy_parent_tests[stat_name] = add_point_queue.GetOrCreateAncestors(
master, bot, suffixed_name, internal_only=internal_only,
unescaped_story_name=unescaped_story_name, **extra_args)
return [
_AddRowsFromData(params, revision, parent_test, legacy_parent_tests,
internal_only),
_AddHistogramFromData(params, revision, test_key, internal_only)]
def _ProcessRowAndHistogram(params):
revision = int(params['revision'])
test_path = params['test_path']
benchmark_description = params['benchmark_description']
data_dict = params['data']
logging.info('Processing: %s', test_path)
hist = histogram_module.Histogram.FromDict(data_dict)
if hist.num_values == 0:
return []
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
benchmark_name = test_path_parts[2]
histogram_name = test_path_parts[3]
if len(test_path_parts) > 4:
rest = '/'.join(test_path_parts[4:])
else:
rest = None
full_test_name = '/'.join(test_path_parts[2:])
internal_only = graph_data.Bot.GetInternalOnlySync(master, bot)
extra_args = GetUnitArgs(hist.unit)
unescaped_story_name = _GetStoryFromDiagnosticsDict(params.get('diagnostics'))
parent_test = add_point_queue.GetOrCreateAncestors(
master, bot, full_test_name, internal_only=internal_only,
unescaped_story_name=unescaped_story_name,
benchmark_description=benchmark_description, **extra_args)
test_key = parent_test.key
statistics_scalars = hist.statistics_scalars
legacy_parent_tests = {}
# TODO(#4213): Stop doing this.
if histogram_helpers.IsLegacyBenchmark(benchmark_name):
statistics_scalars = {}
for stat_name, scalar in statistics_scalars.items():
if histogram_helpers.ShouldFilterStatistic(
histogram_name, benchmark_name, stat_name):
continue
extra_args = GetUnitArgs(scalar.unit)
suffixed_name = '%s/%s_%s' % (
benchmark_name, histogram_name, stat_name)
if rest is not None:
suffixed_name += '/' + rest
legacy_parent_tests[stat_name] = add_point_queue.GetOrCreateAncestors(
master, bot, suffixed_name, internal_only=internal_only,
unescaped_story_name=unescaped_story_name, **extra_args)
return [
_AddRowsFromData(params, revision, parent_test, legacy_parent_tests),
_AddHistogramFromData(params, revision, test_key, internal_only)]
def testGetOrCreateAncestors_CreatesAllExpectedEntities(self):
parent = add_point_queue.GetOrCreateAncestors('ChromiumPerf', 'win7',
'dromaeo/dom/modify')
self.assertEqual('ChromiumPerf/win7/dromaeo/dom/modify',
parent.key.id())
# Check that all the Bot and TestMetadata entities were correctly added.
created_masters = graph_data.Master.query().fetch()
created_bots = graph_data.Bot.query().fetch()
created_tests = graph_data.TestMetadata.query().fetch()
self.assertEqual(1, len(created_masters))
self.assertEqual(1, len(created_bots))
self.assertEqual(3, len(created_tests))
self.assertEqual('ChromiumPerf', created_masters[0].key.id())
self.assertIsNone(created_masters[0].key.parent())
self.assertEqual('win7', created_bots[0].key.id())
self.assertEqual('ChromiumPerf', created_bots[0].key.parent().id())
self.assertEqual('ChromiumPerf/win7/dromaeo',
created_tests[0].key.id())
self.assertIsNone(created_tests[0].parent_test)
self.assertEqual('win7', created_tests[0].bot_name)
self.assertEqual('dom', created_tests[1].test_part1_name)
self.assertEqual('ChromiumPerf/win7/dromaeo',
created_tests[1].parent_test.id())
self.assertIsNone(created_tests[1].bot)
self.assertEqual('ChromiumPerf/win7/dromaeo/dom/modify',
created_tests[2].key.id())
self.assertEqual('ChromiumPerf/win7/dromaeo/dom',
created_tests[2].parent_test.id())
self.assertIsNone(created_tests[2].bot)
def testGetOrCreateAncestors_GetsExistingEntities(self):
master_key = graph_data.Master(id='ChromiumPerf', parent=None).put()
graph_data.Bot(id='win7', parent=master_key).put()
t = graph_data.TestMetadata(id='ChromiumPerf/win7/dromaeo', )
t.UpdateSheriff()
t.put()
t = graph_data.TestMetadata(id='ChromiumPerf/win7/dromaeo/dom')
t.UpdateSheriff()
t.put()
t = graph_data.TestMetadata(id='ChromiumPerf/win7/dromaeo/dom/modify')
t.UpdateSheriff()
t.put()
actual_parent = add_point_queue.GetOrCreateAncestors(
'ChromiumPerf', 'win7', 'dromaeo/dom/modify')
self.assertEqual('ChromiumPerf/win7/dromaeo/dom/modify',
actual_parent.key.id())
# No extra TestMetadata or Bot objects should have been added to the
# database beyond the four that were put in before the _GetOrCreateAncestors
# call.
self.assertEqual(1, len(graph_data.Master.query().fetch()))
self.assertEqual(1, len(graph_data.Bot.query().fetch()))
self.assertEqual(3, len(graph_data.TestMetadata.query().fetch()))
def _ProcessRowAndHistogram(params, bot_whitelist):
revision = int(params['revision'])
test_path = params['test_path']
data_dict = params['data']
logging.info('Processing: %s', test_path)
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = GetUnitArgs(data_dict['unit'])
unescaped_story_name = _GetStoryFromDiagnosticsDict(
params.get('diagnostics'))
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
# https://github.com/catapult-project/catapult/issues/4096
parent_test = add_point_queue.GetOrCreateAncestors(
master,
bot,
test_name,
internal_only,
unescaped_story_name=unescaped_story_name,
**extra_args)
test_key = parent_test.key
return [
_AddRowFromData(params, revision, parent_test, internal_only),
_AddHistogramFromData(params, revision, test_key, internal_only)
]
def testGetOrCreateAncestors_RespectsImprovementDirectionForNewTest(self):
test = add_point_queue.GetOrCreateAncestors(
'M',
'b',
'suite/foo',
units='bogus',
improvement_direction=anomaly.UP)
self.assertEqual(anomaly.UP, test.improvement_direction)
def GetSuiteKey(histograms):
assert len(histograms) > 0
# TODO(eakuefner): Refactor this to coalesce the boilerplate (note that this
# is all also being done in add_histograms_queue's post handler)
master, bot, benchmark = _GetMasterBotBenchmarkFromHistogram(
histograms.GetFirstHistogram())
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
return add_point_queue.GetOrCreateAncestors(master, bot, benchmark,
internal_only).key
def post(self):
"""Adds a single histogram or sparse shared diagnostic to the datastore.
The |data| request parameter can be either a histogram or a sparse shared
diagnostic; the set of diagnostics that are considered sparse (meaning that
they don't normally change on every upload for a given benchmark from a
given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.
See https://goo.gl/lHzea6 for detailed information on the JSON format for
histograms and diagnostics.
Request parameters:
data: JSON encoding of a histogram or shared diagnostic.
revision: a revision, given as an int.
test_path: the test path to which this diagnostic or histogram should be
attached.
"""
datastore_hooks.SetPrivilegedRequest()
data = self.request.get('data')
revision = int(self.request.get('revision'))
test_path = self.request.get('test_path')
data_dict = json.loads(data)
guid = data_dict['guid']
is_diagnostic = 'type' in data_dict
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
test_key = add_point_queue.GetOrCreateAncestors(
master, bot, test_name, internal_only, **extra_args).key
if is_diagnostic:
entity = histogram.SparseDiagnostic(id=guid,
data=data,
test=test_key,
start_revision=revision,
end_revision=revision,
internal_only=internal_only)
else:
entity = histogram.Histogram(id=guid,
data=data,
test=test_key,
revision=revision,
internal_only=internal_only)
AddRow(data_dict, test_key, revision, test_path, internal_only)
entity.put()
def testGetOrCreateAncestors_UpdatesStoppageAlert(self):
testing_common.AddTests(['M'], ['b'], {'suite': {'foo': {}}})
row = testing_common.AddRows('M/b/suite/foo', {123})[0]
test = utils.TestKey('M/b/suite/foo').get()
alert_key = stoppage_alert.CreateStoppageAlert(test, row).put()
test.stoppage_alert = alert_key
test.put()
add_point_queue.GetOrCreateAncestors('M', 'b', 'suite/foo')
self.assertIsNone(test.key.get().stoppage_alert)
self.assertTrue(alert_key.get().recovered)
self.assertIsNotNone(alert_key.get().last_row_timestamp)
def _ProcessRowAndHistogram(params, bot_whitelist):
revision = int(params['revision'])
test_path = params['test_path']
data_dict = params['data']
logging.info('Processing: %s', test_path)
hist = histogram_module.Histogram.FromDict(data_dict)
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = GetUnitArgs(hist.unit)
unescaped_story_name = _GetStoryFromDiagnosticsDict(
params.get('diagnostics'))
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
# https://github.com/catapult-project/catapult/issues/4096
parent_test = add_point_queue.GetOrCreateAncestors(
master,
bot,
test_name,
internal_only=internal_only,
unescaped_story_name=unescaped_story_name,
**extra_args)
test_key = parent_test.key
benchmark_name = test_path_parts[2]
statistics_scalars = hist.statistics_scalars
legacy_parent_tests = {}
if test_name.endswith('_ref'):
test_name = test_name[:-4]
ref_suffix = '_ref'
elif test_name.endswith('/ref'):
test_name = test_name[:-4]
ref_suffix = '/ref'
else:
ref_suffix = ''
# TODO(#4213): Stop doing this.
if _ShouldFilter(test_name, benchmark_name):
statistics_scalars = {'avg': statistics_scalars['avg']}
elif benchmark_name in LEGACY_BENCHMARKS:
statistics_scalars = {}
for stat_name, scalar in statistics_scalars.iteritems():
extra_args = GetUnitArgs(scalar.unit)
suffixed_name = '%s_%s%s' % (test_name, stat_name, ref_suffix)
legacy_parent_tests[stat_name] = add_point_queue.GetOrCreateAncestors(
master,
bot,
suffixed_name,
internal_only=internal_only,
unescaped_story_name=unescaped_story_name,
**extra_args)
return [
_AddRowsFromData(params, revision, parent_test, legacy_parent_tests,
internal_only),
_AddHistogramFromData(params, revision, test_key, internal_only)
]
def post(self):
"""Adds a single histogram or sparse shared diagnostic to the datastore.
The |data| request parameter can be either a histogram or a sparse shared
diagnostic; the set of diagnostics that are considered sparse (meaning that
they don't normally change on every upload for a given benchmark from a
given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.
See https://goo.gl/lHzea6 for detailed information on the JSON format for
histograms and diagnostics.
Request parameters:
data: JSON encoding of a histogram or shared diagnostic.
revision: a revision, given as an int.
test_path: the test path to which this diagnostic or histogram should be
attached.
"""
datastore_hooks.SetPrivilegedRequest()
data = self.request.get('data')
revision = int(self.request.get('revision'))
test_path = self.request.get('test_path')
data_dict = json.loads(data)
guid = data_dict['guid']
is_diagnostic = 'type' in data_dict
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
parent_test = add_point_queue.GetOrCreateAncestors(
master, bot, test_name, internal_only, **extra_args)
test_key = parent_test.key
added_rows = []
monitored_test_keys = []
if is_diagnostic:
entity = histogram.SparseDiagnostic(id=guid,
data=data,
test=test_key,
start_revision=revision,
end_revision=revision,
internal_only=internal_only)
else:
diagnostics = self.request.get('diagnostics')
if diagnostics:
diagnostic_data = json.loads(diagnostics)
diagnostic_entities = []
for diagnostic_datum in diagnostic_data:
# TODO(eakuefner): Pass map of guid to dict to avoid overhead
guid = diagnostic_datum['guid']
diagnostic_entities.append(
histogram.SparseDiagnostic(
id=guid,
data=diagnostic_datum,
test=test_key,
start_revision=revision,
end_revision=sys.maxint,
internal_only=internal_only))
new_guids_to_existing_diagnostics = add_histograms.DeduplicateAndPut(
diagnostic_entities, test_key, revision).iteritems()
# TODO(eakuefner): Move per-histogram monkeypatching logic to Histogram.
hs = histogram_set.HistogramSet()
hs.ImportDicts([data_dict])
# TODO(eakuefner): Share code for replacement logic with add_histograms
for new_guid, existing_diagnostic in new_guids_to_existing_diagnostics:
hs.ReplaceSharedDiagnostic(
new_guid,
diagnostic_ref.DiagnosticRef(
existing_diagnostic['guid']))
data = hs.GetFirstHistogram().AsDict()
entity = histogram.Histogram(id=guid,
data=data,
test=test_key,
revision=revision,
internal_only=internal_only)
row = AddRow(data_dict, test_key, revision, test_path,
internal_only)
added_rows.append(row)
is_monitored = parent_test.sheriff and parent_test.has_rows
if is_monitored:
monitored_test_keys.append(parent_test.key)
entity.put()
tests_keys = [
k for k in monitored_test_keys if not add_point_queue.IsRefBuild(k)
]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(added_rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
ndb.Future.wait_all(futures)
