def _CreateHistogramTasks(suite_path,
histograms,
revision,
benchmark_description,
completion_token=None):
tasks = []
test_paths = set()
for hist in histograms:
diagnostics = FindHistogramLevelSparseDiagnostics(hist)
test_path = '%s/%s' % (suite_path,
histogram_helpers.ComputeTestPath(hist))
# Log the information here so we can see which histograms are being queued.
logging.debug('Queueing: %s', test_path)
if test_path in test_paths:
raise api_request_handler.BadRequestError(
'Duplicate histogram detected: %s' % test_path)
test_paths.add(test_path)
# We create one task per histogram, so that we can get as much time as we
# need for processing each histogram per task.
task_dict = _MakeTaskDict(hist, test_path, revision,
benchmark_description, diagnostics,
completion_token)
tasks.append(_MakeTask([task_dict]))
if completion_token is not None:
completion_token.PopulateMeasurements(test_paths)
return tasks
def _CreateHistogramSetByTestPathDict(self, histograms):
histograms_by_path = collections.defaultdict(list)
for h in histograms:
histograms_by_path[histogram_helpers.ComputeTestPath(h)].append(h)
return histograms_by_path
def testComputeTestPathWithStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORIES.name,
generic_set.GenericSet(['http://story']))
hist = histograms.GetFirstHistogram()
test_path = histogram_helpers.ComputeTestPath(hist)
self.assertEqual('hist/http___story', test_path)
def testComputeTestPathWithIsRefWithoutStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.IS_REFERENCE_BUILD.name,
generic_set.GenericSet([True]))
hist = histograms.GetFirstHistogram()
test_path = histogram_helpers.ComputeTestPath(hist)
self.assertEqual('hist/ref', test_path)
def testComputeTestPathWithGroupingLabel(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORIES.name,
generic_set.GenericSet(['http://story']))
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORY_TAGS.name,
generic_set.GenericSet(
['group:media', 'ignored_tag', 'case:browse']))
hist = histograms.GetFirstHistogram()
test_path = histogram_helpers.ComputeTestPath(hist)
self.assertEqual('hist/browse_media/http___story', test_path)
def _BatchHistogramsIntoTasks(suite_path, histograms, revision,
benchmark_description):
params = []
tasks = []
base_size = _MakeTask([]).size
estimated_size = 0
duplicate_check = set()
for hist in histograms:
diagnostics = FindHistogramLevelSparseDiagnostics(hist)
# TODO(896856): Don't compute full diagnostics, because we need anyway to
# call GetOrCreate here and in the queue.
test_path = '%s/%s' % (suite_path,
histogram_helpers.ComputeTestPath(hist))
# Log the information here so we can see which histograms are being queued.
logging.debug('Queueing: %s', test_path)
if test_path in duplicate_check:
raise api_request_handler.BadRequestError(
'Duplicate histogram detected: %s' % test_path)
duplicate_check.add(test_path)
# TODO(#4135): Batch these better than one per task.
task_dict = _MakeTaskDict(hist, test_path, revision,
benchmark_description, diagnostics)
estimated_size_dict = len(json.dumps(task_dict))
estimated_size += estimated_size_dict
# Creating the task directly and getting the size back is slow, so we just
# keep a running total of estimated task size. A bit hand-wavy but the #
# of histograms per task doesn't need to be perfect, just has to be under
# the max task size.
estimated_total_size = estimated_size * 1.05 + base_size + 1024
if estimated_total_size > taskqueue.MAX_TASK_SIZE_BYTES:
t = _MakeTask(params)
tasks.append(t)
params = []
estimated_size = estimated_size_dict
params.append(task_dict)
if params:
t = _MakeTask(params)
tasks.append(t)
return tasks
def _CreateHistogramTasks(suite_path,
histograms,
revision,
benchmark_description,
completion_token=None):
tasks = []
duplicate_check = set()
measurement_add_futures = []
sheriff_client = sheriff_config_client.GetSheriffConfigClient()
for hist in histograms:
diagnostics = FindHistogramLevelSparseDiagnostics(hist)
test_path = '%s/%s' % (suite_path,
histogram_helpers.ComputeTestPath(hist))
# Log the information here so we can see which histograms are being queued.
logging.debug('Queueing: %s', test_path)
if test_path in duplicate_check:
raise api_request_handler.BadRequestError(
'Duplicate histogram detected: %s' % test_path)
duplicate_check.add(test_path)
# We create one task per histogram, so that we can get as much time as we
# need for processing each histogram per task.
task_dict = _MakeTaskDict(hist, test_path, revision,
benchmark_description, diagnostics,
completion_token)
tasks.append(_MakeTask([task_dict]))
if completion_token is not None:
measurement_add_futures.append(
completion_token.AddMeasurement(
test_path, utils.IsMonitored(sheriff_client, test_path)))
ndb.Future.wait_all(measurement_add_futures)
return tasks
def testComputeTestPathWithoutStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
hist = histograms.GetFirstHistogram()
test_path = histogram_helpers.ComputeTestPath(hist)
self.assertEqual('hist', test_path)
def CreateHistogramSetByTestPathDict(histograms, ignore_grouping_label=False):
histograms_by_path = collections.defaultdict(list)
for h in histograms:
histograms_by_path[histogram_helpers.ComputeTestPath(
h, ignore_grouping_label)].append(h)
return histograms_by_path
