def _CreateHistogramTasks(suite_path, histograms, revision, benchmark_description, completion_token=None): tasks = [] test_paths = set() for hist in histograms: diagnostics = FindHistogramLevelSparseDiagnostics(hist) test_path = '%s/%s' % (suite_path, histogram_helpers.ComputeTestPath(hist)) # Log the information here so we can see which histograms are being queued. logging.debug('Queueing: %s', test_path) if test_path in test_paths: raise api_request_handler.BadRequestError( 'Duplicate histogram detected: %s' % test_path) test_paths.add(test_path) # We create one task per histogram, so that we can get as much time as we # need for processing each histogram per task. task_dict = _MakeTaskDict(hist, test_path, revision, benchmark_description, diagnostics, completion_token) tasks.append(_MakeTask([task_dict])) if completion_token is not None: completion_token.PopulateMeasurements(test_paths) return tasks
def _CreateHistogramSetByTestPathDict(self, histograms): histograms_by_path = collections.defaultdict(list) for h in histograms: histograms_by_path[histogram_helpers.ComputeTestPath(h)].append(h) return histograms_by_path
def testComputeTestPathWithStory(self): hist = histogram_module.Histogram('hist', 'count') histograms = histogram_set.HistogramSet([hist]) histograms.AddSharedDiagnosticToAllHistograms( reserved_infos.STORIES.name, generic_set.GenericSet(['http://story'])) hist = histograms.GetFirstHistogram() test_path = histogram_helpers.ComputeTestPath(hist) self.assertEqual('hist/http___story', test_path)
def testComputeTestPathWithIsRefWithoutStory(self): hist = histogram_module.Histogram('hist', 'count') histograms = histogram_set.HistogramSet([hist]) histograms.AddSharedDiagnosticToAllHistograms( reserved_infos.IS_REFERENCE_BUILD.name, generic_set.GenericSet([True])) hist = histograms.GetFirstHistogram() test_path = histogram_helpers.ComputeTestPath(hist) self.assertEqual('hist/ref', test_path)
def testComputeTestPathWithGroupingLabel(self): hist = histogram_module.Histogram('hist', 'count') histograms = histogram_set.HistogramSet([hist]) histograms.AddSharedDiagnosticToAllHistograms( reserved_infos.STORIES.name, generic_set.GenericSet(['http://story'])) histograms.AddSharedDiagnosticToAllHistograms( reserved_infos.STORY_TAGS.name, generic_set.GenericSet( ['group:media', 'ignored_tag', 'case:browse'])) hist = histograms.GetFirstHistogram() test_path = histogram_helpers.ComputeTestPath(hist) self.assertEqual('hist/browse_media/http___story', test_path)
def _BatchHistogramsIntoTasks(suite_path, histograms, revision, benchmark_description): params = [] tasks = [] base_size = _MakeTask([]).size estimated_size = 0 duplicate_check = set() for hist in histograms: diagnostics = FindHistogramLevelSparseDiagnostics(hist) # TODO(896856): Don't compute full diagnostics, because we need anyway to # call GetOrCreate here and in the queue. test_path = '%s/%s' % (suite_path, histogram_helpers.ComputeTestPath(hist)) # Log the information here so we can see which histograms are being queued. logging.debug('Queueing: %s', test_path) if test_path in duplicate_check: raise api_request_handler.BadRequestError( 'Duplicate histogram detected: %s' % test_path) duplicate_check.add(test_path) # TODO(#4135): Batch these better than one per task. task_dict = _MakeTaskDict(hist, test_path, revision, benchmark_description, diagnostics) estimated_size_dict = len(json.dumps(task_dict)) estimated_size += estimated_size_dict # Creating the task directly and getting the size back is slow, so we just # keep a running total of estimated task size. A bit hand-wavy but the # # of histograms per task doesn't need to be perfect, just has to be under # the max task size. estimated_total_size = estimated_size * 1.05 + base_size + 1024 if estimated_total_size > taskqueue.MAX_TASK_SIZE_BYTES: t = _MakeTask(params) tasks.append(t) params = [] estimated_size = estimated_size_dict params.append(task_dict) if params: t = _MakeTask(params) tasks.append(t) return tasks
def _CreateHistogramTasks(suite_path, histograms, revision, benchmark_description, completion_token=None): tasks = [] duplicate_check = set() measurement_add_futures = [] sheriff_client = sheriff_config_client.GetSheriffConfigClient() for hist in histograms: diagnostics = FindHistogramLevelSparseDiagnostics(hist) test_path = '%s/%s' % (suite_path, histogram_helpers.ComputeTestPath(hist)) # Log the information here so we can see which histograms are being queued. logging.debug('Queueing: %s', test_path) if test_path in duplicate_check: raise api_request_handler.BadRequestError( 'Duplicate histogram detected: %s' % test_path) duplicate_check.add(test_path) # We create one task per histogram, so that we can get as much time as we # need for processing each histogram per task. task_dict = _MakeTaskDict(hist, test_path, revision, benchmark_description, diagnostics, completion_token) tasks.append(_MakeTask([task_dict])) if completion_token is not None: measurement_add_futures.append( completion_token.AddMeasurement( test_path, utils.IsMonitored(sheriff_client, test_path))) ndb.Future.wait_all(measurement_add_futures) return tasks
def testComputeTestPathWithoutStory(self): hist = histogram_module.Histogram('hist', 'count') histograms = histogram_set.HistogramSet([hist]) hist = histograms.GetFirstHistogram() test_path = histogram_helpers.ComputeTestPath(hist) self.assertEqual('hist', test_path)
def CreateHistogramSetByTestPathDict(histograms, ignore_grouping_label=False): histograms_by_path = collections.defaultdict(list) for h in histograms: histograms_by_path[histogram_helpers.ComputeTestPath( h, ignore_grouping_label)].append(h) return histograms_by_path