def _ProcessRowAndHistogram(params, bot_whitelist):
  revision = int(params['revision'])
  test_path = params['test_path']
  benchmark_description = params['benchmark_description']
  data_dict = params['data']

  logging.info('Processing: %s', test_path)

  hist = histogram_module.Histogram.FromDict(data_dict)

  if hist.num_values == 0:
    return []

  test_path_parts = test_path.split('/')
  master = test_path_parts[0]
  bot = test_path_parts[1]
  benchmark_name = test_path_parts[2]
  histogram_name = test_path_parts[3]
  if len(test_path_parts) > 4:
    rest = '/'.join(test_path_parts[4:])
  else:
    rest = None
  full_test_name = '/'.join(test_path_parts[2:])
  internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
  extra_args = GetUnitArgs(hist.unit)

  unescaped_story_name = _GetStoryFromDiagnosticsDict(params.get('diagnostics'))

  # TDOO(eakuefner): Populate benchmark_description once it appears in
  # diagnostics.
  # https://github.com/catapult-project/catapult/issues/4096
  parent_test = add_point_queue.GetOrCreateAncestors(
      master, bot, full_test_name, internal_only=internal_only,
      unescaped_story_name=unescaped_story_name,
      benchmark_description=benchmark_description, **extra_args)
  test_key = parent_test.key

  statistics_scalars = hist.statistics_scalars
  legacy_parent_tests = {}

  # TODO(#4213): Stop doing this.
  if benchmark_name in LEGACY_BENCHMARKS:
    statistics_scalars = {}

  for stat_name, scalar in statistics_scalars.iteritems():
    if _ShouldFilter(histogram_name, benchmark_name, stat_name):
      continue
    extra_args = GetUnitArgs(scalar.unit)
    suffixed_name = '%s/%s_%s' % (
        benchmark_name, histogram_name, stat_name)
    if rest is not None:
      suffixed_name += '/' + rest
    legacy_parent_tests[stat_name] = add_point_queue.GetOrCreateAncestors(
        master, bot, suffixed_name, internal_only=internal_only,
        unescaped_story_name=unescaped_story_name, **extra_args)

  return [
      _AddRowsFromData(params, revision, parent_test, legacy_parent_tests,
                       internal_only),
      _AddHistogramFromData(params, revision, test_key, internal_only)]
示例#2
0
def _ProcessRowAndHistogram(params):
  revision = int(params['revision'])
  test_path = params['test_path']
  benchmark_description = params['benchmark_description']
  data_dict = params['data']

  logging.info('Processing: %s', test_path)

  hist = histogram_module.Histogram.FromDict(data_dict)

  if hist.num_values == 0:
    return []

  test_path_parts = test_path.split('/')
  master = test_path_parts[0]
  bot = test_path_parts[1]
  benchmark_name = test_path_parts[2]
  histogram_name = test_path_parts[3]
  if len(test_path_parts) > 4:
    rest = '/'.join(test_path_parts[4:])
  else:
    rest = None
  full_test_name = '/'.join(test_path_parts[2:])
  internal_only = graph_data.Bot.GetInternalOnlySync(master, bot)
  extra_args = GetUnitArgs(hist.unit)

  unescaped_story_name = _GetStoryFromDiagnosticsDict(params.get('diagnostics'))

  parent_test = add_point_queue.GetOrCreateAncestors(
      master, bot, full_test_name, internal_only=internal_only,
      unescaped_story_name=unescaped_story_name,
      benchmark_description=benchmark_description, **extra_args)
  test_key = parent_test.key

  statistics_scalars = hist.statistics_scalars
  legacy_parent_tests = {}

  # TODO(#4213): Stop doing this.
  if histogram_helpers.IsLegacyBenchmark(benchmark_name):
    statistics_scalars = {}

  for stat_name, scalar in statistics_scalars.items():
    if histogram_helpers.ShouldFilterStatistic(
        histogram_name, benchmark_name, stat_name):
      continue
    extra_args = GetUnitArgs(scalar.unit)
    suffixed_name = '%s/%s_%s' % (
        benchmark_name, histogram_name, stat_name)
    if rest is not None:
      suffixed_name += '/' + rest
    legacy_parent_tests[stat_name] = add_point_queue.GetOrCreateAncestors(
        master, bot, suffixed_name, internal_only=internal_only,
        unescaped_story_name=unescaped_story_name, **extra_args)

  return [
      _AddRowsFromData(params, revision, parent_test, legacy_parent_tests),
      _AddHistogramFromData(params, revision, test_key, internal_only)]
 def testGetOrCreateAncestors_CreatesAllExpectedEntities(self):
     parent = add_point_queue.GetOrCreateAncestors('ChromiumPerf', 'win7',
                                                   'dromaeo/dom/modify')
     self.assertEqual('ChromiumPerf/win7/dromaeo/dom/modify',
                      parent.key.id())
     # Check that all the Bot and TestMetadata entities were correctly added.
     created_masters = graph_data.Master.query().fetch()
     created_bots = graph_data.Bot.query().fetch()
     created_tests = graph_data.TestMetadata.query().fetch()
     self.assertEqual(1, len(created_masters))
     self.assertEqual(1, len(created_bots))
     self.assertEqual(3, len(created_tests))
     self.assertEqual('ChromiumPerf', created_masters[0].key.id())
     self.assertIsNone(created_masters[0].key.parent())
     self.assertEqual('win7', created_bots[0].key.id())
     self.assertEqual('ChromiumPerf', created_bots[0].key.parent().id())
     self.assertEqual('ChromiumPerf/win7/dromaeo',
                      created_tests[0].key.id())
     self.assertIsNone(created_tests[0].parent_test)
     self.assertEqual('win7', created_tests[0].bot_name)
     self.assertEqual('dom', created_tests[1].test_part1_name)
     self.assertEqual('ChromiumPerf/win7/dromaeo',
                      created_tests[1].parent_test.id())
     self.assertIsNone(created_tests[1].bot)
     self.assertEqual('ChromiumPerf/win7/dromaeo/dom/modify',
                      created_tests[2].key.id())
     self.assertEqual('ChromiumPerf/win7/dromaeo/dom',
                      created_tests[2].parent_test.id())
     self.assertIsNone(created_tests[2].bot)
    def testGetOrCreateAncestors_GetsExistingEntities(self):
        master_key = graph_data.Master(id='ChromiumPerf', parent=None).put()
        graph_data.Bot(id='win7', parent=master_key).put()
        t = graph_data.TestMetadata(id='ChromiumPerf/win7/dromaeo', )
        t.UpdateSheriff()
        t.put()

        t = graph_data.TestMetadata(id='ChromiumPerf/win7/dromaeo/dom')
        t.UpdateSheriff()
        t.put()

        t = graph_data.TestMetadata(id='ChromiumPerf/win7/dromaeo/dom/modify')
        t.UpdateSheriff()
        t.put()

        actual_parent = add_point_queue.GetOrCreateAncestors(
            'ChromiumPerf', 'win7', 'dromaeo/dom/modify')
        self.assertEqual('ChromiumPerf/win7/dromaeo/dom/modify',
                         actual_parent.key.id())
        # No extra TestMetadata or Bot objects should have been added to the
        # database beyond the four that were put in before the _GetOrCreateAncestors
        # call.
        self.assertEqual(1, len(graph_data.Master.query().fetch()))
        self.assertEqual(1, len(graph_data.Bot.query().fetch()))
        self.assertEqual(3, len(graph_data.TestMetadata.query().fetch()))
示例#5
0
def _ProcessRowAndHistogram(params, bot_whitelist):
    revision = int(params['revision'])
    test_path = params['test_path']
    data_dict = params['data']

    logging.info('Processing: %s', test_path)

    test_path_parts = test_path.split('/')
    master = test_path_parts[0]
    bot = test_path_parts[1]
    test_name = '/'.join(test_path_parts[2:])
    internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
    extra_args = GetUnitArgs(data_dict['unit'])

    unescaped_story_name = _GetStoryFromDiagnosticsDict(
        params.get('diagnostics'))

    # TDOO(eakuefner): Populate benchmark_description once it appears in
    # diagnostics.
    # https://github.com/catapult-project/catapult/issues/4096
    parent_test = add_point_queue.GetOrCreateAncestors(
        master,
        bot,
        test_name,
        internal_only,
        unescaped_story_name=unescaped_story_name,
        **extra_args)
    test_key = parent_test.key

    return [
        _AddRowFromData(params, revision, parent_test, internal_only),
        _AddHistogramFromData(params, revision, test_key, internal_only)
    ]
 def testGetOrCreateAncestors_RespectsImprovementDirectionForNewTest(self):
     test = add_point_queue.GetOrCreateAncestors(
         'M',
         'b',
         'suite/foo',
         units='bogus',
         improvement_direction=anomaly.UP)
     self.assertEqual(anomaly.UP, test.improvement_direction)
示例#7
0
def GetSuiteKey(histograms):
    assert len(histograms) > 0
    # TODO(eakuefner): Refactor this to coalesce the boilerplate (note that this
    # is all also being done in add_histograms_queue's post handler)
    master, bot, benchmark = _GetMasterBotBenchmarkFromHistogram(
        histograms.GetFirstHistogram())
    bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
    internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
    return add_point_queue.GetOrCreateAncestors(master, bot, benchmark,
                                                internal_only).key
示例#8
0
    def post(self):
        """Adds a single histogram or sparse shared diagnostic to the datastore.

    The |data| request parameter can be either a histogram or a sparse shared
    diagnostic; the set of diagnostics that are considered sparse (meaning that
    they don't normally change on every upload for a given benchmark from a
    given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.

    See https://goo.gl/lHzea6 for detailed information on the JSON format for
    histograms and diagnostics.

    Request parameters:
      data: JSON encoding of a histogram or shared diagnostic.
      revision: a revision, given as an int.
      test_path: the test path to which this diagnostic or histogram should be
          attached.
    """
        datastore_hooks.SetPrivilegedRequest()

        data = self.request.get('data')
        revision = int(self.request.get('revision'))
        test_path = self.request.get('test_path')

        data_dict = json.loads(data)
        guid = data_dict['guid']
        is_diagnostic = 'type' in data_dict

        test_path_parts = test_path.split('/')
        master = test_path_parts[0]
        bot = test_path_parts[1]
        test_name = '/'.join(test_path_parts[2:])
        bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
        internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
        extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
        # TDOO(eakuefner): Populate benchmark_description once it appears in
        # diagnostics.
        test_key = add_point_queue.GetOrCreateAncestors(
            master, bot, test_name, internal_only, **extra_args).key

        if is_diagnostic:
            entity = histogram.SparseDiagnostic(id=guid,
                                                data=data,
                                                test=test_key,
                                                start_revision=revision,
                                                end_revision=revision,
                                                internal_only=internal_only)
        else:
            entity = histogram.Histogram(id=guid,
                                         data=data,
                                         test=test_key,
                                         revision=revision,
                                         internal_only=internal_only)
            AddRow(data_dict, test_key, revision, test_path, internal_only)

        entity.put()
 def testGetOrCreateAncestors_UpdatesStoppageAlert(self):
     testing_common.AddTests(['M'], ['b'], {'suite': {'foo': {}}})
     row = testing_common.AddRows('M/b/suite/foo', {123})[0]
     test = utils.TestKey('M/b/suite/foo').get()
     alert_key = stoppage_alert.CreateStoppageAlert(test, row).put()
     test.stoppage_alert = alert_key
     test.put()
     add_point_queue.GetOrCreateAncestors('M', 'b', 'suite/foo')
     self.assertIsNone(test.key.get().stoppage_alert)
     self.assertTrue(alert_key.get().recovered)
     self.assertIsNotNone(alert_key.get().last_row_timestamp)
示例#10
0
def _ProcessRowAndHistogram(params, bot_whitelist):
    revision = int(params['revision'])
    test_path = params['test_path']
    data_dict = params['data']

    logging.info('Processing: %s', test_path)

    hist = histogram_module.Histogram.FromDict(data_dict)
    test_path_parts = test_path.split('/')
    master = test_path_parts[0]
    bot = test_path_parts[1]
    test_name = '/'.join(test_path_parts[2:])
    internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
    extra_args = GetUnitArgs(hist.unit)

    unescaped_story_name = _GetStoryFromDiagnosticsDict(
        params.get('diagnostics'))

    # TDOO(eakuefner): Populate benchmark_description once it appears in
    # diagnostics.
    # https://github.com/catapult-project/catapult/issues/4096
    parent_test = add_point_queue.GetOrCreateAncestors(
        master,
        bot,
        test_name,
        internal_only=internal_only,
        unescaped_story_name=unescaped_story_name,
        **extra_args)
    test_key = parent_test.key

    benchmark_name = test_path_parts[2]
    statistics_scalars = hist.statistics_scalars
    legacy_parent_tests = {}

    if test_name.endswith('_ref'):
        test_name = test_name[:-4]
        ref_suffix = '_ref'
    elif test_name.endswith('/ref'):
        test_name = test_name[:-4]
        ref_suffix = '/ref'
    else:
        ref_suffix = ''

    # TODO(#4213): Stop doing this.
    if _ShouldFilter(test_name, benchmark_name):
        statistics_scalars = {'avg': statistics_scalars['avg']}
    elif benchmark_name in LEGACY_BENCHMARKS:
        statistics_scalars = {}

    for stat_name, scalar in statistics_scalars.iteritems():
        extra_args = GetUnitArgs(scalar.unit)
        suffixed_name = '%s_%s%s' % (test_name, stat_name, ref_suffix)
        legacy_parent_tests[stat_name] = add_point_queue.GetOrCreateAncestors(
            master,
            bot,
            suffixed_name,
            internal_only=internal_only,
            unescaped_story_name=unescaped_story_name,
            **extra_args)

    return [
        _AddRowsFromData(params, revision, parent_test, legacy_parent_tests,
                         internal_only),
        _AddHistogramFromData(params, revision, test_key, internal_only)
    ]
    def post(self):
        """Adds a single histogram or sparse shared diagnostic to the datastore.

    The |data| request parameter can be either a histogram or a sparse shared
    diagnostic; the set of diagnostics that are considered sparse (meaning that
    they don't normally change on every upload for a given benchmark from a
    given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.

    See https://goo.gl/lHzea6 for detailed information on the JSON format for
    histograms and diagnostics.

    Request parameters:
      data: JSON encoding of a histogram or shared diagnostic.
      revision: a revision, given as an int.
      test_path: the test path to which this diagnostic or histogram should be
          attached.
    """
        datastore_hooks.SetPrivilegedRequest()

        data = self.request.get('data')
        revision = int(self.request.get('revision'))
        test_path = self.request.get('test_path')

        data_dict = json.loads(data)
        guid = data_dict['guid']
        is_diagnostic = 'type' in data_dict

        test_path_parts = test_path.split('/')
        master = test_path_parts[0]
        bot = test_path_parts[1]
        test_name = '/'.join(test_path_parts[2:])
        bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
        internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
        extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
        # TDOO(eakuefner): Populate benchmark_description once it appears in
        # diagnostics.
        parent_test = add_point_queue.GetOrCreateAncestors(
            master, bot, test_name, internal_only, **extra_args)
        test_key = parent_test.key

        added_rows = []
        monitored_test_keys = []

        if is_diagnostic:
            entity = histogram.SparseDiagnostic(id=guid,
                                                data=data,
                                                test=test_key,
                                                start_revision=revision,
                                                end_revision=revision,
                                                internal_only=internal_only)
        else:
            diagnostics = self.request.get('diagnostics')
            if diagnostics:
                diagnostic_data = json.loads(diagnostics)
                diagnostic_entities = []
                for diagnostic_datum in diagnostic_data:
                    # TODO(eakuefner): Pass map of guid to dict to avoid overhead
                    guid = diagnostic_datum['guid']
                    diagnostic_entities.append(
                        histogram.SparseDiagnostic(
                            id=guid,
                            data=diagnostic_datum,
                            test=test_key,
                            start_revision=revision,
                            end_revision=sys.maxint,
                            internal_only=internal_only))
                new_guids_to_existing_diagnostics = add_histograms.DeduplicateAndPut(
                    diagnostic_entities, test_key, revision).iteritems()
                # TODO(eakuefner): Move per-histogram monkeypatching logic to Histogram.
                hs = histogram_set.HistogramSet()
                hs.ImportDicts([data_dict])
                # TODO(eakuefner): Share code for replacement logic with add_histograms
                for new_guid, existing_diagnostic in new_guids_to_existing_diagnostics:
                    hs.ReplaceSharedDiagnostic(
                        new_guid,
                        diagnostic_ref.DiagnosticRef(
                            existing_diagnostic['guid']))
                data = hs.GetFirstHistogram().AsDict()

            entity = histogram.Histogram(id=guid,
                                         data=data,
                                         test=test_key,
                                         revision=revision,
                                         internal_only=internal_only)
            row = AddRow(data_dict, test_key, revision, test_path,
                         internal_only)
            added_rows.append(row)

            is_monitored = parent_test.sheriff and parent_test.has_rows
            if is_monitored:
                monitored_test_keys.append(parent_test.key)

        entity.put()

        tests_keys = [
            k for k in monitored_test_keys if not add_point_queue.IsRefBuild(k)
        ]

        # Updating of the cached graph revisions should happen after put because
        # it requires the new row to have a timestamp, which happens upon put.
        futures = [
            graph_revisions.AddRowsToCacheAsync(added_rows),
            find_anomalies.ProcessTestsAsync(tests_keys)
        ]
        ndb.Future.wait_all(futures)