def _AddRowsFromData(params, revision, parent_test, legacy_parent_tests): data_dict = params['data'] test_key = parent_test.key stat_names_to_test_keys = { k: v.key for k, v in legacy_parent_tests.iteritems() } rows = CreateRowEntities(data_dict, test_key, stat_names_to_test_keys, revision) if not rows: raise ndb.Return() yield ndb.put_multi_async(rows) + [r.UpdateParentAsync() for r in rows] tests_keys = [] is_monitored = parent_test.sheriff and parent_test.has_rows if is_monitored: tests_keys.append(parent_test.key) for legacy_parent_test in legacy_parent_tests.itervalues(): is_monitored = legacy_parent_test.sheriff and legacy_parent_test.has_rows if is_monitored: tests_keys.append(legacy_parent_test.key) tests_keys = [k for k in tests_keys if not add_point_queue.IsRefBuild(k)] # Updating of the cached graph revisions should happen after put because # it requires the new row to have a timestamp, which happens upon put. futures = [ graph_revisions.AddRowsToCacheAsync(rows), find_anomalies.ProcessTestsAsync(tests_keys) ] yield futures
def _AddRowFromData(params, revision, parent_test, internal_only): data_dict = params['data'] test_path = params['test_path'] test_key = parent_test.key row = AddRow(data_dict, test_key, revision, test_path, internal_only) if not row: raise ndb.Return() yield row.put_async() tests_keys = [] is_monitored = parent_test.sheriff and parent_test.has_rows if is_monitored: tests_keys.append(parent_test.key) tests_keys = [k for k in tests_keys if not add_point_queue.IsRefBuild(k)] # Updating of the cached graph revisions should happen after put because # it requires the new row to have a timestamp, which happens upon put. futures = [ graph_revisions.AddRowsToCacheAsync([row]), find_anomalies.ProcessTestsAsync(tests_keys) ] yield futures
def post(self): """Adds a set of points from the post data. Request parameters: data: JSON encoding of a list of dictionaries. Each dictionary represents one point to add. For each dict, one Row entity will be added, and any required TestMetadata or Master or Bot entities will be created. """ datastore_hooks.SetPrivilegedRequest() data = json.loads(self.request.get('data')) _PrewarmGets(data) all_put_futures = [] added_rows = [] parent_tests = [] for row_dict in data: try: new_row, parent_test, put_futures = _AddRow(row_dict) added_rows.append(new_row) parent_tests.append(parent_test) all_put_futures.extend(put_futures) except add_point.BadRequestError as e: logging.error('Could not add %s, it was invalid.', e.message) except datastore_errors.BadRequestError as e: logging.info('While trying to store %s', row_dict) logging.error('Datastore request failed: %s.', e.message) return ndb.Future.wait_all(all_put_futures) client = sheriff_config_client.GetSheriffConfigClient() tests_keys = [] for t in parent_tests: reason = [] subscriptions, _ = client.Match(t.test_path, check=True) if not subscriptions: reason.append('subscriptions') if not t.has_rows: reason.append('has_rows') if IsRefBuild(t.key): reason.append('RefBuild') if reason: logging.info('Skip test: %s reason=%s', t.key, ','.join(reason)) continue logging.info('Process test: %s', t.key) tests_keys.append(t.key) # Updating of the cached graph revisions should happen after put because # it requires the new row to have a timestamp, which happens upon put. futures = [ graph_revisions.AddRowsToCacheAsync(added_rows), find_anomalies.ProcessTestsAsync(tests_keys) ] ndb.Future.wait_all(futures)
def _AddRowsFromData(params, revision, parent_test, legacy_parent_tests): data_dict = params['data'] test_key = parent_test.key stat_names_to_test_keys = { k: v.key for k, v in legacy_parent_tests.items() } rows = CreateRowEntities(data_dict, test_key, stat_names_to_test_keys, revision) if not rows: raise ndb.Return() yield ndb.put_multi_async(rows) + [r.UpdateParentAsync() for r in rows] def IsMonitored(client, test): reason = [] request_sampling_percentage = 1.0 if random.random() < request_sampling_percentage: subscriptions, _ = client.Match(test.test_path, check=True) if not subscriptions: reason.append('subscriptions') elif not test.sheriff: reason.append('sheriff') if not test.has_rows: reason.append('has_rows') if reason: logging.info('Skip test: %s reason=%s', test.key, ','.join(reason)) return False logging.info('Process test: %s', test.key) return True client = sheriff_config_client.GetSheriffConfigClient() tests_keys = [] if IsMonitored(client, parent_test): tests_keys.append(parent_test.key) for legacy_parent_test in legacy_parent_tests.values(): if IsMonitored(client, legacy_parent_test): tests_keys.append(legacy_parent_test.key) tests_keys = [k for k in tests_keys if not add_point_queue.IsRefBuild(k)] # Updating of the cached graph revisions should happen after put because # it requires the new row to have a timestamp, which happens upon put. futures = [ graph_revisions.AddRowsToCacheAsync(rows), find_anomalies.ProcessTestsAsync(tests_keys) ] yield futures
def post(self): """Adds a set of points from the post data. Request parameters: data: JSON encoding of a list of dictionaries. Each dictionary represents one point to add. For each dict, one Row entity will be added, and any required TestMetadata or Master or Bot entities will be created. """ datastore_hooks.SetPrivilegedRequest() data = json.loads(self.request.get('data')) _PrewarmGets(data) bot_whitelist = stored_object.Get(BOT_WHITELIST_KEY) all_put_futures = [] added_rows = [] parent_tests = [] for row_dict in data: try: new_row, parent_test, put_futures = _AddRow( row_dict, bot_whitelist) added_rows.append(new_row) parent_tests.append(parent_test) all_put_futures.extend(put_futures) except add_point.BadRequestError as e: logging.error('Could not add %s, it was invalid.', e.message) except datastore_errors.BadRequestError as e: logging.info('While trying to store %s', row_dict) logging.error('Datastore request failed: %s.', e.message) return ndb.Future.wait_all(all_put_futures) monitored_test_keys = [ t.key for t in parent_tests if t.sheriff and t.has_rows ] tests_keys = [k for k in monitored_test_keys if not IsRefBuild(k)] # Updating of the cached graph revisions should happen after put because # it requires the new row to have a timestamp, which happens upon put. futures = [ graph_revisions.AddRowsToCacheAsync(added_rows), find_anomalies.ProcessTestsAsync(tests_keys) ] ndb.Future.wait_all(futures)
def post(self): """Adds a single histogram or sparse shared diagnostic to the datastore. The |data| request parameter can be either a histogram or a sparse shared diagnostic; the set of diagnostics that are considered sparse (meaning that they don't normally change on every upload for a given benchmark from a given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES. See https://goo.gl/lHzea6 for detailed information on the JSON format for histograms and diagnostics. Request parameters: data: JSON encoding of a histogram or shared diagnostic. revision: a revision, given as an int. test_path: the test path to which this diagnostic or histogram should be attached. """ datastore_hooks.SetPrivilegedRequest() data = self.request.get('data') revision = int(self.request.get('revision')) test_path = self.request.get('test_path') data_dict = json.loads(data) guid = data_dict['guid'] is_diagnostic = 'type' in data_dict test_path_parts = test_path.split('/') master = test_path_parts[0] bot = test_path_parts[1] test_name = '/'.join(test_path_parts[2:]) bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY) internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist) extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit']) # TDOO(eakuefner): Populate benchmark_description once it appears in # diagnostics. parent_test = add_point_queue.GetOrCreateAncestors( master, bot, test_name, internal_only, **extra_args) test_key = parent_test.key added_rows = [] monitored_test_keys = [] if is_diagnostic: entity = histogram.SparseDiagnostic(id=guid, data=data, test=test_key, start_revision=revision, end_revision=revision, internal_only=internal_only) else: diagnostics = self.request.get('diagnostics') if diagnostics: diagnostic_data = json.loads(diagnostics) diagnostic_entities = [] for diagnostic_datum in diagnostic_data: # TODO(eakuefner): Pass map of guid to dict to avoid overhead guid = diagnostic_datum['guid'] diagnostic_entities.append( histogram.SparseDiagnostic( id=guid, data=diagnostic_datum, test=test_key, start_revision=revision, end_revision=sys.maxint, internal_only=internal_only)) new_guids_to_existing_diagnostics = add_histograms.DeduplicateAndPut( diagnostic_entities, test_key, revision).iteritems() # TODO(eakuefner): Move per-histogram monkeypatching logic to Histogram. hs = histogram_set.HistogramSet() hs.ImportDicts([data_dict]) # TODO(eakuefner): Share code for replacement logic with add_histograms for new_guid, existing_diagnostic in new_guids_to_existing_diagnostics: hs.ReplaceSharedDiagnostic( new_guid, diagnostic_ref.DiagnosticRef( existing_diagnostic['guid'])) data = hs.GetFirstHistogram().AsDict() entity = histogram.Histogram(id=guid, data=data, test=test_key, revision=revision, internal_only=internal_only) row = AddRow(data_dict, test_key, revision, test_path, internal_only) added_rows.append(row) is_monitored = parent_test.sheriff and parent_test.has_rows if is_monitored: monitored_test_keys.append(parent_test.key) entity.put() tests_keys = [ k for k in monitored_test_keys if not add_point_queue.IsRefBuild(k) ] # Updating of the cached graph revisions should happen after put because # it requires the new row to have a timestamp, which happens upon put. futures = [ graph_revisions.AddRowsToCacheAsync(added_rows), find_anomalies.ProcessTestsAsync(tests_keys) ] ndb.Future.wait_all(futures)