def _AddRowsFromData(params, revision, parent_test, legacy_parent_tests):
data_dict = params['data']
test_key = parent_test.key
stat_names_to_test_keys = {
k: v.key
for k, v in legacy_parent_tests.iteritems()
}
rows = CreateRowEntities(data_dict, test_key, stat_names_to_test_keys,
revision)
if not rows:
raise ndb.Return()
yield ndb.put_multi_async(rows) + [r.UpdateParentAsync() for r in rows]
tests_keys = []
is_monitored = parent_test.sheriff and parent_test.has_rows
if is_monitored:
tests_keys.append(parent_test.key)
for legacy_parent_test in legacy_parent_tests.itervalues():
is_monitored = legacy_parent_test.sheriff and legacy_parent_test.has_rows
if is_monitored:
tests_keys.append(legacy_parent_test.key)
tests_keys = [k for k in tests_keys if not add_point_queue.IsRefBuild(k)]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
yield futures
def _AddRowFromData(params, revision, parent_test, internal_only):
data_dict = params['data']
test_path = params['test_path']
test_key = parent_test.key
row = AddRow(data_dict, test_key, revision, test_path, internal_only)
if not row:
raise ndb.Return()
yield row.put_async()
tests_keys = []
is_monitored = parent_test.sheriff and parent_test.has_rows
if is_monitored:
tests_keys.append(parent_test.key)
tests_keys = [k for k in tests_keys if not add_point_queue.IsRefBuild(k)]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync([row]),
find_anomalies.ProcessTestsAsync(tests_keys)
]
yield futures
def post(self):
"""Adds a set of points from the post data.
Request parameters:
data: JSON encoding of a list of dictionaries. Each dictionary represents
one point to add. For each dict, one Row entity will be added, and
any required TestMetadata or Master or Bot entities will be created.
"""
datastore_hooks.SetPrivilegedRequest()
data = json.loads(self.request.get('data'))
_PrewarmGets(data)
all_put_futures = []
added_rows = []
parent_tests = []
for row_dict in data:
try:
new_row, parent_test, put_futures = _AddRow(row_dict)
added_rows.append(new_row)
parent_tests.append(parent_test)
all_put_futures.extend(put_futures)
except add_point.BadRequestError as e:
logging.error('Could not add %s, it was invalid.', e.message)
except datastore_errors.BadRequestError as e:
logging.info('While trying to store %s', row_dict)
logging.error('Datastore request failed: %s.', e.message)
return
ndb.Future.wait_all(all_put_futures)
client = sheriff_config_client.GetSheriffConfigClient()
tests_keys = []
for t in parent_tests:
reason = []
subscriptions, _ = client.Match(t.test_path, check=True)
if not subscriptions:
reason.append('subscriptions')
if not t.has_rows:
reason.append('has_rows')
if IsRefBuild(t.key):
reason.append('RefBuild')
if reason:
logging.info('Skip test: %s reason=%s', t.key,
','.join(reason))
continue
logging.info('Process test: %s', t.key)
tests_keys.append(t.key)
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(added_rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
ndb.Future.wait_all(futures)
def _AddRowsFromData(params, revision, parent_test, legacy_parent_tests):
data_dict = params['data']
test_key = parent_test.key
stat_names_to_test_keys = {
k: v.key
for k, v in legacy_parent_tests.items()
}
rows = CreateRowEntities(data_dict, test_key, stat_names_to_test_keys,
revision)
if not rows:
raise ndb.Return()
yield ndb.put_multi_async(rows) + [r.UpdateParentAsync() for r in rows]
def IsMonitored(client, test):
reason = []
request_sampling_percentage = 1.0
if random.random() < request_sampling_percentage:
subscriptions, _ = client.Match(test.test_path, check=True)
if not subscriptions:
reason.append('subscriptions')
elif not test.sheriff:
reason.append('sheriff')
if not test.has_rows:
reason.append('has_rows')
if reason:
logging.info('Skip test: %s reason=%s', test.key, ','.join(reason))
return False
logging.info('Process test: %s', test.key)
return True
client = sheriff_config_client.GetSheriffConfigClient()
tests_keys = []
if IsMonitored(client, parent_test):
tests_keys.append(parent_test.key)
for legacy_parent_test in legacy_parent_tests.values():
if IsMonitored(client, legacy_parent_test):
tests_keys.append(legacy_parent_test.key)
tests_keys = [k for k in tests_keys if not add_point_queue.IsRefBuild(k)]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
yield futures
def post(self):
"""Adds a set of points from the post data.
Request parameters:
data: JSON encoding of a list of dictionaries. Each dictionary represents
one point to add. For each dict, one Row entity will be added, and
any required TestMetadata or Master or Bot entities will be created.
"""
datastore_hooks.SetPrivilegedRequest()
data = json.loads(self.request.get('data'))
_PrewarmGets(data)
bot_whitelist = stored_object.Get(BOT_WHITELIST_KEY)
all_put_futures = []
added_rows = []
parent_tests = []
for row_dict in data:
try:
new_row, parent_test, put_futures = _AddRow(
row_dict, bot_whitelist)
added_rows.append(new_row)
parent_tests.append(parent_test)
all_put_futures.extend(put_futures)
except add_point.BadRequestError as e:
logging.error('Could not add %s, it was invalid.', e.message)
except datastore_errors.BadRequestError as e:
logging.info('While trying to store %s', row_dict)
logging.error('Datastore request failed: %s.', e.message)
return
ndb.Future.wait_all(all_put_futures)
monitored_test_keys = [
t.key for t in parent_tests if t.sheriff and t.has_rows
]
tests_keys = [k for k in monitored_test_keys if not IsRefBuild(k)]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(added_rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
ndb.Future.wait_all(futures)
def post(self):
"""Adds a single histogram or sparse shared diagnostic to the datastore.
The |data| request parameter can be either a histogram or a sparse shared
diagnostic; the set of diagnostics that are considered sparse (meaning that
they don't normally change on every upload for a given benchmark from a
given bot) is shown in add_histograms.SPARSE_DIAGNOSTIC_TYPES.
See https://goo.gl/lHzea6 for detailed information on the JSON format for
histograms and diagnostics.
Request parameters:
data: JSON encoding of a histogram or shared diagnostic.
revision: a revision, given as an int.
test_path: the test path to which this diagnostic or histogram should be
attached.
"""
datastore_hooks.SetPrivilegedRequest()
data = self.request.get('data')
revision = int(self.request.get('revision'))
test_path = self.request.get('test_path')
data_dict = json.loads(data)
guid = data_dict['guid']
is_diagnostic = 'type' in data_dict
test_path_parts = test_path.split('/')
master = test_path_parts[0]
bot = test_path_parts[1]
test_name = '/'.join(test_path_parts[2:])
bot_whitelist = stored_object.Get(add_point_queue.BOT_WHITELIST_KEY)
internal_only = add_point_queue.BotInternalOnly(bot, bot_whitelist)
extra_args = {} if is_diagnostic else GetUnitArgs(data_dict['unit'])
# TDOO(eakuefner): Populate benchmark_description once it appears in
# diagnostics.
parent_test = add_point_queue.GetOrCreateAncestors(
master, bot, test_name, internal_only, **extra_args)
test_key = parent_test.key
added_rows = []
monitored_test_keys = []
if is_diagnostic:
entity = histogram.SparseDiagnostic(id=guid,
data=data,
test=test_key,
start_revision=revision,
end_revision=revision,
internal_only=internal_only)
else:
diagnostics = self.request.get('diagnostics')
if diagnostics:
diagnostic_data = json.loads(diagnostics)
diagnostic_entities = []
for diagnostic_datum in diagnostic_data:
# TODO(eakuefner): Pass map of guid to dict to avoid overhead
guid = diagnostic_datum['guid']
diagnostic_entities.append(
histogram.SparseDiagnostic(
id=guid,
data=diagnostic_datum,
test=test_key,
start_revision=revision,
end_revision=sys.maxint,
internal_only=internal_only))
new_guids_to_existing_diagnostics = add_histograms.DeduplicateAndPut(
diagnostic_entities, test_key, revision).iteritems()
# TODO(eakuefner): Move per-histogram monkeypatching logic to Histogram.
hs = histogram_set.HistogramSet()
hs.ImportDicts([data_dict])
# TODO(eakuefner): Share code for replacement logic with add_histograms
for new_guid, existing_diagnostic in new_guids_to_existing_diagnostics:
hs.ReplaceSharedDiagnostic(
new_guid,
diagnostic_ref.DiagnosticRef(
existing_diagnostic['guid']))
data = hs.GetFirstHistogram().AsDict()
entity = histogram.Histogram(id=guid,
data=data,
test=test_key,
revision=revision,
internal_only=internal_only)
row = AddRow(data_dict, test_key, revision, test_path,
internal_only)
added_rows.append(row)
is_monitored = parent_test.sheriff and parent_test.has_rows
if is_monitored:
monitored_test_keys.append(parent_test.key)
entity.put()
tests_keys = [
k for k in monitored_test_keys if not add_point_queue.IsRefBuild(k)
]
# Updating of the cached graph revisions should happen after put because
# it requires the new row to have a timestamp, which happens upon put.
futures = [
graph_revisions.AddRowsToCacheAsync(added_rows),
find_anomalies.ProcessTestsAsync(tests_keys)
]
ndb.Future.wait_all(futures)
