def _DeleteTestData(test_key):
logging.info('DELETING TEST DATA FOR %s', utils.TestPath(test_key))
futures = []
num_tests_processed = 0
finished = True
descendants = list_tests.GetTestDescendants(test_key)
for descendant in descendants:
rows = graph_data.GetLatestRowsForTest(descendant,
_ROWS_TO_DELETE_AT_ONCE,
keys_only=True)
if rows:
futures.extend(ndb.delete_multi_async(rows))
finished = False
num_tests_processed += 1
if num_tests_processed > _MAX_DELETIONS_PER_TASK:
break
# Only delete TestMetadata entities after all Row entities have been deleted.
if finished:
descendants = ndb.get_multi(descendants)
for descendant in descendants:
_SendNotificationEmail(descendant)
futures.append(descendant.key.delete_async())
ndb.Future.wait_all(futures)
return finished
def _DeleteTestData(test_key, notify):
futures = []
num_tests_processed = 0
more = False
descendants = list_tests.GetTestDescendants(test_key)
for descendant in descendants:
rows = graph_data.GetLatestRowsForTest(descendant,
_ROWS_TO_DELETE_AT_ONCE,
keys_only=True)
if rows:
futures.extend(ndb.delete_multi_async(rows))
more = True
num_tests_processed += 1
if num_tests_processed > _MAX_DELETIONS_PER_TASK:
break
if not more:
more = _DeleteTestHistogramData(descendant)
# Only delete TestMetadata entities after all Row entities have been deleted.
if not more:
descendants = ndb.get_multi(descendants)
for descendant in descendants:
_SendNotificationEmail(descendant, notify)
futures.append(descendant.key.delete_async())
ndb.Future.wait_all(futures)
return not more
def _FetchLatestRows(test, num_points):
"""Does a query for the latest Row entities in the given test.
Args:
test: A TestMetadata entity to fetch Row entities for.
num_points: Number of points to fetch.
Returns:
A list of Row entities, ordered by revision. The number to fetch is limited
to the number that is expected to be processed at once by GASP.
"""
assert utils.IsInternalUser() or not test.internal_only
datastore_hooks.SetSinglePrivilegedRequest()
return list(reversed(graph_data.GetLatestRowsForTest(test.key, num_points)))
def _MigrateTestRows(old_parent_key, new_parent_key):
"""Copies Row entities from one parent to another, deleting old ones.
Args:
old_parent_key: TestMetadata entity key of the test to move from.
new_parent_key: TestMetadata entity key of the test to move to.
Returns:
A tuple of (futures, more)
futures: A list of Future objects for entities being put.
more: Whether or not there's more work to do on the row data.
"""
# In this function we'll build up lists of entities to put and delete
# before returning Future objects for the entities being put and deleted.
rows_to_put = []
rows_to_delete = []
# Add some Row entities to the lists of entities to put and delete.
rows = graph_data.GetLatestRowsForTest(
old_parent_key, _MAX_DATASTORE_PUTS_PER_PUT_MULTI_CALL)
rows_to_put = yield [
_GetOrCreate(graph_data.Row, r, r.key.id(), new_parent_key,
_ROW_EXCLUDE) for r in rows
]
rows_to_delete = [r.key for r in rows]
# Clear the cached revision range selector data for both the old and new
# tests because it will no longer be valid after migration. The cache should
# be updated with accurate data the next time it's set, which will happen
# when someone views the graph.
futures = ndb.put_multi_async(rows_to_put,
use_cache=False,
use_memcache=False)
if rows_to_put:
futures.append(rows_to_put[0].UpdateParentAsync())
futures.extend(ndb.delete_multi_async(rows_to_delete))
futures.append(
graph_revisions.DeleteCacheAsync(utils.TestPath(old_parent_key)))
futures.append(
graph_revisions.DeleteCacheAsync(utils.TestPath(new_parent_key)))
yield futures
raise ndb.Return(bool(rows_to_put))
def _UpdateRevisionMap(revision_map,
parent_test,
rev,
num_points,
start_rev=None,
end_rev=None):
"""Updates a dict of revisions to data point information for one test.
Depending on which arguments are given, there are several ways that
this function can update the dict of revisions:
1. If start_rev and end_rev are given, then revisions in this range
are used. The num_points argument is ignored.
2. Otherwise, if rev is given, then revisions before and after the
specified revision are used.
3. Otherwise, the latest revisions are used.
Args:
revision_map: A dict mapping revision numbers to dicts of point info.
Each point info dict contains information from a Row entity.
parent_test: A TestMetadata entity with Row children.
rev: The middle revision in the revision map (could be None).
num_points: The number of points to include in the revision map.
start_rev: Start revision number (optional).
end_rev: End revision number (optional).
"""
anomaly_annotation_map = _GetAnomalyAnnotationMap(parent_test.key)
assert (datastore_hooks.IsUnalteredQueryPermitted()
or not parent_test.internal_only)
if start_rev and end_rev:
rows = graph_data.GetRowsForTestInRange(parent_test.key, start_rev,
end_rev, True)
elif rev:
assert num_points
rows = graph_data.GetRowsForTestAroundRev(parent_test.key, rev,
num_points, True)
else:
assert num_points
rows = graph_data.GetLatestRowsForTest(parent_test.key,
num_points,
privileged=True)
parent_test_key = parent_test.key.urlsafe()
for row in rows:
if row.revision not in revision_map:
revision_map[row.revision] = {}
revision_map[row.revision][parent_test_key] = _PointInfoDict(
row, anomaly_annotation_map)
def _MigrateTestRows(old_parent_key, new_parent_key):
"""Copies Row entities from one parent to another, deleting old ones.
Args:
old_parent_key: TestMetadata entity key of the test to move from.
new_parent_key: TestMetadata entity key of the test to move to.
Returns:
A dictionary with the following keys:
put_future: A list of Future objects for entities being put.
delete_future: A list of Future objects for entities being deleted.
moved_rows: Whether or not any entities were moved.
"""
# In this function we'll build up lists of entities to put and delete
# before returning Future objects for the entities being put and deleted.
rows_to_put = []
rows_to_delete = []
# Add some Row entities to the lists of entities to put and delete.
rows = graph_data.GetLatestRowsForTest(
old_parent_key, _MAX_DATASTORE_PUTS_PER_PUT_MULTI_CALL)
for row in rows:
rows_to_put.append(
_CreateRenamedEntityIfNotExists(graph_data.Row, row, row.key.id(),
new_parent_key, _ROW_EXCLUDE))
rows_to_delete.append(row.key)
# Clear the cached revision range selector data for both the old and new
# tests because it will no longer be valid after migration. The cache should
# be updated with accurate data the next time it's set, which will happen
# when someone views the graph.
graph_revisions.DeleteCache(utils.TestPath(old_parent_key))
graph_revisions.DeleteCache(utils.TestPath(new_parent_key))
return {
'put_future': ndb.put_multi_async(rows_to_put),
'delete_future': ndb.delete_multi_async(rows_to_delete),
'moved_rows': bool(rows_to_put),
}
