def setUp(self):
datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB)
self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)
self.entities = []
prev = None
for ii in range(0,3):
entity = datastore.Entity("TestKind",
_app="test",
name=str(ii),
parent = prev,
namespace='c')
prev = entity
# have properties with different values bye same property names
entity.update({'aaa': "1111_" + str(ii),
'bbb': "2222"})
self.entities.append(entity.ToPb())
self.keys = ['test/c/TestKind/aaa/1111_0\x00/TestKind:0!',
'test/c/TestKind/bbb/2222\x00/TestKind:0!',
'test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!',
'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!',
'test/c/TestKind/aaa/1111_2\x00/TestKind:0!TestKind:1!TestKind:2!',
'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!TestKind:2!']
self.rkeys = ['test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcf\xff/TestKind:0!',
'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!',
'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xce\xff/TestKind:0!TestKind:1!',
'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!',
'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:0!TestKind:1!TestKind:2!',
'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!TestKind:2!']
def setUp(self):
datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB)
self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)
self.entities = []
prev = None
for ii in range(0,4):
entity = datastore.Entity("TestKind",
_app="test",
name=str(ii),
namespace='b',
parent = prev)
prev = entity
# have properties with different values bye same property names
entity.update({'aaa': "1111_" + str(ii),
'bbb': "2222",
'ccc': "3"*ii})
self.entities.append(entity.ToPb())
self.keys = ['test/b/TestKind:0!',
'test/b/TestKind:0!TestKind:1!',
'test/b/TestKind:0!TestKind:1!TestKind:2!',
'test/b/TestKind:0!TestKind:1!TestKind:2!TestKind:3!']
self.kkeys = ['test/b/TestKind:0!',
'test/b/TestKind:1!TestKind:0!',
'test/b/TestKind:2!TestKind:1!TestKind:0!',
'test/b/TestKind:3!TestKind:2!TestKind:1!TestKind:0!']
def setUp(self):
datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(
DB)
self.app_datastore = datastore_server.DatastoreDistributed(
datastore_batch)
self.entities = []
prev = None
for ii in range(0, 3):
entity = datastore.Entity("TestKind",
_app="test",
name=str(ii),
parent=prev,
namespace='e')
prev = entity
# have properties with different values bye same property names
entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"})
self.entities.append(entity.ToPb())
self.entities2 = []
prev = None
for ii in range(0, 3):
entity = datastore.Entity("TestKind",
_app="test",
name=str(ii),
parent=prev,
namespace='e')
prev = entity
# have properties with different values bye same property names
entity.update({'aaa': "x111_" + str(ii), 'bbb': "x222"})
self.entities2.append(entity.ToPb())
tuples = sorted(
(self.app_datastore.GetTablePrefix(x), x) for x in self.entities)
# keys should be the same for entities and entities2
self.keys = self.app_datastore.GetIndexKVFromTuple(tuples,
reverse=False)
self.keys = [x[0] for x in self.keys]
tuples = sorted(
(self.app_datastore.GetTablePrefix(x), x) for x in self.entities2)
# keys should be the same for entities and entities2
self.keys2 = self.app_datastore.GetIndexKVFromTuple(tuples,
reverse=False)
self.keys2 = [x[0] for x in self.keys2]
def setUp(self):
datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB)
self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)
self.entities = []
self.keys = []
prev = None
for ii in range(0,3):
entity = datastore.Entity("TestKind",
_app="test",
name = str(ii),
parent = prev,
namespace='e')
prev = entity
# have properties with different values bye same property names
entity.update({'aaa': "1111_" + str(ii),
'bbb': "2222"})
self.entities.append(entity.ToPb())
self.keys = [e.key() for e in self.entities]
self.app_datastore.PutEntities(self.entities)
def run_groomer(self):
""" Runs the grooming process. Loops on the entire dataset sequentially
and updates stats, indexes, and transactions.
"""
self.db_access = appscale_datastore_batch.DatastoreFactory.getDatastore(
self.table_name)
self.ds_access = datastore_server.DatastoreDistributed(
datastore_batch=self.db_access, zookeeper=self.zoo_keeper)
logging.info("Groomer started")
start = time.time()
self.reset_statistics()
self.composite_index_cache = {}
tasks = [
{
'id': self.CLEAN_ENTITIES_TASK,
'description': 'clean up entities',
'function': self.clean_up_entities,
'args': []
},
{
'id': self.CLEAN_ASC_INDICES_TASK,
'description': 'clean up ascending indices',
'function': self.clean_up_indexes,
'args': [datastore_pb.Query_Order.ASCENDING]
},
{
'id': self.CLEAN_DSC_INDICES_TASK,
'description': 'clean up descending indices',
'function': self.clean_up_indexes,
'args': [datastore_pb.Query_Order.DESCENDING]
},
{
'id': self.CLEAN_KIND_INDICES_TASK,
'description': 'clean up kind indices',
'function': self.clean_up_kind_indices,
'args': []
},
{
'id': self.CLEAN_LOGS_TASK,
'description': 'clean up old logs',
'function': self.remove_old_logs,
'args': [self.LOG_STORAGE_TIMEOUT]
},
{
'id': self.CLEAN_TASKS_TASK,
'description': 'clean up old tasks',
'function': self.remove_old_tasks_entities,
'args': []
},
{
'id': self.CLEAN_DASHBOARD_TASK,
'description': 'clean up old dashboard items',
'function': self.remove_old_dashboard_data,
'args': []
}
]
groomer_state = self.zoo_keeper.get_node(self.GROOMER_STATE_PATH)
logging.info('groomer_state: {}'.format(groomer_state))
if groomer_state:
self.update_groomer_state(
groomer_state[0].split(self.GROOMER_STATE_DELIMITER))
for task_number in range(len(tasks)):
task = tasks[task_number]
if (len(self.groomer_state) > 0 and self.groomer_state[0] != '' and
self.groomer_state[0] != task['id']):
continue
logging.info('Starting to {}'.format(task['description']))
try:
task['function'](*task['args'])
if task_number != len(tasks) - 1:
next_task = tasks[task_number + 1]
self.update_groomer_state([next_task['id']])
except Exception as exception:
logging.error('Exception encountered while trying to {}:'.
format(task['description']))
logging.exception(exception)
self.update_groomer_state([])
timestamp = datetime.datetime.utcnow()
if not self.update_statistics(timestamp):
logging.error("There was an error updating the statistics")
if not self.update_namespaces(timestamp):
logging.error("There was an error updating the namespaces")
del self.db_access
del self.ds_access
time_taken = time.time() - start
logging.info("Groomer cleaned {0} journal entries".format(
self.journal_entries_cleaned))
logging.info("Groomer checked {0} index entries".format(
self.index_entries_checked))
logging.info("Groomer cleaned {0} index entries".format(
self.index_entries_cleaned))
if self.index_entries_delete_failures > 0:
logging.info("Groomer failed to remove {0} index entries".format(
self.index_entries_delete_failures))
logging.info("Groomer took {0} seconds".format(str(time_taken)))
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) datastore_batch.batch_delete(APP_ID_TABLE, ["a/a"], APP_ID_SCHEMA) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)
