def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0,3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent = prev, namespace='c') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.keys = ['test/c/TestKind/aaa/1111_0\x00/TestKind:0!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!', 'test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!', 'test/c/TestKind/aaa/1111_2\x00/TestKind:0!TestKind:1!TestKind:2!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!TestKind:2!'] self.rkeys = ['test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcf\xff/TestKind:0!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!', 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xce\xff/TestKind:0!TestKind:1!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!', 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:0!TestKind:1!TestKind:2!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!TestKind:2!']
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0,4): entity = datastore.Entity("TestKind", _app="test", name=str(ii), namespace='b', parent = prev) prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222", 'ccc': "3"*ii}) self.entities.append(entity.ToPb()) self.keys = ['test/b/TestKind:0!', 'test/b/TestKind:0!TestKind:1!', 'test/b/TestKind:0!TestKind:1!TestKind:2!', 'test/b/TestKind:0!TestKind:1!TestKind:2!TestKind:3!'] self.kkeys = ['test/b/TestKind:0!', 'test/b/TestKind:1!TestKind:0!', 'test/b/TestKind:2!TestKind:1!TestKind:0!', 'test/b/TestKind:3!TestKind:2!TestKind:1!TestKind:0!']
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = datastore_server.DatastoreDistributed( datastore_batch) self.entities = [] prev = None for ii in range(0, 3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent=prev, namespace='e') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.entities2 = [] prev = None for ii in range(0, 3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent=prev, namespace='e') prev = entity # have properties with different values bye same property names entity.update({'aaa': "x111_" + str(ii), 'bbb': "x222"}) self.entities2.append(entity.ToPb()) tuples = sorted( (self.app_datastore.GetTablePrefix(x), x) for x in self.entities) # keys should be the same for entities and entities2 self.keys = self.app_datastore.GetIndexKVFromTuple(tuples, reverse=False) self.keys = [x[0] for x in self.keys] tuples = sorted( (self.app_datastore.GetTablePrefix(x), x) for x in self.entities2) # keys should be the same for entities and entities2 self.keys2 = self.app_datastore.GetIndexKVFromTuple(tuples, reverse=False) self.keys2 = [x[0] for x in self.keys2]
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch) self.entities = [] self.keys = [] prev = None for ii in range(0,3): entity = datastore.Entity("TestKind", _app="test", name = str(ii), parent = prev, namespace='e') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.keys = [e.key() for e in self.entities] self.app_datastore.PutEntities(self.entities)
def run_groomer(self): """ Runs the grooming process. Loops on the entire dataset sequentially and updates stats, indexes, and transactions. """ self.db_access = appscale_datastore_batch.DatastoreFactory.getDatastore( self.table_name) self.ds_access = datastore_server.DatastoreDistributed( datastore_batch=self.db_access, zookeeper=self.zoo_keeper) logging.info("Groomer started") start = time.time() self.reset_statistics() self.composite_index_cache = {} tasks = [ { 'id': self.CLEAN_ENTITIES_TASK, 'description': 'clean up entities', 'function': self.clean_up_entities, 'args': [] }, { 'id': self.CLEAN_ASC_INDICES_TASK, 'description': 'clean up ascending indices', 'function': self.clean_up_indexes, 'args': [datastore_pb.Query_Order.ASCENDING] }, { 'id': self.CLEAN_DSC_INDICES_TASK, 'description': 'clean up descending indices', 'function': self.clean_up_indexes, 'args': [datastore_pb.Query_Order.DESCENDING] }, { 'id': self.CLEAN_KIND_INDICES_TASK, 'description': 'clean up kind indices', 'function': self.clean_up_kind_indices, 'args': [] }, { 'id': self.CLEAN_LOGS_TASK, 'description': 'clean up old logs', 'function': self.remove_old_logs, 'args': [self.LOG_STORAGE_TIMEOUT] }, { 'id': self.CLEAN_TASKS_TASK, 'description': 'clean up old tasks', 'function': self.remove_old_tasks_entities, 'args': [] }, { 'id': self.CLEAN_DASHBOARD_TASK, 'description': 'clean up old dashboard items', 'function': self.remove_old_dashboard_data, 'args': [] } ] groomer_state = self.zoo_keeper.get_node(self.GROOMER_STATE_PATH) logging.info('groomer_state: {}'.format(groomer_state)) if groomer_state: self.update_groomer_state( groomer_state[0].split(self.GROOMER_STATE_DELIMITER)) for task_number in range(len(tasks)): task = tasks[task_number] if (len(self.groomer_state) > 0 and self.groomer_state[0] != '' and self.groomer_state[0] != task['id']): continue logging.info('Starting to {}'.format(task['description'])) try: task['function'](*task['args']) if task_number != len(tasks) - 1: next_task = tasks[task_number + 1] self.update_groomer_state([next_task['id']]) except Exception as exception: logging.error('Exception encountered while trying to {}:'. format(task['description'])) logging.exception(exception) self.update_groomer_state([]) timestamp = datetime.datetime.utcnow() if not self.update_statistics(timestamp): logging.error("There was an error updating the statistics") if not self.update_namespaces(timestamp): logging.error("There was an error updating the namespaces") del self.db_access del self.ds_access time_taken = time.time() - start logging.info("Groomer cleaned {0} journal entries".format( self.journal_entries_cleaned)) logging.info("Groomer checked {0} index entries".format( self.index_entries_checked)) logging.info("Groomer cleaned {0} index entries".format( self.index_entries_cleaned)) if self.index_entries_delete_failures > 0: logging.info("Groomer failed to remove {0} index entries".format( self.index_entries_delete_failures)) logging.info("Groomer took {0} seconds".format(str(time_taken)))
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(DB) datastore_batch.batch_delete(APP_ID_TABLE, ["a/a"], APP_ID_SCHEMA) self.app_datastore = datastore_server.DatastoreDistributed(datastore_batch)