def test_create_composite_index(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_put_entity").and_return(ASYNC_NONE) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) zk_handle = flexmock(ensure_path=lambda path: None, get=lambda path: (None, flexmock(version=None)), set=lambda path, value, version: None) dd.zookeeper.handle = zk_handle dd = flexmock(dd) index = entity_pb.CompositeIndex() index.set_app_id("appid") index.set_state(2) definition = index.mutable_definition() definition.set_entity_type("kind") definition.set_ancestor(0) prop1 = definition.add_property() prop1.set_name("prop1") prop1.set_direction(1) # ascending prop2 = definition.add_property() prop2.set_name("prop2") prop1.set_direction(1) # ascending index_id = yield dd.create_composite_index("appid", index) assert index_id > 0
def test_apply_txn_changes(self): app = 'guestbook' txn = 1 entity = self.get_new_entity_proto(app, *self.BASIC_ENTITY[1:]) db_batch = flexmock() db_batch.should_receive('get_transaction_metadata').and_return({ 'puts': {entity.key().Encode(): entity.Encode()}, 'deletes': [], 'tasks': [], 'reads': set(), 'start': datetime.datetime.utcnow(), 'is_xg': False, }) db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive('group_updates').and_return([]) db_batch.should_receive('get_indices').and_return([]) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) prefix = dd.get_table_prefix(entity) entity_key = get_entity_key(prefix, entity.key().path()) db_batch.should_receive('batch_get_entity').and_return({entity_key: {}}) db_batch.should_receive('batch_mutate') entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') dd.apply_txn_changes(app, txn)
class InsertAndDeleteIndexesCase(unittest.TestCase): def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0, 3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent=prev, namespace='d') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.keys = ['test/d/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!'] def runTest(self): self.app_datastore.InsertIndexEntries(self.entities) ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, PROPERTY_SCHEMA) assert 'test/d/TestKind:0!TestKind:1!' in \ ret['test/d/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!']['reference'] self.app_datastore.DeleteIndexEntries(self.entities) ret = datastore_batch.batch_get_entity(ASC_PROPERTY_TABLE, self.keys, PROPERTY_SCHEMA) assert 'reference' not in \ ret['test/d/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!'] def tearDown(self): pass
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0, 4): entity = datastore.Entity("TestKind", _app="test", name=str(ii), namespace='b', parent=prev) prev = entity # have properties with different values bye same property names entity.update({ 'aaa': "1111_" + str(ii), 'bbb': "2222", 'ccc': "3" * ii }) self.entities.append(entity.ToPb()) self.keys = [ 'test/b/TestKind:0!', 'test/b/TestKind:0!TestKind:1!', 'test/b/TestKind:0!TestKind:1!TestKind:2!', 'test/b/TestKind:0!TestKind:1!TestKind:2!TestKind:3!' ] self.kkeys = [ 'test/b/TestKind:0!', 'test/b/TestKind:1!TestKind:0!', 'test/b/TestKind:2!TestKind:1!TestKind:0!', 'test/b/TestKind:3!TestKind:2!TestKind:1!TestKind:0!' ]
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0, 3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent=prev, namespace='c') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.keys = [ 'test/c/TestKind/aaa/1111_0\x00/TestKind:0!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!', 'test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!', 'test/c/TestKind/aaa/1111_2\x00/TestKind:0!TestKind:1!TestKind:2!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!TestKind:2!' ] self.rkeys = [ 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcf\xff/TestKind:0!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!', 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xce\xff/TestKind:0!TestKind:1!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!', 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:0!TestKind:1!TestKind:2!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!TestKind:2!' ]
def test_validate_key(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) dd.validate_key(key.key())
def test_delete_entities_txn(self): app = 'guestbook' txn_hash = {'root_key': 1} txn_str = '1'.zfill(ID_KEY_LENGTH) entity = self.get_new_entity_proto(app, *self.BASIC_ENTITY[1:]) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) keys = [entity.key()] prefix = dd.get_table_prefix(entity.key()) entity_key = get_entity_key(prefix, entity.key().path()) encoded_path = str(encode_index_pb(entity.key().path())) txn_keys = [dd._SEPARATOR.join([app, txn_str, '', encoded_path])] txn_values = { txn_keys[0]: { dbconstants.TRANSACTIONS_SCHEMA[0]: dbconstants.TxnActions.DELETE, dbconstants.TRANSACTIONS_SCHEMA[1]: entity_key, dbconstants.TRANSACTIONS_SCHEMA[2]: '' } } flexmock(dd).should_receive('get_root_key').and_return('root_key') db_batch.should_receive('batch_put_entity').with_args( dbconstants.TRANSACTIONS_TABLE, txn_keys, dbconstants.TRANSACTIONS_SCHEMA, txn_values, ttl=TX_TIMEOUT * 2 ) dd.delete_entities_txn(app, keys, txn_hash)
def test_kindless_query(self): query = datastore_pb.Query() ancestor = query.mutable_ancestor() entity_proto1 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop1val", ns="blah") entity_key = entity_proto1.key() get_req = datastore_pb.GetRequest() key = get_req.add_key() key.MergeFrom(entity_key) tombstone1 = {'key': {APP_ENTITY_SCHEMA[0]:TOMBSTONE, APP_ENTITY_SCHEMA[1]: 1}} db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_get_entity").and_return( {"test\x00blah\x00test_kind:nancy\x01": { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } }) db_batch.should_receive("batch_put_entity").and_return(None) entity_proto1 = {'test\x00blah\x00test_kind:nancy\x01':{APP_ENTITY_SCHEMA[0]:entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1}} db_batch.should_receive("range_query").and_return([entity_proto1, tombstone1]).and_return([]) zookeeper = flexmock() zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("is_in_transaction").and_return(False) zookeeper.should_receive("acquire_lock").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) filter_info = { '__key__' : [[0, 0]] } dd.kindless_query(query, filter_info)
def test_insert_composite_indexes(self): composite_index = entity_pb.CompositeIndex() composite_index.set_id(123) composite_index.set_app_id("appid") definition = composite_index.mutable_definition() definition.set_entity_type("kind") prop1 = definition.add_property() prop1.set_name("prop1") prop1.set_direction(1) # ascending prop2 = definition.add_property() prop2.set_name("prop2") prop1.set_direction(1) # ascending ent = self.get_new_entity_proto( "appid", "kind", "entity_name", "prop1", "value", ns="") db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_put_entity").and_return(ASYNC_NONE).once() transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) yield dd.insert_composite_indexes([ent], [composite_index])
def test_dynamic_get(self): entity_proto1 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") zookeeper = flexmock() zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return( {"test\x00blah\x00test_kind:nancy\x01": { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } }) dd = DatastoreDistributed(db_batch, zookeeper) entity_key = entity_proto1.key() get_req = datastore_pb.GetRequest() key = get_req.add_key() key.MergeFrom(entity_key) get_resp = datastore_pb.GetResponse() dd.dynamic_get("test", get_req, get_resp) self.assertEquals(get_resp.entity_size(), 1) # Now test while in a transaction get_resp = datastore_pb.GetResponse() get_req.mutable_transaction().set_handle(1) dd.dynamic_get("test", get_req, get_resp) self.assertEquals(get_resp.entity_size(), 1)
def test_put_entities(self): app_id = 'test' db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) entity_proto1 = self.get_new_entity_proto( app_id, "test_kind", "bob", "prop1name", "prop1val", ns="blah") entity_key1 = 'test\x00blah\x00test_kind:bob\x01' entity_proto2 = self.get_new_entity_proto( app_id, "test_kind", "nancy", "prop1name", "prop2val", ns="blah") entity_key2 = 'test\x00blah\x00test_kind:nancy\x01' entity_list = [entity_proto1, entity_proto2] async_result = gen.Future() async_result.set_result({entity_key1: {}, entity_key2: {}}) db_batch.should_receive('batch_get_entity').and_return(async_result) db_batch.should_receive('normal_batch').and_return(ASYNC_NONE) transaction_manager = flexmock( create_transaction_id=lambda project, xg: 1, delete_transaction_id=lambda project, txid: None, set_groups=lambda project, txid, groups: None) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={app_id: flexmock(indexes_pb=[])}) async_true = gen.Future() async_true.set_result(True) entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire').and_return(async_true) entity_lock.should_receive('release') yield dd.put_entities(app_id, entity_list)
def test_dynamic_put(self): PREFIX = "x\x01" db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zookeeper = flexmock() zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) zookeeper.should_receive("get_transaction_id").and_return(1) entity_proto1 = self.get_new_entity_proto("test", "test_kind", "bob", "prop1name", "prop1val", ns="blah") entity_key1 = 'test\x00blah\x00test_kind:bob\x01' entity_proto2 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") entity_key2 = 'test\x00blah\x00test_kind:nancy\x01' db_batch.should_receive('batch_get_entity').and_return( {entity_key1: {}, entity_key2: {}}) db_batch.should_receive('batch_mutate') dd = DatastoreDistributed(db_batch, zookeeper) putreq_pb = datastore_pb.PutRequest() putreq_pb.add_entity() putreq_pb.mutable_entity(0).MergeFrom(entity_proto1) putreq_pb.add_entity() putreq_pb.mutable_entity(1).MergeFrom(entity_proto2) putresp_pb = datastore_pb.PutResponse() dd.dynamic_put('test', putreq_pb, putresp_pb) self.assertEquals(len(putresp_pb.key_list()), 2)
class GetCase(unittest.TestCase): def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) self.entities = [] self.keys = [] prev = None for ii in range(0, 3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent=prev, namespace='e') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.keys = [e.key() for e in self.entities] self.app_datastore.PutEntities(self.entities) def runTest(self): results, keys = self.app_datastore.FetchKeys(self.keys) self.app_datastore.DeleteEntities(self.keys) results, keys = self.app_datastore.FetchKeys(self.keys) for ii in results: if 'entity' in ii: raise def tearDown(self): pass
def test_apply_txn_changes(self): app = 'guestbook' txn = 1 entity = self.get_new_entity_proto(app, *self.BASIC_ENTITY[1:]) db_batch = flexmock() db_batch.should_receive('get_transaction_metadata').and_return({ 'puts': {entity.key().Encode(): entity.Encode()}, 'deletes': [], 'tasks': [], 'reads': set(), 'start': datetime.datetime.utcnow(), 'is_xg': False, }) db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive('group_updates').and_return([]) db_batch.should_receive('get_indices').and_return([]) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) prefix = dd.get_table_prefix(entity) entity_key = get_entity_key(prefix, entity.key().path()) db_batch.should_receive('batch_get_entity').and_return({entity_key: {}}) db_batch.should_receive('batch_mutate') entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') dd.apply_txn_changes(app, txn)
def test_is_zigzag_merge_join(self): zookeeper = flexmock() zookeeper.should_receive("get_transaction_id").and_return(1) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) query = datastore_pb.Query() dd = DatastoreDistributed(db_batch, zookeeper) db_batch.should_receive("remove_exists_filters").and_return({}) self.assertEquals(dd.is_zigzag_merge_join(query, {}, {}), False) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop2": [(datastore_pb.Query_Filter.EQUAL, "2")]} db_batch.should_receive("remove_exists_filters").and_return(filter_info) self.assertEquals(dd.is_zigzag_merge_join(query, filter_info, []), True) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop1": [(datastore_pb.Query_Filter.EQUAL, "2")]} self.assertEquals(dd.is_zigzag_merge_join(query, filter_info, []), False)
def testFetchKeys(self): entity_proto1 = self.get_new_entity_proto("test", "test_kind", "bob", "prop1name", "prop1val", ns="blah") db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return({'test\x00blah\x00test_kind:bob\x01': {APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1}}).and_return({'test\x00blah\x00test_kind:bob\x01\x000000000002': {JOURNAL_SCHEMA[0]: entity_proto1.Encode()}}) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) self.assertEquals(({'test\x00blah\x00test_kind:bob\x01': {'txnID': 1, 'entity': entity_proto1.Encode()} }, ['test\x00blah\x00test_kind:bob\x01']), dd.fetch_keys([entity_proto1.key()]))
def test_zigzag_merge_join(self): zookeeper = flexmock() zookeeper.should_receive("get_transaction_id").and_return(1) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) query = datastore_pb.Query() dd = DatastoreDistributed(db_batch, zookeeper) flexmock(dd).should_receive("is_zigzag_merge_join").and_return(False) self.assertEquals(dd.zigzag_merge_join(None, None, None), None) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop2": [(datastore_pb.Query_Filter.EQUAL, "2")]} flexmock(query).should_receive("kind").and_return("kind") flexmock(dd).should_receive("get_table_prefix").and_return("prefix") flexmock(dd).should_receive("__apply_filters").and_return([]) flexmock(query).should_receive("limit").and_return(1) self.assertEquals(dd.zigzag_merge_join(query, filter_info, []), None)
def test_put_entities(self): app_id = 'test' db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) entity_proto1 = self.get_new_entity_proto( app_id, "test_kind", "bob", "prop1name", "prop1val", ns="blah") entity_key1 = 'test\x00blah\x00test_kind:bob\x01' entity_proto2 = self.get_new_entity_proto( app_id, "test_kind", "nancy", "prop1name", "prop2val", ns="blah") entity_key2 = 'test\x00blah\x00test_kind:nancy\x01' entity_list = [entity_proto1, entity_proto2] db_batch.should_receive('batch_get_entity').and_return( {entity_key1: {}, entity_key2: {}}) db_batch.should_receive('batch_mutate') transaction_manager = flexmock( create_transaction_id=lambda project, xg: 1, delete_transaction_id=lambda project, txid: None) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') dd.put_entities(app_id, entity_list)
def test_dynamic_put(self): PREFIX = "x\x01" db_batch = flexmock(session=flexmock()) db_batch.should_receive('valid_data_version').and_return(True) entity_proto1 = self.get_new_entity_proto("test", "test_kind", "bob", "prop1name", "prop1val", ns="blah") entity_key1 = 'test\x00blah\x00test_kind:bob\x01' entity_proto2 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") entity_key2 = 'test\x00blah\x00test_kind:nancy\x01' db_batch.should_receive('batch_get_entity').and_return( {entity_key1: {}, entity_key2: {}}) db_batch.should_receive('batch_mutate') dd = DatastoreDistributed(db_batch, self.get_zookeeper()) putreq_pb = datastore_pb.PutRequest() putreq_pb.add_entity() putreq_pb.mutable_entity(0).MergeFrom(entity_proto1) putreq_pb.add_entity() putreq_pb.mutable_entity(1).MergeFrom(entity_proto2) putresp_pb = datastore_pb.PutResponse() entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') flexmock(ScatteredAllocator).should_receive('next').\ and_return(random.randint(1, 500)) dd.dynamic_put('test', putreq_pb, putresp_pb) self.assertEquals(len(putresp_pb.key_list()), 2)
def test_is_zigzag_merge_join(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) query = datastore_pb.Query() transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) db_batch.should_receive("remove_exists_filters").and_return({}) self.assertEquals(dd.is_zigzag_merge_join(query, {}, {}), False) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop2": [(datastore_pb.Query_Filter.EQUAL, "2")]} db_batch.should_receive("remove_exists_filters").and_return(filter_info) self.assertEquals(dd.is_zigzag_merge_join(query, filter_info, []), True) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop1": [(datastore_pb.Query_Filter.EQUAL, "2")]} self.assertEquals(dd.is_zigzag_merge_join(query, filter_info, []), False)
def test_get_composite_index_key(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd = flexmock(dd) composite_index = entity_pb.CompositeIndex() composite_index.set_id(123) composite_index.set_app_id("appid") definition = composite_index.mutable_definition() definition.set_entity_type("kind") prop1 = definition.add_property() prop1.set_name("prop1") prop1.set_direction(1) # ascending prop2 = definition.add_property() prop2.set_name("prop2") prop1.set_direction(1) # ascending ent = self.get_new_entity_proto("appid", "kind", "entity_name", "prop1", "value", ns="") self.assertEquals( dd.get_composite_index_key(composite_index, ent), "appid\x00\x00123\x00\x9avalue\x01\x01\x00\x00kind:entity_name\x01")
def test_zigzag_merge_join(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) query = datastore_pb.Query() transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) flexmock(dd).should_receive("is_zigzag_merge_join").and_return(False) self.assertEquals(dd.zigzag_merge_join(None, None, None), None) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop2": [(datastore_pb.Query_Filter.EQUAL, "2")]} flexmock(query).should_receive("kind").and_return("kind") flexmock(dd).should_receive("get_table_prefix").and_return("prefix") flexmock(dd).should_receive("__apply_filters").and_return([]) flexmock(query).should_receive("limit").and_return(1) self.assertEquals(dd.zigzag_merge_join(query, filter_info, []), None)
def test_delete_entities(self): app_id = 'test' entity_proto1 = self.get_new_entity_proto( app_id, "test_kind", "bob", "prop1name", "prop1val", ns="blah") row_key = "test\x00blah\x00test_kind:bob\x01" row_values = {row_key: {APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: '1'}} async_result = gen.Future() async_result.set_result(row_values) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("get_valid_transaction_id").and_return(1) db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_get_entity").and_return(async_result) db_batch.should_receive('normal_batch').and_return(ASYNC_NONE) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) row_keys = [entity_proto1.key()] yield dd.delete_entities(entity_proto1.key(), 1, row_keys)
def test_delete_composite_index_metadata(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) dd = flexmock(dd) composite_index = entity_pb.CompositeIndex() composite_index.set_id(1) dd.delete_composite_index_metadata("appid", composite_index)
def test_ancestor_query(self): query = datastore_pb.Query() ancestor = query.mutable_ancestor() entity_proto1 = self.get_new_entity_proto( "test", "test_kind", "nancy", "prop1name", "prop1val", ns="blah") entity_key = entity_proto1.key() get_req = datastore_pb.GetRequest() key = get_req.add_key() key.MergeFrom(entity_key) ancestor.MergeFrom(entity_key) async_result = gen.Future() async_result.set_result({ "test\x00blah\x00test_kind:nancy\x01": { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } }) filter_info = [] tombstone1 = {'key': {APP_ENTITY_SCHEMA[0]:TOMBSTONE, APP_ENTITY_SCHEMA[1]: 1}} db_batch = flexmock() db_batch.should_receive('record_reads').and_return(ASYNC_NONE) db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_get_entity").and_return(async_result) entity_proto1 = { 'test\x00blah\x00test_kind:nancy\x01': { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } } async_result_1 = gen.Future() async_result_1.set_result([entity_proto1, tombstone1]) async_result_2 = gen.Future() async_result_2.set_result([]) db_batch.should_receive("range_query").\ and_return(async_result_1).\ and_return(async_result_2) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("is_in_transaction").and_return(False) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) yield dd.ancestor_query(query, filter_info) # Now with a transaction transaction = query.mutable_transaction() transaction.set_handle(2) yield dd.ancestor_query(query, filter_info)
def test_get_table_prefix(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_put_entity").and_return(None) zookeeper = flexmock() zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(dd.get_table_prefix(key), "hello\x00")
def test_delete_composite_index_metadata(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_delete").and_return(ASYNC_NONE) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd = flexmock(dd) composite_index = entity_pb.CompositeIndex() composite_index.set_id(1) yield dd.delete_composite_index_metadata("appid", composite_index)
class ConfigureNamespaceCase(unittest.TestCase): def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) def runTest(self): assert self.app_datastore.GetTablePrefix(("hi", "bye")) == "hi/bye" def tearDown(self): key = self.app_datastore.GetTablePrefix(("hi", "bye"))
class GetPrefixCase(unittest.TestCase): def setUp(self): global datastore_batch datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) def runTest(self): assert self.app_datastore.GetTablePrefix(('hi1', 'bye1')) == "hi1/bye1" def tearDown(self): key = self.app_datastore.GetTablePrefix(("hi1", "bye1"))
def test_validate_key(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) dd.validate_key(key.key())
def test_reverse_path(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) key = "Project:Synapse\x01Module:Core\x01" self.assertEquals(dd.reverse_path(key), "Module:Core\x01Project:Synapse\x01")
def test_rollback_transcation(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zookeeper = flexmock() zookeeper.should_receive("release_lock").and_return(True) zookeeper.should_receive("notify_failed_transaction").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") http_request = commit_request.Encode() self.assertEquals(dd.rollback_transaction("app_id", http_request), (api_base_pb.VoidProto().Encode(), 0, ""))
def test_commit_transaction(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zookeeper = flexmock() zookeeper.should_receive("release_lock").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) flexmock(dd).should_receive('apply_txn_changes') commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") http_request = commit_request.Encode() self.assertEquals(dd.commit_transaction("app_id", http_request), (datastore_pb.CommitResponse().Encode(), 0, ""))
def test_get_table_prefix(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(dd.get_table_prefix(key), "hello\x00")
def test_root_key_from_entity_key(self): zookeeper = flexmock() db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_put_entity").and_return(None) dd = DatastoreDistributed(db_batch, zookeeper) self.assertEquals("test\x00blah\x00test_kind:bob\x01", dd.get_root_key_from_entity_key("test\x00blah\x00test_kind:bob\x01nancy\x01")) entity_proto1 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") self.assertEquals("test\x00blah\x00test_kind:nancy\x01", dd.get_root_key_from_entity_key(entity_proto1.key()))
def test_commit_transaction(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zookeeper = flexmock() zookeeper.should_receive('remove_tx_node') dd = DatastoreDistributed(db_batch, zookeeper) flexmock(dd).should_receive('apply_txn_changes') commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") http_request = commit_request.Encode() self.assertEquals(dd.commit_transaction("app_id", http_request), (datastore_pb.CommitResponse().Encode(), 0, ""))
def test_validate_key(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) dd.validate_key(key.key())
def test_dynamic_delete(self): async_true = gen.Future() async_true.set_result(True) entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire').and_return(async_true) entity_lock.should_receive('release') del_request = flexmock() del_request.should_receive("key_list") del_request.should_receive("has_transaction").never() del_request.should_receive("transaction").never() db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock( create_transaction_id=lambda project, xg: 1, delete_transaction_id=lambda project, txid: None, set_groups=lambda project_id, txid, groups: None) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'guestbook': flexmock(indexes_pb=[])}) yield dd.dynamic_delete("appid", del_request) fake_key = entity_pb.Reference() fake_key.set_app('foo') path = fake_key.mutable_path() element = path.add_element() element.set_type('bar') element.set_id(1) del_request = flexmock() del_request.should_receive("key_list").and_return([fake_key]) del_request.should_receive("has_transaction").and_return(True) transaction = flexmock() transaction.should_receive("handle").and_return(1) del_request.should_receive("transaction").and_return(transaction) del_request.should_receive("has_mark_changes").and_return(False) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'appid': flexmock(indexes_pb=[])}) flexmock(utils).should_receive("get_entity_kind").and_return("kind") db_batch.should_receive('delete_entities_tx').and_return(ASYNC_NONE) yield dd.dynamic_delete("appid", del_request) del_request = flexmock() del_request.should_receive("key_list").and_return([fake_key]) del_request.should_receive("has_transaction").and_return(False) del_request.should_receive("has_mark_changes").and_return(False) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'appid': flexmock(indexes_pb=[])}) flexmock(dd).should_receive("delete_entities").and_return(ASYNC_NONE).once() yield dd.dynamic_delete("appid", del_request)
def test_reverse_path(self): zookeeper = flexmock() zookeeper.should_receive("get_transaction_id").and_return(1) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) dd = DatastoreDistributed(db_batch, zookeeper) key = "Project:Synapse\x01Module:Core\x01" self.assertEquals(dd.reverse_path(key), "Module:Core\x01Project:Synapse\x01")
def test_get_entity_key(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(str(get_entity_key("howdy", key.key().path())), "howdy\x00Item:Bob\x01")
def run(self): """ Starts the main loop of the restore thread. """ datastore_batch = appscale_datastore_batch.\ DatastoreFactory.getDatastore(self.table) transaction_manager = TransactionManager(self.zoo_keeper.handle) self.ds_distributed = DatastoreDistributed( datastore_batch, transaction_manager, zookeeper=self.zoo_keeper) self.dynamic_put_sync = tornado_synchronous( self.ds_distributed.dynamic_put) while True: logging.debug("Trying to get restore lock.") if self.get_restore_lock(): logging.info("Got the restore lock.") self.run_restore() try: self.zoo_keeper.release_lock_with_path(zk.DS_RESTORE_LOCK_PATH) except zk.ZKTransactionException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception))) break else: logging.info("Did not get the restore lock. Another instance may be " "running.") time.sleep(random.randint(1, self.LOCK_POLL_PERIOD))
def test_rollback_transcation(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("release_lock").and_return(True) zookeeper.should_receive("notify_failed_transaction").and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") self.assertEquals( dd.rollback_transaction("app_id", commit_request.handle()), None)
def test_commit_transaction(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) transaction_manager = flexmock( delete_transaction_id=lambda project, txid: None) dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) flexmock(dd).should_receive('apply_txn_changes') commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") http_request = commit_request.Encode() self.assertEquals(dd.commit_transaction("app_id", http_request), (datastore_pb.CommitResponse().Encode(), 0, ""))
def test_apply_txn_changes(self): app = 'guestbook' txn = 1 entity = self.get_new_entity_proto(app, *self.BASIC_ENTITY[1:]) async_metadata = gen.Future() async_metadata.set_result({ 'puts': {entity.key().Encode(): entity.Encode()}, 'deletes': [], 'tasks': [], 'reads': set(), 'start': datetime.datetime.utcnow(), 'is_xg': False, }) db_batch = flexmock() db_batch.should_receive('get_transaction_metadata').\ and_return(async_metadata) db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive('group_updates').and_return([]) transaction_manager = flexmock( delete_transaction_id=lambda project_id, txid: None, set_groups=lambda project_id, txid, groups: None) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'guestbook': flexmock(indexes_pb=[])}) prefix = dd.get_table_prefix(entity) entity_key = get_entity_key(prefix, entity.key().path()) async_result = gen.Future() async_result.set_result({entity_key: {}}) db_batch.should_receive('batch_get_entity').and_return(async_result) db_batch.should_receive('normal_batch').and_return(ASYNC_NONE) async_true = gen.Future() async_true.set_result(True) entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire').and_return(async_true) entity_lock.should_receive('release') yield dd.apply_txn_changes(app, txn)