def test_dynamic_delete(self): async_true = gen.Future() async_true.set_result(True) entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire').and_return(async_true) entity_lock.should_receive('release') del_request = flexmock() del_request.should_receive("key_list") del_request.should_receive("has_transaction").never() del_request.should_receive("transaction").never() db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock( create_transaction_id=lambda project, xg: 1, delete_transaction_id=lambda project, txid: None, set_groups=lambda project_id, txid, groups: None) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'guestbook': flexmock(indexes_pb=[])}) yield dd.dynamic_delete("appid", del_request) fake_key = entity_pb.Reference() fake_key.set_app('foo') path = fake_key.mutable_path() element = path.add_element() element.set_type('bar') element.set_id(1) del_request = flexmock() del_request.should_receive("key_list").and_return([fake_key]) del_request.should_receive("has_transaction").and_return(True) transaction = flexmock() transaction.should_receive("handle").and_return(1) del_request.should_receive("transaction").and_return(transaction) del_request.should_receive("has_mark_changes").and_return(False) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'appid': flexmock(indexes_pb=[])}) flexmock(utils).should_receive("get_entity_kind").and_return("kind") db_batch.should_receive('delete_entities_tx').and_return(ASYNC_NONE) yield dd.dynamic_delete("appid", del_request) del_request = flexmock() del_request.should_receive("key_list").and_return([fake_key]) del_request.should_receive("has_transaction").and_return(False) del_request.should_receive("has_mark_changes").and_return(False) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={'appid': flexmock(indexes_pb=[])}) flexmock(dd).should_receive("delete_entities").and_return(ASYNC_NONE).once() yield dd.dynamic_delete("appid", del_request)
def test_get_index_key_from_params(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) params = ['a','b','c','d','e'] self.assertEquals(get_index_key_from_params(params), "a\x00b\x00c\x00d\x00e")
def test_validate_key(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) dd.validate_key(key.key())
def test_zigzag_merge_join(self): zookeeper = flexmock() zookeeper.should_receive("get_transaction_id").and_return(1) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) query = datastore_pb.Query() dd = DatastoreDistributed(db_batch, zookeeper) flexmock(dd).should_receive("is_zigzag_merge_join").and_return(False) self.assertEquals(dd.zigzag_merge_join(None, None, None), None) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop2": [(datastore_pb.Query_Filter.EQUAL, "2")]} flexmock(query).should_receive("kind").and_return("kind") flexmock(dd).should_receive("get_table_prefix").and_return("prefix") flexmock(dd).should_receive("__apply_filters").and_return([]) flexmock(query).should_receive("limit").and_return(1) self.assertEquals(dd.zigzag_merge_join(query, filter_info, []), None)
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0, 4): entity = datastore.Entity("TestKind", _app="test", name=str(ii), namespace='b', parent=prev) prev = entity # have properties with different values bye same property names entity.update({ 'aaa': "1111_" + str(ii), 'bbb': "2222", 'ccc': "3" * ii }) self.entities.append(entity.ToPb()) self.keys = [ 'test/b/TestKind:0!', 'test/b/TestKind:0!TestKind:1!', 'test/b/TestKind:0!TestKind:1!TestKind:2!', 'test/b/TestKind:0!TestKind:1!TestKind:2!TestKind:3!' ] self.kkeys = [ 'test/b/TestKind:0!', 'test/b/TestKind:1!TestKind:0!', 'test/b/TestKind:2!TestKind:1!TestKind:0!', 'test/b/TestKind:3!TestKind:2!TestKind:1!TestKind:0!' ]
def test_get_composite_index_key(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd = flexmock(dd) composite_index = entity_pb.CompositeIndex() composite_index.set_id(123) composite_index.set_app_id("appid") definition = composite_index.mutable_definition() definition.set_entity_type("kind") prop1 = definition.add_property() prop1.set_name("prop1") prop1.set_direction(1) # ascending prop2 = definition.add_property() prop2.set_name("prop2") prop1.set_direction(1) # ascending ent = self.get_new_entity_proto("appid", "kind", "entity_name", "prop1", "value", ns="") self.assertEquals( dd.get_composite_index_key(composite_index, ent), "appid\x00\x00123\x00\x9avalue\x01\x01\x00\x00kind:entity_name\x01")
def test_get_entity_key(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(str(get_entity_key("howdy", key.key().path())), "howdy\x00Item:Bob\x01")
def test_create_composite_index(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_put_entity").and_return(ASYNC_NONE) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) zk_handle = flexmock(ensure_path=lambda path: None, get=lambda path: (None, flexmock(version=None)), set=lambda path, value, version: None) dd.zookeeper.handle = zk_handle dd = flexmock(dd) index = entity_pb.CompositeIndex() index.set_app_id("appid") index.set_state(2) definition = index.mutable_definition() definition.set_entity_type("kind") definition.set_ancestor(0) prop1 = definition.add_property() prop1.set_name("prop1") prop1.set_direction(1) # ascending prop2 = definition.add_property() prop2.set_name("prop2") prop1.set_direction(1) # ascending index_id = yield dd.create_composite_index("appid", index) assert index_id > 0
def test_is_zigzag_merge_join(self): zookeeper = flexmock() zookeeper.should_receive("get_transaction_id").and_return(1) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return(None) query = datastore_pb.Query() dd = DatastoreDistributed(db_batch, zookeeper) db_batch.should_receive("remove_exists_filters").and_return({}) self.assertEquals(dd.is_zigzag_merge_join(query, {}, {}), False) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop2": [(datastore_pb.Query_Filter.EQUAL, "2")]} db_batch.should_receive("remove_exists_filters").and_return(filter_info) self.assertEquals(dd.is_zigzag_merge_join(query, filter_info, []), True) filter_info = {"prop1":[(datastore_pb.Query_Filter.EQUAL, "1")], "prop1": [(datastore_pb.Query_Filter.EQUAL, "2")]} self.assertEquals(dd.is_zigzag_merge_join(query, filter_info, []), False)
def setUp(self): datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore( DB) self.app_datastore = DatastoreDistributed(datastore_batch) self.entities = [] prev = None for ii in range(0, 3): entity = datastore.Entity("TestKind", _app="test", name=str(ii), parent=prev, namespace='c') prev = entity # have properties with different values bye same property names entity.update({'aaa': "1111_" + str(ii), 'bbb': "2222"}) self.entities.append(entity.ToPb()) self.keys = [ 'test/c/TestKind/aaa/1111_0\x00/TestKind:0!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!', 'test/c/TestKind/aaa/1111_1\x00/TestKind:0!TestKind:1!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!', 'test/c/TestKind/aaa/1111_2\x00/TestKind:0!TestKind:1!TestKind:2!', 'test/c/TestKind/bbb/2222\x00/TestKind:0!TestKind:1!TestKind:2!' ] self.rkeys = [ 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcf\xff/TestKind:0!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!', 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xce\xff/TestKind:0!TestKind:1!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!', 'test/c/TestKind/aaa/\xce\xce\xce\xce\xa0\xcd\xff/TestKind:0!TestKind:1!TestKind:2!', 'test/c/TestKind/bbb/\xcd\xcd\xcd\xcd\xff/TestKind:0!TestKind:1!TestKind:2!' ]
def test_insert_composite_indexes(self): composite_index = entity_pb.CompositeIndex() composite_index.set_id(123) composite_index.set_app_id("appid") definition = composite_index.mutable_definition() definition.set_entity_type("kind") prop1 = definition.add_property() prop1.set_name("prop1") prop1.set_direction(1) # ascending prop2 = definition.add_property() prop2.set_name("prop2") prop1.set_direction(1) # ascending ent = self.get_new_entity_proto( "appid", "kind", "entity_name", "prop1", "value", ns="") db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_put_entity").and_return(ASYNC_NONE).once() transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) yield dd.insert_composite_indexes([ent], [composite_index])
def test_get_entity_kind(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, None) item = Item(name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(get_entity_kind(key), "Item")
def test_dynamic_get(self): entity_proto1 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") zookeeper = flexmock() zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("register_updated_key").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_put_entity").and_return(None) db_batch.should_receive("batch_get_entity").and_return( {"test\x00blah\x00test_kind:nancy\x01": { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } }) db_batch.should_receive('record_reads') dd = DatastoreDistributed(db_batch, zookeeper) entity_key = entity_proto1.key() get_req = datastore_pb.GetRequest() key = get_req.add_key() key.MergeFrom(entity_key) get_resp = datastore_pb.GetResponse() dd.dynamic_get("test", get_req, get_resp) self.assertEquals(get_resp.entity_size(), 1) # Now test while in a transaction get_resp = datastore_pb.GetResponse() get_req.mutable_transaction().set_handle(1) dd.dynamic_get("test", get_req, get_resp) self.assertEquals(get_resp.entity_size(), 1)
def test_kindless_query(self): query = datastore_pb.Query() ancestor = query.mutable_ancestor() entity_proto1 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop1val", ns="blah") entity_key = entity_proto1.key() get_req = datastore_pb.GetRequest() key = get_req.add_key() key.MergeFrom(entity_key) tombstone1 = {'key': {APP_ENTITY_SCHEMA[0]:TOMBSTONE, APP_ENTITY_SCHEMA[1]: 1}} db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_get_entity").and_return( {"test\x00blah\x00test_kind:nancy\x01": { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } }) db_batch.should_receive("batch_put_entity").and_return(None) entity_proto1 = {'test\x00blah\x00test_kind:nancy\x01':{APP_ENTITY_SCHEMA[0]:entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1}} db_batch.should_receive("range_query").and_return([entity_proto1, tombstone1]).and_return([]) zookeeper = flexmock() zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("is_in_transaction").and_return(False) zookeeper.should_receive("acquire_lock").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) filter_info = { '__key__' : [[0, 0]] } dd.kindless_query(query, filter_info)
def test_put_entities(self): app_id = 'test' db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) entity_proto1 = self.get_new_entity_proto( app_id, "test_kind", "bob", "prop1name", "prop1val", ns="blah") entity_key1 = 'test\x00blah\x00test_kind:bob\x01' entity_proto2 = self.get_new_entity_proto( app_id, "test_kind", "nancy", "prop1name", "prop2val", ns="blah") entity_key2 = 'test\x00blah\x00test_kind:nancy\x01' entity_list = [entity_proto1, entity_proto2] async_result = gen.Future() async_result.set_result({entity_key1: {}, entity_key2: {}}) db_batch.should_receive('batch_get_entity').and_return(async_result) db_batch.should_receive('normal_batch').and_return(ASYNC_NONE) transaction_manager = flexmock( create_transaction_id=lambda project, xg: 1, delete_transaction_id=lambda project, txid: None, set_groups=lambda project, txid, groups: None) dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd.index_manager = flexmock( projects={app_id: flexmock(indexes_pb=[])}) async_true = gen.Future() async_true.set_result(True) entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire').and_return(async_true) entity_lock.should_receive('release') yield dd.put_entities(app_id, entity_list)
def test_dynamic_put(self): PREFIX = "x\x01" db_batch = flexmock(session=flexmock()) db_batch.should_receive('valid_data_version').and_return(True) entity_proto1 = self.get_new_entity_proto("test", "test_kind", "bob", "prop1name", "prop1val", ns="blah") entity_key1 = 'test\x00blah\x00test_kind:bob\x01' entity_proto2 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") entity_key2 = 'test\x00blah\x00test_kind:nancy\x01' db_batch.should_receive('batch_get_entity').and_return( {entity_key1: {}, entity_key2: {}}) db_batch.should_receive('batch_mutate') dd = DatastoreDistributed(db_batch, self.get_zookeeper()) putreq_pb = datastore_pb.PutRequest() putreq_pb.add_entity() putreq_pb.mutable_entity(0).MergeFrom(entity_proto1) putreq_pb.add_entity() putreq_pb.mutable_entity(1).MergeFrom(entity_proto2) putresp_pb = datastore_pb.PutResponse() entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') flexmock(ScatteredAllocator).should_receive('next').\ and_return(random.randint(1, 500)) dd.dynamic_put('test', putreq_pb, putresp_pb) self.assertEquals(len(putresp_pb.key_list()), 2)
def test_delete_entities(self): app_id = 'test' entity_proto1 = self.get_new_entity_proto( app_id, "test_kind", "bob", "prop1name", "prop1val", ns="blah") row_key = "test\x00blah\x00test_kind:bob\x01" row_values = {row_key: {APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: '1'}} async_result = gen.Future() async_result.set_result(row_values) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("get_valid_transaction_id").and_return(1) db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_get_entity").and_return(async_result) db_batch.should_receive('normal_batch').and_return(ASYNC_NONE) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) row_keys = [entity_proto1.key()] yield dd.delete_entities(entity_proto1.key(), 1, row_keys)
def run(self): """ Starts the main loop of the restore thread. """ datastore_batch = appscale_datastore_batch.\ DatastoreFactory.getDatastore(self.table) transaction_manager = TransactionManager(self.zoo_keeper.handle) self.ds_distributed = DatastoreDistributed( datastore_batch, transaction_manager, zookeeper=self.zoo_keeper) self.dynamic_put_sync = tornado_synchronous( self.ds_distributed.dynamic_put) while True: logging.debug("Trying to get restore lock.") if self.get_restore_lock(): logging.info("Got the restore lock.") self.run_restore() try: self.zoo_keeper.release_lock_with_path(zk.DS_RESTORE_LOCK_PATH) except zk.ZKTransactionException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception))) break else: logging.info("Did not get the restore lock. Another instance may be " "running.") time.sleep(random.randint(1, self.LOCK_POLL_PERIOD))
def test_apply_txn_changes(self): app = 'guestbook' txn = 1 entity = self.get_new_entity_proto(app, *self.BASIC_ENTITY[1:]) db_batch = flexmock() db_batch.should_receive('get_transaction_metadata').and_return({ 'puts': {entity.key().Encode(): entity.Encode()}, 'deletes': [], 'tasks': [], 'reads': set(), 'start': datetime.datetime.utcnow(), 'is_xg': False, }) db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive('group_updates').and_return([]) db_batch.should_receive('get_indices').and_return([]) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) prefix = dd.get_table_prefix(entity) entity_key = get_entity_key(prefix, entity.key().path()) db_batch.should_receive('batch_get_entity').and_return({entity_key: {}}) db_batch.should_receive('batch_mutate') entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') dd.apply_txn_changes(app, txn)
def test_delete_composite_index_metadata(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_delete").and_return(None) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) dd = flexmock(dd) composite_index = entity_pb.CompositeIndex() composite_index.set_id(1) dd.delete_composite_index_metadata("appid", composite_index)
def test_ancestor_query(self): query = datastore_pb.Query() ancestor = query.mutable_ancestor() entity_proto1 = self.get_new_entity_proto( "test", "test_kind", "nancy", "prop1name", "prop1val", ns="blah") entity_key = entity_proto1.key() get_req = datastore_pb.GetRequest() key = get_req.add_key() key.MergeFrom(entity_key) ancestor.MergeFrom(entity_key) async_result = gen.Future() async_result.set_result({ "test\x00blah\x00test_kind:nancy\x01": { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } }) filter_info = [] tombstone1 = {'key': {APP_ENTITY_SCHEMA[0]:TOMBSTONE, APP_ENTITY_SCHEMA[1]: 1}} db_batch = flexmock() db_batch.should_receive('record_reads').and_return(ASYNC_NONE) db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_get_entity").and_return(async_result) entity_proto1 = { 'test\x00blah\x00test_kind:nancy\x01': { APP_ENTITY_SCHEMA[0]: entity_proto1.Encode(), APP_ENTITY_SCHEMA[1]: 1 } } async_result_1 = gen.Future() async_result_1.set_result([entity_proto1, tombstone1]) async_result_2 = gen.Future() async_result_2.set_result([]) db_batch.should_receive("range_query").\ and_return(async_result_1).\ and_return(async_result_2) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) zookeeper.should_receive("get_valid_transaction_id").and_return(1) zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("is_in_transaction").and_return(False) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) yield dd.ancestor_query(query, filter_info) # Now with a transaction transaction = query.mutable_transaction() transaction.set_handle(2) yield dd.ancestor_query(query, filter_info)
def test_get_table_prefix(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_put_entity").and_return(None) zookeeper = flexmock() zookeeper.should_receive("acquire_lock").and_return(True) zookeeper.should_receive("release_lock").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(dd.get_table_prefix(key), "hello\x00")
def test_delete_composite_index_metadata(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) db_batch.should_receive("batch_delete").and_return(ASYNC_NONE) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, self.get_zookeeper()) dd = flexmock(dd) composite_index = entity_pb.CompositeIndex() composite_index.set_id(1) yield dd.delete_composite_index_metadata("appid", composite_index)
def test_dynamic_delete(self): entity_lock = flexmock(EntityLock) entity_lock.should_receive('acquire') entity_lock.should_receive('release') del_request = flexmock() del_request.should_receive("key_list") del_request.should_receive("has_transaction").never() del_request.should_receive("transaction").never() db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) dd.dynamic_delete("appid", del_request) fake_key = entity_pb.Reference() fake_key.set_app('foo') path = fake_key.mutable_path() element = path.add_element() element.set_type('bar') element.set_id(1) del_request = flexmock() del_request.should_receive("key_list").and_return([fake_key]) del_request.should_receive("has_transaction").and_return(True) transaction = flexmock() transaction.should_receive("handle").and_return(1) del_request.should_receive("transaction").and_return(transaction) del_request.should_receive("has_mark_changes").and_return(False) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) flexmock(utils).should_receive("get_entity_kind").and_return("kind") db_batch.should_receive('delete_entities_tx') dd.dynamic_delete("appid", del_request) del_request = flexmock() del_request.should_receive("key_list").and_return([fake_key]) del_request.should_receive("has_transaction").and_return(False) del_request.should_receive("has_mark_changes").and_return(False) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) flexmock(dd).should_receive("delete_entities").once() dd.dynamic_delete("appid", del_request)
def test_reverse_path(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) key = "Project:Synapse\x01Module:Core\x01" self.assertEquals(dd.reverse_path(key), "Module:Core\x01Project:Synapse\x01")
def test_validate_key(self): zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) dd.validate_key(key.key())
def test_rollback_transcation(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zookeeper = flexmock() zookeeper.should_receive("release_lock").and_return(True) zookeeper.should_receive("notify_failed_transaction").and_return(True) dd = DatastoreDistributed(db_batch, zookeeper) commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") http_request = commit_request.Encode() self.assertEquals(dd.rollback_transaction("app_id", http_request), (api_base_pb.VoidProto().Encode(), 0, ""))
def test_root_key_from_entity_key(self): zookeeper = flexmock() db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) db_batch.should_receive("batch_put_entity").and_return(None) dd = DatastoreDistributed(db_batch, zookeeper) self.assertEquals("test\x00blah\x00test_kind:bob\x01", dd.get_root_key_from_entity_key("test\x00blah\x00test_kind:bob\x01nancy\x01")) entity_proto1 = self.get_new_entity_proto("test", "test_kind", "nancy", "prop1name", "prop2val", ns="blah") self.assertEquals("test\x00blah\x00test_kind:nancy\x01", dd.get_root_key_from_entity_key(entity_proto1.key()))
def test_commit_transaction(self): db_batch = flexmock() db_batch.should_receive('valid_data_version').and_return(True) zookeeper = flexmock() zookeeper.should_receive('remove_tx_node') dd = DatastoreDistributed(db_batch, zookeeper) flexmock(dd).should_receive('apply_txn_changes') commit_request = datastore_pb.Transaction() commit_request.set_handle(123) commit_request.set_app("aaa") http_request = commit_request.Encode() self.assertEquals(dd.commit_transaction("app_id", http_request), (datastore_pb.CommitResponse().Encode(), 0, ""))
def test_get_table_prefix(self): db_batch = flexmock() db_batch.should_receive('valid_data_version_sync').and_return(True) zk_client = flexmock() zk_client.should_receive('add_listener') zookeeper = flexmock(handle=zk_client) transaction_manager = flexmock() dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper) item = Item(key_name="Bob", name="Bob", _app="hello") key = db.model_to_protobuf(item) self.assertEquals(dd.get_table_prefix(key), "hello\x00")