def test_composite_ancestor_index_no_properties(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(1) index.set_entity_type('Yar') stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = [None, None] # 2 writes for the entity. # 4 writes for the indexed properties. # 1 writes for the composite index. self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now use the same entity but give it an ancestor parent_entity = datastore.Entity('parent', id=123, _app=self.app_id) entity = datastore.Entity('Yar', parent=parent_entity.key(), id=123, _app=self.app_id) # 2 writes. entity['this'] = [None, None] # 2 writes for the entity. # 4 writes for the indexed properties. # 2 writes for the composite indices. self.assertEquals( 8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def test_composite_index_no_properties(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(0) index.set_entity_type('Yar') stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) # no properties, and composite index with no properties. entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. # We have the 2 built-in index writes, and one for the entity key in the # composite index despite the fact that there are no proerties defined in # the index. self.assertEquals( 3, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # now with a repeated property entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = [1, 2, 3] # 2 for the entity # 6 for the indexed properties # 1 for the composite index self.assertEquals( 9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def main(): """ Updates a composite index after prompting the user. """ parser = argparse.ArgumentParser(description='Updates composite indexes') parser.add_argument('--type', '-t', default='cassandra', help='The datastore backend type') parser.add_argument('--app_id', '-a', required=True, help='The project ID') parser.add_argument('--all', action='store_true', help='Updates all composite indexes') args = parser.parse_args() datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(args.type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) transaction_manager = TransactionManager(zookeeper.handle) datastore_access = DatastoreDistributed(datastore_batch, transaction_manager, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices_sync(args.app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(args.app_id)) zookeeper.close() return update_composite_index_sync = tornado_synchronous( datastore_access.update_composite_index) if args.all: for index in indices: update_composite_index_sync(args.app_id, index) print('Successfully updated all composite indexes') return selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int( raw_input( 'Select the index you want to update. (1-{}) '.format( len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] update_composite_index_sync(args.app_id, selected_index) zookeeper.close() print('Index successfully updated')
def main(): """ Updates a composite index after prompting the user. """ try: opts, remainder = getopt.getopt(sys.argv[1:], 't:a:', ['type=', 'app_id=']) except getopt.GetoptError: usage() sys.exit(1) db_type = None app_id = None for opt, arg in opts: if opt in ('-t', '--type'): db_type = arg elif opt in ('-a', '--app_id'): app_id = arg if not db_type or not app_id: usage() sys.exit(1) datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(db_type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices(app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(app_id)) zookeeper.close() sys.exit(1) selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int( raw_input( 'Select the index you want to update. (1-{}) '.format( len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] datastore_access.update_composite_index(app_id, selected_index) zookeeper.close()
def main(app_id, db_type): """ Updates a composite index after prompting the user. Args: app_id: A string containing the application ID. db_type: A string specifying which database backend to use. """ datastore_batch = appscale_datastore_batch.DatastoreFactory.\ getDatastore(db_type) zookeeper_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zookeeper_locations) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper) pb_indices = datastore_access.datastore_batch.get_indices(app_id) indices = [datastore_pb.CompositeIndex(index) for index in pb_indices] if len(indices) == 0: print('No composite indices found for app {}'.format(app_id)) zookeeper.close() sys.exit(1) selection = -1 selection_range = range(1, len(indices) + 1) while selection not in selection_range: for number, index in enumerate(indices, start=1): pretty_index = prettify_index(index.definition()) print('{}) {}'.format(number, pretty_index)) try: selection = int( raw_input( 'Select the index you want to update. (1-{}) '.format( len(indices)))) except KeyboardInterrupt: zookeeper.close() sys.exit() selected_index = indices[selection - 1] datastore_access.update_composite_index(app_id, selected_index) zookeeper.close()
def test_composite_ancestor_index(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(1) index.set_entity_type('Yar') prop = index.add_property() prop.set_name('this') prop.set_direction(prop.ASCENDING) prop = index.add_property() prop.set_name('that') prop.set_direction(prop.DESCENDING) stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = 4 entity['that'] = 4 # 2 for the entity # 4 for the indexed properties # 1 for the composite index self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now use the same entity but give it an ancestor parent_entity = datastore.Entity('parent', id=123, _app=self.app_id) entity = datastore.Entity('Yar', parent=parent_entity.key(), id=123, _app=self.app_id) # 2 writes. entity['this'] = 4 entity['that'] = 4 # 2 writes for the entity. # 4 writes for the indexed properties. # 2 writes for the composite indices. self.assertEquals( 8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now use the same entity but give it 2 ancestors. grandparent_entity = datastore.Entity('grandparent', id=123, _app=self.app_id) parent_entity = datastore.Entity('parent', parent=grandparent_entity.key(), id=123, _app=self.app_id) entity = datastore.Entity('Yar', parent=parent_entity.key(), id=123, _app=self.app_id) # 2 writes. entity['this'] = 4 entity['that'] = 4 # 2 writes for the entity. # 4 writes for the indexed properties. # 3 writes for the composite indices. self.assertEquals( 9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now try it with a multi-value prop entity['this'] = [None, None, None] # 2 writes for the entity. # 8 writes for the indexed properties. # 9 writes for the composite indices. self.assertEquals( 19, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # Now try it with 2 multi-value props. entity['that'] = [None, None] # 2 writes for the entity. # 10 writes for the indexed properties. # 18 writes for the composite indices. self.assertEquals( 30, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def test_composite_index(self): ci = datastore_pb.CompositeIndex() ci.set_app_id(datastore_types.ResolveAppId(None)) ci.set_id(0) ci.set_state(ci.WRITE_ONLY) index = ci.mutable_definition() index.set_ancestor(0) index.set_entity_type('Yar') prop = index.add_property() prop.set_name('this') prop.set_direction(prop.ASCENDING) prop = index.add_property() prop.set_name('that') prop.set_direction(prop.DESCENDING) stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') stub.CreateIndex(ci) self.assertEquals(1, len(datastore.GetIndexes())) # no properties, no composite indices. entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. # We only have the 2 built-in index writes because the entity doesn't have # property values for any of the index properties. self.assertEquals( 2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['this'] = 4 # Unindexed property so no additional writes entity.set_unindexed_properties(('this', )) self.assertEquals( 2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = 4 # Unindexed property so no additional writes entity.set_unindexed_properties(('this', 'that')) self.assertEquals( 2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # no indexed property value on 'that' entity.set_unindexed_properties(('that', )) # 2 writes for the entity. # 2 writes for the single indexed property. self.assertEquals( 4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # indexed property value on both 'this' and 'that' entity.set_unindexed_properties(()) # 2 writes for the entity # 4 writes for the indexed properties # 1 writes for the composite index self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # now run tests with null property values entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = None # 2 for the entity # 2 for the single indexed property self.assertEquals( 4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = None # 2 for the entity # 4 for the indexed properties # 1 for the composite index self.assertEquals( 7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) # now run tests with a repeated property entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes. entity['this'] = [1, 2, 3] # 2 for the entity # 6 for the indexed properties self.assertEquals( 8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = None # 2 for the entity # 8 for the indexed properties # 3 for the Composite index self.assertEquals( 13, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity)) entity['that'] = [4, 5] # 2 for the entity # 10 for the indexed properties # 6 for the Composite index self.assertEquals( 18, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))