def test_composite_ancestor_index_no_properties(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(1)
index.set_entity_type('Yar')
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = [None, None]
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 1 writes for the composite index.
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now use the same entity but give it an ancestor
parent_entity = datastore.Entity('parent', id=123, _app=self.app_id)
entity = datastore.Entity('Yar',
parent=parent_entity.key(),
id=123,
_app=self.app_id) # 2 writes.
entity['this'] = [None, None]
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 2 writes for the composite indices.
self.assertEquals(
8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def test_composite_index_no_properties(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(0)
index.set_entity_type('Yar')
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
# no properties, and composite index with no properties.
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
# We have the 2 built-in index writes, and one for the entity key in the
# composite index despite the fact that there are no proerties defined in
# the index.
self.assertEquals(
3, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# now with a repeated property
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = [1, 2, 3]
# 2 for the entity
# 6 for the indexed properties
# 1 for the composite index
self.assertEquals(
9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def main():
""" Updates a composite index after prompting the user. """
parser = argparse.ArgumentParser(description='Updates composite indexes')
parser.add_argument('--type',
'-t',
default='cassandra',
help='The datastore backend type')
parser.add_argument('--app_id', '-a', required=True, help='The project ID')
parser.add_argument('--all',
action='store_true',
help='Updates all composite indexes')
args = parser.parse_args()
datastore_batch = appscale_datastore_batch.DatastoreFactory.\
getDatastore(args.type)
zookeeper_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zookeeper_locations)
transaction_manager = TransactionManager(zookeeper.handle)
datastore_access = DatastoreDistributed(datastore_batch,
transaction_manager,
zookeeper=zookeeper)
pb_indices = datastore_access.datastore_batch.get_indices_sync(args.app_id)
indices = [datastore_pb.CompositeIndex(index) for index in pb_indices]
if len(indices) == 0:
print('No composite indices found for app {}'.format(args.app_id))
zookeeper.close()
return
update_composite_index_sync = tornado_synchronous(
datastore_access.update_composite_index)
if args.all:
for index in indices:
update_composite_index_sync(args.app_id, index)
print('Successfully updated all composite indexes')
return
selection = -1
selection_range = range(1, len(indices) + 1)
while selection not in selection_range:
for number, index in enumerate(indices, start=1):
pretty_index = prettify_index(index.definition())
print('{}) {}'.format(number, pretty_index))
try:
selection = int(
raw_input(
'Select the index you want to update. (1-{}) '.format(
len(indices))))
except KeyboardInterrupt:
zookeeper.close()
sys.exit()
selected_index = indices[selection - 1]
update_composite_index_sync(args.app_id, selected_index)
zookeeper.close()
print('Index successfully updated')
def main():
""" Updates a composite index after prompting the user. """
try:
opts, remainder = getopt.getopt(sys.argv[1:], 't:a:',
['type=', 'app_id='])
except getopt.GetoptError:
usage()
sys.exit(1)
db_type = None
app_id = None
for opt, arg in opts:
if opt in ('-t', '--type'):
db_type = arg
elif opt in ('-a', '--app_id'):
app_id = arg
if not db_type or not app_id:
usage()
sys.exit(1)
datastore_batch = appscale_datastore_batch.DatastoreFactory.\
getDatastore(db_type)
zookeeper_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zookeeper_locations)
datastore_access = DatastoreDistributed(datastore_batch,
zookeeper=zookeeper)
pb_indices = datastore_access.datastore_batch.get_indices(app_id)
indices = [datastore_pb.CompositeIndex(index) for index in pb_indices]
if len(indices) == 0:
print('No composite indices found for app {}'.format(app_id))
zookeeper.close()
sys.exit(1)
selection = -1
selection_range = range(1, len(indices) + 1)
while selection not in selection_range:
for number, index in enumerate(indices, start=1):
pretty_index = prettify_index(index.definition())
print('{}) {}'.format(number, pretty_index))
try:
selection = int(
raw_input(
'Select the index you want to update. (1-{}) '.format(
len(indices))))
except KeyboardInterrupt:
zookeeper.close()
sys.exit()
selected_index = indices[selection - 1]
datastore_access.update_composite_index(app_id, selected_index)
zookeeper.close()
def main(app_id, db_type):
""" Updates a composite index after prompting the user.
Args:
app_id: A string containing the application ID.
db_type: A string specifying which database backend to use.
"""
datastore_batch = appscale_datastore_batch.DatastoreFactory.\
getDatastore(db_type)
zookeeper_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zookeeper_locations)
datastore_access = DatastoreDistributed(datastore_batch,
zookeeper=zookeeper)
pb_indices = datastore_access.datastore_batch.get_indices(app_id)
indices = [datastore_pb.CompositeIndex(index) for index in pb_indices]
if len(indices) == 0:
print('No composite indices found for app {}'.format(app_id))
zookeeper.close()
sys.exit(1)
selection = -1
selection_range = range(1, len(indices) + 1)
while selection not in selection_range:
for number, index in enumerate(indices, start=1):
pretty_index = prettify_index(index.definition())
print('{}) {}'.format(number, pretty_index))
try:
selection = int(
raw_input(
'Select the index you want to update. (1-{}) '.format(
len(indices))))
except KeyboardInterrupt:
zookeeper.close()
sys.exit()
selected_index = indices[selection - 1]
datastore_access.update_composite_index(app_id, selected_index)
zookeeper.close()
def test_composite_ancestor_index(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(1)
index.set_entity_type('Yar')
prop = index.add_property()
prop.set_name('this')
prop.set_direction(prop.ASCENDING)
prop = index.add_property()
prop.set_name('that')
prop.set_direction(prop.DESCENDING)
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = 4
entity['that'] = 4
# 2 for the entity
# 4 for the indexed properties
# 1 for the composite index
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now use the same entity but give it an ancestor
parent_entity = datastore.Entity('parent', id=123, _app=self.app_id)
entity = datastore.Entity('Yar',
parent=parent_entity.key(),
id=123,
_app=self.app_id) # 2 writes.
entity['this'] = 4
entity['that'] = 4
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 2 writes for the composite indices.
self.assertEquals(
8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now use the same entity but give it 2 ancestors.
grandparent_entity = datastore.Entity('grandparent',
id=123,
_app=self.app_id)
parent_entity = datastore.Entity('parent',
parent=grandparent_entity.key(),
id=123,
_app=self.app_id)
entity = datastore.Entity('Yar',
parent=parent_entity.key(),
id=123,
_app=self.app_id) # 2 writes.
entity['this'] = 4
entity['that'] = 4
# 2 writes for the entity.
# 4 writes for the indexed properties.
# 3 writes for the composite indices.
self.assertEquals(
9, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now try it with a multi-value prop
entity['this'] = [None, None, None]
# 2 writes for the entity.
# 8 writes for the indexed properties.
# 9 writes for the composite indices.
self.assertEquals(
19,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# Now try it with 2 multi-value props.
entity['that'] = [None, None]
# 2 writes for the entity.
# 10 writes for the indexed properties.
# 18 writes for the composite indices.
self.assertEquals(
30,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
def test_composite_index(self):
ci = datastore_pb.CompositeIndex()
ci.set_app_id(datastore_types.ResolveAppId(None))
ci.set_id(0)
ci.set_state(ci.WRITE_ONLY)
index = ci.mutable_definition()
index.set_ancestor(0)
index.set_entity_type('Yar')
prop = index.add_property()
prop.set_name('this')
prop.set_direction(prop.ASCENDING)
prop = index.add_property()
prop.set_name('that')
prop.set_direction(prop.DESCENDING)
stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
stub.CreateIndex(ci)
self.assertEquals(1, len(datastore.GetIndexes()))
# no properties, no composite indices.
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
# We only have the 2 built-in index writes because the entity doesn't have
# property values for any of the index properties.
self.assertEquals(
2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['this'] = 4
# Unindexed property so no additional writes
entity.set_unindexed_properties(('this', ))
self.assertEquals(
2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = 4
# Unindexed property so no additional writes
entity.set_unindexed_properties(('this', 'that'))
self.assertEquals(
2, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# no indexed property value on 'that'
entity.set_unindexed_properties(('that', ))
# 2 writes for the entity.
# 2 writes for the single indexed property.
self.assertEquals(
4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# indexed property value on both 'this' and 'that'
entity.set_unindexed_properties(())
# 2 writes for the entity
# 4 writes for the indexed properties
# 1 writes for the composite index
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# now run tests with null property values
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = None
# 2 for the entity
# 2 for the single indexed property
self.assertEquals(
4, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = None
# 2 for the entity
# 4 for the indexed properties
# 1 for the composite index
self.assertEquals(
7, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
# now run tests with a repeated property
entity = datastore.Entity('Yar', id=123, _app=self.app_id) # 2 writes.
entity['this'] = [1, 2, 3]
# 2 for the entity
# 6 for the indexed properties
self.assertEquals(
8, datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = None
# 2 for the entity
# 8 for the indexed properties
# 3 for the Composite index
self.assertEquals(
13,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
entity['that'] = [4, 5]
# 2 for the entity
# 10 for the indexed properties
# 6 for the Composite index
self.assertEquals(
18,
datastore_viewer.DatastoreRequestHandler._get_write_ops(entity))
