def __Init(self):
self.__connection.execute('PRAGMA synchronous = OFF')
self.__connection.executescript(_CORE_SCHEMA)
self.__connection.commit()
c = self.__connection.execute(
'SELECT app_id, name_space FROM Namespaces')
self.__namespaces = set(c.fetchall())
for app_ns in self.__namespaces:
prefix = ('%s!%s' % app_ns).replace('"', '""')
self.__connection.execute(
'INSERT OR IGNORE INTO ScatteredIdCounters VALUES (?, ?)',
(prefix, 1))
self.__connection.commit()
c = self.__connection.execute('SELECT app_id, indexes FROM Apps')
for _, index_proto in c.fetchall():
if not index_proto:
continue
indexes = datastore_pb.CompositeIndices(index_proto)
for index in indexes.index_list():
self._SideLoadIndex(index)
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
resolved_app_id = datastore_types.ResolveAppId(_app)
if hasattr(datastore_pb, 'GetIndicesRequest'):
req = datastore_pb.GetIndicesRequest()
req.set_app_id(resolved_app_id)
else:
req = api_base_pb.StringProto()
req.app_id = MethodType(_StringProtoAppIdGet, req)
req.set_app_id = MethodType(_StringProtoAppIdSet, req)
req.set_app_id(resolved_app_id)
resp = datastore_pb.CompositeIndices()
resp = _Call('GetIndices', req, resp)
return resp.index_list()
def get_indices_request(self, app_id):
""" Gets the indices of the given application.
Args:
app_id: Name of the application.
Returns:
A Tuple of an encoded response, error code, and error explanation.
"""
global datastore_access
response = datastore_pb.CompositeIndices()
try:
indices = datastore_access.get_indexes(app_id)
except (dbconstants.AppScaleDBConnectionError,
dbconstants.InternalError) as error:
logger.exception(
'Internal error while fetching indices for {}'.format(app_id))
raise gen.Return(('', datastore_pb.Error.INTERNAL_ERROR, str(error)))
except dbconstants.BadRequest as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
for index in indices:
new_index = response.add_index()
new_index.MergeFrom(index)
raise gen.Return((response.Encode(), 0, ''))
def test_delete_composite_index_metadata(self):
db_batch = flexmock()
db_batch.should_receive("batch_delete").and_return(None)
dd = DatastoreDistributed(db_batch, self.get_zookeeper())
dd = flexmock(dd)
dd.should_receive("get_meta_data_key").and_return("somekey")
composite_indexes = datastore_pb.CompositeIndices()
dd.delete_composite_index_metadata("appid", composite_indexes)
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
req = api_base_pb.StringProto()
req.set_value(datastore_types.ResolveAppId(_app))
resp = datastore_pb.CompositeIndices()
resp = _Call('GetIndices', req, resp)
return resp.index_list()
def __WriteIndexData(self, conn, app):
"""Writes index data to disk.
Args:
conn: An SQLite connection.
app: The app ID to write indexes for.
"""
indices = datastore_pb.CompositeIndices()
for index in self.GetIndexes(app, True, self._app_id):
indices.index_list().append(index)
conn.execute('UPDATE Apps SET indexes = ? WHERE app_id = ?',
(app, buffer(indices.Encode())))
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
req = api_base_pb.StringProto()
req.set_value(datastore_types.ResolveAppId(_app))
resp = datastore_pb.CompositeIndices()
try:
apiproxy_stub_map.MakeSyncCall('datastore_v3', 'GetIndices', req, resp)
except apiproxy_errors.ApplicationError, err:
raise datastore._ToDatastoreError(err)
def GetIndices(_app=None):
"""Fetches all composite indices in the datastore for this app.
Returns:
list of entity_pb.CompositeIndex
"""
resolved_app_id = datastore_types.ResolveAppId(_app)
req = datastore_pb.GetIndicesRequest()
req.app_id = resolved_app_id
resp = datastore_pb.CompositeIndices()
resp = _Call('GetIndices', req, resp)
return resp.index
def __Init(self):
self.__connection.executescript(_CORE_SCHEMA)
self.__connection.commit()
c = self.__connection.execute(
'SELECT app_id, name_space FROM Namespaces')
self.__namespaces = set(c.fetchall())
c = self.__connection.execute('SELECT app_id, indexes FROM Apps')
for _, index_proto in c.fetchall():
if not index_proto:
continue
indexes = datastore_pb.CompositeIndices(index_proto)
for index in indexes.index_list():
self._SideLoadIndex(index)
def get_indices_request(self, app_id):
""" Gets the indices of the given application.
Args:
app_id: Name of the application.
http_request_data: Stores the protocol buffer request from the
AppServer.
Returns:
A Tuple of an encoded response, error code, and error explanation.
"""
global datastore_access
response = datastore_pb.CompositeIndices()
try:
indices = datastore_access.datastore_batch.get_indices(app_id)
except dbconstants.AppScaleDBConnectionError, dbce:
logger.exception('DB connection error while fetching indices for '
'{}'.format(app_id))
return (response.Encode(), datastore_pb.Error.INTERNAL_ERROR,
"Datastore connection error on get indices request.")
def _get_indexes(cls, ds_access):
""" Retrieves a list of composite indexes for a project.
Args:
ds_access: A DatastoreDistributed client.
Returns:
A list of datastore.Index objects.
"""
request = api_base_pb.StringProto()
request.set_value(ds_access.project_id)
response = datastore_pb.CompositeIndices()
ds_access._Dynamic_GetIndices(request, response)
indexes = []
for index in response.index_list():
props = [(prop.name(), prop.direction())
for prop in index.definition().property_list()]
new_index = datastore.Index(index.id(), index.definition().entity_type(),
index.definition().ancestor(), props)
indexes.append(new_index)
return indexes
def get_indices_request(self, app_id):
""" Gets the indices of the given application.
Args:
app_id: Name of the application.
http_request_data: Stores the protocol buffer request from the
AppServer.
Returns:
A Tuple of an encoded response, error code, and error explanation.
"""
global datastore_access
response = datastore_pb.CompositeIndices()
try:
indices = datastore_access.datastore_batch.get_indices(app_id)
except dbconstants.AppScaleDBConnectionError as error:
logger.exception('DB connection error while fetching indices for '
'{}'.format(app_id))
return '', datastore_pb.Error.INTERNAL_ERROR, str(error)
for index in indices:
new_index = response.add_index()
new_index.ParseFromString(index)
return response.Encode(), 0, ''
def _SetupIndexes(self, _open=open):
"""Ensure that the set of existing composite indexes matches index.yaml.
Create any new indexes, and delete indexes which are no longer required.
Args:
_open: Function used to open a file.
"""
if not self.__root_path:
logging.warning("No index.yaml was loaded.")
return
index_yaml_file = os.path.join(self.__root_path, 'index.yaml')
if (self.__cached_yaml[0] == index_yaml_file
and os.path.exists(index_yaml_file) and
os.path.getmtime(index_yaml_file) == self.__cached_yaml[1]):
requested_indexes = self.__cached_yaml[2]
else:
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
fh = _open(index_yaml_file, 'r')
except (OSError, IOError):
logging.info("Error reading file")
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
finally:
fh.close()
requested_indexes = []
if index_yaml_data is not None:
index_defs = datastore_index.ParseIndexDefinitions(
index_yaml_data)
if index_defs is not None and index_defs.indexes is not None:
requested_indexes = datastore_index.IndexDefinitionsToProtos(
self.__app_id, index_defs.indexes)
self.__cached_yaml = (index_yaml_file, index_yaml_mtime,
requested_indexes)
existing_indexes = datastore_pb.CompositeIndices()
app_str = api_base_pb.StringProto()
app_str.set_value(self.__app_id)
self._Dynamic_GetIndices(app_str, existing_indexes)
requested = dict(
(x.definition().Encode(), x) for x in requested_indexes)
existing = dict((x.definition().Encode(), x)
for x in existing_indexes.index_list())
# Delete any indexes that are no longer requested.
deleted = 0
for key, index in existing.iteritems():
if key not in requested:
self._Dynamic_DeleteIndex(index, api_base_pb.VoidProto())
deleted += 1
# Add existing indexes in the index cache.
for key, index in existing.iteritems():
new_index = entity_pb.CompositeIndex()
new_index.CopyFrom(index)
ent_kind = new_index.definition().entity_type()
if ent_kind in self.__index_cache:
new_indexes = self.__index_cache[ent_kind]
new_indexes.append(new_index)
self.__index_cache[ent_kind] = new_indexes
else:
self.__index_cache[ent_kind] = [new_index]
# Compared the existing indexes to the requested ones and create any
# new indexes requested.
created = 0
for key, index in requested.iteritems():
if key not in existing:
new_index = entity_pb.CompositeIndex()
new_index.CopyFrom(index)
new_index.set_id(
self._Dynamic_CreateIndex(
new_index, api_base_pb.Integer64Proto()).value())
new_index.set_state(entity_pb.CompositeIndex.READ_WRITE)
self._Dynamic_UpdateIndex(new_index, api_base_pb.VoidProto())
created += 1
ent_kind = new_index.definition().entity_type()
if ent_kind in self.__index_cache:
new_indexes = self.__index_cache[ent_kind]
new_indexes.append(new_index)
self.__index_cache[ent_kind] = new_indexes
else:
self.__index_cache[ent_kind] = [new_index]
if created or deleted:
logging.info('Created %d and deleted %d index(es); total %d',
created, deleted, len(requested))