def __Init(self): self.__connection.execute('PRAGMA synchronous = OFF') self.__connection.executescript(_CORE_SCHEMA) self.__connection.commit() c = self.__connection.execute( 'SELECT app_id, name_space FROM Namespaces') self.__namespaces = set(c.fetchall()) for app_ns in self.__namespaces: prefix = ('%s!%s' % app_ns).replace('"', '""') self.__connection.execute( 'INSERT OR IGNORE INTO ScatteredIdCounters VALUES (?, ?)', (prefix, 1)) self.__connection.commit() c = self.__connection.execute('SELECT app_id, indexes FROM Apps') for _, index_proto in c.fetchall(): if not index_proto: continue indexes = datastore_pb.CompositeIndices(index_proto) for index in indexes.index_list(): self._SideLoadIndex(index)
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ resolved_app_id = datastore_types.ResolveAppId(_app) if hasattr(datastore_pb, 'GetIndicesRequest'): req = datastore_pb.GetIndicesRequest() req.set_app_id(resolved_app_id) else: req = api_base_pb.StringProto() req.app_id = MethodType(_StringProtoAppIdGet, req) req.set_app_id = MethodType(_StringProtoAppIdSet, req) req.set_app_id(resolved_app_id) resp = datastore_pb.CompositeIndices() resp = _Call('GetIndices', req, resp) return resp.index_list()
def get_indices_request(self, app_id): """ Gets the indices of the given application. Args: app_id: Name of the application. Returns: A Tuple of an encoded response, error code, and error explanation. """ global datastore_access response = datastore_pb.CompositeIndices() try: indices = datastore_access.get_indexes(app_id) except (dbconstants.AppScaleDBConnectionError, dbconstants.InternalError) as error: logger.exception( 'Internal error while fetching indices for {}'.format(app_id)) raise gen.Return(('', datastore_pb.Error.INTERNAL_ERROR, str(error))) except dbconstants.BadRequest as error: raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error))) for index in indices: new_index = response.add_index() new_index.MergeFrom(index) raise gen.Return((response.Encode(), 0, ''))
def test_delete_composite_index_metadata(self): db_batch = flexmock() db_batch.should_receive("batch_delete").and_return(None) dd = DatastoreDistributed(db_batch, self.get_zookeeper()) dd = flexmock(dd) dd.should_receive("get_meta_data_key").and_return("somekey") composite_indexes = datastore_pb.CompositeIndices() dd.delete_composite_index_metadata("appid", composite_indexes)
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ req = api_base_pb.StringProto() req.set_value(datastore_types.ResolveAppId(_app)) resp = datastore_pb.CompositeIndices() resp = _Call('GetIndices', req, resp) return resp.index_list()
def __WriteIndexData(self, conn, app): """Writes index data to disk. Args: conn: An SQLite connection. app: The app ID to write indexes for. """ indices = datastore_pb.CompositeIndices() for index in self.GetIndexes(app, True, self._app_id): indices.index_list().append(index) conn.execute('UPDATE Apps SET indexes = ? WHERE app_id = ?', (app, buffer(indices.Encode())))
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ req = api_base_pb.StringProto() req.set_value(datastore_types.ResolveAppId(_app)) resp = datastore_pb.CompositeIndices() try: apiproxy_stub_map.MakeSyncCall('datastore_v3', 'GetIndices', req, resp) except apiproxy_errors.ApplicationError, err: raise datastore._ToDatastoreError(err)
def GetIndices(_app=None): """Fetches all composite indices in the datastore for this app. Returns: list of entity_pb.CompositeIndex """ resolved_app_id = datastore_types.ResolveAppId(_app) req = datastore_pb.GetIndicesRequest() req.app_id = resolved_app_id resp = datastore_pb.CompositeIndices() resp = _Call('GetIndices', req, resp) return resp.index
def __Init(self): self.__connection.executescript(_CORE_SCHEMA) self.__connection.commit() c = self.__connection.execute( 'SELECT app_id, name_space FROM Namespaces') self.__namespaces = set(c.fetchall()) c = self.__connection.execute('SELECT app_id, indexes FROM Apps') for _, index_proto in c.fetchall(): if not index_proto: continue indexes = datastore_pb.CompositeIndices(index_proto) for index in indexes.index_list(): self._SideLoadIndex(index)
def get_indices_request(self, app_id): """ Gets the indices of the given application. Args: app_id: Name of the application. http_request_data: Stores the protocol buffer request from the AppServer. Returns: A Tuple of an encoded response, error code, and error explanation. """ global datastore_access response = datastore_pb.CompositeIndices() try: indices = datastore_access.datastore_batch.get_indices(app_id) except dbconstants.AppScaleDBConnectionError, dbce: logger.exception('DB connection error while fetching indices for ' '{}'.format(app_id)) return (response.Encode(), datastore_pb.Error.INTERNAL_ERROR, "Datastore connection error on get indices request.")
def _get_indexes(cls, ds_access): """ Retrieves a list of composite indexes for a project. Args: ds_access: A DatastoreDistributed client. Returns: A list of datastore.Index objects. """ request = api_base_pb.StringProto() request.set_value(ds_access.project_id) response = datastore_pb.CompositeIndices() ds_access._Dynamic_GetIndices(request, response) indexes = [] for index in response.index_list(): props = [(prop.name(), prop.direction()) for prop in index.definition().property_list()] new_index = datastore.Index(index.id(), index.definition().entity_type(), index.definition().ancestor(), props) indexes.append(new_index) return indexes
def get_indices_request(self, app_id): """ Gets the indices of the given application. Args: app_id: Name of the application. http_request_data: Stores the protocol buffer request from the AppServer. Returns: A Tuple of an encoded response, error code, and error explanation. """ global datastore_access response = datastore_pb.CompositeIndices() try: indices = datastore_access.datastore_batch.get_indices(app_id) except dbconstants.AppScaleDBConnectionError as error: logger.exception('DB connection error while fetching indices for ' '{}'.format(app_id)) return '', datastore_pb.Error.INTERNAL_ERROR, str(error) for index in indices: new_index = response.add_index() new_index.ParseFromString(index) return response.Encode(), 0, ''
def _SetupIndexes(self, _open=open): """Ensure that the set of existing composite indexes matches index.yaml. Create any new indexes, and delete indexes which are no longer required. Args: _open: Function used to open a file. """ if not self.__root_path: logging.warning("No index.yaml was loaded.") return index_yaml_file = os.path.join(self.__root_path, 'index.yaml') if (self.__cached_yaml[0] == index_yaml_file and os.path.exists(index_yaml_file) and os.path.getmtime(index_yaml_file) == self.__cached_yaml[1]): requested_indexes = self.__cached_yaml[2] else: try: index_yaml_mtime = os.path.getmtime(index_yaml_file) fh = _open(index_yaml_file, 'r') except (OSError, IOError): logging.info("Error reading file") index_yaml_data = None else: try: index_yaml_data = fh.read() finally: fh.close() requested_indexes = [] if index_yaml_data is not None: index_defs = datastore_index.ParseIndexDefinitions( index_yaml_data) if index_defs is not None and index_defs.indexes is not None: requested_indexes = datastore_index.IndexDefinitionsToProtos( self.__app_id, index_defs.indexes) self.__cached_yaml = (index_yaml_file, index_yaml_mtime, requested_indexes) existing_indexes = datastore_pb.CompositeIndices() app_str = api_base_pb.StringProto() app_str.set_value(self.__app_id) self._Dynamic_GetIndices(app_str, existing_indexes) requested = dict( (x.definition().Encode(), x) for x in requested_indexes) existing = dict((x.definition().Encode(), x) for x in existing_indexes.index_list()) # Delete any indexes that are no longer requested. deleted = 0 for key, index in existing.iteritems(): if key not in requested: self._Dynamic_DeleteIndex(index, api_base_pb.VoidProto()) deleted += 1 # Add existing indexes in the index cache. for key, index in existing.iteritems(): new_index = entity_pb.CompositeIndex() new_index.CopyFrom(index) ent_kind = new_index.definition().entity_type() if ent_kind in self.__index_cache: new_indexes = self.__index_cache[ent_kind] new_indexes.append(new_index) self.__index_cache[ent_kind] = new_indexes else: self.__index_cache[ent_kind] = [new_index] # Compared the existing indexes to the requested ones and create any # new indexes requested. created = 0 for key, index in requested.iteritems(): if key not in existing: new_index = entity_pb.CompositeIndex() new_index.CopyFrom(index) new_index.set_id( self._Dynamic_CreateIndex( new_index, api_base_pb.Integer64Proto()).value()) new_index.set_state(entity_pb.CompositeIndex.READ_WRITE) self._Dynamic_UpdateIndex(new_index, api_base_pb.VoidProto()) created += 1 ent_kind = new_index.definition().entity_type() if ent_kind in self.__index_cache: new_indexes = self.__index_cache[ent_kind] new_indexes.append(new_index) self.__index_cache[ent_kind] = new_indexes else: self.__index_cache[ent_kind] = [new_index] if created or deleted: logging.info('Created %d and deleted %d index(es); total %d', created, deleted, len(requested))