def migrate_composite_index_metadata(cluster, session, zk_client): """ Moves any existing datastore index metadata to ZooKeeper. Args: cluster: A cassandra.cluster.Cluster object. session: A cassandra.cluster.Session object. zk_client: A kazoo.client.KazooClient object. """ keyspace_metadata = cluster.metadata.keyspaces[KEYSPACE] if dbconstants.METADATA_TABLE not in keyspace_metadata.tables: return logging.info('Fetching previously-defined index definitions') results = session.execute( 'SELECT * FROM "{}"'.format(dbconstants.METADATA_TABLE)) indexes_by_project = defaultdict(list) for result in results: try: index_pb = entity_pb.CompositeIndex(result.value) except ProtocolBufferDecodeError: logging.warning('Invalid composite index: {}'.format(result.value)) continue index = DatastoreIndex.from_pb(index_pb) # Assume the index is complete. index.ready = True indexes_by_project[index.project_id].append(index) for project_id, indexes in indexes_by_project.items(): logging.info('Adding indexes for {}'.format(project_id)) merge_indexes(zk_client, project_id, indexes) logging.info('Removing previously-defined index definitions from Cassandra') session.execute('DROP TABLE "{}"'.format(dbconstants.METADATA_TABLE), timeout=SCHEMA_CHANGE_TIMEOUT)
def post(self): """ Handles UpdateIndexes operations. """ project_id = self.get_argument('app_id', None) if project_id is None: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='app_id parameter is required') self.authenticate(project_id, self.ua_client) try: payload = yaml.safe_load(self.request.body) except ParserError: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='Payload must be valid YAML') try: given_indexes = payload['indexes'] except KeyError: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='Payload must contain "indexes"') # If there are no new indexes being added, there's no work to be done. if not given_indexes: return try: given_indexes = [ DatastoreIndex.from_yaml(project_id, index) for index in given_indexes ] except InvalidConfiguration as error: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=six.text_type(error)) merge_indexes(self.zk_client, project_id, given_indexes) logger.info('Updated indexes for {}'.format(project_id))
def post(self): """ Handles UpdateIndexes operations. """ project_id = self.get_argument('app_id', None) if project_id is None: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='app_id parameter is required') self.authenticate(project_id, self.ua_client) try: payload = yaml.safe_load(self.request.body) except ParserError: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='Payload must be valid YAML') try: given_indexes = payload['indexes'] except KeyError: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message='Payload must contain "indexes"') # If there are no new indexes being added, there's no work to be done. if not given_indexes: return try: given_indexes = [DatastoreIndex.from_yaml(project_id, index) for index in given_indexes] except InvalidConfiguration as error: raise CustomHTTPError(HTTPCodes.BAD_REQUEST, message=six.text_type(error)) merge_indexes(self.zk_client, project_id, given_indexes) logger.info('Updated indexes for {}'.format(project_id))
def add_indexes(self, project_id, indexes): """ Adds composite index definitions to a project. Only indexes that do not already exist will be created. Args: project_id: A string specifying a project ID. indexes: An iterable containing index definitions. """ # This is a temporary workaround to get a ZooKeeper client. This method # will not use ZooKeeper in the future. zk_client = self.index_manager.composite_index_manager._zk_client merge_indexes(zk_client, project_id, indexes)