def _AllocateIds(self, reference, size=1, max_id=None): conn = self._GetConnection() try: datastore_stub_util.CheckAppId(reference.app(), self._trusted, self._app_id) datastore_stub_util.Check(not (size and max_id), 'Both size and max cannot be set.') prefix = self._GetTablePrefix(reference) if size: datastore_stub_util.Check(size > 0, 'Size must be greater than 0.') next_id, block_size = self.__id_map.get(prefix, (0, 0)) if not block_size: block_size = (size / 1000 + 1) * 1000 c = conn.execute( 'SELECT next_id FROM IdSeq WHERE prefix = ? LIMIT 1', (prefix, )) next_id = c.fetchone()[0] c = conn.execute( 'UPDATE IdSeq SET next_id = next_id + ? WHERE prefix = ?', (block_size, prefix)) assert c.rowcount == 1 if size > block_size: c = conn.execute( 'SELECT next_id FROM IdSeq WHERE prefix = ? LIMIT 1', (prefix, )) start = c.fetchone()[0] c = conn.execute( 'UPDATE IdSeq SET next_id = next_id + ? WHERE prefix = ?', (size, prefix)) assert c.rowcount == 1 else: start = next_id next_id += size block_size -= size self.__id_map[prefix] = (next_id, block_size) end = start + size - 1 else: datastore_stub_util.Check( max_id >= 0, 'Max must be greater than or equal to 0.') c = conn.execute( 'SELECT next_id FROM IdSeq WHERE prefix = ? LIMIT 1', (prefix, )) start = c.fetchone()[0] if max_id and max_id >= start: c = conn.execute( 'UPDATE IdSeq SET next_id = ? WHERE prefix = ?', (max_id + 1, prefix)) assert c.rowcount == 1 end = max(max_id, start - 1) return (long(start), long(end)) finally: self._ReleaseConnection(conn)
def __AllocateIdsFromBlock(self, conn, prefix, size, id_map, table): datastore_stub_util.Check(size > 0, 'Size must be greater than 0.') next_id, block_size = id_map.get(prefix, (0, 0)) if not block_size: block_size = (size / 1000 + 1) * 1000 c = conn.execute( 'SELECT next_id FROM %s WHERE prefix = ? LIMIT 1' % table, (prefix, )) next_id = c.fetchone()[0] c = conn.execute( 'UPDATE %s SET next_id = next_id + ? WHERE prefix = ?' % table, (block_size, prefix)) assert c.rowcount == 1 if size > block_size: c = conn.execute( 'SELECT next_id FROM %s WHERE prefix = ? LIMIT 1' % table, (prefix, )) start = c.fetchone()[0] c = conn.execute( 'UPDATE %s SET next_id = next_id + ? WHERE prefix = ?' % table, (size, prefix)) assert c.rowcount == 1 else: start = next_id next_id += size block_size -= size id_map[prefix] = (next_id, block_size) end = start + size - 1 return start, end
def _AllocateIds(self, references): conn = self._GetConnection() try: full_keys = [] for key in references: datastore_stub_util.CheckAppId(self._trusted, self._app_id, key.app()) prefix = self._GetTablePrefix(key) last_element = key.path().element_list()[-1] datastore_stub_util.Check(not last_element.has_name(), 'Cannot allocate named key.') if last_element.id(): count, id_space = datastore_stub_util.IdToCounter( last_element.id()) table, _ = self.__id_counter_tables[id_space] self.__AdvanceIdCounter(conn, prefix, count, table) else: count, _ = self.__AllocateIdsFromBlock( conn, prefix, 1, self.__id_map_scattered, 'ScatteredIdCounters') last_element.set_id( datastore_stub_util.ToScatteredId(count)) full_keys.append(key) return full_keys finally: self._ReleaseConnection(conn)
def _AllocateIds(self, reference, size=1, max_id=None): datastore_stub_util.Check(not (size and max_id), 'Both size and max cannot be set.') self.__id_lock.acquire() try: start = self.__next_id if size: datastore_stub_util.Check(size > 0, 'Size must be greater than 0.') self.__next_id += size elif max_id: datastore_stub_util.Check(max_id >=0, 'Max must be greater than or equal to 0.') self.__next_id = max(self.__next_id, max_id + 1) end = self.__next_id - 1 finally: self.__id_lock.release() return (start, end)
def MakeSyncCall(self, service, call, request, response): """The main RPC entry point. service must be 'datastore_v3'.""" self.AssertPbIsInitialized(request) try: apiproxy_stub.APIProxyStub.MakeSyncCall(self, service, call, request, response) except sqlite3.OperationalError, e: datastore_stub_util.Check(e.args[0] == 'database is locked', 'Database is locked.', datastore_pb.Error.TIMEOUT) raise
def _AllocateSequentialIds(self, reference, size=1, max_id=None): datastore_stub_util.Check(not (size and max_id), 'Both size and max cannot be set.') self.__id_lock.acquire() try: id_space = datastore_stub_util.SEQUENTIAL start = self._IdCounter(id_space) if size: datastore_stub_util.Check(size > 0, 'Size must be greater than 0.') self._SetIdCounter(id_space, start + size) elif max_id: datastore_stub_util.Check(max_id >=0, 'Max must be greater than or equal to 0.') self._SetIdCounter(id_space, max(start, max_id + 1)) end = self._IdCounter(id_space) - 1 finally: self.__id_lock.release() return (start, end)
def __AdvanceIdCounter(self, conn, prefix, max_id, table): datastore_stub_util.Check(max_id >= 0, 'Max must be greater than or equal to 0.') c = conn.execute('SELECT next_id FROM %s WHERE prefix = ? LIMIT 1' % table, (prefix,)) start = c.fetchone()[0] if max_id >= start: c = conn.execute('UPDATE %s SET next_id = ? WHERE prefix = ?' % table, (max_id + 1, prefix)) assert c.rowcount == 1 end = max(max_id, start - 1) return start, end
def _GetQueryCursor(self, query, filters, orders, index_list): """Returns a query cursor for the provided query. Args: query: The datastore_pb.Query to run. filters: A list of filters that override the ones found on query. orders: A list of orders that override the ones found on query. index_list: A list of indexes used by the query. Returns: A QueryCursor object. """ if query.has_kind() and query.kind() in self._pseudo_kinds: datastore_stub_util.NormalizeCursors(query, datastore_pb.Query_Order.ASCENDING) cursor = self._pseudo_kinds[query.kind()].Query(query, filters, orders) datastore_stub_util.Check(cursor, 'Could not create query for pseudo-kind') else: orders = datastore_stub_util._GuessOrders(filters, orders) datastore_stub_util.NormalizeCursors(query, orders[0].direction()) filter_info = self.__GenerateFilterInfo(filters, query) order_info = self.__GenerateOrderInfo(orders) for strategy in DatastoreSqliteStub._QUERY_STRATEGIES: result = strategy(self, query, filter_info, order_info) if result: break else: raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'No strategy found to satisfy query.') sql_stmt, params = result conn = self._GetConnection() try: if query.property_name_list(): db_cursor = _ProjectionPartialEntityGenerator( conn.execute(sql_stmt, params)) else: db_cursor = _DedupingEntityGenerator(conn.execute(sql_stmt, params)) dsquery = datastore_stub_util._MakeQuery(query, filters, orders) cursor = datastore_stub_util.ListCursor(query, dsquery, orders, index_list, list(db_cursor)) finally: self._ReleaseConnection(conn) return cursor
def _GetQueryCursor(self, query, filters, orders, index_list): """Returns a query cursor for the provided query. Args: conn: The SQLite connection. query: A datastore_pb.Query protocol buffer. Returns: A QueryCursor object. """ if query.has_kind() and query.kind() in self._pseudo_kinds: cursor = self._pseudo_kinds[query.kind()].Query( query, filters, orders) datastore_stub_util.Check( cursor, 'Could not create query for pseudo-kind') else: orders = datastore_stub_util._GuessOrders(filters, orders) filter_info = self.__GenerateFilterInfo(filters, query) order_info = self.__GenerateOrderInfo(orders) for strategy in DatastoreSqliteStub._QUERY_STRATEGIES: result = strategy(self, query, filter_info, order_info) if result: break else: raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'No strategy found to satisfy query.') sql_stmt, params = result if self.__verbose: logging.info("Executing statement '%s' with arguments %r", sql_stmt, [str(x) for x in params]) conn = self._GetConnection() try: db_cursor = _DedupingEntityIterator( conn.execute(sql_stmt, params)) dsquery = datastore_stub_util._MakeQuery( query, filters, orders) cursor = datastore_stub_util.IteratorCursor( query, dsquery, orders, index_list, db_cursor) finally: self._ReleaseConnection(conn) return cursor
def _AllocateSequentialIds(self, reference, size=1, max_id=None): conn = self._GetConnection() try: datastore_stub_util.CheckAppId(self._trusted, self._app_id, reference.app()) datastore_stub_util.Check(not (size and max_id), 'Both size and max cannot be set.') prefix = self._GetTablePrefix(reference) if size: start, end = self.__AllocateIdsFromBlock( conn, prefix, size, self.__id_map_sequential, 'IdSeq') else: start, end = self.__AdvanceIdCounter(conn, prefix, max_id, 'IdSeq') return long(start), long(end) finally: self._ReleaseConnection(conn)
def _AllocateIds(self, keys): self.__id_lock.acquire() full_keys = [] try: for key in keys: last_element = _FinalElement(key) datastore_stub_util.Check(not last_element.has_name(), 'Cannot allocate named key.') if last_element.id(): self._SetMaxId(last_element.id()) else: id_space = datastore_stub_util.SCATTERED count = self._IdCounter(id_space) last_element.set_id( datastore_stub_util.ToScatteredId(count)) self._SetIdCounter(id_space, count + 1) full_keys.append(key) finally: self.__id_lock.release() return full_keys
def _GetQueryCursor(self, query, filters, orders, index_list, filter_predicate=None): """Returns a query cursor for the provided query. Args: query: The datastore_pb.Query to run. filters: A list of filters that override the ones found on query. orders: A list of orders that override the ones found on query. index_list: A list of indexes used by the query. filter_predicate: an additional filter of type datastore_query.FilterPredicate. This is passed along to implement V4 specific filters without changing the entire stub. Returns: A QueryCursor object. """ if query.has_kind() and query.kind() in self._pseudo_kinds: cursor = self._pseudo_kinds[query.kind()].Query( query, filters, orders) datastore_stub_util.Check( cursor, 'Could not create query for pseudo-kind') else: orders = datastore_stub_util._GuessOrders(filters, orders) filter_info = self.__GenerateFilterInfo(filters, query) order_info = self.__GenerateOrderInfo(orders) for strategy in DatastoreSqliteStub._QUERY_STRATEGIES: result = strategy(self, query, filter_info, order_info) if result: break else: raise apiproxy_errors.ApplicationError( datastore_pb.Error.BAD_REQUEST, 'No strategy found to satisfy query.') sql_stmt, params = result conn = self._GetConnection() try: if query.property_name_list(): db_cursor = _ProjectionPartialEntityGenerator( conn.execute(sql_stmt, params)) else: db_cursor = _DedupingEntityGenerator( conn.execute(sql_stmt, params)) dsquery = datastore_stub_util._MakeQuery( query, filters, orders, filter_predicate) filtered_entities = [r for r in db_cursor] if filter_predicate: filtered_entities = list( filter(filter_predicate, filtered_entities)) cursor = datastore_stub_util.ListCursor( query, dsquery, orders, index_list, filtered_entities) finally: self._ReleaseConnection(conn) return cursor