def _AllocateIds(self, reference, size=1, max_id=None):
conn = self._GetConnection()
try:
datastore_stub_util.CheckAppId(reference.app(), self._trusted,
self._app_id)
datastore_stub_util.Check(not (size and max_id),
'Both size and max cannot be set.')
prefix = self._GetTablePrefix(reference)
if size:
datastore_stub_util.Check(size > 0,
'Size must be greater than 0.')
next_id, block_size = self.__id_map.get(prefix, (0, 0))
if not block_size:
block_size = (size / 1000 + 1) * 1000
c = conn.execute(
'SELECT next_id FROM IdSeq WHERE prefix = ? LIMIT 1',
(prefix, ))
next_id = c.fetchone()[0]
c = conn.execute(
'UPDATE IdSeq SET next_id = next_id + ? WHERE prefix = ?',
(block_size, prefix))
assert c.rowcount == 1
if size > block_size:
c = conn.execute(
'SELECT next_id FROM IdSeq WHERE prefix = ? LIMIT 1',
(prefix, ))
start = c.fetchone()[0]
c = conn.execute(
'UPDATE IdSeq SET next_id = next_id + ? WHERE prefix = ?',
(size, prefix))
assert c.rowcount == 1
else:
start = next_id
next_id += size
block_size -= size
self.__id_map[prefix] = (next_id, block_size)
end = start + size - 1
else:
datastore_stub_util.Check(
max_id >= 0, 'Max must be greater than or equal to 0.')
c = conn.execute(
'SELECT next_id FROM IdSeq WHERE prefix = ? LIMIT 1',
(prefix, ))
start = c.fetchone()[0]
if max_id and max_id >= start:
c = conn.execute(
'UPDATE IdSeq SET next_id = ? WHERE prefix = ?',
(max_id + 1, prefix))
assert c.rowcount == 1
end = max(max_id, start - 1)
return (long(start), long(end))
finally:
self._ReleaseConnection(conn)
def __AllocateIdsFromBlock(self, conn, prefix, size, id_map, table):
datastore_stub_util.Check(size > 0, 'Size must be greater than 0.')
next_id, block_size = id_map.get(prefix, (0, 0))
if not block_size:
block_size = (size / 1000 + 1) * 1000
c = conn.execute(
'SELECT next_id FROM %s WHERE prefix = ? LIMIT 1' % table,
(prefix, ))
next_id = c.fetchone()[0]
c = conn.execute(
'UPDATE %s SET next_id = next_id + ? WHERE prefix = ?' % table,
(block_size, prefix))
assert c.rowcount == 1
if size > block_size:
c = conn.execute(
'SELECT next_id FROM %s WHERE prefix = ? LIMIT 1' % table,
(prefix, ))
start = c.fetchone()[0]
c = conn.execute(
'UPDATE %s SET next_id = next_id + ? WHERE prefix = ?' % table,
(size, prefix))
assert c.rowcount == 1
else:
start = next_id
next_id += size
block_size -= size
id_map[prefix] = (next_id, block_size)
end = start + size - 1
return start, end
def _AllocateIds(self, references):
conn = self._GetConnection()
try:
full_keys = []
for key in references:
datastore_stub_util.CheckAppId(self._trusted, self._app_id,
key.app())
prefix = self._GetTablePrefix(key)
last_element = key.path().element_list()[-1]
datastore_stub_util.Check(not last_element.has_name(),
'Cannot allocate named key.')
if last_element.id():
count, id_space = datastore_stub_util.IdToCounter(
last_element.id())
table, _ = self.__id_counter_tables[id_space]
self.__AdvanceIdCounter(conn, prefix, count, table)
else:
count, _ = self.__AllocateIdsFromBlock(
conn, prefix, 1, self.__id_map_scattered,
'ScatteredIdCounters')
last_element.set_id(
datastore_stub_util.ToScatteredId(count))
full_keys.append(key)
return full_keys
finally:
self._ReleaseConnection(conn)
def _AllocateIds(self, reference, size=1, max_id=None):
datastore_stub_util.Check(not (size and max_id),
'Both size and max cannot be set.')
self.__id_lock.acquire()
try:
start = self.__next_id
if size:
datastore_stub_util.Check(size > 0, 'Size must be greater than 0.')
self.__next_id += size
elif max_id:
datastore_stub_util.Check(max_id >=0,
'Max must be greater than or equal to 0.')
self.__next_id = max(self.__next_id, max_id + 1)
end = self.__next_id - 1
finally:
self.__id_lock.release()
return (start, end)
def MakeSyncCall(self, service, call, request, response):
"""The main RPC entry point. service must be 'datastore_v3'."""
self.AssertPbIsInitialized(request)
try:
apiproxy_stub.APIProxyStub.MakeSyncCall(self, service, call,
request, response)
except sqlite3.OperationalError, e:
datastore_stub_util.Check(e.args[0] == 'database is locked',
'Database is locked.',
datastore_pb.Error.TIMEOUT)
raise
def _AllocateSequentialIds(self, reference, size=1, max_id=None):
datastore_stub_util.Check(not (size and max_id),
'Both size and max cannot be set.')
self.__id_lock.acquire()
try:
id_space = datastore_stub_util.SEQUENTIAL
start = self._IdCounter(id_space)
if size:
datastore_stub_util.Check(size > 0, 'Size must be greater than 0.')
self._SetIdCounter(id_space, start + size)
elif max_id:
datastore_stub_util.Check(max_id >=0,
'Max must be greater than or equal to 0.')
self._SetIdCounter(id_space, max(start, max_id + 1))
end = self._IdCounter(id_space) - 1
finally:
self.__id_lock.release()
return (start, end)
def __AdvanceIdCounter(self, conn, prefix, max_id, table):
datastore_stub_util.Check(max_id >= 0,
'Max must be greater than or equal to 0.')
c = conn.execute('SELECT next_id FROM %s WHERE prefix = ? LIMIT 1'
% table, (prefix,))
start = c.fetchone()[0]
if max_id >= start:
c = conn.execute('UPDATE %s SET next_id = ? WHERE prefix = ?' % table,
(max_id + 1, prefix))
assert c.rowcount == 1
end = max(max_id, start - 1)
return start, end
def _GetQueryCursor(self, query, filters, orders, index_list):
"""Returns a query cursor for the provided query.
Args:
query: The datastore_pb.Query to run.
filters: A list of filters that override the ones found on query.
orders: A list of orders that override the ones found on query.
index_list: A list of indexes used by the query.
Returns:
A QueryCursor object.
"""
if query.has_kind() and query.kind() in self._pseudo_kinds:
datastore_stub_util.NormalizeCursors(query,
datastore_pb.Query_Order.ASCENDING)
cursor = self._pseudo_kinds[query.kind()].Query(query, filters, orders)
datastore_stub_util.Check(cursor,
'Could not create query for pseudo-kind')
else:
orders = datastore_stub_util._GuessOrders(filters, orders)
datastore_stub_util.NormalizeCursors(query, orders[0].direction())
filter_info = self.__GenerateFilterInfo(filters, query)
order_info = self.__GenerateOrderInfo(orders)
for strategy in DatastoreSqliteStub._QUERY_STRATEGIES:
result = strategy(self, query, filter_info, order_info)
if result:
break
else:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No strategy found to satisfy query.')
sql_stmt, params = result
conn = self._GetConnection()
try:
if query.property_name_list():
db_cursor = _ProjectionPartialEntityGenerator(
conn.execute(sql_stmt, params))
else:
db_cursor = _DedupingEntityGenerator(conn.execute(sql_stmt, params))
dsquery = datastore_stub_util._MakeQuery(query, filters, orders)
cursor = datastore_stub_util.ListCursor(query, dsquery, orders,
index_list, list(db_cursor))
finally:
self._ReleaseConnection(conn)
return cursor
def _GetQueryCursor(self, query, filters, orders, index_list):
"""Returns a query cursor for the provided query.
Args:
conn: The SQLite connection.
query: A datastore_pb.Query protocol buffer.
Returns:
A QueryCursor object.
"""
if query.has_kind() and query.kind() in self._pseudo_kinds:
cursor = self._pseudo_kinds[query.kind()].Query(
query, filters, orders)
datastore_stub_util.Check(
cursor, 'Could not create query for pseudo-kind')
else:
orders = datastore_stub_util._GuessOrders(filters, orders)
filter_info = self.__GenerateFilterInfo(filters, query)
order_info = self.__GenerateOrderInfo(orders)
for strategy in DatastoreSqliteStub._QUERY_STRATEGIES:
result = strategy(self, query, filter_info, order_info)
if result:
break
else:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No strategy found to satisfy query.')
sql_stmt, params = result
if self.__verbose:
logging.info("Executing statement '%s' with arguments %r",
sql_stmt, [str(x) for x in params])
conn = self._GetConnection()
try:
db_cursor = _DedupingEntityIterator(
conn.execute(sql_stmt, params))
dsquery = datastore_stub_util._MakeQuery(
query, filters, orders)
cursor = datastore_stub_util.IteratorCursor(
query, dsquery, orders, index_list, db_cursor)
finally:
self._ReleaseConnection(conn)
return cursor
def _AllocateSequentialIds(self, reference, size=1, max_id=None):
conn = self._GetConnection()
try:
datastore_stub_util.CheckAppId(self._trusted, self._app_id,
reference.app())
datastore_stub_util.Check(not (size and max_id),
'Both size and max cannot be set.')
prefix = self._GetTablePrefix(reference)
if size:
start, end = self.__AllocateIdsFromBlock(
conn, prefix, size, self.__id_map_sequential, 'IdSeq')
else:
start, end = self.__AdvanceIdCounter(conn, prefix, max_id,
'IdSeq')
return long(start), long(end)
finally:
self._ReleaseConnection(conn)
def _AllocateIds(self, keys):
self.__id_lock.acquire()
full_keys = []
try:
for key in keys:
last_element = _FinalElement(key)
datastore_stub_util.Check(not last_element.has_name(),
'Cannot allocate named key.')
if last_element.id():
self._SetMaxId(last_element.id())
else:
id_space = datastore_stub_util.SCATTERED
count = self._IdCounter(id_space)
last_element.set_id(
datastore_stub_util.ToScatteredId(count))
self._SetIdCounter(id_space, count + 1)
full_keys.append(key)
finally:
self.__id_lock.release()
return full_keys
def _GetQueryCursor(self,
query,
filters,
orders,
index_list,
filter_predicate=None):
"""Returns a query cursor for the provided query.
Args:
query: The datastore_pb.Query to run.
filters: A list of filters that override the ones found on query.
orders: A list of orders that override the ones found on query.
index_list: A list of indexes used by the query.
filter_predicate: an additional filter of type
datastore_query.FilterPredicate. This is passed along to implement V4
specific filters without changing the entire stub.
Returns:
A QueryCursor object.
"""
if query.has_kind() and query.kind() in self._pseudo_kinds:
cursor = self._pseudo_kinds[query.kind()].Query(
query, filters, orders)
datastore_stub_util.Check(
cursor, 'Could not create query for pseudo-kind')
else:
orders = datastore_stub_util._GuessOrders(filters, orders)
filter_info = self.__GenerateFilterInfo(filters, query)
order_info = self.__GenerateOrderInfo(orders)
for strategy in DatastoreSqliteStub._QUERY_STRATEGIES:
result = strategy(self, query, filter_info, order_info)
if result:
break
else:
raise apiproxy_errors.ApplicationError(
datastore_pb.Error.BAD_REQUEST,
'No strategy found to satisfy query.')
sql_stmt, params = result
conn = self._GetConnection()
try:
if query.property_name_list():
db_cursor = _ProjectionPartialEntityGenerator(
conn.execute(sql_stmt, params))
else:
db_cursor = _DedupingEntityGenerator(
conn.execute(sql_stmt, params))
dsquery = datastore_stub_util._MakeQuery(
query, filters, orders, filter_predicate)
filtered_entities = [r for r in db_cursor]
if filter_predicate:
filtered_entities = list(
filter(filter_predicate, filtered_entities))
cursor = datastore_stub_util.ListCursor(
query, dsquery, orders, index_list, filtered_entities)
finally:
self._ReleaseConnection(conn)
return cursor