def async_queries(queries, limit=100000):
task_runner = AsyncMultiTask()
for query in queries:
task_runner.append(QueryTask(query, limit=limit))
task_runner.run()
return task_runner
def async_queries(queries, limit=100000):
task_runner = AsyncMultiTask()
for query in queries:
task_runner.append(QueryTask(query, limit=limit))
task_runner.run()
return task_runner
def _fetch(cls, requests):
"""
Performs the memcache and data store fetches for requests
whose data isn't in the local cache.
"""
if not requests:
return
cls.listener.on_fetch(requests)
mkeys = requests.keys()
# first, try to fetch the keys from memcache
results = memcache.get_multi(mkeys)
# if any of the keys couldn't be fetched from
# memcache, prepare async tasks to fetch
# their queries from the data store
runner = AsyncMultiTask()
has_tasks = False
# if there are any pending async tasks,
# execute them using the AsyncMultiTask runner.
db_fetches = {}
for mkey, request in requests.items():
if not results.get(mkey):
task = QueryTask(request.query,
limit = request.limit,
client_state = mkey)
runner.append(task)
has_tasks = True
cls.listener.on_db_request(request)
if has_tasks:
runner.run()
cls.listener.on_db_fetch()
for task in runner:
val = task.get_result()
mkey = task.client_state
# if the task only tried to fetch a single object,
# return the first element of the result list
# or none
if task.limit == 1:
if val:
val = val[0]
else:
val = None
results[mkey] = val
db_fetches[mkey] = val
# if any results were fetched from the data store,
# store them in memcache
if db_fetches:
memcache.set_multi(db_fetches)
return results
def _fetch(cls, requests):
"""
Performs the memcache and data store fetches for requests
whose data isn't in the local cache.
"""
if not requests:
return
cls.listener.on_fetch(requests)
mkeys = requests.keys()
# first, try to fetch the keys from memcache
results = memcache.get_multi(mkeys)
# if any of the keys couldn't be fetched from
# memcache, prepare async tasks to fetch
# their queries from the data store
runner = AsyncMultiTask()
has_tasks = False
# if there are any pending async tasks,
# execute them using the AsyncMultiTask runner.
db_fetches = {}
for mkey, request in requests.items():
if not results.get(mkey):
task = QueryTask(request.query,
limit=request.limit,
client_state=mkey)
runner.append(task)
has_tasks = True
cls.listener.on_db_request(request)
if has_tasks:
runner.run()
cls.listener.on_db_fetch()
for task in runner:
val = task.get_result()
mkey = task.client_state
# if the task only tried to fetch a single object,
# return the first element of the result list
# or none
if task.limit == 1:
if val:
val = val[0]
else:
val = None
results[mkey] = val
db_fetches[mkey] = val
# if any results were fetched from the data store,
# store them in memcache
if db_fetches:
memcache.set_multi(db_fetches)
return results
