def _onContinue(self, load):
id = load['id']
query_generator = None
with self.generator_lock:
val = PagedAPIHandler.generator_pool[id]
if val is not None:
query_generator, _ = val
finish = False
load = []
error = None
if query_generator is None:
finish = True
else:
try:
load = next(query_generator)
except StopIteration:
finish = True
with PagedAPIHandler.generator_lock:
del PagedAPIHandler.generator_pool[id]
app_log.log(INFO, 'Successfully finished query %s', id)
except Exception as err:
finish = True
error = str(err)
with PagedAPIHandler.generator_lock:
del PagedAPIHandler.generator_pool[id]
app_log.log(WARN, 'Failed continue fetching for query %s: %s',
id, error)
self.finish({
'finish': json.dumps(finish),
'load': json.dumps(load),
'error': json.dumps(error)
})
def _onCancel(self, load):
job_id = load['id']
error = 'job not found'
with PagedAPIHandler.generator_lock:
val = PagedAPIHandler.generator_pool[job_id]
if val is not None:
_, job, _ = val
self.cancel(job)
del PagedAPIHandler.generator_pool[job_id]
app_log.log(INFO, 'Successfully canceled query %s', job_id)
error = None
self.finish({'id': json.dumps(job_id), 'error': json.dumps(error)})
def clear_generators(self):
with PagedAPIHandler.generator_lock:
job_ids_to_delete = []
for job_id, (generator, job_obj,
last_touch) in PagedAPIHandler.generator_pool.items():
idle_time = time.time() - last_touch
if idle_time >= CLEAR_GENERATORS_MAX_IDILE_SEC:
self.cancel(job_obj)
job_ids_to_delete.append(job_id)
del generator
app_log.log(INFO, 'Deleted {} due to long idle time {}',
job_id, idle_time)
for job_id in job_ids_to_delete:
del PagedAPIHandler.generator_pool[job_id]
def _onStart(self, load):
request_body = load['requestBody']
page_size = load['pageSize']
query_generator = self.query(request_body, page_size)
id = None
error = None
try:
job, id = next(query_generator)
except Exception as err:
error = str(err)
app_log.log(WARN, 'Failed started query: %s', error)
if error is None:
with PagedAPIHandler.generator_lock:
PagedAPIHandler.generator_pool[id] = query_generator, job
app_log.log(INFO, 'Successfully started query %s', id)
self.finish({'id': json.dumps(id), 'error': json.dumps(error)})
