def _onContinue(self, load): id = load['id'] query_generator = None with self.generator_lock: val = PagedAPIHandler.generator_pool[id] if val is not None: query_generator, _ = val finish = False load = [] error = None if query_generator is None: finish = True else: try: load = next(query_generator) except StopIteration: finish = True with PagedAPIHandler.generator_lock: del PagedAPIHandler.generator_pool[id] app_log.log(INFO, 'Successfully finished query %s', id) except Exception as err: finish = True error = str(err) with PagedAPIHandler.generator_lock: del PagedAPIHandler.generator_pool[id] app_log.log(WARN, 'Failed continue fetching for query %s: %s', id, error) self.finish({ 'finish': json.dumps(finish), 'load': json.dumps(load), 'error': json.dumps(error) })
def _onCancel(self, load): job_id = load['id'] error = 'job not found' with PagedAPIHandler.generator_lock: val = PagedAPIHandler.generator_pool[job_id] if val is not None: _, job, _ = val self.cancel(job) del PagedAPIHandler.generator_pool[job_id] app_log.log(INFO, 'Successfully canceled query %s', job_id) error = None self.finish({'id': json.dumps(job_id), 'error': json.dumps(error)})
def clear_generators(self): with PagedAPIHandler.generator_lock: job_ids_to_delete = [] for job_id, (generator, job_obj, last_touch) in PagedAPIHandler.generator_pool.items(): idle_time = time.time() - last_touch if idle_time >= CLEAR_GENERATORS_MAX_IDILE_SEC: self.cancel(job_obj) job_ids_to_delete.append(job_id) del generator app_log.log(INFO, 'Deleted {} due to long idle time {}', job_id, idle_time) for job_id in job_ids_to_delete: del PagedAPIHandler.generator_pool[job_id]
def _onStart(self, load): request_body = load['requestBody'] page_size = load['pageSize'] query_generator = self.query(request_body, page_size) id = None error = None try: job, id = next(query_generator) except Exception as err: error = str(err) app_log.log(WARN, 'Failed started query: %s', error) if error is None: with PagedAPIHandler.generator_lock: PagedAPIHandler.generator_pool[id] = query_generator, job app_log.log(INFO, 'Successfully started query %s', id) self.finish({'id': json.dumps(id), 'error': json.dumps(error)})