def task_set_output(self, s, num_retries=5): for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0): try: return api('v1').job_tasks().update(uuid=self['uuid'], body={ 'output': s, 'success': True, 'progress': 1.0 }).execute() except errors.ApiError as error: if retry.check_http_response_success( error.resp.status) is None and tries_left > 0: logger.debug( "task_set_output: job_tasks().update() raised {}, retrying with {} tries left" .format(repr(error), tries_left)) else: raise
def on_closed(self): if not self.is_closed.is_set(): _logger.warn("Unexpected close. Reconnecting.") for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=15): try: self._setup_event_client() _logger.warn("Reconnect successful.") break except Exception as e: _logger.warn("Error '%s' during websocket reconnect.", e) if tries_left == 0: _logger.exception( "EventClient thread could not contact websocket server.") self.is_closed.set() thread.interrupt_main() return
def current_job(num_retries=5): global _current_job if _current_job: return _current_job for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2): try: job = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute() job = UserDict.UserDict(job) job.tmpdir = os.environ['JOB_WORK'] _current_job = job return job except errors.ApiError as error: if retry.check_http_response_success( error.resp.status) is None and tries_left > 0: logger.debug( "current_job: jobs().get() raised {}, retrying with {} tries left" .format(repr(error), tries_left)) else: raise
def current_task(num_retries=5): global _current_task if _current_task: return _current_task for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2): try: task = api('v1').job_tasks().get( uuid=os.environ['TASK_UUID']).execute() task = UserDict.UserDict(task) task.set_output = types.MethodType(task_set_output, task) task.tmpdir = os.environ['TASK_WORK'] _current_task = task return task except errors.ApiError as error: if retry.check_http_response_success( error.resp.status) is None and tries_left > 0: logger.debug( "current_task: job_tasks().get() raised {}, retrying with {} tries left" .format(repr(error), tries_left)) else: raise
def run(self): if self.last_log_id != None: # Caller supplied the last-seen event ID from a previous # connection skip_old_events = [["id", ">", str(self.last_log_id)]] else: # We need to do a reverse-order query to find the most # recent event ID (see "if not skip_old_events" below). skip_old_events = False self.on_event({'status': 200}) while not self._closing.is_set(): moreitems = False for f in self.filters: for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=self.poll_time): try: if not skip_old_events: # If the caller didn't provide a known # recent ID, our first request will ask # for the single most recent event from # the last 2 hours (the time restriction # avoids doing an expensive database # query, and leaves a big enough margin to # account for clock skew). If we do find a # recent event, we remember its ID but # then discard it (we are supposed to be # returning new/current events, not old # ones). # # Subsequent requests will get multiple # events in chronological order, and # filter on that same cutoff time, or # (once we see our first matching event) # the ID of the last-seen event. skip_old_events = [[ "created_at", ">=", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 7200)) ]] items = self.api.logs().list( order="id desc", limit=1, filters=f + skip_old_events).execute() if items["items"]: skip_old_events = [[ "id", ">", str(items["items"][0]["id"]) ]] items = { "items": [], "items_available": 0, } else: # In this case, either we know the most # recent matching ID, or we know there # were no matching events in the 2-hour # window before subscribing. Either way we # can safely ask for events in ascending # order. items = self.api.logs().list( order="id asc", filters=f + skip_old_events).execute() break except errors.ApiError as error: pass else: tries_left = 0 break if tries_left == 0: _logger.exception( "PollClient thread could not contact API server.") with self._closing_lock: self._closing.set() thread.interrupt_main() return for i in items["items"]: skip_old_events = [["id", ">", str(i["id"])]] with self._closing_lock: if self._closing.is_set(): return try: self.on_event(i) except Exception as e: _logger.exception( "Unexpected exception from event callback.") thread.interrupt_main() if items["items_available"] > len(items["items"]): moreitems = True if not moreitems: self._closing.wait(self.poll_time)
def run(self): self.id = 0 if self.last_log_id != None: self.id = self.last_log_id else: for f in self.filters: for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=self.poll_time): try: items = self.api.logs().list( limit=1, order="id desc", filters=f).execute()['items'] break except errors.ApiError as error: pass else: tries_left = 0 break if tries_left == 0: _logger.exception( "PollClient thread could not contact API server.") with self._closing_lock: self._closing.set() thread.interrupt_main() return if items: if items[0]['id'] > self.id: self.id = items[0]['id'] self.on_event({'status': 200}) while not self._closing.is_set(): max_id = self.id moreitems = False for f in self.filters: for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=self.poll_time): try: items = self.api.logs().list( order="id asc", filters=f + [["id", ">", str(self.id)]]).execute() break except errors.ApiError as error: pass else: tries_left = 0 break if tries_left == 0: _logger.exception( "PollClient thread could not contact API server.") with self._closing_lock: self._closing.set() thread.interrupt_main() return for i in items["items"]: if i['id'] > max_id: max_id = i['id'] with self._closing_lock: if self._closing.is_set(): return try: self.on_event(i) except Exception as e: _logger.exception( "Unexpected exception from event callback.") thread.interrupt_main() if items["items_available"] > len(items["items"]): moreitems = True self.id = max_id if not moreitems: self._closing.wait(self.poll_time)