def task_set_output(self, s, num_retries=5):
for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0):
try:
return api('v1').job_tasks().update(uuid=self['uuid'],
body={
'output': s,
'success': True,
'progress': 1.0
}).execute()
except errors.ApiError as error:
if retry.check_http_response_success(
error.resp.status) is None and tries_left > 0:
logger.debug(
"task_set_output: job_tasks().update() raised {}, retrying with {} tries left"
.format(repr(error), tries_left))
else:
raise
def on_closed(self):
if not self.is_closed.is_set():
_logger.warn("Unexpected close. Reconnecting.")
for tries_left in RetryLoop(num_retries=25,
backoff_start=.1,
max_wait=15):
try:
self._setup_event_client()
_logger.warn("Reconnect successful.")
break
except Exception as e:
_logger.warn("Error '%s' during websocket reconnect.", e)
if tries_left == 0:
_logger.exception(
"EventClient thread could not contact websocket server.")
self.is_closed.set()
thread.interrupt_main()
return
def current_job(num_retries=5):
global _current_job
if _current_job:
return _current_job
for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
try:
job = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute()
job = UserDict.UserDict(job)
job.tmpdir = os.environ['JOB_WORK']
_current_job = job
return job
except errors.ApiError as error:
if retry.check_http_response_success(
error.resp.status) is None and tries_left > 0:
logger.debug(
"current_job: jobs().get() raised {}, retrying with {} tries left"
.format(repr(error), tries_left))
else:
raise
def current_task(num_retries=5):
global _current_task
if _current_task:
return _current_task
for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
try:
task = api('v1').job_tasks().get(
uuid=os.environ['TASK_UUID']).execute()
task = UserDict.UserDict(task)
task.set_output = types.MethodType(task_set_output, task)
task.tmpdir = os.environ['TASK_WORK']
_current_task = task
return task
except errors.ApiError as error:
if retry.check_http_response_success(
error.resp.status) is None and tries_left > 0:
logger.debug(
"current_task: job_tasks().get() raised {}, retrying with {} tries left"
.format(repr(error), tries_left))
else:
raise
def run(self):
if self.last_log_id != None:
# Caller supplied the last-seen event ID from a previous
# connection
skip_old_events = [["id", ">", str(self.last_log_id)]]
else:
# We need to do a reverse-order query to find the most
# recent event ID (see "if not skip_old_events" below).
skip_old_events = False
self.on_event({'status': 200})
while not self._closing.is_set():
moreitems = False
for f in self.filters:
for tries_left in RetryLoop(num_retries=25,
backoff_start=.1,
max_wait=self.poll_time):
try:
if not skip_old_events:
# If the caller didn't provide a known
# recent ID, our first request will ask
# for the single most recent event from
# the last 2 hours (the time restriction
# avoids doing an expensive database
# query, and leaves a big enough margin to
# account for clock skew). If we do find a
# recent event, we remember its ID but
# then discard it (we are supposed to be
# returning new/current events, not old
# ones).
#
# Subsequent requests will get multiple
# events in chronological order, and
# filter on that same cutoff time, or
# (once we see our first matching event)
# the ID of the last-seen event.
skip_old_events = [[
"created_at", ">=",
time.strftime("%Y-%m-%dT%H:%M:%SZ",
time.gmtime(time.time() - 7200))
]]
items = self.api.logs().list(
order="id desc",
limit=1,
filters=f + skip_old_events).execute()
if items["items"]:
skip_old_events = [[
"id", ">",
str(items["items"][0]["id"])
]]
items = {
"items": [],
"items_available": 0,
}
else:
# In this case, either we know the most
# recent matching ID, or we know there
# were no matching events in the 2-hour
# window before subscribing. Either way we
# can safely ask for events in ascending
# order.
items = self.api.logs().list(
order="id asc",
filters=f + skip_old_events).execute()
break
except errors.ApiError as error:
pass
else:
tries_left = 0
break
if tries_left == 0:
_logger.exception(
"PollClient thread could not contact API server.")
with self._closing_lock:
self._closing.set()
thread.interrupt_main()
return
for i in items["items"]:
skip_old_events = [["id", ">", str(i["id"])]]
with self._closing_lock:
if self._closing.is_set():
return
try:
self.on_event(i)
except Exception as e:
_logger.exception(
"Unexpected exception from event callback.")
thread.interrupt_main()
if items["items_available"] > len(items["items"]):
moreitems = True
if not moreitems:
self._closing.wait(self.poll_time)
def run(self):
self.id = 0
if self.last_log_id != None:
self.id = self.last_log_id
else:
for f in self.filters:
for tries_left in RetryLoop(num_retries=25,
backoff_start=.1,
max_wait=self.poll_time):
try:
items = self.api.logs().list(
limit=1, order="id desc",
filters=f).execute()['items']
break
except errors.ApiError as error:
pass
else:
tries_left = 0
break
if tries_left == 0:
_logger.exception(
"PollClient thread could not contact API server.")
with self._closing_lock:
self._closing.set()
thread.interrupt_main()
return
if items:
if items[0]['id'] > self.id:
self.id = items[0]['id']
self.on_event({'status': 200})
while not self._closing.is_set():
max_id = self.id
moreitems = False
for f in self.filters:
for tries_left in RetryLoop(num_retries=25,
backoff_start=.1,
max_wait=self.poll_time):
try:
items = self.api.logs().list(
order="id asc",
filters=f + [["id", ">", str(self.id)]]).execute()
break
except errors.ApiError as error:
pass
else:
tries_left = 0
break
if tries_left == 0:
_logger.exception(
"PollClient thread could not contact API server.")
with self._closing_lock:
self._closing.set()
thread.interrupt_main()
return
for i in items["items"]:
if i['id'] > max_id:
max_id = i['id']
with self._closing_lock:
if self._closing.is_set():
return
try:
self.on_event(i)
except Exception as e:
_logger.exception(
"Unexpected exception from event callback.")
thread.interrupt_main()
if items["items_available"] > len(items["items"]):
moreitems = True
self.id = max_id
if not moreitems:
self._closing.wait(self.poll_time)