def update(self, remote_ip, uptime):
self.last_seen = datetime.utcnow()
if self.last_ip != remote_ip:
LogEntry.log_event(self.key(), 'Info', 'IP changed - new IP: ' + remote_ip)
self.last_ip = remote_ip
if uptime is not None:
if self.update is not None and self.uptime > uptime:
LogEntry.log_event(self.key(), 'Reboot',
'Reboot - Previous uptime: ' + str(timedelta(seconds=self.uptime)))
for action_key in self.reboot_actions:
try:
db.get(action_key).perform_action()
except Exception as exp:
logging.error('Error executing reboot action: ' + str(exp))
self.uptime = uptime
self.put()
# job got back online
if self.status == 'offline':
self.status = 'online'
LogEntry.log_event(self.key(), 'Info', 'Job back online - IP: ' + remote_ip)
# perform all back_online actions
for action_key in self.backonline_actions:
try:
db.get(action_key).perform_action()
except Exception as exp:
logging.error('Error executing backonline action: ' + str(exp))
# delete previous (waiting) task
if self.task_name is not None:
logging.debug('old task: ' + self.task_name)
Queue.delete_tasks(Queue(), Task(name=self.task_name))
task_name = self.name + '_' + datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
# create task to be executed in updated no called in interval minutes
taskqueue.add(name=task_name, url='/task', params={'key': self.key()}, countdown=(self.interval + 2) * 60)
self.task_name = task_name
self.put()
class MessageIterator(object):
"""This iterator will return a batch of messages for a given group.
This iterator should be directly used when trying to avoid the lease
operation inside a transaction, or when other flows are needed.
"""
def __init__(self,
tag,
queue_name,
size,
duration=60,
deadline=10,
auto_delete=True):
"""The generator will yield json deserialized payloads from tasks with
the corresponding tag.
:param tag: :class: `str` Pull queue tag to query against
:param queue_name: :class: `str` Name of PULL queue holding tasks to
lease.
:param size: :class: `int` The number of items to pull at once
:param duration: :class: `int` After this time, the tasks may be leased
again. Tracked in seconds
:param deadline: :class: `int` The time in seconds to wait for the rpc.
:param auto_delete: :class: `bool` Delete tasks when iteration is
complete.
:return: :class: `iterator` of json deserialized payloads
"""
from google.appengine.api.taskqueue import Queue
self.queue_name = queue_name
self.queue = Queue(name=self.queue_name)
self.tag = tag
self.size = size
self.duration = duration
self.auto_delete = auto_delete
self.deadline = deadline
self._messages = []
self._processed_messages = []
self._fetched = False
def fetch_messages(self):
"""Fetch messages from the specified pull-queue.
This should only be called a single time by a given MessageIterator
object. If the MessageIterator is iterated over again, it should
return the originally leased messages.
"""
if self._fetched:
return
start = time.time()
loaded_messages = self.queue.lease_tasks_by_tag(self.duration,
self.size,
tag=self.tag,
deadline=self.deadline)
# If we are within 0.1 sec of our deadline and no messages were
# returned, then we are hitting queue contention issues and this
# should be a DeadlineExceederError.
# TODO: investigate other ways around this, perhaps async leases, etc.
if (not loaded_messages
and round(time.time() - start, 1) >= self.deadline - 0.1):
raise DeadlineExceededError()
self._messages.extend(loaded_messages)
self._fetched = True
logging.debug("Calling fetch messages with %s:%s:%s:%s:%s:%s" % (len(
self._messages), len(loaded_messages), len(
self._processed_messages), self.duration, self.size, self.tag))
def __iter__(self):
"""Initialize this MessageIterator for iteration.
If messages have not been fetched, fetch them. If messages have been
fetched, reset self._messages and self._processed_messages for
re-iteration. The reset is done to prevent deleting messages that were
never applied.
"""
if self._processed_messages:
# If the iterator is used within a transaction, and there is a
# retry we need to re-process the original messages, not new
# messages.
self._messages = list(
set(self._messages) | set(self._processed_messages))
self._processed_messages = []
if not self._messages:
self.fetch_messages()
return self
def next(self):
"""Get the next batch of messages from the previously fetched messages.
If there's no more messages, check if we should auto-delete the
messages and raise StopIteration.
"""
if not self._messages:
if self.auto_delete:
self.delete_messages()
raise StopIteration
message = self._messages.pop(0)
self._processed_messages.append(message)
return json.loads(message.payload)
def delete_messages(self, only_processed=True):
"""Delete the messages previously leased.
Unless otherwise directed, only the messages iterated over will be
deleted.
"""
messages = self._processed_messages
if not only_processed:
messages += self._messages
if messages:
try:
self.queue.delete_tasks(messages)
except Exception:
logging.exception("Error deleting messages")
raise
class MessageIterator(object):
"""This iterator will return a batch of messages for a given group.
This iterator should be directly used when trying to avoid the lease
operation inside a transaction, or when other flows are needed.
"""
def __init__(self, tag, queue_name, size, duration=60, deadline=10,
auto_delete=True):
"""The generator will yield json deserialized payloads from tasks with
the corresponding tag.
:param tag: :class: `str` Pull queue tag to query against
:param queue_name: :class: `str` Name of PULL queue holding tasks to
lease.
:param size: :class: `int` The number of items to pull at once
:param duration: :class: `int` After this time, the tasks may be leased
again. Tracked in seconds
:param deadline: :class: `int` The time in seconds to wait for the rpc.
:param auto_delete: :class: `bool` Delete tasks when iteration is
complete.
:return: :class: `iterator` of json deserialized payloads
"""
from google.appengine.api.taskqueue import Queue
self.queue_name = queue_name
self.queue = Queue(name=self.queue_name)
self.tag = tag
self.size = size
self.duration = duration
self.auto_delete = auto_delete
self.deadline = deadline
self._messages = []
self._processed_messages = []
self._fetched = False
def fetch_messages(self):
"""Fetch messages from the specified pull-queue.
This should only be called a single time by a given MessageIterator
object. If the MessageIterator is iterated over again, it should
return the originally leased messages.
"""
if self._fetched:
return
start = time.time()
loaded_messages = self.queue.lease_tasks_by_tag(
self.duration, self.size, tag=self.tag, deadline=self.deadline)
# If we are within 0.1 sec of our deadline and no messages were
# returned, then we are hitting queue contention issues and this
# should be a DeadlineExceederError.
# TODO: investigate other ways around this, perhaps async leases, etc.
if (not loaded_messages and
round(time.time() - start, 1) >= self.deadline - 0.1):
raise DeadlineExceededError()
self._messages.extend(loaded_messages)
self._fetched = True
logging.debug("Calling fetch messages with %s:%s:%s:%s:%s:%s" % (
len(self._messages), len(loaded_messages),
len(self._processed_messages), self.duration, self.size, self.tag))
def __iter__(self):
"""Initialize this MessageIterator for iteration.
If messages have not been fetched, fetch them. If messages have been
fetched, reset self._messages and self._processed_messages for
re-iteration. The reset is done to prevent deleting messages that were
never applied.
"""
if self._processed_messages:
# If the iterator is used within a transaction, and there is a
# retry we need to re-process the original messages, not new
# messages.
self._messages = list(
set(self._messages) | set(self._processed_messages))
self._processed_messages = []
if not self._messages:
self.fetch_messages()
return self
def next(self):
"""Get the next batch of messages from the previously fetched messages.
If there's no more messages, check if we should auto-delete the
messages and raise StopIteration.
"""
if not self._messages:
if self.auto_delete:
self.delete_messages()
raise StopIteration
message = self._messages.pop(0)
self._processed_messages.append(message)
return json.loads(message.payload)
def delete_messages(self, only_processed=True):
"""Delete the messages previously leased.
Unless otherwise directed, only the messages iterated over will be
deleted.
"""
messages = self._processed_messages
if not only_processed:
messages += self._messages
if messages:
try:
self.queue.delete_tasks(messages)
except Exception:
logging.exception("Error deleting messages")
raise
