def generateHealthReport(consumers, lastCanaryTime):
healthReport = {}
healthReport['last_db_canary'] = secondsSince(lastCanaryTime)
healthReport['uptime'] = getUpdateTime()
healthReport['cpu_times'] = getCPUTimes()
healthReport['cpu_percent'] = getCPUPercent()
healthReport['virtual_memory'] = getVirtualMemory()
healthReport['swap_memory'] = getSwapMemory()
healthReport['disk_usage'] = getDiskUsage()
healthReport['disk_io_counters'] = getDiskIOCounters()
healthReport['net_io_counters'] = getNetworkIOCounters()
healthReport['consumers'] = getConsumers(consumers)
return healthReport
def secondsSinceLastPoll(self):
return secondsSince(self.lastPoll())
def __handleDocChange(self, change):
retry = True
retryCount = 0
maxRetries = 5
while retry:
try:
if "deleted" in change and change["deleted"] == True:
logging.info('[changes] Found a delete')
consumer = self.consumers.getConsumerForTrigger(change['id'])
if consumer != None:
if consumer.desiredState() == Consumer.State.Disabled:
# just remove it from memory
logging.info('[{}] Removing disabled trigger'.format(consumer.trigger))
self.consumers.removeConsumerForTrigger(consumer.trigger)
else:
logging.info('[{}] Shutting down running trigger'.format(consumer.trigger))
consumer.shutdown()
# since we can't use a filter function for the feed (then
# you don't get deletes) we need to manually verify this
# is a valid trigger doc that has changed
elif 'triggerURL' in change['doc']:
logging.info('[changes] Found a change in a trigger document')
document = change['doc']
triggerIsAssignedToMe = self.__isTriggerDocAssignedToMe(document)
if not self.consumers.hasConsumerForTrigger(change["id"]):
if triggerIsAssignedToMe:
logging.info('[{}] Found a new trigger to create'.format(change["id"]))
self.createAndRunConsumer(document)
else:
logging.info("[{}] Found a new trigger, but is assigned to another worker: {}".format(change["id"], document["worker"]))
else:
existingConsumer = self.consumers.getConsumerForTrigger(change["id"])
if existingConsumer.desiredState() == Consumer.State.Running and not self.__isTriggerDocActive(document):
# running trigger should become disabled
# this should be done regardless of which worker the document claims to be assigned to
logging.info('[{}] Existing running trigger should become disabled'.format(change["id"]))
existingConsumer.disable()
elif triggerIsAssignedToMe:
logging.info('[{}] Found a change to an existing trigger'.format(change["id"]))
if existingConsumer.desiredState() == Consumer.State.Dead and self.__isTriggerDocActive(document):
# if a delete occurs followed quickly by a create the consumer might get stuck in a dead state,
# so we need to forcefully delete the process before recreating it.
logging.info('[{}] A create event occurred for a trigger that is shutting down'.format(change["id"]))
if existingConsumer.process.is_alive():
logging.info('[{}] Joining dead process.'.format(existingConsumer.trigger))
existingConsumer.process.join(1)
else:
logging.info('[{}] Process is already dead.'.format(existingConsumer.trigger))
self.consumers.removeConsumerForTrigger(existingConsumer.trigger)
self.createAndRunConsumer(document)
elif existingConsumer.desiredState() == Consumer.State.Disabled and self.__isTriggerDocActive(document):
# disabled trigger has become active
logging.info('[{}] Existing disabled trigger should become active'.format(change["id"]))
self.createAndRunConsumer(document)
else:
# trigger has become reassigned to a different worker
logging.info("[{}] Shutting down trigger as it has been re-assigned to {}".format(change["id"], document["worker"]))
existingConsumer.shutdown()
elif 'canary-timestamp' in change['doc']:
# found a canary - update lastCanaryTime
logging.info('[canary] I found a canary. The last one was {} seconds ago.'.format(secondsSince(self.lastCanaryTime)))
self.lastCanaryTime = datetime.now()
else:
logging.debug('[changes] Found a change for a non-trigger document')
retry = False
except Exception as e:
logging.error('[{}] Exception caught while handling change.'.format(change["id"]))
logging.error(e)
if retry:
retryCount += 1
if retryCount >= maxRetries:
logging.warn('[{}] Maximum number of retries exceeded for failed change.'.format(change["id"]))
retry = False
def run(self):
self.canaryGenerator.start()
while True:
try:
if self.database is not None:
logging.info("Shutting down existing DB client")
self.database.destroy()
logging.info("Starting changes feed")
self.database = Database(timeout=changesFeedTimeout)
self.changes = self.database.changesFeed(timeout=changesFeedTimeout, since=self.lastSequence)
self.lastCanaryTime = datetime.now()
for change in self.changes:
# change could be None because the changes feed will timeout
# if it hasn't detected any changes. This timeout allows us to
# check whether or not the feed is capable of detecting canary
# documents
if change != None:
# Record the sequence in case the changes feed needs to be
# restarted. This way the new feed can pick up right where
# the old one left off.
self.lastSequence = change['seq']
if "deleted" in change and change["deleted"] == True:
logging.info('[changes] Found a delete')
consumer = self.consumers.getConsumerForTrigger(change['id'])
if consumer != None:
if consumer.desiredState() == Consumer.State.Disabled:
# just remove it from memory
logging.info('[{}] Removing disabled trigger'.format(consumer.trigger))
self.consumers.removeConsumerForTrigger(consumer.trigger)
else:
logging.info('[{}] Shutting down running trigger'.format(consumer.trigger))
consumer.shutdown()
# since we can't use a filter function for the feed (then
# you don't get deletes) we need to manually verify this
# is a valid trigger doc that has changed
elif 'triggerURL' in change['doc']:
logging.info('[changes] Found a change in a trigger document')
document = change['doc']
triggerIsAssignedToMe = self.__isTriggerDocAssignedToMe(document)
if not self.consumers.hasConsumerForTrigger(change["id"]):
if triggerIsAssignedToMe:
logging.info('[{}] Found a new trigger to create'.format(change["id"]))
self.createAndRunConsumer(document)
else:
logging.info("[{}] Found a new trigger, but is assigned to another worker: {}".format(change["id"], document["worker"]))
else:
existingConsumer = self.consumers.getConsumerForTrigger(change["id"])
if existingConsumer.desiredState() == Consumer.State.Running and not self.__isTriggerDocActive(document):
# running trigger should become disabled
# this should be done regardless of which worker the document claims to be assigned to
logging.info('[{}] Existing running trigger should become disabled'.format(change["id"]))
existingConsumer.disable()
elif triggerIsAssignedToMe:
logging.info('[{}] Found a change to an existing trigger'.format(change["id"]))
if existingConsumer.desiredState() == Consumer.State.Disabled and self.__isTriggerDocActive(document):
# disabled trigger has become active
logging.info('[{}] Existing disabled trigger should become active'.format(change["id"]))
self.createAndRunConsumer(document)
else:
# trigger has become reassigned to a different worker
logging.info("[{}] Shutting down trigger as it has been re-assigned to {}".format(change["id"], document["worker"]))
existingConsumer.shutdown()
elif 'canary-timestamp' in change['doc']:
# found a canary - update lastCanaryTime
logging.info('[canary] I found a canary. The last one was {} seconds ago.'.format(secondsSince(self.lastCanaryTime)))
self.lastCanaryTime = datetime.now()
else:
logging.debug('[changes] Found a change for a non-trigger document')
except Exception as e:
logging.error('[canary] Exception caught from changes feed. Restarting changes feed...')
logging.error(e)
self.stopChangesFeed()
logging.debug("[changes] I made it out of the changes loop!")
