def filter_backups_due_for_expiration(self, backups):
earliest_date_to_keep = date_minus_seconds(date_now(), self.max_time)
return filter(lambda backup:
backup.created_date < earliest_date_to_keep,
backups)
def _read_next_failed_past_due_task(self):
min_fail_end_date = date_minus_seconds(date_now(), MAX_FAIL_DUE_TIME)
q = { "state": STATE_FAILED,
"engineGuid": self._engine.engine_guid,
"$or": [
{
"plan.nextOccurrence": {"$lte": date_now()}
},
{
"plan": {"$exists": False},
"reschedulable": False,
"endDate": {"$lte": min_fail_end_date}
}
]
}
msg = "Task failed and is past due. Cancelling..."
log_entry = state_change_log_entry(STATE_CANCELED, message=msg)
u = {"$set" : { "state" : STATE_CANCELED},
"$push": {
"logs": log_entry.to_document()
}
}
return self._task_collection.find_and_modify(query=q, update=u,
new=True)
def _read_next_failed_past_due_task(self):
min_fail_end_date = date_minus_seconds(date_now(), MAX_FAIL_DUE_TIME)
q = {
"$or": [
{
"state": State.FAILED,
"engineGuid": self._engine.engine_guid,
"nextRetryDate": None,
"finalRetryDate": {"$lte": date_now()},
"plan.nextOccurrence": {"$lte": date_now()}
},
{
"state": State.FAILED,
"engineGuid": self._engine.engine_guid,
"plan": {"$exists": False},
"nextRetryDate": None,
"finalRetryDate": {"$lte": min_fail_end_date}
}
]
}
msg = "Task failed and is past due. Cancelling..."
log_entry = state_change_log_entry(State.CANCELED, message=msg)
u = {"$set" : { "state" : State.CANCELED},
"$push": {
"logs": log_entry.to_document()
}
}
task = self.task_collection.find_one(query=q)
if task:
return self.task_collection.find_and_modify(query=q, update=u, new=True)
def _notify_on_past_due_scheduled_backups(self):
"""
Send notifications for jobs that has been scheduled for a period
longer than min(half the frequency, 5 hours) of its plan.
If backup does not have a plan (i.e. one off)
then it will check after 60 seconds.
"""
# query for backups whose scheduled date is before current date minus
# than max starvation time
where = (
"(Math.min(%s, (this.plan.schedule.frequencyInSeconds / 2) * 1000) + "
"this.createdDate.getTime()) < new Date().getTime()" %
(MAX_BACKUP_WAIT_TIME * 1000))
one_off_starve_date = date_minus_seconds(date_now(),
ONE_OFF_BACKUP_MAX_WAIT_TIME)
q = {
"state":
STATE_SCHEDULED,
"$or": [
# backups with plans starving query
{
"$and": [{
"plan": {
"$exists": True
}
}, {
"$where": where
}]
},
# One off backups (no plan) starving query
{
"$and": [{
"plan": {
"$exists": False
}
}, {
"createdDate": {
"$lt": one_off_starve_date
}
}]
}
]
}
starving_backups = get_mbs().backup_collection.find(q)
if starving_backups:
msg = ("You have %s scheduled backups that has past the maximum "
"waiting time (%s seconds)." %
(len(starving_backups), MAX_BACKUP_WAIT_TIME))
self.info(msg)
self.info("Sending a notification...")
sbj = "Past due scheduled backups"
get_mbs().send_notification(sbj, msg)
def get_expired_backups(self, plan):
earliest_date_to_keep = date_minus_seconds(date_now(), self.max_time)
q = {
"plan._id": plan.id,
"targetReference": {"$exists": True},
# Filter out backups with targetReference.expiredDate is set
"$or": [
{"targetReference.expiredDate": {"$exists": False}},
{"targetReference.expiredDate": None}
],
"startDate": {
"$lt": earliest_date_to_keep
}
}
backups = get_mbs().backup_collection.find(q)
return backups
def _notify_on_late_in_progress_backups(self):
"""
Send notifications for jobs that have been in progress for a period
longer than a MAX_BACKUP_WAIT_TIME threshold
"""
min_start_date = date_minus_seconds(date_now(), MAX_BACKUP_WAIT_TIME)
q = {"state": STATE_IN_PROGRESS, "startDate": {"$lt": min_start_date}}
late_backups = get_mbs().backup_collection.find(q)
if late_backups:
msg = ("You have %s in-progress backups that has been running for"
" more than the maximum waiting time (%s seconds)." %
(len(late_backups), MAX_BACKUP_WAIT_TIME))
self.info(msg)
self.info("Sending a notification...")
sbj = "Late in-progress backups"
get_mbs().send_notification(sbj, msg)
def get_expired_backups(self, plan):
earliest_date_to_keep = date_minus_seconds(date_now(), self.max_time)
q = {
"plan._id":
plan.id,
"targetReference": {
"$exists": True
},
# Filter out backups with targetReference.expiredDate is set
"$or": [{
"targetReference.expiredDate": {
"$exists": False
}
}, {
"targetReference.expiredDate": None
}],
"startDate": {
"$lt": earliest_date_to_keep
}
}
backups = get_mbs().backup_collection.find(q)
return backups
def is_backup_past_due(self, backup):
max_wait_time = self.get_backup_max_wait_time(backup)
return (backup.state == State.SCHEDULED and
date_minus_seconds(date_now(), max_wait_time) > backup.get_last_scheduled_date())
def max_expire_date_to_delete(self):
return date_minus_seconds(date_now(), self.delete_delay_in_seconds)
def expired_canceled_cutoff_date(self):
return date_minus_seconds(date_now(),
self.expire_canceled_delay_in_seconds)
def get_plan_occurrences_to_retain_as_of(self, plan, dt):
end_date = dt
start_date = date_minus_seconds(end_date, self.max_time)
return plan.schedule.natural_occurrences_between(start_date, end_date)