def filter_backups_due_for_expiration(self, backups):

        earliest_date_to_keep = date_minus_seconds(date_now(), self.max_time)

        return filter(lambda backup:
                      backup.created_date < earliest_date_to_keep,
                      backups)
예제 #2
0
    def _read_next_failed_past_due_task(self):
        min_fail_end_date = date_minus_seconds(date_now(), MAX_FAIL_DUE_TIME)
        q = { "state": STATE_FAILED,
              "engineGuid": self._engine.engine_guid,
              "$or": [
                      {
                      "plan.nextOccurrence": {"$lte": date_now()}
                  },

                      {
                      "plan": {"$exists": False},
                      "reschedulable": False,
                      "endDate": {"$lte": min_fail_end_date}
                  }


              ]
        }

        msg = "Task failed and is past due. Cancelling..."
        log_entry = state_change_log_entry(STATE_CANCELED, message=msg)
        u = {"$set" : { "state" : STATE_CANCELED},
             "$push": {
                 "logs": log_entry.to_document()
             }
        }

        return self._task_collection.find_and_modify(query=q, update=u,
                                                     new=True)
예제 #3
0
    def _read_next_failed_past_due_task(self):
        min_fail_end_date = date_minus_seconds(date_now(), MAX_FAIL_DUE_TIME)
        q = {
            "$or": [
                {
                    "state": State.FAILED,
                    "engineGuid": self._engine.engine_guid,
                    "nextRetryDate": None,
                    "finalRetryDate": {"$lte": date_now()},
                    "plan.nextOccurrence": {"$lte": date_now()}
                },
                {
                    "state": State.FAILED,
                    "engineGuid": self._engine.engine_guid,
                    "plan": {"$exists": False},
                    "nextRetryDate": None,
                    "finalRetryDate": {"$lte": min_fail_end_date}
                }
            ]
        }

        msg = "Task failed and is past due. Cancelling..."
        log_entry = state_change_log_entry(State.CANCELED, message=msg)
        u = {"$set" : { "state" : State.CANCELED},
             "$push": {
                 "logs": log_entry.to_document()
             }
        }

        task = self.task_collection.find_one(query=q)
        if task:
            return self.task_collection.find_and_modify(query=q, update=u, new=True)
    def _notify_on_past_due_scheduled_backups(self):
        """
            Send notifications for jobs that has been scheduled for a period
            longer than min(half the frequency, 5 hours) of its plan.
             If backup does not have a plan (i.e. one off)
             then it will check after 60 seconds.
        """
        # query for backups whose scheduled date is before current date minus
        # than max starvation time

        where = (
            "(Math.min(%s, (this.plan.schedule.frequencyInSeconds / 2) * 1000) + "
            "this.createdDate.getTime()) < new Date().getTime()" %
            (MAX_BACKUP_WAIT_TIME * 1000))
        one_off_starve_date = date_minus_seconds(date_now(),
                                                 ONE_OFF_BACKUP_MAX_WAIT_TIME)
        q = {
            "state":
            STATE_SCHEDULED,
            "$or": [
                # backups with plans starving query
                {
                    "$and": [{
                        "plan": {
                            "$exists": True
                        }
                    }, {
                        "$where": where
                    }]
                },
                # One off backups (no plan) starving query
                {
                    "$and": [{
                        "plan": {
                            "$exists": False
                        }
                    }, {
                        "createdDate": {
                            "$lt": one_off_starve_date
                        }
                    }]
                }
            ]
        }

        starving_backups = get_mbs().backup_collection.find(q)

        if starving_backups:
            msg = ("You have %s scheduled backups that has past the maximum "
                   "waiting time (%s seconds)." %
                   (len(starving_backups), MAX_BACKUP_WAIT_TIME))
            self.info(msg)

            self.info("Sending a notification...")
            sbj = "Past due scheduled backups"
            get_mbs().send_notification(sbj, msg)
예제 #5
0
    def get_expired_backups(self, plan):

        earliest_date_to_keep = date_minus_seconds(date_now(), self.max_time)
        q = {
            "plan._id": plan.id,
            "targetReference": {"$exists": True},
            # Filter out backups with targetReference.expiredDate is set
            "$or": [
                    {"targetReference.expiredDate": {"$exists": False}},
                    {"targetReference.expiredDate": None}
            ],
            "startDate": {
                "$lt": earliest_date_to_keep
            }
        }

        backups = get_mbs().backup_collection.find(q)
        return backups
    def _notify_on_late_in_progress_backups(self):
        """
            Send notifications for jobs that have been in progress for a period
            longer than a MAX_BACKUP_WAIT_TIME threshold
        """

        min_start_date = date_minus_seconds(date_now(), MAX_BACKUP_WAIT_TIME)
        q = {"state": STATE_IN_PROGRESS, "startDate": {"$lt": min_start_date}}

        late_backups = get_mbs().backup_collection.find(q)

        if late_backups:
            msg = ("You have %s in-progress backups that has been running for"
                   " more than the maximum waiting time (%s seconds)." %
                   (len(late_backups), MAX_BACKUP_WAIT_TIME))
            self.info(msg)

            self.info("Sending a notification...")
            sbj = "Late in-progress backups"
            get_mbs().send_notification(sbj, msg)
예제 #7
0
    def get_expired_backups(self, plan):

        earliest_date_to_keep = date_minus_seconds(date_now(), self.max_time)
        q = {
            "plan._id":
            plan.id,
            "targetReference": {
                "$exists": True
            },
            # Filter out backups with targetReference.expiredDate is set
            "$or": [{
                "targetReference.expiredDate": {
                    "$exists": False
                }
            }, {
                "targetReference.expiredDate": None
            }],
            "startDate": {
                "$lt": earliest_date_to_keep
            }
        }

        backups = get_mbs().backup_collection.find(q)
        return backups
예제 #8
0
 def is_backup_past_due(self, backup):
     max_wait_time = self.get_backup_max_wait_time(backup)
     return (backup.state == State.SCHEDULED and
             date_minus_seconds(date_now(), max_wait_time) > backup.get_last_scheduled_date())
 def max_expire_date_to_delete(self):
     return date_minus_seconds(date_now(), self.delete_delay_in_seconds)
 def expired_canceled_cutoff_date(self):
     return date_minus_seconds(date_now(),
                               self.expire_canceled_delay_in_seconds)
 def get_plan_occurrences_to_retain_as_of(self, plan, dt):
     end_date = dt
     start_date = date_minus_seconds(end_date, self.max_time)
     return plan.schedule.natural_occurrences_between(start_date, end_date)