def test_updated_scheduled_next_run(self):
        call_request = CallRequest(itinerary_call)
        interval = datetime.timedelta(minutes=2)
        now = datetime.datetime.now(tz=dateutils.utc_tz())
        old_schedule = dateutils.format_iso8601_interval(interval, now)

        scheduled_id = self.scheduler.add(call_request, old_schedule)

        self.assertNotEqual(scheduled_id, None)

        scheduled_call = self.scheduled_call_collection.find_one({'_id': ObjectId(scheduled_id)})

        self.assertNotEqual(scheduled_call, None)

        old_interval, start_time = dateutils.parse_iso8601_interval(old_schedule)[:2]
        start_time = dateutils.to_naive_utc_datetime(start_time)

        self.assertEqual(scheduled_call['last_run'], None)
        self.assertEqual(scheduled_call['first_run'], start_time + old_interval)
        self.assertEqual(scheduled_call['next_run'], start_time + old_interval)

        interval = datetime.timedelta(minutes=1)
        new_schedule = dateutils.format_iso8601_interval(interval, now)

        self.scheduler.update(scheduled_id, schedule=new_schedule)
        updated_scheduled_call = self.scheduled_call_collection.find_one({'_id': ObjectId(scheduled_id)})

        new_interval = dateutils.parse_iso8601_interval(new_schedule)[0]

        self.assertEqual(updated_scheduled_call['last_run'], None)
        self.assertEqual(updated_scheduled_call['first_run'], start_time + old_interval)
        self.assertEqual(updated_scheduled_call['next_run'], start_time + new_interval)
Exemple #2
0
def is_valid_schedule(schedule):
    """
    Validate an iso8601 interval schedule.
    @param schedule: schedule string to validate
    @return: True if the schedule is valid, False otherwise
    @rtype:  bool
    """
    if not isinstance(schedule, basestring):
        return False
    try:
        dateutils.parse_iso8601_interval(schedule)
    except isodate.ISO8601Error:
        return False
    return True
Exemple #3
0
def _load_repo_extras(repo, repos=None):
    config = get_config()
    repoapi = RepositoryAPI()
    repo["url"] = os.path.join(config.cds.baseurl, repo["relative_path"])

    repo["parent"] = None
    repo["children"] = []
    if repos is None:
        repos = getattr(threading.local(), "repos", dict())

    for repo2 in repos.values():
        if repo2 == repo:
            continue
        elif repo["id"] in repo2["clone_ids"]:
            # the clone_id attribute is broken, but we check it anyway
            # just in case it gets fixed some day
            repo["parent"] = repo2
        elif repo2["id"] in repo["clone_ids"]:
            repo["children"].append(repo2)
        elif (
            repo["source"] and repo["source"]["type"] == "local" and repo["source"]["url"].endswith("/%s" % repo2["id"])
        ):
            # the child syncs from a local repo that ends with
            # /<parent repo id>
            repo["parent"] = repo2
        elif (
            repo2["source"]
            and repo2["source"]["type"] == "local"
            and repo2["source"]["url"].endswith("/%s" % repo["id"])
        ):
            repo["children"].append(repo2)

    repo["keys"] = dict()
    for key in repoapi.listkeys(repo["id"]):
        repo["keys"][os.path.basename(key)] = "%s/%s" % (config.cds.keyurl, key)

    if repo["parent"]:
        repo["updates"] = has_updates(repo)

    if repo["last_sync"] and repo["sync_schedule"]:
        repo["next_sync"] = format_iso8601_datetime(
            parse_iso8601_datetime(repo["last_sync"]) + parse_iso8601_interval(repo["sync_schedule"])[0]
        )
    elif repo["sync_schedule"]:
        repo["next_sync"] = format_iso8601_datetime(parse_iso8601_interval(repo["sync_schedule"])[1])
    else:
        repo["next_sync"] = None

    repo["groupid"].sort()
Exemple #4
0
    def __init__(self,
                 call_request,
                 schedule,
                 failure_threshold=None,
                 last_run=None,
                 enabled=True):
        super(ScheduledCall, self).__init__()

        schedule_tag = resource_tag(dispatch_constants.RESOURCE_SCHEDULE_TYPE,
                                    str(self._id))
        call_request.tags.append(schedule_tag)
        interval, start, runs = dateutils.parse_iso8601_interval(schedule)
        now = datetime.utcnow()
        zero = timedelta(seconds=0)
        start = start and dateutils.to_naive_utc_datetime(start)

        self.serialized_call_request = call_request.serialize()
        self.schedule = schedule
        self.failure_threshold = failure_threshold
        self.consecutive_failures = 0
        self.first_run = start or now
        # NOTE using != because ordering comparison with a Duration is not allowed
        while interval != zero and self.first_run <= now:
            # try to schedule the first run in the future
            self.first_run = dateutils.add_interval_to_datetime(
                interval, self.first_run)
        self.last_run = last_run and dateutils.to_naive_utc_datetime(last_run)
        self.next_run = None  # will calculated and set by the scheduler
        self.remaining_runs = runs
        self.enabled = enabled
Exemple #5
0
 def test_interval_recurrences(self):
     d = datetime.timedelta(hours=4, minutes=2, seconds=59)
     c = 4
     s = dateutils.format_iso8601_interval(d, recurrences=c)
     i, t, r = dateutils.parse_iso8601_interval(s)
     self.assertEqual(d, i)
     self.assertEqual(c, r)
Exemple #6
0
 def test_interval_start_time(self):
     d = datetime.timedelta(minutes=2)
     t = datetime.datetime(year=2014, month=11, day=5, hour=0, minute=23)
     s = dateutils.format_iso8601_interval(d, t)
     i, e, r = dateutils.parse_iso8601_interval(s)
     self.assertEqual(d, i)
     self.assertEqual(t, e)
Exemple #7
0
 def test_interval_recurrences(self):
     d = datetime.timedelta(hours=4, minutes=2, seconds=59)
     c = 4
     s = dateutils.format_iso8601_interval(d, recurrences=c)
     i, t, r = dateutils.parse_iso8601_interval(s)
     self.assertEqual(d, i)
     self.assertEqual(c, r)
Exemple #8
0
 def test_interval_start_time(self):
     d = datetime.timedelta(minutes=2)
     t = datetime.datetime(year=2014, month=11, day=5, hour=0, minute=23)
     s = dateutils.format_iso8601_interval(d, t)
     i, e, r = dateutils.parse_iso8601_interval(s)
     self.assertEqual(d, i)
     self.assertEqual(t, e)
Exemple #9
0
 def update(self, schedule_id, **schedule_updates):
     """
     Update a scheduled call request
     Valid schedule updates:
      * call_request
      * schedule
      * failure_threshold
      * remaining_runs
      * enabled
     @param schedule_id: id of the schedule for the call request
     @type  schedule_id: str
     @param schedule_updates: updates for scheduled call
     @type  schedule_updates: dict
     """
     if isinstance(schedule_id, basestring):
         schedule_id = ObjectId(schedule_id)
     scheduled_call_collection = ScheduledCall.get_collection()
     if scheduled_call_collection.find_one(schedule_id) is None:
         raise pulp_exceptions.MissingResource(schedule=str(schedule_id))
     validate_schedule_updates(schedule_updates)
     call_request = schedule_updates.pop('call_request', None)
     if call_request is not None:
         schedule_updates['serialized_call_request'] = call_request.serialize()
     schedule = schedule_updates.get('schedule', None)
     if schedule is not None:
         interval, start, runs = dateutils.parse_iso8601_interval(schedule)
         schedule_updates.setdefault('remaining_runs', runs) # honor explicit update
         # XXX (jconnor) it'd be nice to update the next_run if the schedule
         # has changed, but it requires mucking with the internals of the
         # of the scheduled call instance, which is all encapsulated in the
         # ScheduledCall constructor
         # the next_run field will be correctly updated after the next run
     scheduled_call_collection.update({'_id': schedule_id}, {'$set': schedule_updates}, safe=True)
Exemple #10
0
def interval_iso6801_validator(x):
    """
    Validates that a user-entered value is a correct iso8601 date with
    an interval. This call will raise an exception to be passed to the CLI
    framework if it is invalid; there is no return otherwise.

    :param x: input value to be validated
    :type  x: str
    """

    # These are meant to be used with okaara which expects either ValueError or
    # TypeError for a graceful failure, so catch any parsing errors and raise
    # the appropriate new error.
    try:
        dateutils.parse_iso8601_interval(x)
    except Exception:
        raise ValueError(_('value must be a valid iso8601 string with an interval'))
Exemple #11
0
def is_valid_schedule(schedule):
    """
    Validate an iso8601 interval schedule.
    @param schedule: schedule string to validate
    @return: True if the schedule is valid, False otherwise
    @rtype:  bool
    """
    if not isinstance(schedule, basestring):
        return False

    try:
        dateutils.parse_iso8601_interval(schedule)

    except isodate.ISO8601Error:
        return False

    return True
Exemple #12
0
def convert_schedule(save_func, call):
    """
    Converts one scheduled call from the old schema to the new

    :param save_func:   a function that takes one parameter, a dictionary that
                        represents the scheduled call in its new schema. This
                        function should save the call to the database.
    :type  save_func:   function
    :param call:        dictionary representing the scheduled call in its old
                        schema
    :type  call:        dict
    """
    call.pop('call_exit_states', None)
    call['total_run_count'] = call.pop('call_count')

    call['iso_schedule'] = call['schedule']
    interval, start_time, occurrences = dateutils.parse_iso8601_interval(
        call['schedule'])
    # this should be a pickled instance of celery.schedules.schedule
    call['schedule'] = pickle.dumps(schedule(interval))

    call_request = call.pop('serialized_call_request')
    # we are no longer storing these pickled.
    # these are cast to a string because python 2.6 sometimes fails to
    # deserialize json from unicode.
    call['args'] = pickle.loads(str(call_request['args']))
    call['kwargs'] = pickle.loads(str(call_request['kwargs']))
    # keeping this pickled because we don't really know how to use it yet
    call['principal'] = call_request['principal']
    # this always get calculated on-the-fly now
    call.pop('next_run', None)
    first_run = call['first_run'].replace(tzinfo=dateutils.utc_tz())
    call['first_run'] = dateutils.format_iso8601_datetime(first_run)
    last_run = call.pop('last_run')
    if last_run:
        last_run_at = last_run.replace(tzinfo=dateutils.utc_tz())
        call['last_run_at'] = dateutils.format_iso8601_datetime(last_run_at)
    else:
        call['last_run_at'] = None
    call['task'] = NAMES_TO_TASKS[call_request['callable_name']]

    # this is a new field that is used to determine when the scheduler needs to
    # re-read the collection of schedules.
    call['last_updated'] = time.time()

    # determine if this is a consumer-related schedule, which we can only identify
    # by the consumer resource tag. If it is, save that tag value in the new
    # "resource" field, which is the new way that we will identify the
    # relationship between a schedule and some other object. This is not
    # necessary for repos, because we have a better method above for identifying
    # them (move_scheduled_syncs).
    tags = call_request.get('tags', [])
    for tag in tags:
        if tag.startswith('pulp:consumer:'):
            call['resource'] = tag
            break

    save_func(call)
def convert_schedule(save_func, call):
    """
    Converts one scheduled call from the old schema to the new

    :param save_func:   a function that takes one parameter, a dictionary that
                        represents the scheduled call in its new schema. This
                        function should save the call to the database.
    :type  save_func:   function
    :param call:        dictionary representing the scheduled call in its old
                        schema
    :type  call:        dict
    """
    call.pop('call_exit_states', None)
    call['total_run_count'] = call.pop('call_count')

    call['iso_schedule'] = call['schedule']
    interval, start_time, occurrences = dateutils.parse_iso8601_interval(call['schedule'])
    # this should be a pickled instance of celery.schedules.schedule
    call['schedule'] = pickle.dumps(schedule(interval))

    call_request = call.pop('serialized_call_request')
    # we are no longer storing these pickled.
    # these are cast to a string because python 2.6 sometimes fails to
    # deserialize json from unicode.
    call['args'] = pickle.loads(str(call_request['args']))
    call['kwargs'] = pickle.loads(str(call_request['kwargs']))
    # keeping this pickled because we don't really know how to use it yet
    call['principal'] = call_request['principal']
    # this always get calculated on-the-fly now
    call.pop('next_run', None)
    first_run = call['first_run'].replace(tzinfo=dateutils.utc_tz())
    call['first_run'] = dateutils.format_iso8601_datetime(first_run)
    last_run = call.pop('last_run')
    if last_run:
        last_run_at = last_run.replace(tzinfo=dateutils.utc_tz())
        call['last_run_at'] = dateutils.format_iso8601_datetime(last_run_at)
    else:
        call['last_run_at'] = None
    call['task'] = NAMES_TO_TASKS[call_request['callable_name']]

    # this is a new field that is used to determine when the scheduler needs to
    # re-read the collection of schedules.
    call['last_updated'] = time.time()

    # determine if this is a consumer-related schedule, which we can only identify
    # by the consumer resource tag. If it is, save that tag value in the new
    # "resource" field, which is the new way that we will identify the
    # relationship between a schedule and some other object. This is not
    # necessary for repos, because we have a better method above for identifying
    # them (move_scheduled_syncs).
    tags = call_request.get('tags', [])
    for tag in tags:
        if tag.startswith('pulp:consumer:'):
            call['resource'] = tag
            break

    save_func(call)
Exemple #14
0
 def test_interval_full(self):
     i1 = datetime.timedelta(hours=100)
     t1 = datetime.datetime(year=2, month=6, day=20, hour=2, minute=22, second=46)
     r1 = 5
     s = dateutils.format_iso8601_interval(i1, t1, r1)
     i2, t2, r2 = dateutils.parse_iso8601_interval(s)
     self.assertEqual(i1, i2)
     self.assertEqual(t1, t2)
     self.assertEqual(r1, r2)
Exemple #15
0
 def test_interval_full(self):
     i1 = datetime.timedelta(hours=100)
     t1 = datetime.datetime(year=2, month=6, day=20, hour=2, minute=22, second=46)
     r1 = 5
     s = dateutils.format_iso8601_interval(i1, t1, r1)
     i2, t2, r2 = dateutils.parse_iso8601_interval(s)
     self.assertEqual(i1, i2)
     self.assertEqual(t1, t2)
     self.assertEqual(r1, r2)
Exemple #16
0
def interval_iso6801_validator(x):
    """
    Validates that a user-entered value is a correct iso8601 date with
    an interval.

    :param x: input value to be validated
    :type  x: str

    :raise ValueError: if the input is not a valid iso8601 string
    """

    # These are meant to be used with okaara which expects either ValueError or
    # TypeError for a graceful failure, so catch any parsing errors and raise
    # the appropriate new error.
    try:
        dateutils.parse_iso8601_interval(x)
    except Exception:
        raise ValueError(_('value must be a valid iso8601 string with an interval'))
Exemple #17
0
def interval_iso6801_validator(x):
    """
    Validates that a user-entered value is a correct iso8601 date with
    an interval.

    :param x: input value to be validated
    :type  x: str

    :raise ValueError: if the input is not a valid iso8601 string
    """

    # These are meant to be used with okaara which expects either ValueError or
    # TypeError for a graceful failure, so catch any parsing errors and raise
    # the appropriate new error.
    try:
        dateutils.parse_iso8601_interval(x)
    except Exception:
        raise ValueError(_('value must be a valid iso8601 string with an interval'))
Exemple #18
0
    def test_future(self, mock_time):
        mock_time.return_value = 1389307330.966561
        call = ScheduledCall('2014-01-19T17:15Z/PT1H', 'pulp.tasks.dosomething')

        next_run = call.calculate_next_run()

        # make sure the next run is equal to the specified first run.
        # don't want to compare a generated ISO8601 string directly, because there
        # could be subtle variations that are valid but break string equality.
        self.assertEqual(dateutils.parse_iso8601_interval(call.iso_schedule)[1],
                         dateutils.parse_iso8601_datetime(next_run))
Exemple #19
0
def _calculate_next_run(scheduled_call):
    # rip-off from scheduler module
    if scheduled_call['remaining_runs'] == 0:
        return None
    last_run = scheduled_call['last_run']
    if last_run is None:
        return scheduled_call['first_run']
    now = datetime.utcnow()
    interval = dateutils.parse_iso8601_interval(scheduled_call['schedule'])[0]
    next_run = last_run
    while next_run < now:
        next_run = dateutils.add_interval_to_datetime(interval, next_run)
    return next_run
Exemple #20
0
def _calculate_next_run(scheduled_call):
    # rip-off from scheduler module
    if scheduled_call['remaining_runs'] == 0:
        return None
    last_run = scheduled_call['last_run']
    if last_run is None:
        return scheduled_call['first_run']
    now = datetime.utcnow()
    interval = dateutils.parse_iso8601_interval(scheduled_call['schedule'])[0]
    next_run = last_run
    while next_run < now:
        next_run = dateutils.add_interval_to_datetime(interval, next_run)
    return next_run
Exemple #21
0
def update(schedule_id, delta):
    """
    Updates the schedule with unique ID schedule_id. This only allows updating
    of fields in ScheduledCall.USER_UPDATE_FIELDS.

    :param schedule_id: a unique ID for a schedule
    :type  schedule_id: basestring
    :param delta:       a dictionary of keys with values that should be modified
                        on the schedule.
    :type  delta:       dict

    :return:    instance of ScheduledCall representing the post-update state
    :rtype      ScheduledCall

    :raise  exceptions.UnsupportedValue
    :raise  exceptions.MissingResource
    """
    unknown_keys = set(delta.keys()) - ScheduledCall.USER_UPDATE_FIELDS
    if unknown_keys:
        raise exceptions.UnsupportedValue(list(unknown_keys))

    delta['last_updated'] = time.time()

    # bz 1139703 - if we update iso_schedule, update the pickled object as well
    if 'iso_schedule' in delta:
        interval, start_time, occurrences = dateutils.parse_iso8601_interval(
            delta['iso_schedule'])
        delta['schedule'] = pickle.dumps(CelerySchedule(interval))

        # set first_run and next_run so that the schedule update will take effect
        new_schedule_call = ScheduledCall(delta['iso_schedule'],
                                          'dummytaskname')
        delta['first_run'] = new_schedule_call.first_run
        delta['next_run'] = new_schedule_call.next_run

    try:
        spec = {'_id': ObjectId(schedule_id)}
    except InvalidId:
        # During schedule update, MissingResource should be raised even if
        # schedule_id is invalid object_id.
        raise exceptions.MissingResource(schedule_id=schedule_id)
    schedule = ScheduledCall.get_collection().find_and_modify(
        query=spec, update={'$set': delta}, safe=True, new=True)
    if schedule is None:
        raise exceptions.MissingResource(schedule_id=schedule_id)
    return ScheduledCall.from_db(schedule)
Exemple #22
0
def update(schedule_id, delta):
    """
    Updates the schedule with unique ID schedule_id. This only allows updating
    of fields in ScheduledCall.USER_UPDATE_FIELDS.

    :param schedule_id: a unique ID for a schedule
    :type  schedule_id: basestring
    :param delta:       a dictionary of keys with values that should be modified
                        on the schedule.
    :type  delta:       dict

    :return:    instance of ScheduledCall representing the post-update state
    :rtype      ScheduledCall

    :raise  exceptions.UnsupportedValue
    :raise  exceptions.MissingResource
    """
    unknown_keys = set(delta.keys()) - ScheduledCall.USER_UPDATE_FIELDS
    if unknown_keys:
        raise exceptions.UnsupportedValue(list(unknown_keys))

    delta['last_updated'] = time.time()

    # bz 1139703 - if we update iso_schedule, update the pickled object as well
    if 'iso_schedule' in delta:
        interval, start_time, occurrences = dateutils.parse_iso8601_interval(delta['iso_schedule'])
        delta['schedule'] = pickle.dumps(CelerySchedule(interval))

        # set first_run and next_run so that the schedule update will take effect
        new_schedule_call = ScheduledCall(delta['iso_schedule'], 'dummytaskname')
        delta['first_run'] = new_schedule_call.first_run
        delta['next_run'] = new_schedule_call.next_run

    try:
        spec = {'_id': ObjectId(schedule_id)}
    except InvalidId:
        # During schedule update, MissingResource should be raised even if
        # schedule_id is invalid object_id.
        raise exceptions.MissingResource(schedule_id=schedule_id)
    schedule = ScheduledCall.get_collection().find_and_modify(
        query=spec, update={'$set': delta}, new=True)
    if schedule is None:
        raise exceptions.MissingResource(schedule_id=schedule_id)
    return ScheduledCall.from_db(schedule)
Exemple #23
0
    def update(self, schedule_id, **schedule_updates):
        """
        Update a scheduled call request

        Valid schedule updates:
         * call_request
         * schedule
         * failure_threshold
         * remaining_runs
         * enabled

        @param schedule_id: id of the schedule for the call request
        @type  schedule_id: str
        @param schedule_updates: updates for scheduled call
        @type  schedule_updates: dict
        """
        if isinstance(schedule_id, basestring):
            schedule_id = ObjectId(schedule_id)

        if self.scheduled_call_collection.find_one(schedule_id) is None:
            raise pulp_exceptions.MissingResource(schedule=str(schedule_id))

        validate_schedule_updates(schedule_updates)

        call_request = schedule_updates.pop('call_request', None)

        if call_request is not None:
            schedule_updates['serialized_call_request'] = call_request.serialize()

        schedule = schedule_updates.get('schedule', None)

        if schedule is not None:
            interval, start, runs = dateutils.parse_iso8601_interval(schedule)
            schedule_updates.setdefault('remaining_runs', runs) # honor explicit update
            # XXX (jconnor) it'd be nice to update the next_run if the schedule
            # has changed, but it requires mucking with the internals of the
            # of the scheduled call instance, which is all encapsulated in the
            # ScheduledCall constructor
            # the next_run field will be correctly updated after the next run

        self.scheduled_call_collection.update({'_id': schedule_id}, {'$set': schedule_updates}, safe=True)
Exemple #24
0
    def update(self, schedule_id, **updated_schedule_options):
        """
        Update and existing scheduled call.

        Supported update schedule options:

         * call_request: new itinerary call request instance
         * schedule: new ISO8601 interval string
         * failure_threshold: new failure threshold integer
         * remaining_runs: new remaining runs count integer
         * enabled: new enabled flag boolean

        :param schedule_id: unique identifier of the scheduled call
        :type  schedule_id: str or pulp.server.compat.ObjectID
        :param updated_schedule_options: updated options for this scheduled call
        :raises: pulp.server.exceptions.MissingResource if the corresponding scheduled call does not exist
        :raises: pulp.server.exceptions.UnsupportedValue if unsupported schedule options are passed in
        :raises: pulp.server.exceptions.InvalidValue if any of the options are invalid
        """

        if isinstance(schedule_id, basestring):
            schedule_id = ObjectId(schedule_id)

        if self.scheduled_call_collection.find_one(schedule_id) is None:
            raise pulp_exceptions.MissingResource(schedule=str(schedule_id))

        validate_updated_schedule_options(updated_schedule_options)

        call_request = updated_schedule_options.pop('call_request', None)

        if isinstance(call_request, call.CallRequest):
            updated_schedule_options['serialized_call_request'] = call_request.serialize()

        schedule = updated_schedule_options.get('schedule', None)

        if schedule is not None:
            runs = dateutils.parse_iso8601_interval(schedule)[2]
            updated_schedule_options.setdefault('remaining_runs', runs) # honor explicit update
            updated_schedule_options['next_run'] = self.calculate_first_run(schedule)

        self.scheduled_call_collection.update({'_id': schedule_id}, {'$set': updated_schedule_options}, safe=True)
Exemple #25
0
    def calculate_next_run(self, scheduled_call):
        """
        Calculate the next run datetime of a scheduled call
        @param scheduled_call: scheduled call to schedule
        @type  scheduled_call: dict
        @return: datetime of scheduled call's next run or None if there is no next run
        @rtype:  datetime.datetime or None
        """
        if scheduled_call['remaining_runs'] == 0:
            return None

        last_run = scheduled_call['last_run']
        if last_run is None:
            return scheduled_call['first_run'] # this was calculated by the model constructor

        now = datetime.datetime.utcnow()
        interval = dateutils.parse_iso8601_interval(scheduled_call['schedule'])[0]
        next_run = last_run
        while next_run < now:
            next_run += interval
        return next_run
Exemple #26
0
    def __init__(self, call_request, schedule, failure_threshold=None, last_run=None, enabled=True):
        super(ScheduledCall, self).__init__()

        schedule_tag = resource_tag(dispatch_constants.RESOURCE_SCHEDULE_TYPE, str(self._id))
        call_request.tags.append(schedule_tag)
        interval, start, runs = dateutils.parse_iso8601_interval(schedule)
        now = datetime.utcnow()
        zero = timedelta(seconds=0)
        start = start and dateutils.to_naive_utc_datetime(start)

        self.serialized_call_request = call_request.serialize()
        self.schedule = schedule
        self.failure_threshold = failure_threshold
        self.consecutive_failures = 0
        self.first_run = start or now
        # NOTE using != because ordering comparison with a Duration is not allowed
        while interval != zero and self.first_run <= now:
            # try to schedule the first run in the future
            self.first_run = dateutils.add_interval_to_datetime(interval, self.first_run)
        self.last_run = last_run and dateutils.to_naive_utc_datetime(last_run)
        self.next_run = None # will calculated and set by the scheduler
        self.remaining_runs = runs
        self.enabled = enabled
Exemple #27
0
    def calculate_next_run(self, scheduled_call):
        """
        Calculate the next run datetime of a scheduled call
        @param scheduled_call: scheduled call to schedule
        @type  scheduled_call: dict
        @return: datetime of scheduled call's next run or None if there is no next run
        @rtype:  datetime.datetime or None
        """
        if scheduled_call['remaining_runs'] == 0:
            return None

        last_run = scheduled_call['last_run']
        if last_run is None:
            return scheduled_call['first_run'] # this was calculated by the model constructor

        now = datetime.datetime.utcnow()
        interval = dateutils.parse_iso8601_interval(scheduled_call['schedule'])[0]
        next_run = last_run

        while next_run < now:
            next_run = dateutils.add_interval_to_datetime(interval, next_run)

        return next_run
Exemple #28
0
def _is_valid_schedule(schedule):
    """
    Test that a schedule string is in the ISO8601 interval format

    :param schedule: schedule string
    :type schedule: str
    :return: True if the schedule is in the ISO8601 format, False otherwise
    :rtype:  bool
    """

    if not isinstance(schedule, basestring):
        return False

    try:
        interval, start_time, runs = dateutils.parse_iso8601_interval(schedule)

    except isodate.ISO8601Error:
        return False

    if runs is not None and runs <= 0:
        return False

    return True
Exemple #29
0
def _is_valid_schedule(schedule):
    """
    Test that a schedule string is in the ISO8601 interval format

    :param schedule: schedule string
    :type schedule: str
    :return: True if the schedule is in the ISO8601 format, False otherwise
    :rtype:  bool
    """

    if not isinstance(schedule, basestring):
        return False

    try:
        interval, start_time, runs = dateutils.parse_iso8601_interval(schedule)

    except isodate.ISO8601Error:
        return False

    if runs is not None and runs <= 0:
        return False

    return True
Exemple #30
0
    def _insert_scheduled_v2_repo(self, repo_id, schedule):
        importer_id = ObjectId()
        schedule_id = ObjectId()

        importer_doc = {'importer_id': importer_id,
                        'importer_type_id': yum_repos.YUM_IMPORTER_TYPE_ID,
                        'scheduled_syncs': [str(schedule_id)]}
        self.tmp_test_db.database.repo_importers.update({'repo_id': repo_id}, {'$set': importer_doc}, safe=True)

        call_request = CallRequest(sync_with_auto_publish_itinerary, [repo_id], {'overrides': {}})
        interval, start, recurrences = dateutils.parse_iso8601_interval(schedule)
        scheduled_call_doc = {'_id': schedule_id,
                              'id': str(schedule_id),
                              'serialized_call_request': call_request.serialize(),
                              'schedule': schedule,
                              'failure_threshold': None,
                              'consecutive_failures': 0,
                              'first_run': start or datetime.datetime.utcnow(),
                              'next_run': None,
                              'last_run': None,
                              'remaining_runs': recurrences,
                              'enabled': True}
        scheduled_call_doc['next_run'] = all_repos._calculate_next_run(scheduled_call_doc)
        self.tmp_test_db.database.scheduled_calls.insert(scheduled_call_doc, safe=True)
Exemple #31
0
    def calculate_first_run(schedule):
        """
        Given a schedule in ISO8601 interval format, calculate the first time
        the schedule should be run.

        This method make a best effort to calculate a time in the future.

        :param schedule: ISO8601 interval schedule
        :type  schedule: str
        :return: when the schedule should be run for the first time
        :rtype:  datetime.datetime
        """

        now = datetime.datetime.utcnow()
        interval, start = dateutils.parse_iso8601_interval(schedule)[0:2]

        first_run = dateutils.to_naive_utc_datetime(start) if start else now

        # the "zero time" handles the really off case where the schedule is a
        # start time and a single run instead of something recurring
        while interval != ZERO_TIME and first_run <= now:
            first_run = dateutils.add_interval_to_datetime(interval, first_run)

        return first_run
Exemple #32
0
    def calculate_next_run(scheduled_call):
        """
        Given a schedule call, calculate when it should be run next.

        :param scheduled_call: scheduled call
        :type  scheduled_call: bson.BSON or pulp.server.db.model.dispatch.ScheduledCall
        :return: when the scheduled call should be run next
        :rtype:  datetime.datetime
        """

        last_run = scheduled_call['last_run']

        if last_run is None:
            return scheduled_call['first_run']

        now = datetime.datetime.utcnow()
        interval = dateutils.parse_iso8601_interval(scheduled_call['schedule'])[0]

        next_run = last_run

        while next_run < now:
            next_run = dateutils.add_interval_to_datetime(interval, next_run)

        return next_run
Exemple #33
0
    def __init__(self, call_request, schedule, failure_threshold=None, last_run=None, enabled=True):
        super(ScheduledCall, self).__init__()

        # add custom scheduled call tag to call request
        schedule_tag = resource_tag(dispatch_constants.RESOURCE_SCHEDULE_TYPE, str(self._id))
        call_request.tags.append(schedule_tag)

        self.serialized_call_request = call_request.serialize()

        self.schedule = schedule
        self.enabled = enabled

        self.failure_threshold = failure_threshold
        self.consecutive_failures = 0

        # scheduling fields
        self.first_run = None # will be calculated and set by the scheduler
        self.last_run = last_run and dateutils.to_naive_utc_datetime(last_run)
        self.next_run = None # will be calculated and set by the scheduler
        self.remaining_runs = dateutils.parse_iso8601_interval(schedule)[2]

        # run-time call group metadata for tracking success or failure
        self.call_count = 0
        self.call_exit_states = []
Exemple #34
0
    def _insert_scheduled_v2_repo(self, repo_id, schedule):
        importer_id = ObjectId()
        schedule_id = ObjectId()

        importer_doc = {'importer_id': importer_id,
                        'importer_type_id': yum_repos.YUM_IMPORTER_TYPE_ID,
                        'scheduled_syncs': [str(schedule_id)]}
        self.tmp_test_db.database.repo_importers.update({'repo_id': repo_id}, {'$set': importer_doc}, safe=True)

        call_request = CallRequest(sync_with_auto_publish_itinerary, [repo_id], {'overrides': {}})
        interval, start, recurrences = dateutils.parse_iso8601_interval(schedule)
        scheduled_call_doc = {'_id': schedule_id,
                              'id': str(schedule_id),
                              'serialized_call_request': call_request.serialize(),
                              'schedule': schedule,
                              'failure_threshold': None,
                              'consecutive_failures': 0,
                              'first_run': start or datetime.datetime.utcnow(),
                              'next_run': None,
                              'last_run': None,
                              'remaining_runs': recurrences,
                              'enabled': True}
        scheduled_call_doc['next_run'] = all_repos._calculate_next_run(scheduled_call_doc)
        self.tmp_test_db.database.scheduled_calls.insert(scheduled_call_doc, safe=True)
Exemple #35
0
    def __init__(self,
                 iso_schedule,
                 task,
                 total_run_count=0,
                 next_run=None,
                 schedule=None,
                 args=None,
                 kwargs=None,
                 principal=None,
                 last_updated=None,
                 consecutive_failures=0,
                 enabled=True,
                 failure_threshold=None,
                 last_run_at=None,
                 first_run=None,
                 remaining_runs=None,
                 id=None,
                 tags=None,
                 name=None,
                 options=None,
                 resource=None):
        """
        :param iso_schedule:        string representing the schedule in ISO8601 format
        :type  iso_schedule:        basestring
        :param task:                the task that should be run on a schedule. This
                                    can be an instance of a celery task or the name
                                    of the task, as taken from a task's "name" attribute
        :type  task:                basestring or celery.Task
        :param total_run_count:     total number of times this schedule has run
        :type  total_run_count:     int
        :param next_run:            ignored, because it is always re-calculated at instantiation
        :param schedule:            pickled instance of celery.schedules.schedule,
                                    representing the schedule that should be run.
                                    This is optional.
        :type  schedule:            basestring or None
        :param args:                list of arguments that should be passed to the
                                    task's apply_async function as its "args" argument
        :type  args:                list
        :param kwargs:              dict of keyword arguments that should be passed to the task's
                                    apply_async function as its "kwargs" argument
        :type  kwargs:              dict
        :param principal:           pickled instance of pulp.server.db.model.auth.User
                                    representing the pulp user who the task
                                    should be run as. This is optional.
        :type  principal:           basestring or None
        :param last_updated:        timestamp for the last time this schedule was updated in the
                                    database as seconds since the epoch
        :type  last_updated:        float
        :param consecutive_failures:    number of times this task has failed consecutively. This
                                        gets reset to zero if the task succeeds.
        :type  consecutive_failures:    int
        :param enabled:             boolean indicating whether this schedule should be actively run
                                    by the scheduler. If False, the schedule will be ignored.
        :type  enabled:             bool
        :param failure_threshold:   number of consecutive failures after which this task should be
                                    automatically disabled. Because these tasks run asynchronously,
                                    they may finish in a different order than they were queued in.
                                    Thus, it is possible that n consecutive failures will be
                                    reported by jobs that were not queued consecutively. So do not
                                    depend on the queuing order when using this feature. If this
                                    value is 0, no automatic disabling will occur.
        :type  failure_threshold:   int
        :param last_run_at:         ISO8601 string representing when this schedule last ran.
        :type  last_run_at:         basestring
        :param first_run:           ISO8601 string or datetime instance (in UTC timezone)
                                    representing when this schedule should run or should have been
                                    run for the first time. If the schedule has a specified date and
                                    time to start, this will be that value. If not, the value from
                                    the first time the schedule was actually run will be used.
        :type  first_run:           basestring or datetime.datetime or NoneType
        :param remaining_runs:      number of runs remaining until this schedule will be
                                    automatically disabled.
        :type  remaining_runs:      int or NoneType
        :param id:                  unique ID used by mongodb to identify this schedule
        :type  id:                  basestring
        :param tags:                ignored, but allowed to exist as historical
                                    data for now
        :param name:                ignored, because the "id" value is used for this now. The value
                                    is here for backward compatibility.
        :param options:             dictionary that should be passed to the apply_async function as
                                    its "options" argument.
        :type  options:             dict
        :param resource:            optional string indicating a unique resource that should be used
                                    to find this schedule. For example, to find all schedules for a
                                    given repository, a resource string will be derived for that
                                    repo, and this collection will be searched for that resource
                                    string.
        :type  resource:            basestring
        """
        if id is None:
            # this creates self._id and self.id
            super(ScheduledCall, self).__init__()
            self._new = True
        else:
            self.id = id
            self._id = ObjectId(id)
            self._new = False

        if hasattr(task, 'name'):
            task = task.name

        # generate this if it wasn't passed in
        if schedule is None:
            interval, start_time, occurrences = dateutils.parse_iso8601_interval(
                iso_schedule)
            schedule = pickle.dumps(CelerySchedule(interval))

        # generate this if it wasn't passed in
        principal = principal or factory.principal_manager().get_principal()

        self.args = args or []
        self.consecutive_failures = consecutive_failures
        self.enabled = enabled
        self.failure_threshold = failure_threshold
        self.iso_schedule = iso_schedule
        self.kwargs = kwargs or {}
        self.last_run_at = last_run_at
        self.last_updated = last_updated or time.time()
        self.name = id
        self.options = options or {}
        self.principal = principal
        self.resource = resource
        self.schedule = schedule
        self.task = task
        self.total_run_count = total_run_count

        if first_run is None:
            # get the date and time from the iso_schedule value, and if it does not have a date and
            # time, use the current date and time
            self.first_run = dateutils.format_iso8601_datetime(
                dateutils.parse_iso8601_interval(iso_schedule)[1]
                or datetime.utcnow().replace(tzinfo=isodate.UTC))
        elif isinstance(first_run, datetime):
            self.first_run = dateutils.format_iso8601_datetime(first_run)
        else:
            self.first_run = first_run
        if remaining_runs is None:
            self.remaining_runs = dateutils.parse_iso8601_interval(
                iso_schedule)[2]
        else:
            self.remaining_runs = remaining_runs

        self.next_run = self.calculate_next_run()
Exemple #36
0
 def test_interval(self):
     d = datetime.timedelta(hours=1)
     s = dateutils.format_iso8601_interval(d)
     i, t, r = dateutils.parse_iso8601_interval(s)
     self.assertTrue(d == i)
Exemple #37
0
 def test_interval(self):
     d = datetime.timedelta(hours=1)
     s = dateutils.format_iso8601_interval(d)
     i, t, r = dateutils.parse_iso8601_interval(s)
     self.assertTrue(d == i)
Exemple #38
0
def _sync_schedules(v1_database, v2_database, report):
    v1_repo_collection = v1_database.repos
    v2_repo_importer_collection = v2_database.repo_importers
    v2_scheduled_call_collection = v2_database.scheduled_calls

    # ugly hack to find out which repos have already been scheduled
    # necessary because $size is not a meta-query and doesn't support $gt, etc
    repos_without_schedules = v2_repo_importer_collection.find(
        {'scheduled_syncs': {'$size': 0}}, fields=['repo_id'])

    repo_ids_without_schedules = [r['repo_id'] for r in repos_without_schedules]

    repos_with_schedules = v2_repo_importer_collection.find(
        {'repo_id': {'$nin': repo_ids_without_schedules}}, fields=['repo_id'])

    repo_ids_with_schedules = [r['repo_id'] for r in repos_with_schedules]

    repos_to_schedule = v1_repo_collection.find(
        {'id': {'$nin': repo_ids_with_schedules}, 'sync_schedule': {'$ne': None}},
        fields=['id', 'sync_schedule', 'sync_options', 'last_sync'])

    for repo in repos_to_schedule:

        if repo['id'] not in repo_ids_without_schedules:
            report.error('Repository [%s] not found in the v2 database.'
                         'sync scheduling being canceled.' % repo['id'])
            return False

        args = [repo['id']]
        kwargs = {'overrides': {}}
        call_request = CallRequest(sync_with_auto_publish_itinerary, args, kwargs, principal=SystemUser())

        scheduled_call_document = {
            '_id': ObjectId(),
            'id': None,
            'serialized_call_request': None,
            'schedule': repo['sync_schedule'],
            'failure_threshold': None,
            'consecutive_failures': 0,
            'first_run': None,
            'last_run': None,
            'next_run': None,
            'remaining_runs': None,
            'enabled': True}

        scheduled_call_document['id'] = str(scheduled_call_document['_id'])

        schedule_tag = resource_tag(dispatch_constants.RESOURCE_SCHEDULE_TYPE, scheduled_call_document['id'])
        call_request.tags.append(schedule_tag)

        scheduled_call_document['serialized_call_request'] = call_request.serialize()

        if isinstance(repo['sync_options'], dict):
            scheduled_call_document['failure_threshold'] = repo['sync_options'].get('failure_threshold', None)

        interval, start, recurrences = dateutils.parse_iso8601_interval(scheduled_call_document['schedule'])

        scheduled_call_document['first_run'] = start or datetime.utcnow()
        scheduled_call_document['remaining_runs'] = recurrences
        scheduled_call_document['next_run'] = _calculate_next_run(scheduled_call_document)

        if repo['last_sync'] is not None:
            scheduled_call_document['last_run'] = dateutils.to_naive_utc_datetime(dateutils.parse_iso8601_datetime(repo['last_sync']))

        v2_scheduled_call_collection.insert(scheduled_call_document, safe=True)
        v2_repo_importer_collection.update({'repo_id': repo['id']},
                                           {'$push': {'scheduled_syncs': scheduled_call_document['id']}},
                                           safe=True)

    return True
Exemple #39
0
    def __init__(self, iso_schedule, task, total_run_count=0, next_run=None,
                 schedule=None, args=None, kwargs=None, principal=None, last_updated=None,
                 consecutive_failures=0, enabled=True, failure_threshold=None,
                 last_run_at=None, first_run=None, remaining_runs=None, id=None,
                 tags=None, name=None, options=None, resource=None):
        """
        :param iso_schedule:        string representing the schedule in ISO8601 format
        :type  iso_schedule:        basestring
        :param task:                the task that should be run on a schedule. This
                                    can be an instance of a celery task or the name
                                    of the task, as taken from a task's "name" attribute
        :type  task:                basestring or celery.Task
        :param total_run_count:     total number of times this schedule has run
        :type  total_run_count:     int
        :param next_run:            ignored, because it is always re-calculated at instantiation
        :param schedule:            pickled instance of celery.schedules.schedule,
                                    representing the schedule that should be run.
                                    This is optional.
        :type  schedule:            basestring or None
        :param args:                list of arguments that should be passed to the
                                    task's apply_async function as its "args" argument
        :type  args:                list
        :param kwargs:              dict of keyword arguments that should be passed to the task's
                                    apply_async function as its "kwargs" argument
        :type  kwargs:              dict
        :param principal:           pickled instance of pulp.server.db.model.auth.User
                                    representing the pulp user who the task
                                    should be run as. This is optional.
        :type  principal:           basestring or None
        :param last_updated:        timestamp for the last time this schedule was updated in the
                                    database as seconds since the epoch
        :type  last_updated:        float
        :param consecutive_failures:    number of times this task has failed consecutively. This
                                        gets reset to zero if the task succeeds.
        :type  consecutive_failures:    int
        :param enabled:             boolean indicating whether this schedule should be actively run
                                    by the scheduler. If False, the schedule will be ignored.
        :type  enabled:             bool
        :param failure_threshold:   number of consecutive failures after which this task should be
                                    automatically disabled. Because these tasks run asynchronously,
                                    they may finish in a different order than they were queued in.
                                    Thus, it is possible that n consecutive failures will be
                                    reported by jobs that were not queued consecutively. So do not
                                    depend on the queuing order when using this feature. If this
                                    value is 0, no automatic disabling will occur.
        :type  failure_threshold:   int
        :param last_run_at:         ISO8601 string representing when this schedule last ran.
        :type  last_run_at:         basestring
        :param first_run:           ISO8601 string or datetime instance (in UTC timezone)
                                    representing when this schedule should run or should have been
                                    run for the first time. If the schedule has a specified date and
                                    time to start, this will be that value. If not, the value from
                                    the first time the schedule was actually run will be used.
        :type  first_run:           basestring or datetime.datetime or NoneType
        :param remaining_runs:      number of runs remaining until this schedule will be
                                    automatically disabled.
        :type  remaining_runs:      int or NoneType
        :param id:                  unique ID used by mongodb to identify this schedule
        :type  id:                  basestring
        :param tags:                ignored, but allowed to exist as historical
                                    data for now
        :param name:                ignored, because the "id" value is used for this now. The value
                                    is here for backward compatibility.
        :param options:             dictionary that should be passed to the apply_async function as
                                    its "options" argument.
        :type  options:             dict
        :param resource:            optional string indicating a unique resource that should be used
                                    to find this schedule. For example, to find all schedules for a
                                    given repository, a resource string will be derived for that
                                    repo, and this collection will be searched for that resource
                                    string.
        :type  resource:            basestring
        """
        if id is None:
            # this creates self._id and self.id
            super(ScheduledCall, self).__init__()
            self._new = True
        else:
            self.id = id
            self._id = ObjectId(id)
            self._new = False

        if hasattr(task, 'name'):
            task = task.name

        # generate this if it wasn't passed in
        if schedule is None:
            interval, start_time, occurrences = dateutils.parse_iso8601_interval(iso_schedule)
            schedule = pickle.dumps(CelerySchedule(interval))

        # generate this if it wasn't passed in
        principal = principal or factory.principal_manager().get_principal()

        self.args = args or []
        self.consecutive_failures = consecutive_failures
        self.enabled = enabled
        self.failure_threshold = failure_threshold
        self.iso_schedule = iso_schedule
        self.kwargs = kwargs or {}
        self.last_run_at = last_run_at
        self.last_updated = last_updated or time.time()
        self.name = id
        self.options = options or {}
        self.principal = principal
        self.resource = resource
        self.schedule = schedule
        self.task = task
        self.total_run_count = total_run_count

        if first_run is None:
            # get the date and time from the iso_schedule value, and if it does not have a date and
            # time, use the current date and time
            self.first_run = dateutils.format_iso8601_datetime(
                dateutils.parse_iso8601_interval(iso_schedule)[1] or
                datetime.utcnow().replace(tzinfo=isodate.UTC))
        elif isinstance(first_run, datetime):
            self.first_run = dateutils.format_iso8601_datetime(first_run)
        else:
            self.first_run = first_run
        if remaining_runs is None:
            self.remaining_runs = dateutils.parse_iso8601_interval(iso_schedule)[2]
        else:
            self.remaining_runs = remaining_runs

        self.next_run = self.calculate_next_run()
Exemple #40
0
def _sync_schedules(v1_database, v2_database, report):
    v1_repo_collection = v1_database.repos
    v2_repo_importer_collection = v2_database.repo_importers
    v2_scheduled_call_collection = v2_database.scheduled_calls

    # ugly hack to find out which repos have already been scheduled
    # necessary because $size is not a meta-query and doesn't support $gt, etc
    repos_without_schedules = v2_repo_importer_collection.find(
        {'scheduled_syncs': {
            '$size': 0
        }}, fields=['repo_id'])

    repo_ids_without_schedules = [
        r['repo_id'] for r in repos_without_schedules
    ]

    repos_with_schedules = v2_repo_importer_collection.find(
        {'repo_id': {
            '$nin': repo_ids_without_schedules
        }}, fields=['repo_id'])

    repo_ids_with_schedules = [r['repo_id'] for r in repos_with_schedules]

    repos_to_schedule = v1_repo_collection.find(
        {
            'id': {
                '$nin': repo_ids_with_schedules
            },
            'sync_schedule': {
                '$ne': None
            }
        },
        fields=['id', 'sync_schedule', 'sync_options', 'last_sync'])

    for repo in repos_to_schedule:

        if repo['id'] not in repo_ids_without_schedules:
            report.error('Repository [%s] not found in the v2 database.'
                         'sync scheduling being canceled.' % repo['id'])
            return False

        args = [repo['id']]
        kwargs = {'overrides': {}}
        call_request = CallRequest(sync_with_auto_publish_itinerary,
                                   args,
                                   kwargs,
                                   principal=SystemUser())

        scheduled_call_document = {
            '_id': ObjectId(),
            'id': None,
            'serialized_call_request': None,
            'schedule': repo['sync_schedule'],
            'failure_threshold': None,
            'consecutive_failures': 0,
            'first_run': None,
            'last_run': None,
            'next_run': None,
            'remaining_runs': None,
            'enabled': True
        }

        scheduled_call_document['id'] = str(scheduled_call_document['_id'])

        schedule_tag = resource_tag(dispatch_constants.RESOURCE_SCHEDULE_TYPE,
                                    scheduled_call_document['id'])
        call_request.tags.append(schedule_tag)

        scheduled_call_document[
            'serialized_call_request'] = call_request.serialize()

        if isinstance(repo['sync_options'], dict):
            scheduled_call_document['failure_threshold'] = repo[
                'sync_options'].get('failure_threshold', None)

        interval, start, recurrences = dateutils.parse_iso8601_interval(
            scheduled_call_document['schedule'])

        scheduled_call_document['first_run'] = start or datetime.utcnow()
        scheduled_call_document['remaining_runs'] = recurrences
        scheduled_call_document['next_run'] = _calculate_next_run(
            scheduled_call_document)

        if repo['last_sync'] is not None:
            scheduled_call_document[
                'last_run'] = dateutils.to_naive_utc_datetime(
                    dateutils.parse_iso8601_datetime(repo['last_sync']))

        v2_scheduled_call_collection.insert(scheduled_call_document, safe=True)
        v2_repo_importer_collection.update(
            {'repo_id': repo['id']},
            {'$push': {
                'scheduled_syncs': scheduled_call_document['id']
            }},
            safe=True)

    return True