def increment_failure_count(schedule_id): """ Increment the number of consecutive failures, and if it has met or exceeded the threshold, disable the schedule. :param schedule_id: ID of the schedule whose count should be incremented :type schedule_id: str """ try: spec = {'_id': ObjectId(schedule_id)} except InvalidId: raise exceptions.InvalidValue(['schedule_id']) delta = { '$inc': {'consecutive_failures': 1}, '$set': {'last_updated': time.time()}, } schedule = ScheduledCall.get_collection().find_and_modify( query=spec, update=delta, new=True) if schedule: scheduled_call = ScheduledCall.from_db(schedule) if scheduled_call.failure_threshold is None or not scheduled_call.enabled: return if scheduled_call.consecutive_failures >= scheduled_call.failure_threshold: _logger.info(_('disabling schedule %(id)s with %(count)d consecutive failures') % { 'id': schedule_id, 'count': scheduled_call.consecutive_failures }) delta = {'$set': { 'enabled': False, 'last_updated': time.time(), }} ScheduledCall.get_collection().update(spec, delta)
def tearDown(self): super(SchedulerTests, self).tearDown() ScheduledCall.get_collection().drop() self.scheduler = None dispatch_factory.coordinator = self._coordinator_factory self._coordinator_factory = None dispatch_factory._SCHEDULER = None
def delete_by_resource(resource): """ Deletes all schedules for the specified resource :param resource: string indicating a unique resource :type resource: basestring """ ScheduledCall.get_collection().remove({'resource': resource}, safe=True)
def delete(schedule_id): """ Deletes the schedule with unique ID schedule_id :param schedule_id: a unique ID for a schedule :type schedule_id: basestring """ try: ScheduledCall.get_collection().remove({'_id': ObjectId(schedule_id)}, safe=True) except InvalidId: raise exceptions.InvalidValue(['schedule_id'])
def remove(self, schedule_id): """ Remove a scheduled call request @param schedule_id: id of the schedule for the call request @type schedule_id: str """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) if ScheduledCall.get_collection().find_one(schedule_id) is None: raise pulp_exceptions.MissingResource(schedule=str(schedule_id)) scheduled_call_collection = ScheduledCall.get_collection() scheduled_call_collection.remove({'_id': schedule_id}, safe=True)
def reset_failure_count(schedule_id): """ Reset the consecutive failure count on a schedule to 0, presumably because it ran successfully. :param schedule_id: ID of the schedule whose count should be reset :type schedule_id: str """ spec = {'_id': ObjectId(schedule_id)} delta = {'$set': { 'consecutive_failures': 0, 'last_updated': time.time(), }} ScheduledCall.get_collection().update(spec=spec, document=delta)
def test_remove(self): call_request = CallRequest(call) schedule_id = self.scheduler.add(call_request, SCHEDULE_START_TIME) self.scheduler.remove(schedule_id) collection = ScheduledCall.get_collection() scheduled_call = collection.find_one({'_id': schedule_id}) self.assertTrue(scheduled_call is None)
def update(self, schedule_id, **schedule_updates): """ Update a scheduled call request Valid schedule updates: * call_request * schedule * failure_threshold * remaining_runs * enabled @param schedule_id: id of the schedule for the call request @type schedule_id: str @param schedule_updates: updates for scheduled call @type schedule_updates: dict """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) scheduled_call_collection = ScheduledCall.get_collection() if scheduled_call_collection.find_one(schedule_id) is None: raise pulp_exceptions.MissingResource(schedule=str(schedule_id)) validate_schedule_updates(schedule_updates) call_request = schedule_updates.pop('call_request', None) if call_request is not None: schedule_updates['serialized_call_request'] = call_request.serialize() schedule = schedule_updates.get('schedule', None) if schedule is not None: interval, start, runs = dateutils.parse_iso8601_interval(schedule) schedule_updates.setdefault('remaining_runs', runs) # honor explicit update # XXX (jconnor) it'd be nice to update the next_run if the schedule # has changed, but it requires mucking with the internals of the # of the scheduled call instance, which is all encapsulated in the # ScheduledCall constructor # the next_run field will be correctly updated after the next run scheduled_call_collection.update({'_id': schedule_id}, {'$set': schedule_updates}, safe=True)
def update(schedule_id, delta): """ Updates the schedule with unique ID schedule_id. This only allows updating of fields in ScheduledCall.USER_UPDATE_FIELDS. :param schedule_id: a unique ID for a schedule :type schedule_id: basestring :param delta: a dictionary of keys with values that should be modified on the schedule. :type delta: dict :return: instance of ScheduledCall representing the post-update state :rtype ScheduledCall :raise exceptions.UnsupportedValue :raise exceptions.MissingResource """ unknown_keys = set(delta.keys()) - ScheduledCall.USER_UPDATE_FIELDS if unknown_keys: raise exceptions.UnsupportedValue(list(unknown_keys)) delta['last_updated'] = time.time() try: spec = {'_id': ObjectId(schedule_id)} except InvalidId: raise exceptions.InvalidValue(['schedule_id']) schedule = ScheduledCall.get_collection().find_and_modify( query=spec, update={'$set': delta}, safe=True, new=True) if schedule is None: raise exceptions.MissingResource(schedule_id=schedule_id) return ScheduledCall.from_db(schedule)
def add(self, call_request, schedule, **schedule_options): """ Add a scheduled call request Valid schedule options: * failure_threshold: max number of consecutive failures, before scheduled call is disabled, None means no max * last_run: datetime of the last run of the call request or None if no last run * enabled: boolean flag if the scheduled call is enabled or not @param call_request: call request to schedule @type call_request: pulp.server.dispatch.call.CallRequest @param schedule: iso8601 formatted interval schedule @type schedule: str @param schedule_options: keyword options for this schedule @type schedule_options: dict @return: schedule id if successfully scheduled or None otherwise @rtype: str or None """ validate_schedule_options(schedule, schedule_options) scheduled_call = ScheduledCall(call_request, schedule, **schedule_options) next_run = self.calculate_next_run(scheduled_call) if next_run is None: return None scheduled_call_collection = ScheduledCall.get_collection() scheduled_call['next_run'] = next_run scheduled_call_collection.insert(scheduled_call, safe=True) return str(scheduled_call['_id'])
def test_add_no_runs(self): call_request = CallRequest(call) schedule_id = self.scheduler.add(call_request, SCHEDULE_0_RUNS) self.assertTrue(schedule_id is None) collection = ScheduledCall.get_collection() cursor = collection.find() self.assertTrue(cursor.count() == 0)
def find_by_repo_list(repo_id_list): """ Returns serialized versions of all importers for given repos. Any IDs that do not refer to valid repos are ignored and will not raise an error. @param repo_id_list: list of importer IDs to fetch @type repo_id_list: list of str @return: list of serialized importers @rtype: list of dict """ spec = {'repo_id': {'$in': repo_id_list}} projection = {'scratchpad': 0} importers = list(RepoImporter.get_collection().find(spec, projection)) # Process any scheduled syncs and get schedule details using schedule id for importer in importers: scheduled_sync_ids = importer.get('scheduled_syncs', None) if scheduled_sync_ids is not None: scheduled_sync_details = list( ScheduledCall.get_collection().find( {"id": { "$in": scheduled_sync_ids }})) importer['scheduled_syncs'] = [ s["schedule"] for s in scheduled_sync_details ] return importers
def update_last_run(self, scheduled_call, call_report=None): """ Update the metadata for a scheduled call that has been run @param scheduled_call: scheduled call to be updated @type scheduled_call: dict @param call_report: call report from last run, if available @type call_report: CallReport instance or None """ schedule_id = scheduled_call['_id'] update = {} # use scheduled time instead of current to prevent schedule drift delta = update.setdefault('$set', {}) delta['last_run'] = scheduled_call['next_run'] # if we finished in an error state, make sure we haven't crossed the threshold state = getattr(call_report, 'state', None) if state == dispatch_constants.CALL_ERROR_STATE: inc = update.setdefault('$inc', {}) inc['consecutive_failures'] = 1 failure_threshold = scheduled_call['failure_threshold'] consecutive_failures = scheduled_call['consecutive_failures'] + 1 if failure_threshold and failure_threshold <= consecutive_failures: delta = update.setdefault('$set', {}) delta['enabled'] = False msg = _('Scheduled task [%s] disabled after %d consecutive failures') _LOG.error(msg % (schedule_id, consecutive_failures)) else: delta = update.setdefault('$set', {}) delta['consecutive_failures'] = 0 # decrement the remaining runs, if we're tracking that if scheduled_call['remaining_runs'] is not None: inc = update.setdefault('$inc', {}) inc['remaining_runs'] = -1 scheduled_call_collection = ScheduledCall.get_collection() scheduled_call_collection.update({'_id': schedule_id}, update, safe=True)
def reset_failure_count(schedule_id): """ Reset the consecutive failure count on a schedule to 0, presumably because it ran successfully. :param schedule_id: ID of the schedule whose count should be reset :type schedule_id: str """ try: spec = {'_id': ObjectId(schedule_id)} except InvalidId: raise exceptions.InvalidValue(['schedule_id']) delta = {'$set': { 'consecutive_failures': 0, 'last_updated': time.time(), }} ScheduledCall.get_collection().update(spec=spec, document=delta)
def __init__(self, dispatch_interval=30): self.dispatch_interval = dispatch_interval self.scheduled_call_collection = ScheduledCall.get_collection() self.__exit = False self.__lock = threading.RLock() self.__condition = threading.Condition(self.__lock) self.__dispatcher = None
def get_enabled(): """ Get schedules that are enabled, that is, their "enabled" attribute is True :return: pymongo cursor of ScheduledCall database objects :rtype: pymongo.cursor.Cursor """ criteria = Criteria(filters={'enabled': True}) return ScheduledCall.get_collection().query(criteria)
def setUp(self): super(SchedulerTests, self).setUp() pickling.initialize() self.scheduler = Scheduler() # replace the coordinator so we do not actually execute tasks self._coordinator_factory = dispatch_factory.coordinator dispatch_factory.coordinator = mock.Mock() # NOTE we are not starting the scheduler self.scheduled_call_collection = ScheduledCall.get_collection()
def remove(self, schedule_id): """ Remove a scheduled call request @param schedule_id: id of the schedule for the call request @type schedule_id: str """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) scheduled_call_collection = ScheduledCall.get_collection() scheduled_call_collection.remove({'_id': schedule_id}, safe=True)
def disable(self, schedule_id): """ Disable a scheduled call request without removing it @param schedule_id: id of the schedule for the call request @type schedule_id: str """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) scheduled_call_collection = ScheduledCall.get_collection() update = {'$set': {'enabled': False}} scheduled_call_collection.update({'_id': schedule_id}, update, safe=True)
def enable(self, schedule_id): """ Enable a previously disabled scheduled call request @param schedule_id: id of the schedule for the call request @type schedule_id: str """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) scheduled_call_collection = ScheduledCall.get_collection() update = {'$set': {'enabled': True}} scheduled_call_collection.update({'_id': schedule_id}, update, safe=True)
def scheduler_complete_callback(call_request, call_report): """ Call back for call request results and rescheduling """ scheduler = dispatch_factory.scheduler() scheduled_call_collection = ScheduledCall.get_collection() schedule_id = call_report.schedule_id scheduled_call = scheduled_call_collection.find_one({'_id': ObjectId(schedule_id)}) scheduler.update_last_run(scheduled_call, call_report) scheduler.update_next_run(scheduled_call)
def test_disable_enable(self): call_request = CallRequest(call) schedule_id = self.scheduler.add(call_request, SCHEDULE_3_RUNS) collection = ScheduledCall.get_collection() scheduled_call = collection.find_one({'_id': ObjectId(schedule_id)}) self.assertTrue(scheduled_call['enabled']) self.scheduler.disable(schedule_id) scheduled_call = collection.find_one({'_id': ObjectId(schedule_id)}) self.assertFalse(scheduled_call['enabled']) self.scheduler.enable(schedule_id) scheduled_call = collection.find_one({'_id': ObjectId(schedule_id)}) self.assertTrue(scheduled_call['enabled'])
def scheduler_complete_callback(call_request, call_report): """ Call back for call request results and rescheduling """ scheduled_call_collection = ScheduledCall.get_collection() schedule_id = call_report.schedule_id scheduled_call = scheduled_call_collection.find_one({'_id': ObjectId(schedule_id)}) if scheduled_call is None: # schedule was deleted while call was running return scheduler = dispatch_factory.scheduler() scheduler.update_last_run(scheduled_call, call_report)
def get_by_resource(resource): """ Get schedules by resource :param resource: unique ID for a lockable resource :type resource: basestring :return: iterator of ScheduledCall instances :rtype: iterator """ criteria = Criteria(filters={'resource': resource}) schedules = ScheduledCall.get_collection().query(criteria) return itertools.imap(ScheduledCall.from_db, schedules)
def remove(self, schedule_id): """ Remove a scheduled call request @param schedule_id: id of the schedule for the call request @type schedule_id: str """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) if ScheduledCall.get_collection().find_one(schedule_id) is None: raise pulp_exceptions.MissingResource(schedule=str(schedule_id)) self.scheduled_call_collection.remove({'_id': schedule_id}, safe=True)
def find(self, *tags): """ Find the scheduled call requests for the given call request tags @param tags: call request tags @type tags: list @return: possibly empty list of scheduled call report dictionaries @rtype: list """ scheduled_call_collection = ScheduledCall.get_collection() query = {'serialized_call_request.tags': {'$all': tags}} scheduled_calls = scheduled_call_collection.find(query) reports = [scheduled_call_to_report_dict(s) for s in scheduled_calls] return reports
def test_complete_callback(self): scheduled_call_request = CallRequest(itinerary_call) schedule_id = self.scheduler.add(scheduled_call_request, SCHEDULE_3_RUNS) run_call_request = scheduled_call_request.call()[0] run_call_report = CallReport.from_call_request(run_call_request) run_call_report.schedule_id = schedule_id scheduler_complete_callback(run_call_request, run_call_report) collection = ScheduledCall.get_collection() scheduled_call = collection.find_one({'_id': ObjectId(schedule_id)}) self.assertNotEqual(scheduled_call['last_run'], None)
def get(schedule_ids): """ Get schedules by ID :param schedule_ids: a list of schedule IDs :type schedule_ids: list :return: iterator of ScheduledCall instances :rtype: iterator """ object_ids = map(ObjectId, schedule_ids) criteria = Criteria(filters={'_id': {'$in': object_ids}}) schedules = ScheduledCall.get_collection().query(criteria) return itertools.imap(ScheduledCall.from_db, schedules)
def test_create_schedule(self): sync_options = {'override_config': {}} schedule_data = {'schedule': 'R1/P1DT'} schedule_id = self.schedule_manager.create_sync_schedule(self.repo_id, self.importer_type_id, sync_options, schedule_data) collection = ScheduledCall.get_collection() schedule = collection.find_one(ObjectId(schedule_id)) self.assertFalse(schedule is None) self.assertTrue(schedule_id == str(schedule['_id'])) schedule_list = self._importer_manager.list_sync_schedules(self.repo_id) self.assertTrue(schedule_id in schedule_list)
def __init__(self, dispatch_interval=DEFAULT_DISPATCH_INTERVAL): """ :param dispatch_interval: scheduler dispatch interval :type dispatch_interval: int """ assert dispatch_interval > 0 self.dispatch_interval = dispatch_interval self.scheduled_call_collection = ScheduledCall.get_collection() self.__exit = False self.__lock = threading.RLock() self.__condition = threading.Condition(self.__lock) self.__dispatch_thread = None
def test_complete_callback_missing_schedule(self): scheduled_call_request = CallRequest(itinerary_call) schedule_id = self.scheduler.add(scheduled_call_request, SCHEDULE_3_RUNS) run_call_request = scheduled_call_request.call()[0] run_call_report = CallReport.from_call_request(run_call_request) run_call_report.schedule_id = schedule_id collection = ScheduledCall.get_collection() collection.remove({'_id': ObjectId(schedule_id)}, safe=True) try: scheduler_complete_callback(run_call_request, run_call_report) except: self.fail()
def update_next_run(self, scheduled_call): """ Update the metadata for a scheduled call that will be run again @param scheduled_call: scheduled call to be updated @type scheduled_call: dict """ scheduled_call_collection = ScheduledCall.get_collection() schedule_id = scheduled_call['_id'] next_run = self.calculate_next_run(scheduled_call) if next_run is None: # remove the scheduled call if there are no more scheduled_call_collection.remove({'_id': schedule_id}, safe=True) return update = {'$set': {'next_run': next_run}} scheduled_call_collection.update({'_id': schedule_id}, update, safe=True)
def get_updated_since(seconds): """ Get schedules that are enabled, that is, their "enabled" attribute is True, and that have been updated since the timestamp represented by "seconds". :param seconds: seconds since the epoch :param seconds: float :return: pymongo cursor of ScheduledCall database objects :rtype: pymongo.cursor.Cursor """ criteria = Criteria(filters={ 'enabled': True, 'last_updated': {'$gt': seconds}, }) return ScheduledCall.get_collection().query(criteria)
def get(self, schedule_id): """ Get the call request and the schedule for the given schedule id @param schedule_id: id of the schedule for the call request @type schedule_id: str @return: scheduled call report dictionary @rtype: dict """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) scheduled_call_collection = ScheduledCall.get_collection() scheduled_call = scheduled_call_collection.find_one({'_id': schedule_id}) if scheduled_call is None: raise pulp_exceptions.MissingResource(schedule=str(schedule_id)) report = scheduled_call_to_report_dict(scheduled_call) return report
def get(schedule_ids): """ Get schedules by ID :param schedule_ids: a list of schedule IDs :type schedule_ids: list :return: iterator of ScheduledCall instances :rtype: iterator """ try: object_ids = map(ObjectId, schedule_ids) except InvalidId: raise exceptions.InvalidValue(['schedule_ids']) criteria = Criteria(filters={'_id': {'$in': object_ids}}) schedules = ScheduledCall.get_collection().query(criteria) return itertools.imap(ScheduledCall.from_db, schedules)
def update(schedule_id, delta): """ Updates the schedule with unique ID schedule_id. This only allows updating of fields in ScheduledCall.USER_UPDATE_FIELDS. :param schedule_id: a unique ID for a schedule :type schedule_id: basestring :param delta: a dictionary of keys with values that should be modified on the schedule. :type delta: dict :return: instance of ScheduledCall representing the post-update state :rtype ScheduledCall :raise exceptions.UnsupportedValue :raise exceptions.MissingResource """ unknown_keys = set(delta.keys()) - ScheduledCall.USER_UPDATE_FIELDS if unknown_keys: raise exceptions.UnsupportedValue(list(unknown_keys)) delta['last_updated'] = time.time() # bz 1139703 - if we update iso_schedule, update the pickled object as well if 'iso_schedule' in delta: interval, start_time, occurrences = dateutils.parse_iso8601_interval( delta['iso_schedule']) delta['schedule'] = pickle.dumps(CelerySchedule(interval)) # set first_run and next_run so that the schedule update will take effect new_schedule_call = ScheduledCall(delta['iso_schedule'], 'dummytaskname') delta['first_run'] = new_schedule_call.first_run delta['next_run'] = new_schedule_call.next_run try: spec = {'_id': ObjectId(schedule_id)} except InvalidId: # During schedule update, MissingResource should be raised even if # schedule_id is invalid object_id. raise exceptions.MissingResource(schedule_id=schedule_id) schedule = ScheduledCall.get_collection().find_and_modify( query=spec, update={'$set': delta}, safe=True, new=True) if schedule is None: raise exceptions.MissingResource(schedule_id=schedule_id) return ScheduledCall.from_db(schedule)
def delete(schedule_id): """ Deletes the schedule with unique ID schedule_id :param schedule_id: a unique ID for a schedule :type schedule_id: basestring """ try: spec = {'_id': ObjectId(schedule_id)} except InvalidId: raise exceptions.InvalidValue(['schedule_id']) schedule = ScheduledCall.get_collection().find_and_modify( query=spec, remove=True, safe=True) if schedule is None: raise exceptions.MissingResource(schedule_id=schedule_id)
def delete(schedule_id): """ Deletes the schedule with unique ID schedule_id :param schedule_id: a unique ID for a schedule :type schedule_id: basestring """ try: spec = {'_id': ObjectId(schedule_id)} except InvalidId: # During schedule deletion, MissingResource should be raised even if # schedule_id is invalid object_id. raise exceptions.MissingResource(schedule_id=schedule_id) schedule = ScheduledCall.get_collection().find_and_modify( query=spec, remove=True) if schedule is None: raise exceptions.MissingResource(schedule_id=schedule_id)
def test_delete_schedule(self): publish_options = {'override_config': {}} schedule_data = {'schedule': 'R1/P1DT'} schedule_id = self.schedule_manager.create_publish_schedule(self.repo_id, self.distributor_id, publish_options, schedule_data) collection = ScheduledCall.get_collection() schedule = collection.find_one(ObjectId(schedule_id)) self.assertFalse(schedule is None) self.schedule_manager.delete_publish_schedule(self.repo_id, self.distributor_id, schedule_id) schedule = collection.find_one(ObjectId(schedule_id)) self.assertTrue(schedule is None) schedule_list = self._distributor_manager.list_publish_schedules(self.repo_id, self.distributor_id) self.assertFalse(schedule_id in schedule_list)
def clean(self): super(ScheduleTests, self).clean() Repo.get_collection().remove(safe=True) RepoDistributor.get_collection().remove(safe=True) RepoImporter.get_collection().remove(safe=True) ScheduledCall.get_collection().remove(safe=True)