def test_populate_heap(self, _when): scheduler = mScheduler(app=self.app) scheduler.update_from_dict( {'foo': {'schedule': mocked_schedule(True, 10)}} ) scheduler.populate_heap() assert scheduler._heap == [event_t(1, 5, scheduler.schedule['foo'])]
def tick(self, event_t=event_t, min=min, heappop=heapq.heappop, heappush=heapq.heappush): """ Make the tick function thread safe Run a tick - one iteration of the scheduler. Executes on due task per call Returns: float: preferred delay in seconds for next call. """ with thread_lock: adjust = self.adjust max_interval = self.max_interval if self._heap is None or not self.schedules_equal( self.old_schedulers, self.schedule): self.old_schedulers = copy.copy(self.schedule) print('Repopulating the heap') self.populate_heap() H = self._heap if not H: return max_interval # event_t = namedtuple('event_t', ('time', 'priority', 'entry')) event = H[0] entry = event[2] is_due, next_time_to_run = self.is_due(entry) if is_due: verify = heappop(H) if verify is event: next_entry = self.reserve(entry) self.apply_entry(entry, producer=self.producer) heappush( H, event_t(self._when(next_entry, next_time_to_run), event[1], next_entry)) return 0 else: heappush(H, verify) return min(verify[0], max_interval) # Temporarily spin in a tight loop until the # @beat_init.connect callback occurs and calls run on the # (yahoo_panoptes.framework.plugins.scheduler.)PanoptesPluginScheduler # which calls update (-> merge_inplace) on the cached schedule. if self.SCHEDULE_POPULATED is False: return min(adjust(next_time_to_run), 0.01) return min(adjust(next_time_to_run) or max_interval, max_interval)
def get_new_tasks(self): debug('DatabaseScheduler: remove non exist data from beat') self.delete_out_dated_task() debug('DatabaseScheduler: decide whether have new record or not') def _when(entry, next_time_to_run): mktime = time.mktime return (mktime(entry.schedule.now().timetuple()) + (self.adjust(next_time_to_run) or 0)) if self._heap is None: self._heap = [] if self._schedule is None: self._schedule = {} query = self.Model.get_new_add(list(self._schedule.keys())) if query.count() > 0: info('DatabaseScheduler: found new record in database') s = {} for model in query: try: tmp_entry = self.Entry(model, app=self.app) self._heap.append( event_t( _when(tmp_entry, tmp_entry.is_due()[1]) or 0, 5, tmp_entry)) s[model.name] = tmp_entry model.modified = False model.save() except Exception as tmp: logger.exception('get_new_tasks %r', tmp) if s: self._schedule.update(s)
def _discard_overdue_tasks(self, heappop, heappush): """ Discards overdued tasks on first `tick` function call. It emulates executing overdued task, uses standart `populate_heap` function where overdued tasks have higher priority and therefore they are first in heap. :return: preferred delay in seconds for next tick call. :rtype: ``int`` """ if (self._heap is None or not self.schedules_equal(self.old_schedulers, self.schedule)): self.old_schedulers = copy.copy(self.schedule) self.populate_heap() events_heap = self._heap if not events_heap: return self.max_interval for _i in range(len(events_heap)): event = events_heap[0] entry = event[2] is_due, next_time_to_run = self.is_due(entry) if not is_due: break verify = heappop(events_heap) if verify is event: next_entry = self.reserve(entry) self._tasks_since_sync += 1 if self.should_sync(): self._do_sync() logger.info(f'{entry.task} task discarded!') heappush( events_heap, event_t(self._when(next_entry, next_time_to_run), event[1], next_entry)) return 0