def _reconstitute_job(self, job_state): schedule = job_state['schedule'] schedule.pop('coalesce', None) next_run_time = job_state['next_run_time'] if next_run_time == 1: # This is hacky. We need to subtract more than value of misfire_grace_time # so that the job will be missed right after loading it for the first time # after doing fresh install instead of being executed. next_run_time = int(time.time() - 1200) job = Job( id=job_state['id'], func="__main__:job", trigger=CronTrigger(**schedule), name=job_state['name'], args=[job_state['task']] + job_state['args'], scheduler=self._scheduler, executor='default', next_run_time=utc_timestamp_to_datetime(next_run_time), kwargs={ 'id': job_state['id'], 'name': job_state['name'], 'hidden': job_state.get('hidden', False), 'protected': job_state.get('protected', False) } ) job.coalesce = True job.max_instances = 1 job.misfire_grace_time = 600 job._jobstore_alias = self._alias return job
def _reconstitute_job(self, job_state): schedule = job_state['schedule'] schedule.pop('coalesce', None) job = Job( id=job_state['id'], func="__main__:job", trigger=CronTrigger(**schedule), name=job_state['name'], args=[job_state['task']] + job_state['args'], scheduler=self._scheduler, executor='default', next_run_time=utc_timestamp_to_datetime(job_state['next_run_time']), kwargs={ 'id': job_state['id'], 'name': job_state['name'], 'hidden': job_state.get('hidden', False), 'protected': job_state.get('protected', False) } ) job.coalesce = True job.max_instances = 1 job.misfire_grace_time = 0 job._jobstore_alias = self._alias return job
def _update_scheduler_status(scheduler, config): now = datetime.now() working_hours = parse_working_hours(config) jobs = scheduler.get_jobs() work = False for start, end in working_hours: if start <= (now.hour * 60 + now.minute) <= end: work = True if not work: for j in jobs: if j.name == 'partial' and \ j.func.func.__name__ in excluded_job_names: continue j.pause() else: # slack post message limit for job_id, times_ in g.items(): if times_ > config.max_alert - 1: job = Job(scheduler, job_id) job.pause() stoped[job_id] = (job, now) g[job_id] = 0 for job_id in list(stoped): job, time_ = stoped[job_id] if time_ + timedelta(minutes=config.pause_time) <= now: job.resume() del stoped[job_id] for j in jobs: if j.name == 'partial' and \ j.func.func.__name__ in excluded_job_names: continue if j.id not in stoped: j.resume()
async def _remove_and_silence_alarm(self, alarm_job: Job, reminder_text: str): # We found a reminder with an alarm. Remove it from the dict of current # alarms ALARMS.pop((self.room.room_id, reminder_text.upper()), None) if SCHEDULER.get_job(alarm_job.id): # Silence the alarm job alarm_job.remove()
def __init__(self, *args, **kwargs): if len(args) > 1 and hasattr(args[1], '__call__') or 'func' in kwargs: models.Model.__init__(self) Job.__init__(self, *args, **kwargs) else: models.Model.__init__(self, *args, **kwargs) self.instances = 0 self._lock = Lock() self.func = ref_to_obj(self.func_ref)
def add_job(self, key, jobname, trigger, func, args, kwargs, jobstore='default', **options): job = Job(trigger, func, args or [], kwargs or {}, options.pop('misfire_grace_time', self.misfire_grace_time), options.pop('coalesce', self.coalesce), name=jobname, **options) job.key = key if not self.running: self._pending_jobs.append((job, jobstore)) else: self._real_add_job(job, jobstore, True) return job
def construct_job(job: Job, scheduler, alias="beer_garden"): """Convert a Beergarden job to an APScheduler one.""" if job is None: return None trigger = construct_trigger(job.trigger_type, job.trigger) next_run_time = utc.localize(job.next_run_time) if job.next_run_time else None ap_job = APJob.__new__(APJob) ap_job._scheduler = scheduler ap_job._jobstore_alias = alias ap_job.__setstate__( { "id": job.id, "func": "beer_garden.scheduler:run_job", "trigger": trigger, "executor": "default", "args": (), "kwargs": {"request_template": job.request_template, "job_id": job.id}, "name": job.name, "misfire_grace_time": job.misfire_grace_time, "coalesce": job.coalesce, "max_instances": job.max_instances, "next_run_time": next_run_time, } ) return ap_job
def add_job(self, trigger, func, args, kwargs, jobstore='default', **options): """ Adds the given job to the job list and notifies the scheduler thread. Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class (see :ref:`job_options`). :param trigger: trigger that determines when ``func`` is called :param func: callable to run at the given time :param args: list of positional arguments to call func with :param kwargs: dict of keyword arguments to call func with :param jobstore: alias of the job store to store the job in :rtype: :class:`~apscheduler.job.Job` """ job = Job(trigger, func, args or [], kwargs or {}, options.pop('misfire_grace_time', self.misfire_grace_time), options.pop('coalesce', self.coalesce), **options) if not self.running: self._pending_jobs.append((job, jobstore)) logger.info('Adding job tentatively -- it will be properly ' 'scheduled when the scheduler starts') else: self._real_add_job(job, jobstore, True) return job
def _reconstitute_job(self, job_state): job_state = pickle.loads(job_state) job = Job.__new__(Job) job.__setstate__(job_state) job._scheduler = self._scheduler job._jobstore_alias = self._alias return job
def _modify_job(background_job: Job, job_modification: Dict[Text, Union[bool, Dict, Set]]) -> None: changes = {} job_id = background_job.id run_immediately = job_modification.pop("run_immediately", False) if run_immediately: changes["next_run_time"] = datetime.now() logger.debug(f"Running job with id '{job_id}' immediately.") # Set keyword arguments to call scheduled job function with changes["kwargs"] = _get_merged_job_kwargs(background_job, job_modification) background_job.modify(**changes) logger.debug(f"Modifying job with id '{background_job.id}'.")
def add(self): job_id = self._get_job_id() print(job_id) pid = self.pid if pid is not None: # if scheduler is running, stop it. self.stop() SCHEDULER.add_job( Job( get_forecast, # trigger=None, # args=None, # kwargs=None, id=job_id, # name=None, # misfire_grace_time=undefined, # coalesce=undefined, # max_instances=undefined, # next_run_time=undefined, # jobstore='default', # executor='default', # replace_existing=False, # **trigger_args )) if pid is not None: self.start() # fork and exit parent
def db_to_scheduler(document, scheduler, alias="beer_garden"): """Convert a database job to a scheduler's job.""" job = APJob.__new__(APJob) if document.next_run_time: next_run_time = utc.localize(document.next_run_time) else: next_run_time = None state = { "id": document.id, "func": "brew_view.scheduler:run_job", "trigger": construct_trigger(document.trigger_type, document.trigger), "executor": "default", "args": (), "kwargs": { "request_template": document.request_template, "job_id": str(document.id), }, "name": document.name, "misfire_grace_time": document.misfire_grace_time, "coalesce": document.coalesce, "max_instances": document.max_instances, "next_run_time": next_run_time, } job.__setstate__(state) job._scheduler = scheduler job._jobstore_alias = alias return job
def add_job(self, trigger, func, args, kwargs, jobstore='default', **options): """ Adds the given job to the job list and notifies the scheduler thread. The ``func`` argument can be given either as a callable object or a textual reference in the ``package.module:some.object`` format, where the first half (separated by ``:``) is an importable module and the second half is a reference to the callable object, relative to the module. Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class (see :ref:`job_options`). :param trigger: trigger that determines when ``func`` is called :param func: callable (or a textual reference to one) to run at the given time :param args: list of positional arguments to call func with :param kwargs: dict of keyword arguments to call func with :param jobstore: alias of the job store to store the job in :rtype: :class:`~apscheduler.job.Job` """ job = Job(trigger, func, args or [], kwargs or {}, options.pop('misfire_grace_time', self.misfire_grace_time), options.pop('coalesce', self.coalesce), **options) if not self.running: self._pending_jobs.append((job, jobstore)) logger.info( 'Adding job tentatively -- it will be properly scheduled when the scheduler starts' ) else: self._real_add_job(job, jobstore, True) return job
def db_to_scheduler(document, scheduler, alias='beer_garden'): """Convert a database job to a scheduler's job.""" job = APJob.__new__(APJob) if document.next_run_time: next_run_time = utc.localize(document.next_run_time) else: next_run_time = None state = { 'id': document.id, 'func': 'brew_view.scheduler.runner:run_job', 'trigger': construct_trigger(document.trigger_type, document.trigger), 'executor': 'default', 'args': (), 'kwargs': { 'request_template': document.request_template, 'job_id': str(document.id), }, 'name': document.name, 'misfire_grace_time': document.misfire_grace_time, 'coalesce': document.coalesce, 'max_instances': 3, 'next_run_time': next_run_time, } job.__setstate__(state) job._scheduler = scheduler job._jobstore_alias = alias return job
def _encode_job(self, job: Job): job_state = job.__getstate__() job_state["kwargs"]["config"] = None job_state["kwargs"]["bot"] = None # new_kwargs = job_state["kwargs"] # new_kwargs["config"] = None # new_kwargs["bot"] = None # job_state["kwargs"] = new_kwargs encoded = base64.b64encode( pickle.dumps(job_state, self.pickle_protocol)) out = { "_id": job.id, "next_run_time": datetime_to_utc_timestamp(job.next_run_time), "job_state": encoded.decode("ascii"), } job_state["kwargs"]["config"] = self.config job_state["kwargs"]["bot"] = self.bot # new_kwargs = job_state["kwargs"] # new_kwargs["config"] = self.config # new_kwargs["bot"] = self.bot # job_state["kwargs"] = new_kwargs # log.debug(f"Encoding job id: {job.id}\n" # f"Encoded as: {out}") return out
async def _decode_job(self, in_job): if in_job is None: return None job_state = in_job["job_state"] job_state = pickle.loads(base64.b64decode(job_state)) if job_state["args"]: # Backwards compatibility on args to kwargs job_state["kwargs"] = {**job_state["args"][0]} job_state["args"] = [] job_state["kwargs"]["config"] = self.config job_state["kwargs"]["bot"] = self.bot # new_kwargs = job_state["kwargs"] # new_kwargs["config"] = self.config # new_kwargs["bot"] = self.bot # job_state["kwargs"] = new_kwargs job = Job.__new__(Job) job.__setstate__(job_state) job._scheduler = self._scheduler job._jobstore_alias = self._alias # task_name, guild_id = _disassemble_job_id(job.id) # task = Task(task_name, guild_id, self.config) # await task.load_from_config() # save_task_objects.append(task) # # job.func = task.execute # log.debug(f"Decoded job id: {job.id}\n" # f"Decoded as {job_state}") return job
def test_adding_a_jobstore_adds_all_jobs_in_it(self, mock_notify_jobstore_event, mock_notify_job_event, _): watcher = SchedulerWatcher(self.scheduler) jobstore = MemoryJobStore() jobstore.add_job(Job(scheduler=self.scheduler, id='job_1', next_run_time=datetime.now() + timedelta(days=1))) jobstore.add_job(Job(scheduler=self.scheduler, id='job_2', next_run_time=datetime.now() + timedelta(days=2))) self.assertEqual(0, len(watcher.jobs)) self.scheduler.add_jobstore(jobstore, alias='in_memory_2') self.assertIn('in_memory_2', watcher.jobstores, 'Watcher should have the new jobstore tracked') self.assertEqual(2, len(watcher.jobs), 'Watcher should add all jobs in the newly added jobstore') self.assertTrue(all([job_id in watcher.jobs for job_id in ['job_1', 'job_2']])) self.assertEqual(2, mock_notify_job_event.call_count) mock_notify_jobstore_event.assert_called_once()
def create(**kwargs): kwargs.setdefault("scheduler", Mock(BaseScheduler, timezone=timezone)) job_kwargs = job_defaults.copy() job_kwargs.update(kwargs) job_kwargs["trigger"] = BlockingScheduler()._create_trigger( job_kwargs.pop("trigger"), job_kwargs.pop("trigger_args")) job_kwargs.setdefault("next_run_time", None) return Job(**job_kwargs)
def test_one_job_fails_to_load(self): global dummy_job2, dummy_job_temp job1 = Job(self.trigger, dummy_job, [], {}, 1, False) job2 = Job(self.trigger, dummy_job2, [], {}, 1, False) job3 = Job(self.trigger, dummy_job3, [], {}, 1, False) for job in job1, job2, job3: job.next_run_time = self.trigger_date self.jobstore.add_job(job) dummy_job_temp = dummy_job2 del dummy_job2 try: self.jobstore.load_jobs() eq_(len(self.jobstore.jobs), 2) finally: dummy_job2 = dummy_job_temp del dummy_job_temp
def _reconstitute_job(self, job_state): from apscheduler.job import Job job_state['jobstore'] = self job = Job.__new__(Job) job.__setstate__(job_state) job._scheduler = self._scheduler job._jobstore_alias = self._alias return job
def create(**kwargs): kwargs.setdefault('scheduler', Mock(BaseScheduler, timezone=timezone)) job_kwargs = job_defaults.copy() job_kwargs.update(kwargs) job_kwargs['trigger'] = BlockingScheduler()._create_trigger(job_kwargs.pop('trigger'), job_kwargs.pop('trigger_args')) job_kwargs.setdefault('next_run_time', None) return Job(**job_kwargs)
def _reconstitute_job(self, job_state): job_state = pickle.loads(job_state) job_state['jobstore'] = self job = Job.__new__(Job) job.__setstate__(job_state) job._scheduler = self._scheduler # pylint: disable=protected-access job._jobstore_alias = self._alias # pylint: disable=protected-access return job
def init(self): tz = self.store.timezone trigger = IntervalTrigger({}, minutes=5, timezone=tz) url = 'http://t.cn' now = datetime.now(tz) self.job1 = Job(1, trigger, url) self.job1.compute_next_run_time(now) self.job2 = Job(2, trigger, url) self.job2.compute_next_run_time(now) self.store.add_job(self.job1) self.store.add_job(self.job2) self.now = now self.url = url self.trigger = trigger
def test_jobs_equal(self): assert self.job == self.job job2 = Job(SimpleTrigger(self.RUNTIME), lambda: None, [], {}, 1, False) assert self.job != job2 job2.id = self.job.id = 123 eq_(self.job, job2) assert self.job != 'bleh'
def add_job(self, job: AppSchedulerJob): with transaction.atomic(): try: return DjangoJob.objects.create( id=job.id, next_run_time=get_django_internal_datetime(job.next_run_time), job_state=pickle.dumps(job.__getstate__(), self.pickle_protocol), ) except IntegrityError: raise ConflictingIdError(job.id)
def _reconstitute_job(self, row): ''' code gen by shell cmd: cat a | awk -F '=' '{print $1}' | cut -c5- | awk '{ print "job."$1" = row."$1}' what in file a is the wm_jobs_t create statement which can be found in the current source code file ''' conf = JobConf() conf.id = row.id conf.cmd = row.cmd conf.cron_str = row.cron_str conf.name = row.name conf.desc = row.desc conf.mails = row.mails conf.phones = row.phones conf.team = row.team conf.owner = row.owner conf.hosts = row.hosts conf.host_strategy = row.host_strategy conf.restore_strategy = row.restore_strategy conf.retry_strategy = row.retry_strategy conf.error_strategy = row.error_strategy conf.exist_strategy = row.exist_strategy conf.running_timeout_s = row.running_timeout_s conf.status = row.status conf.modify_time = row.modify_time conf.modify_user = row.modify_user conf.create_time = row.create_time conf.create_user = row.create_user conf.start_date = row.start_date conf.end_date = row.end_date conf.oupput_match_reg = row.oupput_match_reg conf.next_run_time = row.next_run_time job = Job.__new__(Job) job.conf = conf job.id = job.conf.id job._scheduler = self._scheduler job._jobstore_alias = self._alias job.trigger = self._create_trigger_by_conf(job) t = apscheduler.util.local_timestamp_to_datetime( conf.next_run_time) if conf.next_run_time > 0 else None t = apscheduler.util.convert_to_ware_datetime(t, get_localzone(), 'conf.next_run_time') state = { 'version': 1, 'conf': conf, 'id': conf.id, 'name': conf.name, 'next_run_time': t, } job.__setstate__(state) return job
def _prepare_job(job: APSJob) -> APSJob: """ Erase all unpickable data from telegram.ext.Job Args: job (:obj:`apscheduler.job`): The job to be processed. """ # make new job which is copy of actual job cause # modifying actual job also modifies jobs in threadpool # executor which are currently running/going to run and # we'll get incorrect argument instead of CallbackContext. prepped_job = APSJob.__new__(APSJob) prepped_job.__setstate__(job.__getstate__()) # remove CallbackContext from job args since # it includes refrences to dispatcher which # is unpickleable. we'll recreate CallbackContext # in _reconstitute_job method. if isinstance(job.args[0], CallbackContext): tg_job = job.args[0].job # APScheduler stores args as tuple. prepped_job.args = (tg_job.name, tg_job.context) return prepped_job
def load_jobs(self): jobs = [] for row in self.engine.execute(select([self.jobs_t])): try: job = Job.__new__(Job) job_dict = dict(row.items()) job.__setstate__(job_dict) jobs.append(job) except Exception: job_name = job_dict.get("name", "(unknown)") logger.exception('Unable to restore job "%s"', job_name) self.jobs = jobs
def load_jobs(self): jobs = [] for job_dict in itervalues(self.store): try: job = Job.__new__(Job) job.__setstate__(job_dict) jobs.append(job) except Exception: job_name = job_dict.get('name', '(unknown)') logger.exception('Unable to restore job "%s"', job_name) self.jobs = jobs
def test_constructor(job_id): with patch('apscheduler.job.Job._modify') as _modify: scheduler_mock = MagicMock(BaseScheduler) job = Job(scheduler_mock, id=job_id) assert job._scheduler is scheduler_mock assert job._jobstore_alias is None modify_kwargs = _modify.call_args[1] if job_id is None: assert len(modify_kwargs['id']) == 32 else: assert modify_kwargs['id'] == job_id
def load_jobs(self): jobs = [] for row in self.engine.execute(select([self.jobs_t])): try: job = Job.__new__(Job) job_dict = dict(row.items()) job.__setstate__(job_dict) jobs.append(job) except Exception: job_name = job_dict.get('name', '(unknown)') logger.exception('Unable to restore job "%s"', job_name) self.jobs = jobs
def _reconstitute_job(self, job_state): try: job_state = pickle.loads(job_state) except UnicodeDecodeError: # Unpickle py2 objects job_state = pickle.loads(job_state, encoding='latin1') job_state['jobstore'] = self job = Job.__new__(Job) job.__setstate__(job_state) job._scheduler = self._scheduler job._jobstore_alias = self._alias return job
def _reconstitute_job(self, job_state): schedule = job_state['schedule'] schedule.pop('coalesce', None) next_run_time = job_state['next_run_time'] if next_run_time == 1: # This is hacky. We need to subtract more than value of misfire_grace_time # so that the job will be missed right after loading it for the first time # after doing fresh install instead of being executed. next_run_time = int(time.time() - 1200) job = Job(id=job_state['id'], func="__main__:job", trigger=CronTrigger(**schedule), name=job_state['name'], args=[job_state['task']] + job_state['args'], scheduler=self._scheduler, executor='default', next_run_time=utc_timestamp_to_datetime(next_run_time), kwargs={ 'id': job_state['id'], 'name': job_state['name'], 'hidden': job_state.get('hidden', False), 'protected': job_state.get('protected', False) }) job.coalesce = True job.max_instances = 1 job.misfire_grace_time = 600 job._jobstore_alias = self._alias return job
def _reconstitute_job(self, row): ''' code gen by shell cmd: cat a | awk -F '=' '{print $1}' | cut -c5- | awk '{ print "job."$1" = row."$1}' what in file a is the wm_jobs_t create statement which can be found in the current source code file ''' conf = JobConf() conf.id = row.id conf.cmd = row.cmd conf.cron_str = row.cron_str conf.name = row.name conf.desc = row.desc conf.mails = row.mails conf.phones = row.phones conf.team = row.team conf.owner = row.owner conf.hosts = row.hosts conf.host_strategy = row.host_strategy conf.restore_strategy = row.restore_strategy conf.retry_strategy = row.retry_strategy conf.error_strategy = row.error_strategy conf.exist_strategy = row.exist_strategy conf.running_timeout_s = row.running_timeout_s conf.status = row.status conf.modify_time = row.modify_time conf.modify_user = row.modify_user conf.create_time = row.create_time conf.create_user = row.create_user conf.start_date = row.start_date conf.end_date = row.end_date conf.oupput_match_reg = row.oupput_match_reg conf.next_run_time = row.next_run_time job = Job.__new__(Job) job.conf = conf job.id = job.conf.id job._scheduler = self._scheduler job._jobstore_alias = self._alias job.trigger = self._create_trigger_by_conf(job) t = apscheduler.util.local_timestamp_to_datetime(conf.next_run_time) if conf.next_run_time > 0 else None t = apscheduler.util.convert_to_ware_datetime(t, get_localzone(), 'conf.next_run_time' ) state = { 'version': 1, 'conf': conf, 'id': conf.id, 'name': conf.name, 'next_run_time': t, } job.__setstate__(state) return job
def test_real_add_job(self, scheduler, job_exists, replace_existing, wakeup): job = Job(scheduler, id='foo', func=lambda: None, args=(), kwargs={}, next_run_time=None) jobstore = MagicMock( BaseJobStore, _alias='bar', add_job=MagicMock( side_effect=ConflictingIdError('foo') if job_exists else None)) scheduler.wakeup = MagicMock() scheduler._job_defaults = { 'misfire_grace_time': 3, 'coalesce': False, 'max_instances': 6 } scheduler._dispatch_event = MagicMock() scheduler._jobstores = {'bar': jobstore} # Expect and exception if the job already exists and we're not trying to replace it if job_exists and not replace_existing: pytest.raises(ConflictingIdError, scheduler._real_add_job, job, 'bar', replace_existing, wakeup) return scheduler._real_add_job(job, 'bar', replace_existing, wakeup) # Check that the undefined values were replaced with scheduler defaults assert job.misfire_grace_time == 3 assert job.coalesce is False assert job.max_instances == 6 assert job.next_run_time is None if job_exists: jobstore.update_job.assert_called_once_with(job) else: assert not jobstore.update_job.called if wakeup: scheduler.wakeup.assert_called_once_with() else: assert not scheduler.wakeup.called assert job._jobstore_alias == 'bar' assert scheduler._dispatch_event.call_count == 1 event = scheduler._dispatch_event.call_args[0][0] assert event.code == EVENT_JOB_ADDED assert event.job_id == 'foo'
def _db_to_job(self, row): if row['trigger_type'] == 'date': trigger = DateTrigger(run_date=row['run_date']) if row['trigger_type'] == 'cron': keys = row['crontab'].split(',')[1] values = row['crontab'].split(',')[0].split(' ') cronMapRev = {v: k for k, v in cronMap.items()} crontab = {cronMapRev[k]: values[i] for i, k in enumerate(keys)} trigger = CronTrigger(**crontab) if row['trigger_type'] == 'interval': trigger = IntervalTrigger(seconds=row['interval']) job = Job.__new__(Job) job.__setstate__({ 'id': row['id'], 'name': row['name'], 'func': row['func'], 'args': json.loads(row['args']) if row['args'] else [], 'kwargs': json.loads(row['kwargs']) if row['kwargs'] else {}, 'version': 1, 'trigger': trigger, 'executor': row['executor'], 'start_date': row['start_date'], 'end_date': row['end_date'], 'next_run_time': utc_timestamp_to_datetime(row['next_run_time'].timestamp()), 'coalesce': row['coalesce'], 'misfire_grace_time': row['misfire_grace_time'], 'max_instances': row['max_instances'], 'jobstore': self, }) job._scheduler = self._scheduler job._jobstore_alias = self._alias print(job._scheduler) print(job._jobstore_alias) return job
def load_jobs(self): jobs = [] for job_dict in self.collection.find(): try: job = Job.__new__(Job) job_dict['id'] = job_dict.pop('_id') job_dict['trigger'] = pickle.loads(job_dict['trigger']) job_dict['args'] = pickle.loads(job_dict['args']) job_dict['kwargs'] = pickle.loads(job_dict['kwargs']) job.__setstate__(job_dict) jobs.append(job) except Exception: job_name = job_dict.get('name', '(unknown)') logger.exception('Unable to restore job "%s"', job_name) self.jobs = jobs
def load_jobs(self): jobs = [] for row in self.engine.execute(self.jobs_t.select()): try: job = Job.__new__(Job) job_dict = dict(row.items()) job.__setstate__(job_dict) jobs.append(job) except Exception as e: print e traceback.print_exc(e) job_name = job_dict.get('name', '(unknown)') logger.exception('Unable to restore job "%s"', job_name) self.jobs = jobs return jobs
def get_job(self, id): select = self.jobs_t.select().where(self.jobs_t.c.id == id) try: row = self.engine.execute(select).fetchone() except Exception as e: #todo logger.exception(e) if row: try: job = Job.__new__(Job) job_dict = dict(row.items()) job.__setstate__(job_dict) return job except Exception: job_name = job_dict.get('name', 'unknown') logger.exception("Unable to restore job '%s'", job_name) return None
def pop(self, now): item = self.redis.rpop(self.key) if item is None: return 0, '', None try: job_id, change_type, job_str = item.split('||') job_id = int(job_id) if job_str == 'None': job = None else: job_state = pickle.loads(job_str) job = Job.__new__(Job) job.__setstate__(job_state) job.compute_next_run_time(now) return job_id, change_type, job except: logger=logging.getLogger('cron.backend') logger.exception('sync item invalid') return 0, ''
def load_jobs(self): #continue standart execution jobs = [] for job_dict in self.collection.find({'crecord_type': 'schedule'}): try: job = Job.__new__(Job) if job_dict['aaa_owner'] != 'root': if job_dict['kwargs']['task'] != 'task_reporting': raise ValueError("User %s isn\'t allow to run task %s" % (job_dict['aaa_owner'],job_dict['kwargs']['task'])) #keep memory of id job_dict_id = job_dict['_id'] job_dict['id'] = job_dict.pop('_id') if job_dict.has_key('runs'): job_dict['runs'] = job_dict['runs'] else: job_dict['runs'] = 0 job_dict['coalesce'] = False #try to get interval try: if job_dict['interval'] != None: job_dict['trigger'] = IntervalTrigger(timedelta(**job_dict['interval'])) except Exception, err: pass #try to get simple try: if job_dict['date'] != None: job_dict['trigger'] = SimpleTrigger( datetime(*job_dict['date'])) except Exception, err: pass #try to get crontab try: if job_dict['cron'] != None: job_dict['trigger'] = CronTrigger(**job_dict['cron']) except Exception, err: pass
def load_jobs(self): jobs = [] keys = self.redis.keys(self.key_prefix + "*") pipeline = self.redis.pipeline() for key in keys: pipeline.hgetall(key) results = pipeline.execute() for job_dict in results: job_state = {} try: job = Job.__new__(Job) job_state = pickle.loads(job_dict["job_state".encode()]) job_state["runs"] = long(job_dict["runs".encode()]) dateval = job_dict["next_run_time".encode()].decode() job_state["next_run_time"] = datetime.strptime(dateval, "%Y-%m-%dT%H:%M:%S") job.__setstate__(job_state) jobs.append(job) except Exception: job_name = job_state.get("name", "(unknown)") logger.exception('Unable to restore job "%s"', job_name) self.jobs = jobs
def add_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', executor='default', replace_existing=False, conf=None, **trigger_args): """ add_job(func, trigger=None, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, \ coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', \ executor='default', replace_existing=False, **trigger_args) Adds the given job to the job list and wakes up the scheduler if it's already running. Any option that defaults to ``undefined`` will be replaced with the corresponding default value when the job is scheduled (which happens when the scheduler is started, or immediately if the scheduler is already running). The ``func`` argument can be given either as a callable object or a textual reference in the ``package.module:some.object`` format, where the first half (separated by ``:``) is an importable module and the second half is a reference to the callable object, relative to the module. The ``trigger`` argument can either be: #. the alias name of the trigger (e.g. ``date``, ``interval`` or ``cron``), in which case any extra keyword arguments to this method are passed on to the trigger's constructor #. an instance of a trigger class :param func: callable (or a textual reference to one) to run at the given time :param str|apscheduler.triggers.base.BaseTrigger trigger: trigger that determines when ``func`` is called :param list|tuple args: list of positional arguments to call func with :param dict kwargs: dict of keyword arguments to call func with :param str|unicode id: explicit identifier for the job (for modifying it later) :param str|unicode name: textual description of the job :param int misfire_grace_time: seconds after the designated run time that the job is still allowed to be run :param bool coalesce: run once instead of many times if the scheduler determines that the job should be run more than once in succession :param int max_instances: maximum number of concurrently running instances allowed for this job :param datetime next_run_time: when to first run the job, regardless of the trigger (pass ``None`` to add the job as paused) :param str|unicode jobstore: alias of the job store to store the job in :param str|unicode executor: alias of the executor to run the job with :param bool replace_existing: ``True`` to replace an existing job with the same ``id`` (but retain the number of runs from the existing one) :rtype: Job """ job_kwargs = { 'trigger': trigger, 'executor': executor, 'func': func, 'args': tuple(args) if args is not None else (), 'kwargs': dict(kwargs) if kwargs is not None else {}, 'id': id, 'name': name, 'misfire_grace_time': misfire_grace_time, 'coalesce': coalesce, 'max_instances': max_instances, 'next_run_time': next_run_time } job_kwargs = dict((key, value) for key, value in six.iteritems(job_kwargs) if value is not undefined) job = Job(self, **job_kwargs) job.conf = conf # Don't really add jobs to job store before the scheduler is up and running with self._jobstore_lock: if not self.running: self._pending_jobs.append((job, jobstore, replace_existing)) logging.info('Adding job tentatively -- it will be properly scheduled when the scheduler starts') else: logging.info('Adding job really, it will be start as soon as the settings. ') self._real_add_job(job, jobstore, replace_existing, True) return job
from apscheduler.jobstores.sqlalchemy_store import SQLAlchemyJobStore from dateutil.tz import gettz from datetime import datetime, timedelta from apscheduler.triggers import IntervalTrigger, DateTrigger from apscheduler.scripts import HttpScript, CommandScript if __name__ == '__main__': queue = HotQueue('job_changes') script = HttpScript(url='http://baidu.comm') store = SQLAlchemyJobStore(url='sqlite:////tmp/task.db', tablename='tasks') #script = CommandScript(command='ping -c 3 www.baidu.com') local_tz = gettz('Asia/Chongqing') defaults = {'timezone': local_tz} trigger = IntervalTrigger(defaults, seconds=60) #trigger = DateTrigger(defaults, run_date=datetime(2013,12,11, 8, 11)) job = Job(name=u'BaiduCurlWithWrongUrl', script=script, trigger=trigger) #print job.run() now = datetime.now(local_tz) next_run_time = job.compute_next_run_time(now) print job.get_run_times(now+timedelta(seconds=60)) if next_run_time: print "add job" print job store.add_job(job) queue.put({'opt_type':'add', 'job_id':job.id})
from dateutil.tz import gettz from apscheduler.job import Job from apscheduler.triggers import IntervalTrigger from apscheduler.scripts import HttpScript t = IntervalTrigger({}, minutes=5, timezone=gettz('Asia/Chongqing')) s = HttpScript(url='http://baidu.com') job = Job(t, s) print job.__getstate__()
def load_jobs(self): #continue standart execution jobs = [] for job_dict in self.collection.find({'crecord_type': 'schedule'}): try: job = Job.__new__(Job) if job_dict['aaa_owner'] != 'account.root': if job_dict['kwargs']['task'] != 'task_reporting': raise ValueError("User %s isn\'t allow to run task %s" % (job_dict['aaa_owner'],job_dict['kwargs']['task'])) #keep memory of id job_dict_id = job_dict['_id'] job_dict['id'] = job_dict.pop('_id') if job_dict.has_key('runs'): job_dict['runs'] = job_dict['runs'] else: job_dict['runs'] = 0 job_dict['coalesce'] = False #try to get interval interval = job_dict.get('interval') if interval is not None: job_dict[TRIGGER] = IntervalTrigger(timedelta(**interval)) else: #try to get simple date = job_dict.get('date') if date is not None: job_dict[TRIGGER] = SimpleTrigger( datetime(*date)) else: #try to get crontab cron = job_dict.get('cron') if cron is not None: job_dict[TRIGGER] = CronTrigger(**cron) if TRIGGER not in job_dict: raise ValueError("No interval, nor date, nor cron is given in task %s".format(job_dict['crecord_name'])) job_dict['next_run_time'] = job_dict['trigger'].get_next_fire_time(datetime.now()) job_dict['args'] = job_dict['args'] job_dict['kwargs'] = job_dict['kwargs'] job_dict['max_runs'] = None job_dict['max_instances'] = 3 job_dict['name'] = job_dict['crecord_name'] job_dict['misfire_grace_time'] = 1 job_dict['func_ref'] = 'apschedulerlibs.aps_to_celery:launch_celery_task' job.__setstate__(job_dict) jobs.append(job) #change flag to true self.collection.update({'_id':job_dict_id},{"$set":{'loaded':True, 'next_run_time': job_dict['next_run_time']}},True) except Exception: job_name = job_dict.get('name', '(unknown)') logger.exception('Unable to restore job "%s"', job_name) logger.info(' + %s jobs loaded' % len(jobs)) self.jobs = jobs
class JobStoreTest(TestCase): def setUp(self): self.redis = StrictRedis(**settings.REDISES['default']) self.store = JobStore(self.redis, 'Asia/Chongqing') self.store.clear() self.redis.delete('active_jobs_pool') self.init() def tearDown(self): self.redis.delete('active_jobs_pool') self.store.clear() def init(self): tz = self.store.timezone trigger = IntervalTrigger({}, minutes=5, timezone=tz) url = 'http://t.cn' now = datetime.now(tz) self.job1 = Job(1, trigger, url) self.job1.compute_next_run_time(now) self.job2 = Job(2, trigger, url) self.job2.compute_next_run_time(now) self.store.add_job(self.job1) self.store.add_job(self.job2) self.now = now self.url = url self.trigger = trigger def testLoadJob(self): Task.objects.all().delete() task = self.createTask() task.save() job = self.store.load_job(task.id, self.now) self.assertEqual(self.url, job.func) def testLoadJobs(self): Task.objects.all().delete() self.store.clear() for i in range(3): task = self.createTask() task.save() self.redis.sadd('active_jobs_pool', task.id) self.assertEqual(3, self.store.count()) self.assertEqual(0, len(self.store.jobs)) self.store.load_jobs() self.assertEqual(3, len(self.store.jobs)) def testAddJob(self): self.assertEqual(2, self.store.count()) job = Job(3, self.trigger, self.url) job.compute_next_run_time(self.now) self.store.add_job(job) self.assertEqual(3, self.store.count()) def testRemoveJob(self): self.assertEqual(2, self.store.count()) self.store.remove_job(id=1) self.assertEqual(1, self.store.count()) self.store.remove_job(id=3) self.assertEqual(1, self.store.count()) def createTask(self): import json task = Task() task.run_time = json.dumps({'minutes':10}) task.name = 'name' task.run_entry = self.url return task def testUpdateJob(self): task = self.createTask() task.name = 'name1' task.save() self.store.clear() self.redis.sadd(self.store.active_jobs_key, task.id) self.store.load_jobs() job = self.store.find_job(task.id) self.assertEqual(task.name, job.name) task.name = 'name2' task.save() self.store.update_job(id=task.id) job = self.store.find_job(task.id) self.assertEqual(task.name, job.name) def testFindJob(self): self.assertEqual(2, self.store.count()) job = self.store.find_job(1) self.assertEqual(1, job.id) self.assertIsNone(self.store.find_job(3)) def testHasJob(self): self.assertTrue(self.store.has_job(1)) self.assertFalse(self.store.has_job(3))
def testAddJob(self): self.assertEqual(2, self.store.count()) job = Job(3, self.trigger, self.url) job.compute_next_run_time(self.now) self.store.add_job(job) self.assertEqual(3, self.store.count())
def setup(self): self.trigger = SimpleTrigger(self.RUNTIME) self.job = Job(self.trigger, dummyfunc, [], {}, 1, False)
class TestJob(object): RUNTIME = datetime(2010, 12, 13, 0, 8, 0) def setup(self): self.trigger = SimpleTrigger(self.RUNTIME) self.job = Job(self.trigger, dummyfunc, [], {}, 1, False) def test_compute_next_run_time(self): self.job.compute_next_run_time(self.RUNTIME - timedelta(microseconds=1)) eq_(self.job.next_run_time, self.RUNTIME) self.job.compute_next_run_time(self.RUNTIME) eq_(self.job.next_run_time, self.RUNTIME) self.job.compute_next_run_time(self.RUNTIME + timedelta(microseconds=1)) eq_(self.job.next_run_time, None) def test_compute_run_times(self): expected_times = [self.RUNTIME + timedelta(seconds=1), self.RUNTIME + timedelta(seconds=2)] self.job.trigger = IntervalTrigger(timedelta(seconds=1), self.RUNTIME) self.job.compute_next_run_time(expected_times[0]) eq_(self.job.next_run_time, expected_times[0]) run_times = self.job.get_run_times(self.RUNTIME) eq_(run_times, []) run_times = self.job.get_run_times(expected_times[0]) eq_(run_times, [expected_times[0]]) run_times = self.job.get_run_times(expected_times[1]) eq_(run_times, expected_times) def test_max_runs(self): self.job.max_runs = 1 self.job.runs += 1 self.job.compute_next_run_time(self.RUNTIME) eq_(self.job.next_run_time, None) def test_eq_num(self): # Just increasing coverage here assert not self.job == 'dummyfunc' def test_getstate(self): state = self.job.__getstate__() eq_(state, dict(trigger=self.trigger, func_ref='testjob:dummyfunc', name='dummyfunc', args=[], kwargs={}, misfire_grace_time=1, coalesce=False, max_runs=None, max_instances=1, runs=0)) def test_setstate(self): trigger = SimpleTrigger('2010-12-14 13:05:00') state = dict(trigger=trigger, name='testjob.dummyfunc', func_ref='testjob:dummyfunc', args=[], kwargs={}, misfire_grace_time=2, max_runs=2, coalesce=True, max_instances=2, runs=1) self.job.__setstate__(state) eq_(self.job.trigger, trigger) eq_(self.job.func, dummyfunc) eq_(self.job.max_runs, 2) eq_(self.job.coalesce, True) eq_(self.job.max_instances, 2) eq_(self.job.runs, 1) assert not hasattr(self.job, 'func_ref') assert isinstance(self.job._lock, lock_type) def test_jobs_equal(self): assert self.job == self.job job2 = Job(SimpleTrigger(self.RUNTIME), lambda: None, [], {}, 1, False) assert self.job != job2 job2.id = self.job.id = 123 eq_(self.job, job2) assert self.job != 'bleh' def test_instances(self): self.job.max_instances = 2 eq_(self.job.instances, 0) self.job.add_instance() eq_(self.job.instances, 1) self.job.add_instance() eq_(self.job.instances, 2) assert_raises(MaxInstancesReachedError, self.job.add_instance) self.job.remove_instance() eq_(self.job.instances, 1) self.job.remove_instance() eq_(self.job.instances, 0) assert_raises(AssertionError, self.job.remove_instance) def test_repr(self): self.job.compute_next_run_time(self.RUNTIME) eq_(repr(self.job), "<Job (name=dummyfunc, " "trigger=<SimpleTrigger (run_date=datetime.datetime(2010, 12, 13, 0, 8))>)>") eq_(str(self.job), "dummyfunc (trigger: date[2010-12-13 00:08:00], " "next run at: 2010-12-13 00:08:00)")
from hotqueue import HotQueue from apscheduler.job import Job from apscheduler.jobstores.sqlalchemy_store import SQLAlchemyJobStore from apscheduler.triggers import IntervalTrigger from apscheduler.scripts import HttpScript if __name__ == '__main__': script = HttpScript(url='http://baidu.com') local_tz = gettz('Asia/Chongqing') defaults = {'timezone': local_tz} trigger = IntervalTrigger(defaults, seconds=3) store = SQLAlchemyJobStore(url='sqlite:////tmp/task.db', tablename='tasks') job = store.get_job(3) if not job: job = Job(id=3, name='BaiduCheck', script=script, trigger=trigger) store.add_job(job) print job job.trigger = IntervalTrigger(defaults, seconds=5) store.update_job(job) queue = HotQueue('job_changes') queue.put({'job_id':job.id, 'opt_type':'update'})