import os, time from datetime import datetime from apscheduler.schedulers.background import BackgroundScheduler def myjob(): print(datetime.now()) if __name__ == '__main__': scheduler = BackgroundScheduler() scheduler.start() job = scheduler.add_job(myjob, 'interval', seconds=1, id='myjob') print(job) jobs = scheduler.get_jobs() print(jobs) try: time.sleep(5) print('pause job') scheduler.pause_job('myjob') time.sleep(5) print('resume job') scheduler.resume_job('myjob') print('reschedule job ...') scheduler.reschedule_job('myjob', trigger='cron', second='*/5') time.sleep(10) except (KeyboardInterrupt, SystemExit): scheduler.shutdown()
def encode_compute(): scheduler = BackgroundScheduler() scheduler.start() message = request.form['message'] image = request.files['img'] if not message or not image: return redirect(url_for('steganography_web.encode')) filename = 'website/static/images/' + str( session['uid']) + image.filename.split('.')[0] + '.png' image.save(filename) session['steganography_image'] = filename binary_string = encode_string(message) + encode_string(END_OF_ENCODE) image = encode_image(filename, binary_string) time = datetime.datetime.now() + datetime.timedelta(minutes=4) if scheduler.get_job(filename): scheduler.reschedule_job(job_id=filename, trigger='date', run_date=time) print(f'job rescheduled for {time}') else: scheduler.add_job(delete_file, args=[filename], trigger='date', run_date=time, id=filename) print(f'job scheduled for {time}') send_days_left_text() return redirect(url_for('steganography_web.encode', show='True'))
def test8(): """定时执行任务,修改作业""" scheduler = BackgroundScheduler() job1 = scheduler.add_job(my_job, 'interval', args=('123', ), seconds=2, id='my_job_id', max_instances=2) # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; print('开始scheduler') scheduler.start() time.sleep(2) print('再添加一个实例') job = scheduler.add_job(tick, 'interval', seconds=2, id='id2') # add_job函数这里有一个max_instances参数用于指定当前工作同时运行的实例数量, time.sleep(4) job.modify( max_instances=6, name='Alternate name' ) # 上一个工作正在运行而未结束,那么下一个就认为失败, 那么此时可以为一个特定的作业设置最大数量的调度程序;可以为一个特定的作业设置最大数量的调度程序 # scheduler.reschedule_job('my_job_id', trigger='cron', minute='*/5') # 每5分钟执行一次,具体参见cron表达式 print('修改执行时间间隔') scheduler.reschedule_job('id2', trigger='cron', second='*/1') # 每1秒执行一次,具体参见cron表达式 time.sleep(3) print('ok')
class Scheduler: def __init__(self): self.jobs = {} self.scheduler = BackgroundScheduler({ 'apscheduler.job_defaults.coalesce': 'true', }) def addJob(self, policy, job_function, job_name): # Check that a job with the same name has not already be registered if self.jobs.has_key(job_name): raise Exception('A job named "' + job_name + '" has already been scheduled') # Get the period trigger to use trigger = self._getTrigger(policy) # Add the job to the scheduler j = self.scheduler.add_job(job_function, name=job_name, max_instances=1, trigger=trigger) getLogger( __name__).debug('Job "' + job_name + '" has been added to scheduler, it has the id ' + j.id + '. It is scheduled every ' + policy) # Store job if so that we can update it if needed self.jobs[job_name] = j.id def rescheduleJob(self, policy, job_name): # Check that the job with job_name well exist if not (self.jobs.has_key(job_name)): raise Exception( 'Job named "' + job_name + '" can not be rescheduled because it is not registered in scheduler' ) # Get the period trigger to use trigger = self._getTrigger(policy) # Reschedule the job with the new trigger getLogger(__name__).debug('Reschedule job "' + job_name + '" having id ' + self.jobs.get(job_name)) self.scheduler.reschedule_job(self.jobs.get(job_name), trigger=trigger) def start(self): self.scheduler.start() getLogger(__name__).info('Start scheduler') def stop(self): self.scheduler.shutdown() getLogger(__name__).info('Stop scheduler') def _getTrigger(self, policy): comp = re.compile("^([0-9]*)([smhd])?$") match = comp.match(policy) policy = match.groups() if (policy[0] == '' or policy[0] == None): raise Exception( "The periodicity of your task is not well defined.") else: period = int(policy[0]) return IntervalTrigger(seconds=period)
class ApsScheduler(Scheduler): def __init__(self): self._scheduler = BackgroundScheduler() def start(self): self._scheduler.add_jobstore(DjangoJobStore(), 'default') self._scheduler.start() def add_job(self, func: Callable, func_kwargs: dict, cron_schedule: CronSchedule) -> str: job = self._scheduler.add_job(func=func, kwargs=func_kwargs, trigger='cron', start_date=cron_schedule.start_date, end_date=cron_schedule.end_date, year=cron_schedule.year, month=cron_schedule.month, day=cron_schedule.day, week=cron_schedule.week, day_of_week=cron_schedule.day_of_week, second=cron_schedule.second, minute=cron_schedule.minute, hour=cron_schedule.hour) return job.id def modify(self, cron_schedule: CronSchedule): for job_id in self._get_all_affected_job_ids(cron_schedule.pk): self._scheduler.reschedule_job( job_id, trigger='cron', start_date=cron_schedule.start_date, end_date=cron_schedule.end_date, year=cron_schedule.year, month=cron_schedule.month, day=cron_schedule.day, week=cron_schedule.week, day_of_week=cron_schedule.day_of_week, second=cron_schedule.second, minute=cron_schedule.minute, hour=cron_schedule.hour) def delete(self, cron_schedule_id: int): for job_id in self._get_all_affected_job_ids(cron_schedule_id): self._scheduler.remove_job(job_id) def delete_job(self, job_id): self._scheduler.remove_job(job_id) @staticmethod def _get_all_affected_job_ids(cron_schedule_id): objs = Timelapse.objects.filter(schedule_id=cron_schedule_id) for obj in objs: assert isinstance(obj, Timelapse) yield obj.schedule_job_id
def new_season(idS): # 3 temps # fermer la précédente saisons # lancer la nouvelle saison # lancer la saison dans 1 an Dday, Dmonth, Dyear, dict_Dates = parse_dates() idS_old = 0 if idS == 1: idS_old = 4 else: idS_old = idS - 1 print("Nouvelle saison en cours de lancement...") es, nb_invest, bank_tot = end_season() if es: # Calcul du multiplicateur try: m_x = bank_tot / nb_invest print("m_x : {}".format(m_x)) except ValueError: m_x = 10 m_n = dict_Dates["mult"] print("m_n : {}".format(m_n)) if m_n == 0: m_n = 2 try: mult = math.log2(m_n) * math.log10(m_x) print("mult : {}".format(mult)) except ValueError: mult = 1 # Ecriture de la nouvelle année et du nouveau mult dans le fichier dict_Dates[idS_old] = dict_Dates[idS_old][:6] + str(Dyear[idS_old] + 1) dict_Dates["mult"] = mult dict_Dates["total"] = dict_Dates["total"] + 1 path = "core/saisons.json" with open(path, 'w', encoding='utf-8') as json_file: json.dump(dict_Dates, json_file, indent=4) # Reprogrammation de la nouvelle date scheduler = BackgroundScheduler() scheduler.reschedule_job('S' + str(idS_old), trigger='date', run_date=date(int(dict_Dates[idS_old][6:]), Dmonth[int(idS_old)], Dday[int(idS_old)])) else: print("\n\n\n ...Impossible de mettre fin à la précédente saison.")
class Scheduler: def __init__(self): self.jobs = {} self.scheduler = BackgroundScheduler({ 'apscheduler.job_defaults.coalesce': 'true', }) def addJob(self, policy, job_function, job_name): # Check that a job with the same name has not already be registered if self.jobs.has_key(job_name): raise Exception ('A job named "' + job_name + '" has already been scheduled') # Get the period trigger to use trigger = self._getTrigger(policy) # Add the job to the scheduler j = self.scheduler.add_job(job_function, name = job_name, max_instances = 1, trigger=trigger) getLogger(__name__).debug('Job "' + job_name +'" has been added to scheduler, it has the id ' + j.id + '. It is scheduled every ' + policy) # Store job if so that we can update it if needed self.jobs[job_name] = j.id def rescheduleJob(self, policy, job_name): # Check that the job with job_name well exist if not(self.jobs.has_key(job_name)): raise Exception ('Job named "' + job_name + '" can not be rescheduled because it is not registered in scheduler') # Get the period trigger to use trigger = self._getTrigger(policy) # Reschedule the job with the new trigger getLogger(__name__).debug('Reschedule job "' + job_name +'" having id ' + self.jobs.get(job_name)) self.scheduler.reschedule_job(self.jobs.get(job_name), trigger=trigger) def start(self): self.scheduler.start() getLogger(__name__).info('Start scheduler') def stop(self): self.scheduler.shutdown() getLogger(__name__).info('Stop scheduler') def _getTrigger(self, policy): comp = re.compile("^([0-9]*)([smhd])?$") match = comp.match(policy) policy = match.groups() if (policy[0] == '' or policy[0] == None): raise Exception ("The periodicity of your task is not well defined.") else: period = int(policy[0]) return IntervalTrigger(seconds=period)
class ManageSchedule(ManageObject,Device): def __init__(self): """Constructeur de notre classe""" self._timezone = Device.config['timezone'] self._connexion_mode = Device.config['connexion_mode'] self._scheduler = '' self._DateToCompare = '' self._Now = '' self.DateFileModification ='' def _Get_Timezone (self): return self._timezone def _Set_Scheduler_With_Timezone (self,_timezone): self._scheduler = BackgroundScheduler(timezone=_timezone,replace_existing=True) def _Set_DateToCompare(self): self._DateToCompare = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') def _Get_DateFileModification(self,file): return datetime.datetime.fromtimestamp(os.stat(file).st_mtime).strftime('%Y-%m-%d %H:%M:%S') def LaunchSchedule(self,): self._Set_Scheduler_With_Timezone(self._timezone) self.DefaultJob() self._scheduler.start() def updateSchedule(self,_file,_state,_type,id_sched): if self._DateToCompare =='': self.UpdateObject(_file,_state,_type) self._Set_DateToCompare() if self._DateToCompare > self._Get_DateFileModification(_file): print('la date a comparer est plus grande je ne fais rien') else: print('la date a comparer est plus petite je fais qqchose') self._scheduler.reschedule_job(id_sched, trigger='cron', minute='*/1') self.UpdateObject(_file,_state,_type) self._Set_DateToCompare() self._scheduler.reschedule_job(id_sched, trigger='cron',second='*') def DefaultJob(self): self._scheduler.add_job(self.updateSchedule,args=['main.json','enable','component','sched_component'], trigger='cron', second='*',id='sched_component') self._scheduler.add_job(self.updateSchedule,args=['main.json','enable','plugins','sched_plugin'], trigger='cron', second='*',id='sched_plugin')
def mandelbrot(): scheduler = BackgroundScheduler() scheduler.start() try: session['uid'] except: session['uid'] = uuid.uuid4() if 'r' in request.values and 'i' in request.values: real = float(request.values['r']) imaginary = float(request.values['i']) image = create(real, imaginary) filename = 'website/static/images/' + str( session['uid']) + 'fractal.png' # image.save('static/images/fractal.png') image.save(filename) time = datetime.datetime.now() + datetime.timedelta(seconds=10) if scheduler.get_job(filename): scheduler.reschedule_job(job_id=filename, trigger='date', run_date=time) print(f'job rescheduled for {time}') else: scheduler.add_job(delete_file, args=[filename], trigger='date', run_date=time, id=filename) print(f'job scheduled for {time}') return render_template("mandelbrot.html", image='/' + filename[8:] + '?' + str(rand.randint(1000)), real=real, imag=imaginary) else: return render_template("mandelbrot.html", image='/static/images/defaultFractal.png', real=0, imag=0)
def test8(): """定时执行任务,修改作业""" scheduler = BackgroundScheduler() job1 = scheduler.add_job(my_job, 'interval', args=('123',),seconds=2, id='my_job_id',max_instances=2) # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; print('开始scheduler') scheduler.start() time.sleep(2) print('再添加一个实例') job = scheduler.add_job(tick, 'interval', seconds=2,id='id2') # add_job函数这里有一个max_instances参数用于指定当前工作同时运行的实例数量, time.sleep(4) job.modify(max_instances=6, name='Alternate name') # 上一个工作正在运行而未结束,那么下一个就认为失败, 那么此时可以为一个特定的作业设置最大数量的调度程序;可以为一个特定的作业设置最大数量的调度程序 # scheduler.reschedule_job('my_job_id', trigger='cron', minute='*/5') # 每5分钟执行一次,具体参见cron表达式 print('修改执行时间间隔') scheduler.reschedule_job('id2', trigger='cron', second='*/1') # 每1秒执行一次,具体参见cron表达式 time.sleep(3) print('ok')
class Scheduler: def startup_scheduler(self): self.scheduler = BackgroundScheduler() logger.info("----INIT SCHEDULE OBJECT-----") self.scheduler.start() self.scheduler.add_job(auto_backup_job, trigger="cron", hour="3", max_instances=1) settings = SiteSettings.get_site_settings() time = cron_parser(settings.webhooks.webhookTime) self.webhook = self.scheduler.add_job( post_webhooks, trigger="cron", name="webhooks", hour=time.hours, minute=time.minutes, max_instances=1, ) logger.info(self.scheduler.print_jobs()) def reschedule_webhooks(self): """ Reads the site settings database entry to reschedule the webhooks task Called after each post to the webhooks endpoint. """ settings = SiteSettings.get_site_settings() time = cron_parser(settings.webhooks.webhookTime) self.scheduler.reschedule_job( self.webhook.id, trigger="cron", hour=time.hours, minute=time.minutes, ) logger.info(self.scheduler.print_jobs())
def add_scheduler(): jobstores = {'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')} executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } job_defaults = {'coalesce': False, 'max_instances': 3} scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) scheduler.start() scheduler.reschedule_job(job_id='covid_job', trigger='interval', minutes=app.config['REQUEST_INTERVAL']) scheduler.reschedule_job(job_id='currencies_job', trigger='interval', minutes=app.config['REQUEST_INTERVAL']) scheduler.print_jobs()
def update_game_thread(): log.info("开始启动定时更新游戏线程...") pre_update_time = DEFAULT_GAME_UPDATE_TIME update_job_id = "update_game" parse_time(pre_update_time) # 启动定时调度器框架 scheduler = BackgroundScheduler(logger=log, timezone="Asia/Shanghai") scheduler.add_job(update_game, trigger="cron", id=update_job_id, **parse_time(pre_update_time)) scheduler.start() SLEEP_TIME = 30 while True: while True: try: # 获取当前游戏更新时间 time_str = DeviceGameService.get_game_update_time(cache_client) if time_str == pre_update_time: log.info("当前定时更新游戏时间: {}".format(time_str)) break log.info("当前游戏更新时间发生变更: pre = {} cur = {}".format( pre_update_time, time_str)) pre_update_time = time_str scheduler.reschedule_job(update_job_id, trigger="cron", **parse_time(time_str)) except Exception as e: log.error("游戏更新调度周期异常:") log.exception(e) break time.sleep(SLEEP_TIME)
class Scheduler: def __init__(self): self.sched = BackgroundScheduler() self.sched.start() self.job_id = '' def __del__(self): self.sched.shutdown() def shutdown(self): self.sched.shutdown() def stop(self, job_id): try: self.sched.remove_job(job_id) except JobLookupError as error: print(f"fail to stop Scheduler : {error}.") return def restart(self, job_id, run_time): #run_time format : %H:%M hour, minute = map(int, run_time.split(':')) self.sched.reschedule_job(job_id, trigger='cron', hour=hour, minute=minute) def add_job(self, func_name, func_param, run_dayOfWeek, run_time, job_name): #run_time format : %H:%M hour, minute = map(int, run_time.split(':')) self.sched.add_job(func_name, args=func_param, trigger='cron', day_of_week=run_dayOfWeek, hour=hour, minute=minute, id=job_name, name=job_name)
class JobScheduler(ISingleton): @inject def __init__( self, database_config: DatabaseConfig, database_session_manager: DatabaseSessionManager, ): self.database_session_manager: DatabaseSessionManager = database_session_manager self.database_config: DatabaseConfig = database_config self.scheduler: BackgroundScheduler = None def run(self): self.run_scheduler() print("job_process started") def run_scheduler(self): jobstores = { 'default': SQLAlchemyJobStore(url=self.database_config.connection_string, tablename='ApSchedulerJobsTable', engine=self.database_session_manager.engine, metadata=IocManager.Base.metadata, tableschema='Aps') } executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } job_defaults = {'coalesce': False, 'max_instances': 5} self.scheduler = BackgroundScheduler(daemon=True, jobstores=jobstores, executors=executors, job_defaults=job_defaults) JobSchedulerEvent.job_scheduler_type = JobScheduler self.scheduler.add_listener(JobSchedulerEvent.listener_finish, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) self.scheduler.add_listener(JobSchedulerEvent.listener_job_added, EVENT_JOB_ADDED) self.scheduler.add_listener(JobSchedulerEvent.listener_job_submitted, EVENT_JOB_SUBMITTED) self.scheduler.add_listener(JobSchedulerEvent.listener_job_removed, EVENT_JOB_REMOVED) self.scheduler.add_listener( JobSchedulerEvent.listener_all_jobs_removed, EVENT_ALL_JOBS_REMOVED) self.scheduler.add_listener( JobSchedulerEvent.listener_job_others, EVENT_JOB_MODIFIED | EVENT_JOB_MISSED | EVENT_JOB_MAX_INSTANCES) self.scheduler.add_listener( JobSchedulerEvent.listener_scheduler_other_events, EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN | EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED | EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED) self.scheduler.start() print('To clear the alarms, delete the example.sqlite file.') print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) def add_job_with_date(self, job_function, run_date, args=None, kwargs=None) -> Job: # if run_date is None: # run_date = datetime.now() + timedelta(seconds=10) job: Job = self.scheduler.add_job(job_function, 'date', run_date=run_date, misfire_grace_time=30000, args=args, kwargs=kwargs) return job def add_job_with_cron(self, job_function, cron: CronTrigger, args=None, kwargs=None) -> Job: # if cron.start_date is not None and cron.start_date < datetime.now().astimezone(get_localzone()): # cron.start_date = None # if cron.end_date is not None and cron.end_date < datetime.now().astimezone(get_localzone()): # cron.end_date = None job: Job = self.scheduler.add_job(job_function, cron, misfire_grace_time=15, args=args, kwargs=kwargs) return job def remove_job(self, job_id): self.scheduler.remove_job(job_id) def modify_job(self, job_id, jobstore=None, **changes): return self.scheduler.modify_job(job_id, jobstore, **changes) def reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args): return self.scheduler.reschedule_job(job_id, jobstore, trigger, **trigger_args) def pause_job(self, job_id, jobstore=None): return self.scheduler.pause_job(job_id, jobstore) def resume_job(self, job_id, jobstore=None): return self.scheduler.resume_job(job_id, jobstore) def remove_job(self, job_id, jobstore=None): self.scheduler.remove_job(job_id, jobstore) def get_job(self, job_id): return self.scheduler.get_job(job_id) def get_jobs(self, jobstore=None): return self.scheduler.get_jobs(jobstore)
class SchedulerManager(object): # __metaclass__ = ABCMeta global _mongoclient def __init__(self): self.jobstores = { 'mongo': MongoDBJobStore(collection='job1', database='saasjob', client=_mongoclient), 'default': MemoryJobStore() } self.executors = { 'default': ThreadPoolExecutor(1), 'processpool': ProcessPoolExecutor(1) } self.job_defaults = { 'coalesce': False, 'misfire_grace_time': 1, 'max_instances': 1 } self._sched = BackgroundScheduler(jobstores=self.jobstores, executors=self.executors, job_defaults=self.job_defaults) # 添加 任务提交 事件监听 self._sched.add_listener(self.when_job_submitted, EVENT_JOB_SUBMITTED) # 添加 任务执行完成 事件监听 self._sched.add_listener(self.when_job_executed, EVENT_JOB_EXECUTED) # 添加 任务异常退出 事件监听 self._sched.add_listener(self.when_job_crashed, EVENT_JOB_ERROR) self._jobs = {} self._jobhandlers = {} # format, key: jobid, value: jobhandler self._jobs_key = ["name", "func", "args", "kwargs"] self.start() def cmd_valid(self, cmd): cmd = cmd.strip() if cmd.startswith("python"): return True else: return False def get_job_trigger(self, _job): # ('trigger', <CronTrigger (second='4', timezone='Asia/Shanghai')>) _trigger = self._get_job_attr(_job, "trigger") # options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default] if _trigger: return dict([(f.name, f.__str__()) for f in _trigger.fields if not f.is_default]) else: return {} # 获取job属性 def _get_job_attr(self, _job, attr): try: result = eval("_job.%s" % attr) return result except: import traceback print(traceback.print_exc()) return None def when_job_submitted(self, event): try: job_id = event.job_id if job_id not in self._jobhandlers and job_id in self._jobhandlers: self._jobhandlers.setdefault(job_id, JobHandler(self._jobs[job_id])) jobhandler = self._jobhandlers[event.job_id] jobhandler.when_job_submitted() print("%s submitted at %s" % (event.job_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))) except: import traceback print(traceback.print_exc()) def when_job_executed(self, event): try: job_id = event.job_id if job_id not in self._jobhandlers: self._jobhandlers.setdefault(job_id, JobHandler(self._jobs[job_id])) jobhandler = self._jobhandlers[event.job_id] jobhandler.when_job_executed() print("%s executed at %s" % (event.job_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))) except: import traceback print(traceback.print_exc()) def when_job_crashed(self, event): try: if event.exception: job_id = event.job_id if job_id not in self._jobhandlers: self._jobhandlers.setdefault( job_id, JobHandler(self._jobs[job_id])) jobhandler = self._jobhandlers[event.job_id] jobhandler.when_job_crashed() print("%s crashed at %s" % (event.job_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))) except: import traceback print(traceback.print_exc()) # 添加例行任务,crontab 格式 def addCron(self, cmd, **params): try: create_jobid = uuid.uuid4().hex if not self.cmd_valid(cmd): return {"errinfo": "wrong cmd"} jobcmdobj = JobCmd(cmd) data = params.get("data", {}) jobcmdobj.set_jobid(create_jobid) s = params.get("second", None) if params.get("second", None) != "*" else None m = params.get("minute", None) if params.get("minute", None) != "*" else None h = params.get("hour", None) if params.get("hour", None) != "*" else None d = params.get("day", None) if params.get("day", None) != "*" else None dw = params.get( "day_of_week", None) if params.get("day_of_week", None) != "*" else None mnth = params.get( "month", None) if params.get("month", None) != "*" else None y = params.get("year", None) if params.get("year", None) != "*" else None _job = self._sched.add_job(jobcmdcallable, 'cron', year=y, month=mnth, day=d, day_of_week=dw, hour=h, minute=m, second=s, args=[jobcmdobj, data], executor="processpool", jobstore="mongo", id=create_jobid) self._jobhandlers.setdefault(create_jobid, JobHandler(_job)) # 保存 job 属性 return {"job_id": create_jobid} except: import traceback print(traceback.print_exc(), cmd, params) return False # 修改 job 属性 def modifyJobAttr(self, job_id, **changes): try: _job = self._sched.modify_job(job_id=job_id, **changes) self._jobs[job_id] = _job if job_id in self._jobhandlers: self._jobhandlers[job_id].job = _job else: self._jobhandlers.setdefault(job_id, JobHandler(_job)) return True except: import traceback print(traceback.print_exc(), job_id, changes) return False def modifyJobData(self, job_id, data): try: args = self._get_job_attr(self._jobhandlers[job_id].job, "args") # args_copy = [item for item in args] for key in data: args[1][key] = data[key] _job = self._sched.modify_job(job_id, args=args) self._jobs[job_id] = _job if job_id in self._jobhandlers: self._jobhandlers[job_id].job = _job else: self._jobhandlers.setdefault(job_id, JobHandler(_job)) return True except: import traceback print(traceback.print_exc(), job_id, data) return False # 修改执行时间,crontab 格式 def modifyJobFreq(self, job_id, cronargs): try: _job = self._sched.reschedule_job(job_id, trigger='cron', **cronargs) self._jobs[job_id] = _job if job_id in self._jobhandlers: self._jobhandlers[job_id].job = _job else: self._jobhandlers.setdefault(job_id, JobHandler(_job)) return True except: import traceback print(traceback.print_exc(), job_id, cronargs) return False # 删除 job def removeFromCron(self, job_id): try: self._sched.remove_job(job_id) if job_id in self._jobhandlers: self._jobhandlers.pop(job_id) if job_id in self._jobs: self._jobs.pop(job_id) return True except: import traceback print(traceback.print_exc(), job_id) return False def job_exists(self, job_id): if job_id in self._jobhandlers or job_id in self._jobs: if job_id not in self._jobhandlers and job_id in self._jobs: self._jobhandlers[job_id] = JobHandler(self._jobs[job_id]) elif job_id in self._jobhandlers and job_id not in self._jobs: self._jobs[job_id] = self._jobhandlers[job_id].job return True else: return False # 根据 job id 查询任务信息 def findCronJob(self, job_ids): result = [] _keys = [ "cmd", "create_stamp", "is_running", "start_stamp", "hope_runtime", "is_success", "is_pause", "status", "name", "desc", "allowmodify" ] for job_id in job_ids: print("job_exists", self.job_exists(job_id)) if self.job_exists(job_id): _jobhander = self._jobhandlers[job_id] job_info = _jobhander.jobhandlerattr cron_trigger = {} # cron_trigger = self.get_cron_trigger(_jobhander.job) tmp = {} tmp["job_id"] = job_id if job_info["is_running"]: execute_time = time.time() - job_info["start_stamp"] tmp["running_time"] = round(execute_time, 3) else: tmp["running_time"] = round(job_info["hope_runtime"], 3) for key in _keys: v = job_info.get(key, None) if key == "is_running": tmp["finished"] = False if job_info[ "is_running"] else True else: tmp[key] = v if tmp["finished"]: tmp["completed_per"] = 1.0 else: tmp["completed_per"] = round( tmp["running_time"] / max([tmp["running_time"], tmp["hope_runtime"]]), 3) # del tmp["hope_runtime"] # del tmp["is_success"] # del tmp["is_pause"] tmp.pop("hope_runtime") tmp.pop("is_success") tmp.pop("is_pause") _result = dict(tmp, **cron_trigger) print("_result", _result) if _result["status"] == 3: _result["completed_per"] = 0 _result["running_time"] = 0 _result["start_stamp"] = None result.append(_result) else: result.append({"job_id": job_id, "errinfo": "no exists"}) return result def getAllJobInfo(self): try: result = self.findCronJob( set(self._jobhandlers.keys()) | set(self._jobs.keys())) return result except: import traceback print(traceback.print_exc()) return False def start_addition(self): for _job in self._sched.get_jobs(): job_id = self._get_job_attr(_job, "id") self._jobs.setdefault(job_id, _job) def start(self): try: self._sched.start() self._sched.pause() self.start_addition() self._sched.resume() return True except: import traceback print(traceback.print_exc()) return False def stop(self, iswait=True): try: self._sched.shutdown(wait=iswait) self._jobhandlers.clear() return True except: import traceback print(traceback.print_exc()) return False def pause_job(self, job_id): try: self._sched.pause_job(job_id=job_id) self._jobhandlers[job_id].ispause = True self._jobhandlers[job_id].status = 3 self._jobhandlers[job_id].isrunning = False return True except: import traceback print(traceback.print_exc()) return False def resume_job(self, job_id): try: self._sched.resume_job(job_id=job_id) self._jobhandlers[job_id].ispause = False self._jobhandlers[job_id].status = 1 return True except: import traceback print(traceback.print_exc()) return False
class DataCollector: def __init__(self): self.logger = get_logger('data-collector') self.scheduler = BackgroundScheduler(timezone="Asia/Seoul") self.scheduler.start() self.templates = dict() self.__global_store = dict() self.job_broker = Celery('routine-jobs', broker=BROKER_URL, backend=CELERY_RESULT_BACKEND) # ========================================================================= def add_job_schedules(self, schedule_templates: list): for schedule_template in schedule_templates: schedule_name, trigger = operator.itemgetter( 'schedule_name', 'trigger')(schedule_template) # schedule name can't be duplicated. schedule_names = [ x['schedule_name'] for x in self.get_schedule_jobs() ] if schedule_name in schedule_names: msg = f'The schedule name \'{schedule_name}\' is already assigned.' self.logger.error(msg) raise ExceptionScheduleReduplicated(msg) self._add_job_schedule(schedule_name, trigger_type=trigger['type'], trigger_setting=trigger['setting']) # store the schedule template self.templates[schedule_name] = schedule_template self.__global_store[schedule_name] = {'_gv': dict()} # ========================================================================= def _add_job_schedule(self, key, trigger_type, trigger_setting): if trigger_type == 'crontab' and 'crontab' in trigger_setting: crontab = self.crontab_add_second(trigger_setting['crontab']) trigger_type = 'cron' trigger_setting = {**trigger_setting, **crontab} del trigger_setting['crontab'] arguments = dict(func=self.request_data, args=(key, ), id=key, trigger=trigger_type) arguments = {**arguments, **trigger_setting} self.scheduler.pause() try: self.scheduler.add_job(**arguments) finally: self.scheduler.resume() # ========================================================================= def remove_job_schedule(self, schedule_name: str): self.get_schedule_job(schedule_name) self.scheduler.remove_job(schedule_name) try: del self.templates[schedule_name] del self.__global_store[schedule_name] except KeyError: # it should be failing to collect data. such as not connecting. pass return # ========================================================================= def modify_job_schedule(self, schedule_name, trigger_type, trigger_args): if trigger_type == 'crontab' and 'crontab' in trigger_args: crontab = self.crontab_add_second(trigger_args['crontab']) trigger = 'cron' setting = {**trigger_args, **crontab} del setting['crontab'] else: trigger = trigger_type setting = trigger_args job = self.scheduler.get_job(schedule_name) job.reschedule(trigger, **setting) self.templates[schedule_name]['trigger'] = dict(type=trigger_type, setting=trigger_args) # ========================================================================= @staticmethod def get_python_module(code, name): module = types.ModuleType(name) exec(code, module.__dict__) return module # ========================================================================= @staticmethod def insert_number_each_line(data: str): result = list() data = data.split('\n') for (number, line) in enumerate(data): result.append(f'{number+1:04} {line}') return '\n'.join(result) # ========================================================================= @staticmethod def filter_dict(dict_to_filter, thing_with_kwargs): sig = inspect.signature(thing_with_kwargs) filter_keys = [ param.name for param in sig.parameters.values() if param.kind == param.POSITIONAL_OR_KEYWORD ] filtered_dict = { filter_key: dict_to_filter[filter_key] for filter_key in filter_keys } return filtered_dict # ========================================================================= def _source(self, name, setting): source_type, code, arguments = operator.itemgetter( 'type', 'code', 'arguments')(setting) module = DataCollector.get_python_module(code, name) try: _gv = self.__global_store[name] arguments = {**arguments, **_gv} filterd_arguments = DataCollector.filter_dict( arguments, module.main) data = module.main(**filterd_arguments) except Exception as e: code = DataCollector.insert_number_each_line(code) self.logger.error(f'{e}\ncode: \n{code}') raise return data # ========================================================================= def request_data(self, schedule_name): schedule = self.templates[schedule_name] if schedule_name not in self.templates: msg = f'The template "{schedule_name}" ' \ f'is not in the main template store' self.logger.error(msg) raise KeyError(msg) # checking use flag if not schedule['use']: self.logger.info(f'{schedule_name} is disabled.') return # source data = self._source(schedule_name, schedule['source']) if data is None: message = f'[{schedule_name}] The user function returned None.' self.logger.warning(message) # works # calling function for each works with arguments via celery for work in schedule['works']: work_type, arguments = operator.itemgetter('type', 'arguments')(work) self.job_broker.send_task(work_type, args=(data, ), kwargs=arguments) # sending events to the event # PING # event emitting data = json.dumps(data) event = { 'name': schedule_name, 'event': { 'type': 'data-collector', 'schedule_name': schedule_name }, 'data': data } try: self.emit_event(schedule_name, event) except (urllib3.exceptions.MaxRetryError, requests.exceptions.ConnectionError) as e: self.logger.error(f'Connection Error: Failed to emit events.') except Exception as e: import traceback traceback.print_exc() return # ========================================================================= def emit_event(self, name: str, event: dict): with requests.Session() as s: api = EVENT_COLLECTOR_URL + '/api/v1/events/emit' response = s.post(api, json=event) if response.status_code != 200: raise Exception(f'code: {response.status_code}\n' f'messages: [{name}] - {response.reason}') data = json.loads(response.text) self.logger.info(f'[{name}] emitted a event.') # ========================================================================= def remove_job_schedule(self, _id: str): self.scheduler.remove_job(_id) del self.data[_id] return # ========================================================================= def modify_job_schedule(self, _id, seconds): self.scheduler.reschedule_job(_id, trigger='interval', seconds=seconds) # ========================================================================= def get_schedule_jobs(self): jobs = self.scheduler.get_jobs() if not jobs: return jobs result = list() for job in jobs: schedule_name = job.id next_run_time = job.next_run_time template_data = self.templates[schedule_name] template_data['next_run_time'] = next_run_time result.append(template_data) return result
class config: __white = (255, 255, 255) __red = (255, 0, 0) __color = (0, 255, 255) __colorInv = __red __colorFile = "/home/pi/SmartMirror/data/configFiles/colorFile.txt" __brightnessFile = "/home/pi/SmartMirror/data/configFiles/brightnessFile.txt" __fontFile = "/home/pi/SmartMirror/data/configFiles/fontFile.txt" __speedFile = "/home/pi/SmartMirror/data/configFiles/speedFile.txt" __viewFile = "/home/pi/SmartMirror/data/configFiles/viewFile.txt" __sleepFile = "/home/pi/SmartMirror/data/configFiles/sleepTimer.txt" __fileIO = fileIO() __defFontName = "Serif" def __init__(self): self.__vcgencmd = Vcgencmd() self.__pwm = pigpio.pi() self.__valChange = ["None", "None", "None"] self.__fonts = [0] * 8 self.__cSize = 0 self.__scheduler = BackgroundScheduler() self.__scheduler.start() self.__sleepF1 = True self.__sleepF2 = True self.__stayOffFlag = False def getStayOffFlag(self): return self.__stayOffFlag def setStayOffFlag(self, state): self.__stayOffFlag = state def restartDim(self): sleep = self.__fileIO.simpleRead(self.__sleepFile, multiLine=True) try: self.__scheduler.reschedule_job("dim", trigger='interval', minutes=int(sleep[2])) except: print(datetime.now().strftime("%H:%M:%S"), "IN config::restartDim: cannot reschedule job!") return -1 def setColor(self): color, colorInv = self.__fileIO.simpleRead(self.__colorFile, separator=":") self.__color = [ int(color[i:i + 2], 16) for i in range(1, len(color) - 1, 2) ] self.__colorInv = [ int(colorInv[i:i + 2], 16) for i in range(1, len(colorInv) - 1, 2) ] if (self.__color == self.__white): self.__colorInv = self.__red return self.__color, self.__colorInv def setSleepTime(self): sleep = self.__fileIO.simpleRead(self.__sleepFile, multiLine=True) if (len(sleep) > 4): if (self.__valChange[0] != sleep[0] or self.__valChange[1] != sleep[1]): try: self.__sleepF1 = True self.__scheduler.remove_job('dpoff') self.__scheduler.remove_job('dpon') except: print(datetime.now().strftime("%H:%M:%S"), "IN config::setSleepTime: jobs dont exist!") if (self.__valChange[2] != sleep[2]): try: self.__sleepF2 = True self.__scheduler.remove_job("dim") except: print(datetime.now().strftime("%H:%M:%S"), "IN config::setSleepTime: jobs dont exist!") if (sleep[3] in "False" and self.__sleepF1): self.__sleepF1 = False hr = int(sleep[0].split(":")[0]) mi = int(sleep[0].split(":")[1]) self.__scheduler.add_job(self.displayOff, "cron", hour=hr, minute=mi, id='dpoff') hr = int(sleep[1].split(":")[0]) mi = int(sleep[1].split(":")[1]) self.__scheduler.add_job(self.displayOn, "cron", hour=hr, minute=mi, id='dpon') elif (sleep[3] in "True" and not self.__sleepF1): self.__sleepF1 = True self.__scheduler.remove_job('dpoff') self.__scheduler.remove_job('dpon') self.__vcgencmd.display_power_on(2) if (sleep[4] in "False" and self.__sleepF2): self.__sleepF2 = False self.__scheduler.add_job( lambda: self.__pwm.set_PWM_dutycycle(18, 0), 'interval', minutes=int(sleep[2]), id='dim') elif (sleep[4] in "True" and not self.__sleepF2): self.__sleepF2 = True self.__scheduler.remove_job('dim') self.__vcgencmd.display_power_on(2) self.__pwm.set_PWM_dutycycle( 18, int(self.__fileIO.simpleRead(self.__brightnessFile))) self.displayOn() self.__valChange = [sleep[0], sleep[1], sleep[2]] def displayOn(self): self.__vcgencmd.display_power_on(2) brightness = self.__fileIO.simpleRead(self.__brightnessFile) self.__pwm.set_PWM_dutycycle(18, int(brightness)) self.__stayOffFlag = False def displayOff(self): self.__pwm.set_PWM_dutycycle(18, 0) self.__stayOffFlag = True def readFontFromFile(self): font = self.__fileIO.simpleRead(self.__fontFile, multiLine=True) font[0] = font[0].split(",")[0] font[1] = int(font[1]) if (len(font) == 2): font.append(False) font.append(False) elif (len(font) == 3): if ('Bold' in font[2]): font[2] = True font.append(False) else: font[2] = False font.append(True) elif (len(font) == 4): font[2] = True font[3] = True return font def getFont(self, index): return self.__fonts[index] def getDefFont(self): return self.__defFont def getMetFont(self): return self.__metFont def setCSize(self, size): self.__cSize = size def getSpeed(self): speed = self.__fileIO.simpleRead(self.__speedFile) if ("Non" in speed): return 0 else: return int(speed) * 1000 def getView(self): return self.__fileIO.simpleRead(self.__viewFile, multiLine=True)[0] def getAllFonts(self): fontName, fontSize, bold, italic = self.readFontFromFile() try: self.__defFont = pygame.font.SysFont(self.__defFontName, fontSize - 12, bold=bold, italic=italic) self.__metFont = pygame.font.SysFont(self.__defFontName, fontSize - 4, bold=bold, italic=italic) self.__fonts[0] = pygame.font.SysFont(fontName, fontSize + self.__cSize, bold=bold, italic=italic) #calendar self.__fonts[1] = pygame.font.SysFont(fontName, fontSize + 0, bold=bold, italic=italic) #date self.__fonts[2] = pygame.font.SysFont(fontName, fontSize + 32, bold=bold, italic=italic) #time self.__fonts[3] = pygame.font.SysFont(fontName, fontSize - 12, bold=bold, italic=italic) #events self.__fonts[4] = pygame.font.SysFont(fontName, fontSize - 12, bold=bold, italic=italic) #news self.__fonts[5] = pygame.font.SysFont(fontName, fontSize - 4, bold=bold, italic=italic) #weather self.__fonts[6] = pygame.font.SysFont(fontName, fontSize - 12, bold=bold, italic=italic) #hour self.__fonts[7] = pygame.font.SysFont(fontName, fontSize - 18, bold=bold, italic=italic) #hour except: self.__defFont = pygame.font.Font(self.__defFontName, fontSize - 12, bold=bold, italic=italic) self.__metFont = pygame.font.Font(self.__defFontName, fontSize - 4, bold=bold, italic=italic) self.__fonts[0] = pygame.font.Font(fontName, fontSize, bold=bold, italic=italic) self.__fonts[1] = pygame.font.Font(fontName, fontSize + 0, bold=bold, italic=italic) self.__fonts[2] = pygame.font.Font(fontName, fontSize + 32, bold=bold, italic=italic) self.__fonts[3] = pygame.font.Font(fontName, fontSize - 12, bold=bold, italic=italic) self.__fonts[4] = pygame.font.Font(fontName, fontSize - 12, bold=bold, italic=italic) self.__fonts[5] = pygame.font.Font(fontName, fontSize - 4, bold=bold, italic=italic) self.__fonts[6] = pygame.font.Font(fontName, fontSize - 12, bold=bold, italic=italic) self.__fonts[7] = pygame.font.Font(fontName, fontSize - 18, bold=bold, italic=italic) return self.__fonts
class Reconciler(object): RECONCILE_JOB_ID = "broccoli.worker_reconcile" def __init__( self, worker_config_store: WorkerConfigStore, worker_queue: WorkerQueue, pause_workers: bool, ): self.worker_config_store = worker_config_store self.worker_queue = worker_queue self.pause_workers = pause_workers self.reconcile_scheduler = BlockingScheduler() self.reconcile_scheduler.add_job( self.reconcile, id=self.RECONCILE_JOB_ID, trigger="interval", seconds=10 ) self.trigger_scheduler = BackgroundScheduler() def start(self): # Less verbose logging from apscheduler logging.getLogger("apscheduler").setLevel(logging.ERROR) # start trigger scheduler before reconcile scheduler otherwise triggers might not be actually added self.trigger_scheduler.start() self.reconcile_scheduler.start() def stop(self): self.trigger_scheduler.shutdown(wait=False) self.reconcile_scheduler.shutdown(wait=False) def reconcile(self): if self.pause_workers: logger.info("Workers have been globally paused") return actual_worker_ids = set( map(lambda j: j.id, self.trigger_scheduler.get_jobs()) ) # type: Set[str] desired_workers = self.worker_config_store.get_all() desired_worker_ids = set(desired_workers.keys()) # type: Set[str] self.remove_workers( actual_worker_ids=actual_worker_ids, desired_worker_ids=desired_worker_ids ) self.add_workers( actual_worker_ids=actual_worker_ids, desired_worker_ids=desired_worker_ids, desired_workers=desired_workers, ) self.configure_workers( actual_worker_ids=actual_worker_ids, desired_worker_ids=desired_worker_ids, desired_workers=desired_workers, ) def remove_workers(self, actual_worker_ids: Set[str], desired_worker_ids: Set[str]): removed_worker_ids = actual_worker_ids - desired_worker_ids if not removed_worker_ids: logger.debug("No worker to remove") return logger.info(f"Going to remove workers with id {removed_worker_ids}") for removed_worker_id in removed_worker_ids: self.trigger_scheduler.remove_job(removed_worker_id) def add_workers( self, actual_worker_ids: Set[str], desired_worker_ids: Set[str], desired_workers: Dict[str, WorkerMetadata], ): added_worker_ids = desired_worker_ids - actual_worker_ids if not added_worker_ids: logger.debug("No workers to add") return logger.info(f"Going to add workers with id {added_worker_ids}") for added_worker_id in added_worker_ids: self.add_worker(added_worker_id, desired_workers) def add_worker( self, added_worker_id: str, desired_workers: Dict[str, WorkerMetadata] ): worker_metadata = desired_workers[added_worker_id] def _trigger(): logger.info(f"Enqueuing worker {added_worker_id}") self.worker_queue.enqueue( WorkerPayload( type="worker", module_name=worker_metadata.module_name, args=worker_metadata.args, ) ) self.trigger_scheduler.add_job( _trigger, id=added_worker_id, trigger="interval", seconds=worker_metadata.interval_seconds, ) def configure_workers( self, actual_worker_ids: Set[str], desired_worker_ids: Set[str], desired_workers: Dict[str, WorkerMetadata], ): same_worker_ids = actual_worker_ids.intersection(desired_worker_ids) for worker_id in same_worker_ids: desired_interval_seconds = desired_workers[worker_id].interval_seconds actual_interval_seconds = self.trigger_scheduler.get_job( worker_id ).trigger.interval.seconds if desired_interval_seconds != actual_interval_seconds: logger.info( f"Going to reconfigure worker interval with id {worker_id} to {desired_interval_seconds} " f"seconds" ) self.trigger_scheduler.reschedule_job( job_id=worker_id, trigger="interval", seconds=desired_interval_seconds, )
class Schedule(Object): def __init__(self): self._init_scheduler() def _init_scheduler(self): ''' init schedule agent ''' jobstores = { 'default': MongoDBJobStore(collection=self.col, database=self.database, client=self.client), } executors = { 'default': ThreadPoolExecutor(10), 'processpool': ProcessPoolExecutor(2) } job_defaults = {'coalesce': False, 'max_instances': 4} self.scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) self.scheduler.start() def add_job(self, *args, **kwargs): ''' add schedule job add_job(func, trigger=None, args=None, kwargs=None, id=None, \ name=None, misfire_grace_time=undefined, coalesce=undefined, \ max_instances=undefined, next_run_time=undefined, \ jobstore='default', executor='default', \ replace_existing=False, **trigger_args) ''' self.scheduler.add_job(*args, **kwargs) self.running() # def add_interval_job(self, task, job_id=None, args=None, schedule=None): def add_interval_job(self, *args, **kwargs): ''' add schedule job kwargs: args = [] :call function args kwargs = {} :call function kwargs id = 'job id' :job id trigger :(e.g. ``date``, ``interval`` or ``cron``) args: task day=1, minute=20 is equivalent to year='*', month='*', day=1, week='*', day_of_week='*', hour='*', minute=20, second=0. The job will then execute on the first day of every month on every year at 20 minutes of every hour. The code examples below should further illustrate this behavior. http://apscheduler.readthedocs.io/en/latest/modules/triggers/cron.html?highlight=add_job ''' job_id = kwargs.get('id') if not job_id: raise ValueError('The job id not provided.') job = self.get_schedule_jobs(job_id) if not job: self.scheduler.add_job(*args, **kwargs) else: self.scheduler.reschedule_job(*args, **kwargs) self.running() def running(self): ''' start scheduler ''' if self.scheduler.state != 1: self.scheduler.start() def remove_job(self, job_id=None): ''' remove job by id ''' if not job_id: raise ValueError('Job id not provided for remove schedule task.') if not self.jobstore: raise RuntimeError( 'Job store for persiste schedule task not registered yet.') if self.get_schedule_jobs(job_id): self.scheduler.remove_job(job_id, jobstore=self.jobstore) self.running() def get_schedule_jobs(self, job_id=None): ''' get schedule jobs ''' jobs = [] if not self.jobstore: raise RuntimeError( 'Job store for persiste schedule task not registered yet.') if not job_id: jobs = self.scheduler.get_jobs(jobstore=self.jobstore) else: jobs = self.scheduler.get_job(job_id, jobstore=self.jobstore) return jobs
class Stats(RyuApp): OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION] _CONTEXTS = { 'dpset': DPSet, 'wsgi': WSGIApplication, } def __init__(self, *args, **kwargs): super(Stats, self).__init__(*args, **kwargs) # logger settings self.logger.setLevel(logging.INFO) self.logger.info('Application %s initialized', __name__) # routing self.mac_to_port = {} # dpset instance self.dpset = kwargs['dpset'] # setup wsgi wsgi = kwargs['wsgi'] wsgi.register(StatsRestApi, { StatsRestApi.controller_instance_name: self, StatsRestApi.dpset_instance_name: self.dpset }) # init scheduler self.interval = STATS_INTERVAL self.sched = BackgroundScheduler() self.sched.start() logging.getLogger('apscheduler.executors.default').setLevel(logging.WARNING) logging.getLogger('apscheduler.scheduler').propagate = False # init place for PNDA watchdog try: os.mkdir('out') except OSError: pass # stats logs to file try: os.mkdir('stats') except OSError: pass def set_interval(self, interval): self.interval = interval self.change_sched_interval(interval) def change_sched_interval(self, interval): self.logger.debug("Rescheduling stat request to %i seconds", interval) for s in self.sched.get_jobs(): self.logger.debug('rescheduling job %s', s.id) it = IntervalTrigger(seconds=interval) self.sched.reschedule_job(s.id, trigger=it) def send_flow_stats_request(self, datapath): # https://osrg.github.io/ryu-book/en/html/traffic_monitor.html self.logger.debug('Sending flow stat request to sw: %016x%i', datapath.id) ofproto = datapath.ofproto parser = datapath.ofproto_parser req = parser.OFPFlowStatsRequest(datapath) datapath.send_msg(req) with open('stats/flow_stats_req.txt', 'a') as file: file.write(str(int(time.time())) + ' ' +str(ofproto.OFP_FLOW_STATS_SIZE) + ' ' +str(req) + '\n') def send_port_stats_request(self, datapath): self.logger.debug('Sending flow stat request to sw: %016x%i', datapath.id) ofproto = datapath.ofproto parser = datapath.ofproto_parser req = parser.OFPPortStatsRequest(datapath, 0, ofproto.OFPP_ANY) datapath.send_msg(req) with open('stats/port_stats_req.txt', 'a') as file: file.write(str(int(time.time())) + ' ' +str(ofproto.OFP_PORT_STATS_SIZE) + ' '+ str(req) + '\n') @set_ev_cls(ofp_event.EventOFPFlowStatsReply, MAIN_DISPATCHER) def flow_stats_reply_handler(self, ev): body = ev.msg.body flow_stats = [] for stat in sorted([flow for flow in body if flow.priority == 1], key=lambda flow: (flow.match['in_port'], flow.match['eth_dst'])): flow_stats.append('table_id=%s ' 'duration_sec=%d duration_nsec=%d ' 'priority=%d ' 'idle_timeout=%d hard_timeout=%d flags=0x%04x ' 'cookie=%d packet_count=%d byte_count=%d ' 'match=%s instructions=%s' % (stat.table_id, stat.duration_sec, stat.duration_nsec, stat.priority, stat.idle_timeout, stat.hard_timeout, stat.flags, stat.cookie, stat.packet_count, stat.byte_count, stat.match, stat.instructions)) data = { "origin": "flow_stats", "timestamp": time.time(), "switch_id": ev.msg.datapath.id, "duration_sec": stat.duration_sec, "duration_nsec": stat.duration_nsec, "src_mac": stat.match['eth_src'], "dst_mac": stat.match['eth_dst'], "byte_count": stat.byte_count, "packet_count": stat.packet_count, "in_port": stat.match['in_port'] } with open('out/flow_stats.out', 'a') as file: file.write(json.dumps(data) + '\n') self.logger.debug('FlowStats for switch %i: %s', ev.msg.datapath.id, flow_stats) with open('stats/flow_stats_reply.txt', 'a') as file: file.write(str(int(time.time())) + ' ' +str(ev.msg.msg_len) + '\n') @set_ev_cls(ofp_event.EventOFPPortStatsReply, MAIN_DISPATCHER) def port_stats_reply_handler(self, ev): body = ev.msg.body port_stats = [] for stat in sorted(body, key=attrgetter('port_no')): port_stats.append("port_no=%d " "rx_packets=%d " "rx_bytes=%d " "rx_errors=%d " "tx_packets=%d " "tx_bytes=%d " "tx_errors=%d " % (stat.port_no, stat.rx_packets, stat.rx_bytes, stat.rx_errors, stat.tx_packets, stat.tx_bytes, stat.tx_errors) ) data = { "origin": "port_stats", "timestamp": time.time(), "switch_id": ev.msg.datapath.id, "port_no": stat.port_no, "rx_packets": stat.rx_packets, "rx_bytes": stat.rx_bytes, "rx_errors": stat.rx_errors, "tx_packets": stat.tx_packets, "tx_bytes": stat.tx_bytes, "tx_errors": stat.tx_errors } with open('out/port_stats.out', 'a') as file: file.write(json.dumps(data) + '\n') self.logger.debug('PortStats for switch %i: %s', ev.msg.datapath.id, port_stats) with open('stats/port_stats_reply.txt', 'a') as file: file.write(str(int(time.time())) + ' ' + str(ev.msg.msg_len) + '\n') @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER) def switch_features_handler(self, ev): datapath = ev.msg.datapath ofproto = datapath.ofproto parser = datapath.ofproto_parser # install table-miss flow entry # # We specify NO BUFFER to max_len of the output action due to # OVS bug. At this moment, if we specify a lesser number, e.g., # 128, OVS will send Packet-In with invalid buffer_id and # truncated packet data. In that case, we cannot output packets # correctly. The bug has been fixed in OVS v2.1.0. match = parser.OFPMatch() actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER, ofproto.OFPCML_NO_BUFFER)] self.add_flow(datapath, 0, match, actions, 0) # Add job to scheduler self.logger.debug('Starting scheduler for dp =%i', ev.msg.datapath.id) # ## flow stats self.sched.add_job(self.send_flow_stats_request, 'interval', seconds=STATS_INTERVAL, start_date=START_DATE, args=[ev.msg.datapath]) ## port stats self.sched.add_job(self.send_port_stats_request, 'interval', seconds=STATS_INTERVAL, start_date=START_DATE, args=[ev.msg.datapath]) def add_flow(self, datapath, priority, match, actions, idle_timeout, buffer_id=None): ofproto = datapath.ofproto parser = datapath.ofproto_parser inst = [parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)] if buffer_id: mod = parser.OFPFlowMod(datapath=datapath, buffer_id=buffer_id, priority=priority, match=match, instructions=inst,idle_timeout=idle_timeout) else: mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, instructions=inst,idle_timeout=idle_timeout) datapath.send_msg(mod) @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER) def _packet_in_handler(self, ev): # If you hit this you might want to increase # the "miss_send_length" of your switch if ev.msg.msg_len < ev.msg.total_len: self.logger.debug("packet truncated: only %s of %s bytes", ev.msg.msg_len, ev.msg.total_len) msg = ev.msg datapath = msg.datapath ofproto = datapath.ofproto parser = datapath.ofproto_parser in_port = msg.match['in_port'] pkt = packet.Packet(msg.data) eth = pkt.get_protocols(ethernet.ethernet)[0] if eth.ethertype == ether_types.ETH_TYPE_LLDP: # ignore lldp packet return dst = eth.dst src = eth.src # self.logger.info("packet in dpid = %s src = %s dst = %s in_port = %s", dpid, src, dst, in_port) # learn a mac address to avoid FLOOD next time. self.mac_to_port[dpid][src] = in_port if dst in self.mac_to_port[dpid]: out_port = self.mac_to_port[dpid][dst] else: out_port = ofproto.OFPP_FLOOD actions = [parser.OFPActionOutput(out_port)] # install a flow to avoid packet_in next time if out_port != ofproto.OFPP_FLOOD: match = parser.OFPMatch(in_port=in_port, eth_dst=dst, eth_src=src) # verify if we have a valid buffer_id, if yes avoid to send both # flow_mod & packet_out if msg.buffer_id != ofproto.OFP_NO_BUFFER: self.add_flow(datapath, 1, match, actions, 3, msg.buffer_id) return else: self.add_flow(datapath, 1, match, actions, 3) data = None if msg.buffer_id == ofproto.OFP_NO_BUFFER: data = msg.data out = parser.OFPPacketOut(datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port, actions=actions, data=data) datapath.send_msg(out)
class SchedulerEdge(object): core = None def __init__(self, parent): #instância do objeto e inicia o escalonador self.core = parent self.scheduler = BackgroundScheduler( ) # atribui um agendador background self.scheduler.start() # inicia o agendador #self.create_job_check_persistence() #self.check_scheduler_reactivave() def add_job(self, jsonObject): # cria uma nova tarefa no escalonador #print(type(jsonObject['status'])) jsonObject['collect_to_rule'] = False if (jsonObject['status'] == 'True' or jsonObject['status'] == True): if jsonObject['id'] != '0': #print("ADICIONOU SENSOR") try: #print('klhashjkah') self.scheduler.add_job(self.function, jsonObject['modo'], second=jsonObject['second'], minute=jsonObject['minute'], hour=jsonObject['hour'], day=jsonObject['day'], month=jsonObject['month'], year=jsonObject['year'], id=str(jsonObject['id']), args=[jsonObject], max_instances=50, misfire_grace_time=120) except: # Utilizado quando tem uma tarefa com ID para reescalonar self.scheduler.reschedule_job(jsonObject['id'], trigger='cron', second=jsonObject['second'], minute=jsonObject['minute'], hour=jsonObject['hour'], day=jsonObject['day'], month=jsonObject['month'], year=jsonObject['year']) else: self.scheduler.add_job(self.check_persistence, jsonObject['modo'], second=jsonObject['second'], minute=jsonObject['minute'], hour=jsonObject['hour'], day=jsonObject['day'], month=jsonObject['month'], year=jsonObject['year'], id=jsonObject['id'], max_instances=1, misfire_grace_time=120) def remove_job( self, jsonObject ): # id_tarefa - É ID do sensor/atuador a ser removido do CRON self.scheduler.remove_job(jsonObject['id']) def function(self, jsonObject): # response - É JSON passado como argumento #print(jsonObject['id']) #print("SENSOR ADD"+jsonObject['id_sensor']) object_events = Event_Treatment(self.core) object_events.event(jsonObject) def check_persistence(self): # Modificar print("Tentou publicar a persistencia") persistence_publisher = Publisher(self.core) persistence_publisher.start() #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # Adiciona uma TAREFA no CRON, tornando resposavél pela publicação no contexto # quando não ocorreu com sucesso este ato no módulo de gathering. def create_job_check_persistence( self): # Cria um JSON no formato exato que SCHEDULER # irá utilizar. QQ tratamento deve ocorrer aqui. job = {} job['modo'] = "cron" job['id'] = "0" job['status'] = "True" job['second'] = "*/30" #job['second'] = "0" #job['minute'] = "*/10" job['minute'] = "*" job['hour'] = "*" job['day'] = "*" job['week'] = "*" job['month'] = "*" job['year'] = "*" self.add_job(job) def check_scheduler_reactivave(self): # print("check_scheduler_reactivave") try: #print("check_scheduler_reactivave TRY") jsonSchedules = self.core.API_access("get", "schedules").json() print(jsonSchedules) for schedule in jsonSchedules: schedule['modo'] = 'cron' print(schedule) self.add_job(schedule) except Exception as inst: #print("check_scheduler_reactivave EXCEPTION") #print(type(inst)) time.sleep(10) self.check_scheduler_reactivave() except: print("PODE SER ERRO DE ACESSO POR CAUSA DE TOKEN INVALIDO")
class SchedulerCollectData(): def __init__(self, exe_function, live_freq, history_freq): # ex: {"4":{"ports":[280,281],"mempools":[1,2]}} # {"4":{"ports":[280,281]}} self.live_dev = { } #{1:{'devices':[{'device_id':1}]}, 2:{'devices':[{'device_id':2}]}} #{d1:{rec_message}, d2:{rec_message}} self.all_dev = { } #{4:{'devices':[{'device_id':4, 'result':0}]}, 5:{'devices':[{'device_id':5, 'result':0}]},6:{'devices':[{'device_id':6, 'result':0}]}, 7:{'devices':[{'device_id':7, 'result':0}]}} # {d1:{rec_message}, d2:{rec_message}} self.event_dev = {} self.scheduler = BackgroundScheduler({ 'apscheduler.executors.default': { 'class': 'apscheduler.executors.pool:ThreadPoolExecutor', 'max_workers': '30' } }) self.exe_function = exe_function self.pool = ThreadPool(processes=multiprocessing.cpu_count()) self.live_freq = int(live_freq) self.history_freq = int(history_freq) self.send_live = {} self.jobdir = '/opt/observium/qrmobservium/common/scheduler/jobs/livedata' self.scheduler.add_job(self._period_function, trigger='interval', args=('live', ), seconds=live_freq, id="live_period") self.scheduler.add_job(self._job_function, trigger='interval', args=('job', ), seconds=3, id="job_poll") self.scheduler.start() def start(self): try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(1) except (KeyboardInterrupt, SystemExit): self.scheduler.shutdown( ) # Not strictly necessary if daemonic mode is enabled but should be done if possible def add_dev(self, msg): #msg = {"4":{"ports":[280,281]}} for dev_id in msg: print 'add_dev : %s' % dev_id if dev_id not in self.live_dev: self.live_dev[dev_id] = msg[dev_id].copy() print 'live_list %s' % self.live_dev #dev_id = int(msg['devices'][0]['device_id']) #if dev_id not in self.live_dev: # msg.pop('get_event_devices', None) # self.all_dev[dev_id] = msg.copy() #print "all list:" + simplejson.dumps(self.all_dev) def remove_dev(self, msg): dev_id = int(msg['devices'][0]['device_id']) self.all_dev.pop(dev_id, None) self.live_dev.pop(dev_id, None) self.event_dev.pop(dev_id, None) def add_event_dev(self, msg): self.event_dev = {} for dev in msg["devices"]: self.event_dev[dev["device_id"]] = dev #print "event list:" + simplejson.dumps(self.event_dev) def remove_event_dev(self, msg): dev_id = int(msg['devices'][0]['device_id']) self.event_dev.pop(dev_id, None) def _period_function(self, schedule_type): global d_time results = [] print 'debug %s' % multiprocessing.cpu_count() if schedule_type == "live": #print "enter live_function:" + str(datetime.datetime.now()) d_time = int(time.time()) for key, value in self.live_dev.iteritems(): print '_period %s %s d_time %s' % (key, value, d_time) #value =self.live_dev.values() for port_id in value['ports']: #print 'kk %s' % port_id self.pool.apply_async(self.exe_function, args=( key, port_id, )) def _job_function(self, schedule_type): if not os.path.isdir(self.jobdir): print 'path not exist' return device_ids = os.listdir(self.jobdir) if len(device_ids) <= 0: print 'no devices' for device_id in self.live_dev.keys(): for port in self.live_dev[device_id]['ports']: self.live_dev[device_id]['ports'].remove(port) del self.live_dev[device_id] return for device_id in self.live_dev.keys(): if device_id not in device_ids: self.live_dev.pop(device_id, None) for device_id in device_ids: msg = {} constructport = {} portlist = [] ports = os.listdir(self.jobdir + "/" + device_id) if len(ports) <= 0: if not device_id in self.live_dev: return for port in self.live_dev[device_id]['ports']: if port not in ports: self.live_dev[device_id]['ports'].remove(port) print 'del ports %s' % self.live_dev[device_id]['ports'] return portlist = [int(port) for port in ports] #msg[device_id] = constructport if device_id in self.live_dev: cur_ports = self.live_dev[device_id]['ports'] else: cur_ports = [] diff_ports = list(set(portlist) - set(cur_ports)) if len(diff_ports) == 0 and len(portlist) >= len(cur_ports): continue for port in diff_ports: if port not in portlist: portlist.append(port) constructport['ports'] = portlist msg[device_id] = constructport self.live_dev[device_id] = msg[device_id].copy() print '_job_function live_list %s' % self.live_dev def change_live_freq(self, sec): self.live_freq = sec self.scheduler.reschedule_job(job_id="live_period", trigger='interval', seconds=self.live_freq) def change_history_freq(self, minute): self.history_freq = minute
class Gardener: """ Gardener manages garden according schedule and collected sensor data. * Garden - Controls HW I/O - simple stateless servant for single thread use. * sensors: temperature (TODO: water level, light density, ...) * relays: pump, fan, fogger * Records - Via scheduler collects and store sensors data + current garden state. * Web server shows * current garden state (TODO) * light version of sensor data history * next planned maintenance action (TODO) * buttons for manipulation with garden """ def __init__(self): self.garden = Garden() self.records = Records(sensors=self.garden.sensors) self.scheduler = BackgroundScheduler({ 'apscheduler.executors.default': { 'class': 'apscheduler.executors.pool:ThreadPoolExecutor', 'max_workers': '1' } }) def reschedule_job(self, job_id): period_minutes = self.compute_period( job_id, self.garden.city_temperature.value) last_job_run = self.garden.get_last_job_run(job_id) next_job_run = max((self.get_asap_schedule(), last_job_run + timedelta(minutes=period_minutes))) self.scheduler.reschedule_job(job_id, trigger='cron', minute="*/{}".format(period_minutes), start_date=str(next_job_run)) def sensors_refresh(self): old_temperature = self.garden.city_temperature.value self.garden.sensors_refresh() new_temperature = self.garden.city_temperature.value if old_temperature != new_temperature and new_temperature: self.reschedule_job('FOGGING') self.reschedule_job('WATERING') def send_sms_report(self): message = 'I am alive.' for sensor in self.garden.sensors: message += " {}:{}".format(sensor.name, str(sensor.value)) message += " f:{}/{} w:{}/{}".format( self.get_job_period("FOGGING"), self.garden.get_job_run_count("FOGGING"), self.get_job_period("WATERING"), self.garden.get_job_run_count("WATERING")) utils.sms.send_sms(message) def working_loop(self): # shared cross threads threading.gardener = self # default schedule cron_params = { 'trigger': 'cron', 'misfire_grace_time': MISFIRE_GRACE_TIME } self.scheduler.add_job(self.garden.watering, trigger='date') self.scheduler.add_job(self.garden.watering, minute='*/20', id='WATERING', **cron_params) self.scheduler.add_job(self.garden.fogging, minute='*/3', id='FOGGING', **cron_params) # sensors maintenance self.scheduler.add_job(self.sensors_refresh, minute='*/10', **cron_params) self.scheduler.add_job(self.records.write_values, minute='*/10', kwargs={'file': config.SensorData.FULL_FILE}, **cron_params) self.scheduler.add_job(self.records.write_values, hour='*', kwargs={'file': config.SensorData.WEB_FILE}, **cron_params) self.scheduler.add_job( self.records.trim_records, week='*', # show on web only latest 30 days kwargs={ 'file': config.SensorData.WEB_FILE, 'count': 24 * 7 * 4 }, **cron_params) self.scheduler.add_job(self.send_sms_report, hour='12', **cron_params) # TODO: create more oxygen when high temperature via extra long pumping cycle? # network maintenance self.scheduler.add_job(utils.network.check_and_fix, hour='*', kwargs={ 'address': config.RouterAddress, 'network': 'wlan0' }, **cron_params) self.scheduler.add_job(utils.system.reboot, hour='0', **cron_params) logging.info('Starting scheduler.') self.scheduler.start() # web server needs main thread for its signal handling logging.info('Starting web server.') web_server.run(**config.WebServer) self.scheduler.shutdown() def get_job_period(self, job_id): trigger = self.scheduler.get_job(job_id).trigger period = re.search(r"cron\[minute='\*/(\d+)'\]", str(trigger)) return int(period.group(1)) if period else 0 def get_job_next_run_time(self, job_id): return self.scheduler.get_job(job_id).next_run_time def start_job(self, job_id): # FIXME job = self.scheduler.get_job(job_id) @staticmethod def get_asap_schedule(): return datetime.now() + timedelta(seconds=2) @staticmethod def compute_period(job_id, temperature): if job_id == 'FOGGING': return int(2 * 4 * 60 / (temperature - 4)**1.5) if 4 < temperature < 27 else INFINITE_MINUTES elif job_id == 'WATERING': return int(4 * 24 * 60 / (temperature - 4)**2) if 4 < temperature < 27 else INFINITE_MINUTES else: assert 0
class Scheduler(Flask): days = { "0": "sun", "1": "mon", "2": "tue", "3": "wed", "4": "thu", "5": "fri", "6": "sat", "7": "sun", "*": "*", } seconds = {"seconds": 1, "minutes": 60, "hours": 3600, "days": 86400} def __init__(self): super().__init__(__name__) with open(Path.cwd().parent / "setup" / "scheduler.json", "r") as file: self.settings = load(file) dictConfig(self.settings["logging"]) self.configure_scheduler() self.register_routes() @staticmethod def aps_date(date): if not date: return date = datetime.strptime(date, "%d/%m/%Y %H:%M:%S") return datetime.strftime(date, "%Y-%m-%d %H:%M:%S") def configure_scheduler(self): self.scheduler = BackgroundScheduler(self.settings["config"]) self.scheduler.start() def register_routes(self): @self.route("/delete_job/<job_id>", methods=["POST"]) def delete_job(job_id): if self.scheduler.get_job(job_id): self.scheduler.remove_job(job_id) return jsonify(True) @self.route("/next_runtime/<task_id>") def next_runtime(task_id): job = self.scheduler.get_job(task_id) if job and job.next_run_time: return jsonify(job.next_run_time.strftime("%Y-%m-%d %H:%M:%S")) return jsonify("Not Scheduled") @self.route("/schedule", methods=["POST"]) def schedule(): if request.json["mode"] in ("resume", "schedule"): result = self.schedule_task(request.json["task"]) if not result: return jsonify({"alert": "Cannot schedule in the past."}) else: return jsonify({ "response": "Task resumed.", "active": True }) else: try: self.scheduler.pause_job(request.json["task"]["id"]) return jsonify({"response": "Task paused."}) except JobLookupError: return jsonify( {"alert": "There is no such job scheduled."}) @self.route("/time_left/<task_id>") def time_left(task_id): job = self.scheduler.get_job(task_id) if job and job.next_run_time: delta = job.next_run_time.replace(tzinfo=None) - datetime.now() hours, remainder = divmod(delta.seconds, 3600) minutes, seconds = divmod(remainder, 60) days = f"{delta.days} days, " if delta.days else "" return jsonify(f"{days}{hours}h:{minutes}m:{seconds}s") return jsonify("Not Scheduled") @staticmethod def run_service(task_id): post( f"{getenv('ENMS_ADDR')}/rest/run_task", json=task_id, auth=HTTPBasicAuth(getenv("ENMS_USER"), getenv("ENMS_PASSWORD")), verify=int(getenv("VERIFY_CERTIFICATE", 1)), ) def schedule_task(self, task): if task["scheduling_mode"] == "cron": crontab = task["crontab_expression"].split() crontab[-1] = ",".join(self.days[day] for day in crontab[-1].split(",")) trigger = {"trigger": CronTrigger.from_crontab(" ".join(crontab))} elif task["frequency"]: trigger = { "trigger": "interval", "start_date": self.aps_date(task["start_date"]), "end_date": self.aps_date(task["end_date"]), "seconds": int(task["frequency"]) * self.seconds[task["frequency_unit"]], } else: trigger = { "trigger": "date", "run_date": self.aps_date(task["start_date"]) } if not self.scheduler.get_job(task["id"]): job = self.scheduler.add_job( id=str(task["id"]), replace_existing=True, func=self.run_service, args=[task["id"]], **trigger, ) else: job = self.scheduler.reschedule_job(str(task["id"]), **trigger) return job.next_run_time > datetime.now(job.next_run_time.tzinfo)
class Collector: def __init__(self): self.logger = get_logger('collector') self.device_info = None self.job_order_queue = None self.scheduler = BackgroundScheduler(timezone="Asia/Seoul") self.scheduler.start() self.data = dict() self.data['__last_fetch'] = dict() self.queue = deque(maxlen=60) # ========================================================================= def add_job_schedule_by_template_file(self, file_path): with open(file_path, 'r') as f: templates = yaml.safe_load(f) for key in templates: name = key seconds, code, template, use, description = operator.itemgetter( 'seconds', 'code', 'template', 'use', 'description' )(templates[key]) comm_type = templates[key]['comm']['type'] host, port = operator.itemgetter('host', 'port')( templates[key]['comm']['setting']) kw_argument = dict(code=code, name=name, interval_second=seconds, template=template, ip=host, port=port, description=description, use=use, comm_type=comm_type) self.add_job_schedule(**kw_argument) # ========================================================================= @staticmethod def get_python_module(code, name, ip, port): def indent(text, amount, ch=' '): import textwrap return textwrap.indent(text, amount * ch) code = CONTEXT.format(ip=ip, port=port, code=indent(code, 4)) module = types.ModuleType(name) exec(code, module.__dict__) return module # ========================================================================= def add_job_schedule(self, code: str, name: str, interval_second: int, template, ip, port, description, use, comm_type): module = self.get_python_module(code, name, ip, port) parameters = name, module, template comm = {'comm_typ': comm_type, 'setting': {'host': ip, 'port': port}} self.scheduler.pause() try: self.scheduler.add_job( self.request_data, args=parameters, kwargs={'code': code, 'use': use, 'description': description, 'comm': comm, 'interval_second': interval_second}, id=name, trigger='interval', seconds=interval_second) finally: self.scheduler.resume() # ========================================================================= def remove_job_schedule(self, _id: str): self.scheduler.remove_job(_id) del self.data[_id] return # ========================================================================= def modify_job_schedule(self, _id, seconds): self.scheduler.reschedule_job(_id, trigger='interval', seconds=seconds) # ========================================================================= def request_data(self, name, module, template, **kwargs): data = module._main() result = get_json_data_with_template(data, template=template) result['hex'] = data.hex(' ') if name not in self.data: self.data[name] = deque(maxlen=60) self.data['__last_fetch'][name] = [result] self.data[name].append(result) self.data[name].rotate() return result # ========================================================================= def get_schedule_jobs(self): jobs = self.scheduler.get_jobs() if not jobs: return jobs result = list() for job in jobs: _, _, template = job.args code, description, use, comm, seconds = operator.itemgetter( 'code', 'description', 'use', 'comm', 'interval_second')(job.kwargs) result.append( dict(id=job.id, code=code, template=template, description=description, use=use, comm=comm, seconds=seconds)) return result
class SchedulerService(rpyc.Service): def __init__(self, **config): self._scheduler = BackgroundScheduler() self._scheduler.configure(**config) self._scheduler.start() self.logger = logging.getLogger("Heartbeat.core") self.logger.info("Heartbeat Core Initalized") def on_connect(self, conn): # code that runs when a connection is created # (to init the service, if needed) self.logger.info("----------Begin New Client----------") self.logger.info(conn) self.logger.info("----------End New Client----------") def on_disconnect(self, conn): # code that runs after the connection has already closed # (to finalize the service, if needed) self.logger.info("----------Begin Goodbye Client----------") self.logger.info(conn) self.logger.info("----------End Goodbye Client----------") def exposed_add_job(self, func, *args, **kwargs): self.logger.info("----------Begin New Job----------") self.logger.info("Function: %s", str(func)) self.logger.info("*args: %s", str(args)) self.logger.info("**kwargs: %s", str(dict(kwargs))) self.logger.info("----------Eng New Job----------") return self._scheduler.add_job(func, *args, **kwargs) def exposed_modify_job(self, job_id, jobstore=None, **changes): return self._scheduler.modify_job(job_id, jobstore, **changes) def exposed_reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args): return self._scheduler.reschedule_job(job_id, jobstore, trigger, **trigger_args) def exposed_pause_job(self, job_id, jobstore=None): return self._scheduler.pause_job(job_id, jobstore) def exposed_resume_job(self, job_id, jobstore=None): return self._scheduler.resume_job(job_id, jobstore) def exposed_remove_job(self, job_id, jobstore=None): self._scheduler.remove_job(job_id, jobstore) def exposed_get_job(self, job_id, jobstore=None): return self._scheduler.get_job(job_id, jobstore=jobstore) def exposed_get_jobs(self, jobstore=None): results = self._scheduler.get_jobs(jobstore) return results def exposed_get_tasks(self): """Return a list of schedule-able function""" tasks = [] for module_file in os.listdir( os.path.join(os.path.dirname(__file__), "task")): if module_file == "__init__.py" or module_file[-3:] != ".py": continue module_name = "server.task.{}".format(module_file[:-3]) module = importlib.import_module(module_name) if not hasattr(module, "__all__"): continue for function_name in module.__all__: function = getattr(module, function_name) if not callable(function): continue parameters = inspect.signature(function).parameters parameters_str = ", ".join( [str(val) for key, val in parameters.items()]) tasks.append("{}:{}({})".format(module_name, function_name, parameters_str)) return tasks
class SpecificTimeReporter(object): """ This class is used to get the real-time market data at specific time everyday from yahoo finance database, the accessed data will not be saved at local, please use this with my Stock Data Reader class """ def __init__(self, function): """ :param function: function of the arranged job, in this case, it should be the getCurrentMarketData function """ self._scheduler = None self.function = function self.count = 1 self._all_job = {} self.start() def start(self): """ start the reporter :return: None """ self._scheduler = BackgroundScheduler() self._scheduler.start() def convertInt2Time(self, hour, minute, second): """ You do not need to call this method, you can treat this as a private method :param hour: integer ranging from 0 to 23 :param minute: integer ranging from 0 to 59 :param second: integer ranging from 0 to 59 :return: string format of time """ ans = "" if hour < 10: ans = ans + "0" + str(hour) else: ans = ans + str(hour) if minute < 10: ans = ans + "0" + str(minute) else: ans = ans + str(minute) if second < 10: ans = ans + "0" + str(second) else: ans = ans + str(second) return ans def addJob(self, hour, minute, second, *args): """ add a reporter :param hour: integer ranging from 0 to 23 :param minute: integer ranging from 0 to 59 :param second: integer ranging from 0 to 59 :param args: tickerList,like:["AAPL","IBM","JPM"] :return: None """ timeString = self.convertInt2Time(hour, minute, second) if timeString not in self._all_job: self._all_job[timeString] = str(self.count) self._scheduler.add_job(self.function, trigger='cron', hour=hour, minute=minute, second=second, args=args, id=str(self.count)) self.count = self.count + 1 else: self._scheduler.reschedule_job(self._all_job[timeString], trigger='cron', hour=hour, minute=minute, second=second) def removeJob(self, hour, minute, second): """ remove a reporter :param hour: integer ranging from 0 to 23 :param minute: integer ranging from 0 to 59 :param second: integer ranging from 0 to 59 :return: None """ timeString = self.convertInt2Time(hour, minute, second) if timeString not in self._all_job: warnings.warn("Job not found!") else: self._scheduler.remove_job(self._all_job[timeString]) def removeAllJobs(self): """ remove all reporters :return: None """ self._scheduler.remove_all_jobs() def pause(self): """ pause all reporters :return: None """ self._scheduler.pause() def resume(self): """ resume the paused reporters :return: None """ self._scheduler.resume() def getAllJobs(self): """ print the information of all reporters :return: None """ self._scheduler.print_jobs() def shutDown(self): """ shut down all reporters :return: None """ self._scheduler.shutdown()
class Scheduler(object): def __init__(self): self.cron_job_args = [ 'year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second', 'start_date', 'end_date' ] self.scheduler = BackgroundScheduler( timezone=getattr(config, 'TIME_ZONE', 'Asia/Shanghai')) self.jobs = {} self.already_init = False def init(self): if not self.already_init: for job in Job.query.filter_by(enabled=True).all(): self.add_job(job) self.scheduler.start() self.already_init = True def __parse_args(self, trigger, trigger_args): if trigger == 'cron': args = { k: v for k, v in zip(self.cron_job_args, trigger_args.split(';')) if v } # 周需要单独处理,0 对应周一,与页面上的说明不一致 day_of_week = int( args['day_of_week']) if args.get('day_of_week') else None if day_of_week == 0: args['day_of_week'] = 6 elif day_of_week is not None: args['day_of_week'] = day_of_week - 1 return args elif trigger == 'interval': return {'seconds': int(trigger_args)} elif trigger == 'date': return {'run_date': trigger_args} else: raise ValueError('未知的调度策略: %r' % trigger) def add_job(self, job): job_id = str(job.id) args = self.__parse_args(job.trigger, job.trigger_args) instance = self.scheduler.add_job(agent, job.trigger, id=job_id, args=(job.id, job.command_user, job.command, job.targets), **args) self.jobs[job_id] = instance def valid_job_trigger(self, trigger, trigger_args): try: args = self.__parse_args(trigger, trigger_args) job = self.scheduler.add_job(agent, trigger, args=(None, None, None, None), next_run_time=None, **args) job.remove() return True except ValueError: return False def remove_job(self, job_id): job_id = str(job_id) if self.scheduler.get_job(job_id): self.scheduler.remove_job(job_id) def update_job(self, job): job_id = str(job.id) if self.scheduler.get_job(job_id): args = self.__parse_args(job.trigger, job.trigger_args) self.scheduler.reschedule_job(job_id, trigger=job.trigger, **args) elif job.enabled: self.add_job(job)
class MelsecplcWorker(BaseWorker): melsecPlc_Setting = settings['manager']['workers']['melsecplc'] gateway_id = None device_name = None if 'topic_subscription' in melsecPlc_Setting: _, _, gateway_id, device_name, cmd_type, cmd = melsecPlc_Setting[ 'topic_subscription'].split('/') # Default values devices = {} # Payload that should be send when device is available available_payload = 'home' # type: str # Payload that should be send when device is unavailable unavailable_payload = 'not_home' # type: str # After what time (in seconds) we should inform that device is available (default: 0 seconds) available_timeout = 0 # type: float # After what time (in seconds) we should inform that device is unavailable (default: 60 seconds) unavailable_timeout = 60 # type: float scan_timeout = 10. # type: float scan_passive = True # type: str or bool start_to_collect = False #Global variable to control start to collect or not flag_started = False #Flag to control BleDeviceStatus object creation count = 0 # Defined Puhlish MQTT Topic ReadData_Topic = "/IEW/{gateway}/{device}/ReplyData" HeartBeat_Topic = "/IEW/{gateway}/{device}/Status/HeartBeat" Alarm_Topic = "/IEW/{gateway}/{device}/Status/Alarm" ReadData_job_id = '{}_interval_job'.format("ReadData") Version = "1.0" Status = "Init" Job_queue = queue(maxsize=10) _LOGGER.info("Version = " + Version) _LOGGER.info("MelsecPLC Class Initial Funished") _LOGGER.info("Setting Gateway_ID = " + gateway_id + ", Device_ID = " + device_name) _LOGGER.info("ReadData_Topic = " + ReadData_Topic) _LOGGER.info("HeartBeat_Topic = " + HeartBeat_Topic) def __init__(self, command_timeout, **kwargs): super(MelsecplcWorker, self).__init__(command_timeout, **kwargs) self._scheduler = BackgroundScheduler(timezone=utc) self._scheduler.add_job(self.Read_PLC_Data, 'interval', seconds=10, id=self.ReadData_job_id) self._scheduler.start() # Apschedule 不要紀錄 Logger logging.getLogger('apscheduler.executors.default').propagate = False self.Status = "Init" _LOGGER.info("MelsecplcWorker --> starts = " + self.Status) self.report_alarm('0000', 'Info', 'MelsecplcWorker is Initial') def run(self, mqtt): while True: time.sleep(1) if self.Job_queue.empty() == True: continue else: try: SendOutMQTT = self.Job_queue.get() mqtt.publish(SendOutMQTT) except Exception as ee: logger.log_exception( _LOGGER, 'MelsecplcWorker --> Publish Error Msg : %s', str(ee)) def create_devices(self): if not self.start_to_collect: self.last_status = [] for device_name, dev_info in self.devices.items(): _LOGGER.info( "MelsecplcWorker --> create_devices : device_name = " + device_name) for d in dev_info: _LOGGER.info( "MelsecplcWorker --> create_devices : ip_addr = " + d['IP_ADDR']) _LOGGER.info( "MelsecplcWorker --> create_devices : port_id = " + d['PORT_ID']) self.last_status = [ PLCDevice(self, d['IP_ADDR'], d['PORT_ID'], device_name) ] self.start_to_collect = True def set_stop_flag(self): self.start_to_collect = False self.flag_started = False def Read_PLC_Data(self): if self.start_to_collect and self.flag_started: self.count += 1 if self.count > 65535: self.count = 1 _LOGGER.debug("MelsecplcWorker --> ReadPLC Data enters count = " + str(self.count)) sendout_topic = self.ReadData_Topic.replace( "{gateway}", self.gateway_id).replace("{device}", self.device_name) # 成功讀取狀態改為Run try: for status in self.last_status: json_msg = status.read_data(self.addr_array) status.set_status(status is not None) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) _LOGGER.info("MelsecplcWorker --> Report Read PLC Data: " + json_msg) self.Status = "Run" except Exception as ee: logger.log_exception( _LOGGER, 'MelsecplcWorker --> Read Data Excepting. Msg : %s', str(ee)) self.Status = "Down" # Read Data Error Send Error to MQTT else: _LOGGER.info( "MelsecplcWorker --> status_update: Waiting for Collect Command!" ) def read_payload_cmd_start(self, device_name, payload, topic): try: cmd_start = {} cmd_start = json.loads(payload) _LOGGER.info( "MelsecplcWorker --> read_payload_cmd_start: payload = " + payload) _LOGGER.info( "MelsecplcWorker --> read_payload_cmd_start: cmd_start = " + str(cmd_start)) _LOGGER.info( "MelsecplcWorker --> read_payload_cmd_start: cmd_start['Device_Info] = " + str(cmd_start['Device_Info'])) sendout_topic = topic + "/Ack" if cmd_start['Cmd_Type'] == "Start": connect_status = 'OK' self.devices.update({device_name: cmd_start['Device_Info']}) # 連線設定完成 if not self.start_to_collect: self.create_devices() for status in self.last_status: if status.p_device == None: connect_status = 'NG' ret_json = {} ret_json.update({'Cmd_Result': connect_status}) ret_json.update({'Trace_ID': cmd_start['Trace_ID']}) json_msg = json.dumps(ret_json) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) if connect_status == 'NG': self.Status = "Down" self.report_alarm('1001', 'ERROR', 'Connected PLC Faild') else: self.Status = "Ready" self.report_alarm('0001', 'INFO', 'Connected PLC successed') else: ret_json = {} ret_json.update({'Cmd_Result': "NG"}) ret_json.update({'Trace_ID': cmd_start['Trace_ID']}) json_msg = json.dumps(ret_json) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) # 補送Alarm to MQTT self.report_alarm('1002', 'ERROR', 'Cmd_Type not Start') except Exception as ee: logger.log_exception(_LOGGER, 'Cmd Start Exception Error Msg : %s', str(ee)) self.Status = "Down" ret_json = {} ret_json.update({'Cmd_Result': "NG"}) ret_json.update({'Trace_ID': cmd_start['Trace_ID']}) json_msg = json.dumps(ret_json) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) # 補送Alarm to MQTT self.report_alarm( '1003', 'ERROR', 'Cmd_Type Start happened exception error :' + str(ee)) def read_payload_cmd_readdata(self, device_name, payload, topic): try: cmd_read = {} cmd_read = json.loads(payload) _LOGGER.info( "MelsecplcWorker --> read_payload_cmd_parameter: cmd_parameter = " + str(cmd_read)) sendout_topic = topic + "/Ack" if cmd_read['Cmd_Type'] == "Collect": self.addr_array = cmd_read['Address_Info'] interval = int(cmd_read['Report_Interval']) self.flag_started = False self._scheduler.pause() self._scheduler.reschedule_job(job_id=self.ReadData_job_id, trigger='interval', seconds=interval) self._scheduler.resume() self.flag_started = True ret_json = {} ret_json.update({'Cmd_Result': "OK"}) ret_json.update({'Trace_ID': cmd_read['Trace_ID']}) json_msg = json.dumps(ret_json) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) else: ret_json = {} ret_json.update({'Cmd_Result': "NG"}) ret_json.update({'Trace_ID': cmd_read['Trace_ID']}) json_msg = json.dumps(ret_json) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) _LOGGER.error( "MelsecplcWorker --> read_payload_cmd_NG tpye not Collect, cmd_read_type = " + str(cmd_read['Cmd_Type'])) # 補送Alarm to host self.report_alarm('1004', 'ERROR', 'Type not Collect') except Exception as ee: logger.log_exception(_LOGGER, 'Cmd Start Exception Error Msg : %s', str(ee)) ret_json = {} ret_json.update({'Cmd_Result': "NG"}) ret_json.update({'Trace_ID': cmd_read['Trace_ID']}) json_msg = json.dumps(ret_json) self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) self.Status = "Down" self.report_alarm( '1005', 'ERROR', 'Cmd_Type Collect happened exception error :' + str(ee)) def read_payload_parameter_request(self, device_name, payload): parameter_request = {} parameter_request = json.loads(payload) _LOGGER.info( "MelsecplcWorker --> read_payload_parameter_request: parameter_request = " + parameter_request) def cmd_stop(self, value): if value == KILL_ME: sys.exit("END") elif value == STOP_COLLECT: self._scheduler.pause() for status in self.last_status: self.set_stop_flag() self.last_status = None def status_update(self): _LOGGER.info("MelsecplcWorker --> Heartbit Report") now = datetime.datetime.now() sendout_topic = self.HeartBeat_Topic.replace( "{gateway}", self.gateway_id).replace("{device}", self.device_name) HB_json = {} HB_json.update({'Version': self.Version}) HB_json.update({'Status': self.Status}) HB_json.update({'HBDatetime': now.strftime("%Y%m%d%H%M%S%f")[:-3]}) json_msg = json.dumps(HB_json) _LOGGER.debug("Heartbit Report : " + json_msg) ret = [] messages = [] messages.append(MqttMessage(topic=sendout_topic, payload=json_msg)) ret += messages return ret def report_alarm(self, code, level, desc): now = datetime.datetime.now() report_alarm_Topic = self.Alarm_Topic.replace( "{gateway}", self.gateway_id).replace("{device}", self.device_name) Alarm_json = {} Alarm_json.update({'AlarmCode': code}) Alarm_json.update({'AlarmLevel': level}) Alarm_json.update({'AlarmApp': 'WORKER'}) Alarm_json.update({'DateTime': now.strftime("%Y%m%d%H%M%S%f")[:-3]}) Alarm_json.update({'AlarmDesc': desc}) json_msg = json.dumps(Alarm_json) self.Job_queue.put( [MqttMessage(topic=report_alarm_Topic, payload=json_msg)]) _LOGGER.error("MelsecplcWorker --> Report Alarm to Host Msg : " + str(json_msg)) def on_command(self, topic, value): value = value.decode('utf-8')\ # 設定改在其他地方 _, _, gateway_id, device_name, cmd_type, cmd = topic.split('/') if cmd_type == "Cmd": _LOGGER.info("MelsecplcWorker --> on_command: topic = " + topic) _LOGGER.info("MelsecplcWorker --> on_command: gateway_id = " + gateway_id) _LOGGER.info("MelsecplcWorker --> on_command: device_name = " + device_name) _LOGGER.info("MelsecplcWorker --> on_command: cmd_type = " + cmd_type) _LOGGER.info("MelsecplcWorker --> on_command: cmd = " + cmd) if cmd == "Start": self.read_payload_cmd_start(device_name, value, topic) elif cmd == "ReadData": self.read_payload_cmd_readdata(device_name, value, topic) elif cmd == "Stop": self.cmd_stop(value) _LOGGER.error("MelsecplcWorker --> on_command = " + cmd + "Stop Collect Data") self.Status = "Idle" elif cmd == "OTA": cmd_OTA = json.loads(value) if cmd_OTA["Cmd"] == "OTA": OTA_json = {} OTA_json.update({'Trace_ID': cmd_OTA["Trace_ID"]}) OTA_json.update({'Version': self.Version}) OTA_json.update({'Status': self.Status}) OTA_json.update( {'Datetime': now.strftime("%Y%m%d%H%M%S%f")[:-3]}) OTA_json.update({'ProcrssID': os.getpid()}) json_msg = json.dumps(OTA_json) sendout_topic = topic + "/Ack" self.Job_queue.put( [MqttMessage(topic=sendout_topic, payload=json_msg)]) time.sleep(5) self.cmd_stop("kill") elif cmd_type == "Parameter": if cmd == "Request": self.read_payload_parameter_request(device_name, value)
""" Demonstrates how to use the blocking scheduler to schedule a job that executes on 3 second intervals. """ from datetime import datetime import os import time from apscheduler.schedulers.blocking import BlockingScheduler from apscheduler.schedulers.background import BackgroundScheduler def tick(): print('Tick! The time is: %s' % datetime.now()) if __name__ == '__main__': #scheduler = BlockingScheduler() scheduler = BackgroundScheduler() scheduler.add_job(tick, 'interval', seconds=3, id='test') print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) try: scheduler.start() scheduler.reschedule_job('test', trigger='cron', minute='*/1') time.sleep(1000) except (KeyboardInterrupt, SystemExit): pass
class Planner: def __init__(self, db_url): job_defaults = dict(misfire_grace_time=180, coalesce=True, max_instances=1, next_run_time=None) self.scheduler = BackgroundScheduler(timezone='UTC', job_defaults=job_defaults) self.scheduler.add_jobstore('sqlalchemy', url=db_url + '?application_name=aps') self.scheduler.start() log.info('started planner scheduler') for idx, j in enumerate(self.get_jobs()): log.info( '%d. name:%s trigger:%s func:%s args:%s kwargs:%s next:%s', idx + 1, j['name'], j['trigger'], j['func'], j['args'], j['kwargs'], j['next_run_time']) def _job_to_dict(self, job): if isinstance(job.trigger, IntervalTrigger): trigger = 'interval' elif isinstance(job.trigger, DateTrigger): trigger = 'date' elif isinstance(job.trigger, CronTrigger): trigger = 'cron' else: trigger = 'unknown' next_run_time = "" if job.next_run_time: next_run_time = job.next_run_time.strftime("%Y-%m-%dT%H:%M:%SZ") return dict(id=job.id, name=job.name, func=job.func_ref, args=job.args, kwargs=job.kwargs, trigger=trigger, next_run_time=next_run_time) def add_job( self, func=None, trigger=None, args=None, kwargs=None, job_id=None, name=None, misfire_grace_time=None, # pylint: disable=too-many-arguments coalesce=True, max_instances=None, next_run_time=None, replace_existing=False, trigger_args=None): if trigger_args is None: trigger_args = {} all_kw_args = dict(args=args, kwargs=kwargs, id=job_id, name=name, replace_existing=replace_existing) if misfire_grace_time is not None: all_kw_args['misfire_grace_time'] = misfire_grace_time if coalesce is not None: all_kw_args['coalesce'] = coalesce if max_instances is not None: all_kw_args['max_instances'] = max_instances if next_run_time is not None: all_kw_args['next_run_time'] = next_run_time if trigger in ['interval', 'repo_interval']: trigger = IntervalTrigger(**trigger_args) elif trigger == 'date': trigger = DateTrigger(**trigger_args) elif trigger == 'cron': trigger = CronTrigger(**trigger_args) else: raise Exception('unknown trigger type %s' % trigger) all_kw_args['trigger'] = trigger log.info('add_job args: %s', all_kw_args) try: job = self.scheduler.add_job(func, **all_kw_args) log.info('add_job job:%s', job) except Exception: log.exception('some problem') raise return self._job_to_dict(job) def get_jobs(self): try: jobs = [] for j in self.scheduler.get_jobs(): jobs.append(self._job_to_dict(j)) except Exception: log.exception('some problem') raise return jobs def reschedule_job(self, job_id=None, trigger=None, trigger_args=None): if trigger_args is None: trigger_args = {} log.info('reschedule_job args: %s %s %s', job_id, trigger, trigger_args) try: job = self.scheduler.reschedule_job(job_id, trigger=trigger, **trigger_args) log.info('reschedule_job job:%s', job) except Exception: log.exception('some problem') raise return self._job_to_dict(job) def remove_job(self, job_id=None): log.info('remove_job arg: %s', job_id) try: self.scheduler.remove_job(job_id) log.info('remove_job done') except Exception: log.exception('some problem')
class Scheduler: TRIGGERS = { "trig_5minutes": { "id": "trig_5minutes", "name": "Every five minutes", "options": [], "schema": {}, "trigger_args": lambda args: dict(minute="*/5"), "from_trigger": lambda trig: [] }, "trig_hourly": { "id": "trig_hourly", "name": "Each hour", "options": [], "schema": {}, "trigger_args": lambda args: dict(hour="*"), "from_trigger": lambda trig: [] }, "trig_daily": { "id": "trig_daily", "name": "Each day", "options": [], "schema": {}, "trigger_args": lambda args: dict(day="*"), "from_trigger": lambda trig: [] }, "trig_weekday": { "id": "trig_weekday", "name": "Each weekday", "options": [{ "id": i, "name": el, "active": True } for i, el in enumerate("Mon Tue Wed Thu Fri Sat Sun".split())], "schema": { "id": { "type": "integer", "coerce": int, "min": 0, "max": 6 }, "name": { "type": "string", "required": False }, "active": { "type": "boolean", "coerce": utility.coerce_bool, "required": True } }, "trigger_args": lambda args: dict(day_of_week=",".join(str(a) for a in args)), "from_trigger": lambda trig: [int(d) for d in str(trig.fields[4]).split(",")] }, "trig_monthly": { "id": "trig_monthly", "name": "Each month", "options": [{ "id": i + 1, "name": el, "active": True } for i, el in enumerate(("Jan Feb Mar Apr May Jun " "Jul Aug Sep Oct Nov Dec").split())], "schema": { "id": { "type": "integer", "coerce": int, "min": 0, "max": 12 }, "name": { "type": "string", "required": False }, "active": { "type": "boolean", "coerce": utility.coerce_bool, "required": True } }, "trigger_args": lambda args: dict(month=",".join(str(a) for a in args)), "from_trigger": lambda trig: [int(d) for d in str(trig.fields[1]).split(",")] }, } """Predefined triggers and their argument checks.""" def __init__(self, elastic, crawler_dir="crawlers", crawler_args={}, **cron_defaults): """Initializes the scheduler by binding it to it's elasticsearch db. Args: elastic (elasticsearch.Elasticsearh): The es-client to save the crawling jobs in. crawler_dir (str): the directory, where the crawlers will be found. Defaults to "crawlers". job_defaults (dict): a dictionary of keyword arguments for the schedulers job_defaults. **cron_defaults (dict): a dictionary of keyword arguments for the schedulers job_defaults. Returns: Scheduler: a fresh Scheduler instance. """ jobstores = { "default": { "type": "memory" }, "elastic": InjectorJobStore(kwargs=crawler_args, client=elastic) } executors = { "default": ThreadPoolExecutor(10), "processpool": ProcessPoolExecutor(10) } job_defaults = { "misfire_grace_time": 5 * 60, # 5min "coalesce": True, } self.cron_defaults = utility.DefaultDict( { # standard is every day at 00:00:00 "hour": 0, "minute": 0, "second": 0 }, **cron_defaults) self.scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) self.crawlers = _detect_crawlers() # set up the validator schema. self.job_validator = cerberus.Validator(SCHEMATA["job"]({ "trigger_ids": list(self.TRIGGERS) }), allow_unknown=True) self.scheduler.start() def upsert_job(self, job_dict, **runtime_args): """Adds or updates a job using the provided user_input. If an id field is present in the dict, the job is updated, otherwise a new one is created. Args: job_dict (dict): user input for a job, as defined in `SCHEMATA`. **runtime_args (dict): additional runtime arguments for the crawler. Returns: apscheduler.job.Job: a new job Object. """ if not self.job_validator.validate(job_dict): raise (AssertionError(str(self.job_validator.errors))) doc = utility.SDA(job_dict) job = self.crawlers.get(doc["crawler.id"], None) # default to the SearchPlugin, and give the search name as argument. if job is None: inst = { "args": ("SearchPlugin", runtime_args), "kwargs": dict(search_id=doc["crawler.id"]) } else: inst = {"args": (doc["crawler.id"], runtime_args), "kwargs": {}} trigger = self._make_trigger(doc["schedule"]) if doc["id"]: self.scheduler.modify_job(doc["id"], jobstore="elastic", func=_run_plugin, name=doc["name.name"], **inst) new_job = self.scheduler.reschedule_job(doc["id"], jobstore="elastic", trigger=trigger) else: # use the crawler id as name, when the job is created. new_job = self.scheduler.add_job(_run_plugin, jobstore="elastic", trigger=trigger, name=doc["crawler.id"], **inst) return new_job def get_triggers(self): """Returns a list of triggers, that are predefined in the system. Returns: list: a list of tuples, holding id and name for each trigger. """ return [{ "id": v["id"], "name": v["name"], "options": v["options"] } for v in self.TRIGGERS.values()] def sync_jobs(self, joblist): """Synchronize the current jobs with a given list of jobs. This means, that all jobs not included in the list will be removed, existing ones will be updated and new ones will be added to the scheduler. Args: joblist (list): a list of jobs in the format of the schema. Returns: bool: whether this operation was successful or not. """ logger.debug("Syncing job lists ...") current_jobs = self.get_jobs() jobs_to_keep = {j["id"] for j in joblist if j.get("id")} # remove old jobs for job in current_jobs: if job["id"] not in jobs_to_keep: self.scheduler.remove_job(job["id"], jobstore="elastic") # update and add jobs for job in joblist: self.upsert_job(job) return True def _make_trigger(self, trigger_doc): """Creates a trigger from a given dictionary of user input.""" # we can assume, that an id for the trigger is given in the input. cur_trigger = self.TRIGGERS[trigger_doc["id"]] option_validator = cerberus.Validator(cur_trigger["schema"]) args = [ o["id"] for o in trigger_doc["options"] if option_validator(o) and o["active"] ] trigger_args = cur_trigger["trigger_args"](args) return CronTrigger(**trigger_args) def _serialize_trigger(self, trigger): """Serializes a trigger into a json array, as defined in TRIGGERS.""" # since we only have a defined set of triggers, the following is # possible. mapping = [(v["trigger_args"]([]).keys(), k) for k, v in self.TRIGGERS.items()] trigger_doc = None result = {} for keys, name in mapping: # all keys for the mapping need to be defined. def_keys = [f.name for f in trigger.fields if not f.is_default] if all([(key in def_keys) for key in keys]): trigger_doc = self.TRIGGERS[name] break if not trigger_doc: return result result["name"] = trigger_doc["name"] result["id"] = trigger_doc["id"] args = set(trigger_doc["from_trigger"](trigger)) # copy the list of options (otherwise this leads to nasty side effects) options = [dict(**item) for item in trigger_doc["options"]] for option in options: option["active"] = option["id"] in args result["options"] = options return result def get_jobs(self): """Returns a list of jobs that are scheduled in the system. Returns: list: a list of job-dicts, holding the id and the runtimes. """ jobs = self.scheduler.get_jobs() joblist = [] for job in jobs: joblist.append({ "id": job.id, "name": { "name": job.name }, "crawler": { "id": job.args[0] }, "schedule": self._serialize_trigger(job.trigger), "next_run": { "name": job.next_run_time } }) logger.debug(f"Retrieved {len(joblist)} jobs from the jobstore.") return joblist def run_job(self, job_id): """Runs the job with the specified id immediately. Args: job_id: the id of the job that should be run. Returns: bool: whether running the job succeeded or not. """ logger.debug(f"Running job '{job_id}' directly.") cur_job = self.scheduler.get_job(job_id, jobstore="elastic") if cur_job is None: return False cur_job.func(*cur_job.args, **cur_job.kwargs) return True