def test6(): """定时执行任务,暂停,恢复, 实例化""" start_time = time.time() scheduler = BackgroundScheduler() job = scheduler.add_job(my_job, 'interval', args=('123',),seconds=1, id='my_job_id') # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; print("作业id:{},作业名字:{},作业参数:{},作业函数:{},触发条件:{}".format(job.id, job.name, job.args, job.func, job.trigger)) scheduler.start() # 程序运行到这里,任务没有运行完也会往后执行,既执行后面的任务,又执行这个任务。 print('运行到了这里1') while (scheduler.state): if time.time() - start_time >5: print('暂停作业') #scheduler.pause() # 暂停作业: job.pause() # 暂停单个实例 break print('恢复作业') if time.time() - start_time >5: #scheduler.resume() # 恢复作业 job.resume() # 恢复单个实例 time.sleep(4) print('当前任务列表:{}'.format(scheduler.get_jobs())) # 获得调度作业的列表,可以使用 get_jobs() 来完成,它会返回所有的job实例 scheduler.get_job('my_job_id') # 获取id为my_job_id的作业实例 scheduler.print_jobs() # 输出所有格式化的作业列表。 print('移除作业') # scheduler.remove_job('my_job_id') # 移除id为my_job_id的作业 # scheduler.remove_all_jobs() # 移除所有的作业 job.remove() # 移除单个实例的作业
def test6(): """定时执行任务,暂停,恢复, 实例化""" start_time = time.time() scheduler = BackgroundScheduler() job = scheduler.add_job( my_job, 'interval', args=('123', ), seconds=1, id='my_job_id') # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; print("作业id:{},作业名字:{},作业参数:{},作业函数:{},触发条件:{}".format( job.id, job.name, job.args, job.func, job.trigger)) scheduler.start() # 程序运行到这里,任务没有运行完也会往后执行,既执行后面的任务,又执行这个任务。 print('运行到了这里1') while (scheduler.state): if time.time() - start_time > 5: print('暂停作业') #scheduler.pause() # 暂停作业: job.pause() # 暂停单个实例 break print('恢复作业') if time.time() - start_time > 5: #scheduler.resume() # 恢复作业 job.resume() # 恢复单个实例 time.sleep(4) print('当前任务列表:{}'.format( scheduler.get_jobs())) # 获得调度作业的列表,可以使用 get_jobs() 来完成,它会返回所有的job实例 scheduler.get_job('my_job_id') # 获取id为my_job_id的作业实例 scheduler.print_jobs() # 输出所有格式化的作业列表。 print('移除作业') # scheduler.remove_job('my_job_id') # 移除id为my_job_id的作业 # scheduler.remove_all_jobs() # 移除所有的作业 job.remove() # 移除单个实例的作业
def test3(): """定时执行任务,暂停,恢复""" start_time = time.time() scheduler = BackgroundScheduler() scheduler.add_job( my_job, 'interval', args=('123', ), seconds=1, id='my_job_id') # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; scheduler.start() # 程序运行到这里,任务没有运行完也会往后执行,既执行后面的任务,又执行这个任务。 print('运行到了这里1') while (scheduler.state): if time.time() - start_time > 5: print('暂停作业') scheduler.pause() # 暂停作业: break print('恢复作业') if time.time() - start_time > 5: scheduler.resume() # time.sleep(4) print('当前任务列表:{}'.format( scheduler.get_jobs())) # 获得调度作业的列表,可以使用 get_jobs() 来完成,它会返回所有的job实例 scheduler.get_job('my_job_id') # 获取id为my_job_id的作业实例 scheduler.print_jobs() # 输出所有格式化的作业列表。 print('移除作业') # scheduler.remove_job('my_job_id') # 移除id为my_job_id的作业 scheduler.remove_all_jobs() # 移除所有的作业
class AutoReserve(): def __init__(self): self.scheduler = BackgroundScheduler(timezone="Asia/Seoul") self.jobid = 1 self.jobdb = {} def addGolfJob(self, ccName, BackFunc, user_id, user_pw, schedule, target_date, target_time, day=21): print('autoReserve in Jayuro', schedule.hour(), schedule.minute(), schedule.second(), target_date, target_time, day) print(self.jobid) while self.scheduler.get_job(self.jobid): print(self.jobid, self.scheduler.get_job(self.jobid)) self.jobid += 1 result = self.scheduler.add_job(BackFunc,'cron', args=[user_id, user_pw, target_date, target_time, day], week='1-53', day_of_week='0-6', \ hour=schedule.hour(), minute=schedule.minute(), second=schedule.second(), id=str(self.jobid)) print("result:", result) self.jobdb[self.jobid]= ccName if self.scheduler.state == 1: #apscheduler.schedulers.base.STATE_RUNNING print('Scheduler is running') elif self.scheduler.state == 2: print('Scheduler is paused') elif self.scheduler.state == 0: print('Scheduler is stopped') self.scheduler.start() return self.jobid def autoStop(self): print('autoStop') for jj in self.scheduler.get_jobs(): print(jj) self.scheduler.remove_all_jobs() self.jobdb.clear() self.jobid = 1 print(self.scheduler.get_jobs()) def autoInfo(self): print('Auto information') print(self.scheduler.get_jobs()) for jj in self.scheduler.get_jobs(): print(jj) for key in self.jobdb: print(key, self.jobdb[key]) def printlog(self, target_date, target_time): now = datetime.datetime.now() print(str(now) + str(target_date + "," + target_time)) # print("Running main process............... : " + str(datetime.datetime.now(timezone('Asia/Seoul')))) def getJobInfo(self): return self.jobdb
class AdvancedScheduleManager: def __init__(self): self.s = BackgroundScheduler() self.s.start() def reset_venue(self, bot, venue): print("reset venue") if self.s.get_job(venue) != None: print("remove venue") self.s.remove_job(venue) self.s.add_job(bot.send_non_update_venues, 'date', run_date=datetime.now() + timedelta(seconds=3600), kwargs={"venue": venue}, id=venue) def test_scheduler(self, bot): self.s.add_job(bot.pprint, 'date', run_date=datetime.now() + timedelta(seconds=3), kwargs={"text": "Advanced schedule執行"}, id="test") def get_queue(self): return len(self.s.get_jobs())
class Scheduler: scheduler = None def __init__(self) -> None: super().__init__() self.scheduler = BackgroundScheduler() self.scheduler.start() self.print_schedules() def __del__(self): self.shutdown() def shutdown(self): self.scheduler.shutdown() def schedule_event(self, aid, dt, event, args): scheduled = self.scheduler.add_job(func=event, trigger='date', id=aid, run_date=dt, args=args) return scheduled.id def update_event(self, aid, dt, event, args): scheduled = self.scheduler.get_job(job_id=aid) if scheduled: return scheduled.id else: return self.schedule_event(aid=aid, dt=dt, event=event, args=args) def print_schedules(self): return self.scheduler.print_jobs()
def encode_compute(): scheduler = BackgroundScheduler() scheduler.start() message = request.form['message'] image = request.files['img'] if not message or not image: return redirect(url_for('steganography_web.encode')) filename = 'website/static/images/' + str( session['uid']) + image.filename.split('.')[0] + '.png' image.save(filename) session['steganography_image'] = filename binary_string = encode_string(message) + encode_string(END_OF_ENCODE) image = encode_image(filename, binary_string) time = datetime.datetime.now() + datetime.timedelta(minutes=4) if scheduler.get_job(filename): scheduler.reschedule_job(job_id=filename, trigger='date', run_date=time) print(f'job rescheduled for {time}') else: scheduler.add_job(delete_file, args=[filename], trigger='date', run_date=time, id=filename) print(f'job scheduled for {time}') send_days_left_text() return redirect(url_for('steganography_web.encode', show='True'))
def initialize_scheduler(): """Initialize the task scheduler. This method configures the global scheduler, checks the loaded tasks, and ensures they are all scheduled. """ global SCHEDULER # If the scheduler is not enabled, clear it and exit. This prevents any # unexpected database session issues. if not CONF.scheduler.enable: if SCHEDULER: SCHEDULER.remove_all_jobs() SCHEDULER = None LOG.info("Scheduler is not enabled.") return # Use SQLAlchemy as a Job store. jobstores = { 'default': SQLAlchemyJobStore(engine=get_engine()) } # Two executors: The default is for all plugins. The second one is for # the scheduler manager, which makes sure this scheduler instance is # aware of all of our plugins. executors = { 'default': ThreadPoolExecutor(10), 'manager': ThreadPoolExecutor(1), } # Allow executions to coalesce. See https://apscheduler.readthedocs.org/en # /latest/userguide.html#missed-job-executions-and-coalescing job_defaults = { 'coalesce': True, 'max_instances': 1, 'replace_existing': True } # This will automatically create the table. SCHEDULER = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) SCHEDULER.start() atexit.register(shutdown_scheduler) # Make sure we load in the update_scheduler job. If it exists, # we remove/update it to make sure any code changes get propagated. if SCHEDULER.get_job(SCHEDULE_MANAGER_ID): SCHEDULER.remove_job(SCHEDULE_MANAGER_ID) SCHEDULER.add_job( update_scheduler, id=SCHEDULE_MANAGER_ID, trigger=IntervalTrigger(minutes=1), executor='manager', replace_existing=True )
class Scheduler: def __init__(self): self.sched = BackgroundScheduler() self.sched.start() self.job_id = '' def __del__(self): self.shutdown() def shutdown(self): self.sched.shutdown() def add_schdule(self, job, trigger, job_id, args=None, seconds=None, minutes=None, hours=None, day=None, day_of_week=None): if trigger == 'interval': self.sched.add_job(job, trigger=trigger, id=job_id, args=args, seconds=int(seconds)) elif trigger == 'cron': self.sched.add_job(job, type, id=job_id, args=args, second=seconds, minute=minutes, hour=hours, day=day, day_of_week=day_of_week) self.sched.get_job().modify()
def initialize_scheduler(): """Initialize the task scheduler. This method configures the global scheduler, checks the loaded tasks, and ensures they are all scheduled. """ global SCHEDULER # If the scheduler is not enabled, clear it and exit. This prevents any # unexpected database session issues. if not CONF.scheduler.enable: if SCHEDULER: SCHEDULER.remove_all_jobs() SCHEDULER = None LOG.info("Scheduler is not enabled.") return # Use SQLAlchemy as a Job store. jobstores = {'default': SQLAlchemyJobStore(engine=get_engine())} # Two executors: The default is for all plugins. The second one is for # the scheduler manager, which makes sure this scheduler instance is # aware of all of our plugins. executors = { 'default': ThreadPoolExecutor(10), 'manager': ThreadPoolExecutor(1), } # Allow executions to coalesce. See https://apscheduler.readthedocs.org/en # /latest/userguide.html#missed-job-executions-and-coalescing job_defaults = { 'coalesce': True, 'max_instances': 1, 'replace_existing': True } # This will automatically create the table. SCHEDULER = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) SCHEDULER.start() atexit.register(shutdown_scheduler) # Make sure we load in the update_scheduler job. If it exists, # we remove/update it to make sure any code changes get propagated. if SCHEDULER.get_job(SCHEDULE_MANAGER_ID): SCHEDULER.remove_job(SCHEDULE_MANAGER_ID) SCHEDULER.add_job(update_scheduler, id=SCHEDULE_MANAGER_ID, trigger=IntervalTrigger(minutes=1), executor='manager', replace_existing=True)
class Aggregator(): """Store data over time from Storj.""" def __init__(self): """Constructor.""" self.api = StorjApi() self.scheduler = BackgroundScheduler() self.scheduler.start() def _store_data(self, node_id): """Store the data for `node_id`.""" try: info = self.api.get_contact_info(node_id) print(info) except ApiException as exception: print('Error retrieving information: {}'.format(exception.message)) def start(self, node_id): """Start storing data.""" if not self.scheduler.get_job(job_id=node_id): self.scheduler.add_job(func=self._store_data, trigger='interval', id=node_id, seconds=1, args=[node_id]) return True return False def stop(self, node_id): """Stop storing data.""" job = self.scheduler.get_job(job_id=node_id) if job: job.remove() return True return False
class TaskScheduler: def __init__(self): jobs_database_name = 'jobs.sqlite' jobstores = { # 'default': SQLAlchemyJobStore(url=F'sqlite:///{jobs_database_name}') } executors = { 'default': { 'type': 'threadpool', 'max_workers': 20 }, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = {'coalesce': False, 'max_instances': 3} self.scheduler = BackgroundScheduler() self.scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) def get_task(self, task_id): return self.scheduler.get_job(task_id) def get_tasks_ids(self): task_ids = [] jobs = self.scheduler.get_jobs() for j in jobs: task_ids.append(j.id) return task_ids def start_scheduler(self): self.scheduler.start() def stop_scheduler(self): self.scheduler.shutdown() def add_task(self, task_func, interval_minutes, args, task_id): print('Adding an interval task') self.scheduler.add_job(task_func, IntervalTrigger(minutes=interval_minutes), args=args, id=str(task_id)) print('Adding the interval task finished') def remove_task(self, id): print(F'Removing task (Id: {id})') self.scheduler.remove_job(id) print('The task removed.')
def test3(): """定时执行任务,暂停,恢复""" start_time = time.time() scheduler = BackgroundScheduler() scheduler.add_job(my_job, 'interval', args=('123',),seconds=1, id='my_job_id') # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; scheduler.start() # 程序运行到这里,任务没有运行完也会往后执行,既执行后面的任务,又执行这个任务。 print('运行到了这里1') while (scheduler.state): if time.time() - start_time >5: print('暂停作业') scheduler.pause() # 暂停作业: break print('恢复作业') if time.time() - start_time >5: scheduler.resume() # time.sleep(4) print('当前任务列表:{}'.format(scheduler.get_jobs())) # 获得调度作业的列表,可以使用 get_jobs() 来完成,它会返回所有的job实例 scheduler.get_job('my_job_id') # 获取id为my_job_id的作业实例 scheduler.print_jobs() # 输出所有格式化的作业列表。 print('移除作业') # scheduler.remove_job('my_job_id') # 移除id为my_job_id的作业 scheduler.remove_all_jobs() # 移除所有的作业
class ScheduleController: def __init__(self): self.scheduler = BackgroundScheduler() # self.scheduler.add_job() def run_schedules(self): try: self.scheduler.start() # Shut down the scheduler when exiting the app atexit.register(lambda: self.scheduler.shutdown()) except Exception as e: print( "*** ScheduleController.run_schedules occurred an exception: {}" .format([e, e.with_traceback])) def get_pending_jobs(self, job_id=None): """ Get list of jobs or get job by job_id :param job_id: string | optional if it's not none returns job which equals job_id :returns: List or Dict """ try: if job_id is not None: return self.scheduler.get_job(job_id) else: return self.scheduler.get_jobs() except Exception as e: pass def register_schedule(self, callback, trigger_timer, job_id, name=None): """ Void method Schedule registerer hook :param callback: function :param trigger_timer: the alias name of the trigger (e.g. ``date``, ``interval`` or ``cron``) :param job_id: string | Schedule job id :param name: string | optional name of job """ if callable(callback) is False: return False else: self.scheduler.add_job( func=callback, trigger=trigger_timer, id=job_id, name=name if name is not None else str(job_id) + " Schedule registered without name", replace_existing=True)
class Scheduler: timezone = settings.TIME_ZONE def __init__(self): self.scheduler = BackgroundScheduler(timezone=self.timezone, executors={'default': ThreadPoolExecutor(30)}) def _dispatch(self, task_id, tp, targets, extra, threshold, quiet): Detection.objects.filter(pk=task_id).update(latest_run_time=human_datetime()) rds_cli = get_redis_connection() for t in json.loads(targets): rds_cli.rpush(MONITOR_WORKER_KEY, json.dumps([task_id, tp, t, extra, threshold, quiet])) connections.close_all() def _init(self): self.scheduler.start() for item in Detection.objects.filter(is_active=True): now = datetime.now() trigger = IntervalTrigger(minutes=int(item.rate), timezone=self.timezone) self.scheduler.add_job( self._dispatch, trigger, id=str(item.id), args=(item.id, item.type, item.targets, item.extra, item.threshold, item.quiet), next_run_time=now + timedelta(seconds=randint(0, 60)) ) connections.close_all() def run(self): rds_cli = get_redis_connection() self._init() rds_cli.delete(settings.MONITOR_KEY) logging.warning('Running monitor') while True: _, data = rds_cli.brpop(settings.MONITOR_KEY) task = AttrDict(json.loads(data)) if task.action in ('add', 'modify'): trigger = IntervalTrigger(minutes=int(task.rate), timezone=self.timezone) self.scheduler.add_job( self._dispatch, trigger, id=str(task.id), args=(task.id, task.type, task.targets, task.extra, task.threshold, task.quiet), replace_existing=True ) elif task.action == 'remove': job = self.scheduler.get_job(str(task.id)) if job: job.remove()
def _handle_next_queue_item(scheduler: BackgroundScheduler, job_information: Dict[Text, Any]) -> None: job_id = job_information.pop(JOB_ID_KEY) existing_job: Optional[Job] = scheduler.get_job(job_id) should_cancel_job = job_information.pop(CANCEL_JOB_KEY, False) if should_cancel_job and existing_job: scheduler.remove_job(job_id) return if existing_job: _modify_job(existing_job, job_information) elif job_id == background_dump_service.BACKGROUND_DUMPING_JOB_ID: _add_job_to_dump_files(scheduler, job_information) else: logger.warning(f"Did not find a scheduled job with id '{job_id}'.")
def init(): printer.set_handler(print_handler) progressbar.set_handler(progressbar_handler) scheduler = BackgroundScheduler(daemon=True) load_min_err_rev_dic() job = scheduler.get_job("auto_check") if job is not None: scheduler.remove_job("auto_check") # 定时9点, 12点, 15点, 18点, 21点的时候开始检查。 job = scheduler.add_job(auto_check, 'cron', hour='6, 12, 15, 18, 21', id="auto_check") scheduler.start()
class Scheduler(object): def __init__(self): self._scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults) self._scheduler.add_jobstore('redis', jobs_key='crontpy.jobs', run_times_key='crontpy.run_times') @property def running(self): return self._scheduler.running def start(self): self._scheduler.start() def shutdown(self, wait=True): self._scheduler.shutdown(wait) def pause(self): self._scheduler.pause() def resume(self): self._scheduler.resume() def get_jobs(self): return self._scheduler.get_jobs() def get_job(self, jid): return self._scheduler.get_job(job_id=jid) def run_job(self, jid): job = self.get_job(jid) if not job: raise Exception('job id:{0} not found'.format(jid)) job.func(*job.args, **job.kwargs) def resume_job(self, jid): self._scheduler.resume_job(job_id=jid) def pause_job(self, jid): self._scheduler.pause_job(job_id=jid) def modify_job(self, jid, **changes): return self._scheduler.modify_job(job_id=jid, **changes) def delete_job(self, jid): self._scheduler.remove_job(job_id=jid)
class Scheduler: def __init__(self): self.scheduler = BackgroundScheduler() self.scheduler.start() self.jobs = {} # schedule util cannot track job metadata # trigger_time_str format (in UTC): HH:MM:SS def schedule_job(self, trigger_time_str, job_func, job_name, job_args=[], job_kwargs={}): if not re.match(TIME_REGEX, trigger_time_str): raise ValueError( f'invalid time format, expecting HH:MM:SS (in UTC), got: {trigger_time_str}' ) # IntervalTrigger will schedule event every 24 hrs beginning at strat date BUT NOT in the past trigger = IntervalTrigger(hours=24, start_date=f'2021-01-01 {trigger_time_str}', timezone=pytz.utc) job = self.scheduler.add_job(job_func, args=job_args, kwargs=job_kwargs, trigger=trigger, name=job_name) self.jobs[job.id] = { 'job_name': job_name, 'job_args': job_args, 'job_kwargs': job_kwargs } def unschedule_job(self, job_id): if job_id not in self.jobs: raise ValueError( f'Scheduler.unschedule_job: job_id not found: {job_id}') self.scheduler.remove_job(job_id) del self.jobs[job_id] def get_scheduled_jobs(self): for job_id, job_info in self.jobs.items(): job_info['next_run_time'] = str( self.scheduler.get_job(job_id).next_run_time) return self.jobs
def job_event_handler(sched: BackgroundScheduler, err_count: list[int | float], event: JobExecutionEvent): if (time.time() - err_count[1]) > 60: err_count[0] = 0 err_count[0] += 1 err_count[1] = time.time() job: apscheduler.job.Job = sched.get_job(event.job_id) now = datetime.datetime.now() now = now.strftime("%Y-%m-%d %H:%M:%S") try: if event.exception: pushstr = "{}\n{}\n第{}次出现异常\n异常任务: {}\nTraceback (most recent call last):\n{}\n{}\n".format( str(now), str(get_self_dir()[2]), str(err_count[0]), str(job.name), str(event.traceback), str(event.exception), ) else: pushstr = "{}\n{}\n第{}次出现异常\n异常任务: {}\n任务被跳过\n原定执行时间: \n{}\n".format( str(now), str(get_self_dir()[2]), str(err_count[0]), str(job.name), str(event.scheduled_run_time), ) except Exception: pushstr = "{}\n{}\n第{}次出现异常\n异常任务: 未知\n原定执行时间: \n{}\n".format( str(now), str(get_self_dir()[2]), str(err_count[0]), str(event.scheduled_run_time), ) if err_count[0] >= 3: sched.pause() pushstr += "短时间内出现3次异常, 定时任务已暂停" print(pushstr) if "pushkey" in push_option: print(dayepao_push(pushstr, **push_option))
class SwTimer(object): def __init__(self): self.scheduler = BackgroundScheduler() def getJob(self,jobId): return self.scheduler.get_job(jobId) def stop_job(self,jobId): self.scheduler.remove_job(jobId) def add_interval(self,jobId,jobFunc,sec=0,*args): #self.scheduler.add_job(jobFunc, 'interval',seconds=sec,id=jobId,args=args) self.scheduler.add_job(jobFunc, 'cron', second='*/%d'%sec,id=jobId,args=args) def run(self): self.scheduler.start()
def mandelbrot(): scheduler = BackgroundScheduler() scheduler.start() try: session['uid'] except: session['uid'] = uuid.uuid4() if 'r' in request.values and 'i' in request.values: real = float(request.values['r']) imaginary = float(request.values['i']) image = create(real, imaginary) filename = 'website/static/images/' + str( session['uid']) + 'fractal.png' # image.save('static/images/fractal.png') image.save(filename) time = datetime.datetime.now() + datetime.timedelta(seconds=10) if scheduler.get_job(filename): scheduler.reschedule_job(job_id=filename, trigger='date', run_date=time) print(f'job rescheduled for {time}') else: scheduler.add_job(delete_file, args=[filename], trigger='date', run_date=time, id=filename) print(f'job scheduled for {time}') return render_template("mandelbrot.html", image='/' + filename[8:] + '?' + str(rand.randint(1000)), real=real, imag=imaginary) else: return render_template("mandelbrot.html", image='/static/images/defaultFractal.png', real=0, imag=0)
def setRoomEndDate(sender, instance, **kwargs): ''' Set the timeout of the room for the extraction! ''' # ToDo: reload all event on startup! # Add the job to the scheduler (only if not present) scheduler = BackgroundScheduler() if instance.job_id in [i.id for i in scheduler.get_jobs()] and instance.end_date > timezone.now(): # The event is already present job = scheduler.get_job(instance.job_id) if job.trigger.run_date != make_aware(instance.end_date): # The event is different job.trigger.run_date = instance.end_date print(f"Event (is changed) for room {instance} will start at {instance.end_date}") elif instance.end_date > timezone.now(): # Creation of a new event print(f"Event for room {instance} will start at {instance.end_date}") job = scheduler.add_job(xmasg.xmasg_extraction, 'date', run_date=instance.end_date, args=[instance.id]) # https://apscheduler.readthedocs.io/en/stable/modules/triggers/date.html#module-apscheduler.triggers.date scheduler.start() # Update on DB Room.objects.filter(pk=instance.id).update(job_id=job.id) # update should not launch this function again
class Runner: state = RunnerStateEnum.STOPPED def __init__(self): self.state = RunnerStateEnum.RUNNING self.scheduler = BackgroundScheduler(logger=logger) self.jobs = {} def add_job(self, id: str, func: Callable, **kwargs): logger.debug(f"Adding job to queue: {id}") self.jobs[id] = self.scheduler.add_job(func, **kwargs) def remove_job(self, id: str): logger.debug(f"Removing job from queue: {id}") if not self.scheduler.get_job(id): logger.warning(f"Unable to remove job {id}") return self.scheduler.remove_job(id) def start(self): self.scheduler.start() def stop(self): self.scheduler.shutdown()
class Scheduler: timezone = settings.TIME_ZONE def __init__(self): self.scheduler = BackgroundScheduler(timezone=self.timezone) self.scheduler.add_listener(self._handle_event, ) @classmethod def parse_trigger(cls, trigger, trigger_args): if trigger == 'interval': return IntervalTrigger(seconds=int(trigger_args), timezone=cls.timezone) elif trigger == 'date': return DateTrigger(run_date=trigger_args, timezone=cls.timezone) else: raise TypeError(f'unknown schedule policy: {trigger!r}') def _handle_event(self, event): obj = SimpleLazyObject( lambda: Task.objects.filter(pk=event.job_id).first()) if event.code == events.EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('schedule', '1', '调度器已关闭', '调度器意外关闭,你可以在github上提交issue') elif event.code == events.EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('schedule', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时') elif event.code == events.EVENT_JOB_ERROR: logger.info( f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}' ) Notify.make_notify('schedule', '1', f'{obj.name} - 执行异常', f'{event.exception}') elif event.code == events.EVENT_JOB_EXECUTED: if event.retval: score = 0 for item in event.retval: score += 1 if item[1] else 0 Task.objects.filter(pk=event.job_id).update( latest_status=2 if score == len(event.retval) else 1 if score else 0, latest_run_time=human_datetime(event.scheduled_run_time), latest_output=json.dumps(event.retval)) if score != 0 and time.time() - counter.get(event.job_id, 0) > 3600: counter[event.job_id] = time.time() Notify.make_notify('schedule', '1', f'{obj.name} - 执行失败', '请在任务计划中查看失败详情') def _init_builtin_jobs(self): self.scheduler.add_job(auto_clean_records, 'cron', hour=0, minute=0) def _init(self): self.scheduler.start() self._init_builtin_jobs() for task in Task.objects.filter(is_active=True): trigger = self.parse_trigger(task.trigger, task.trigger_args) self.scheduler.add_job( dispatch, trigger, id=str(task.id), args=(task.command, json.loads(task.targets)), ) def run(self): rds_cli = get_redis_connection() self._init() rds_cli.delete(settings.SCHEDULE_KEY) logger.info('Running scheduler') while True: _, data = rds_cli.blpop(settings.SCHEDULE_KEY) task = AttrDict(json.loads(data)) if task.action in ('add', 'modify'): trigger = self.parse_trigger(task.trigger, task.trigger_args) self.scheduler.add_job(dispatch, trigger, id=str(task.id), args=(task.command, task.targets), replace_existing=True) elif task.action == 'remove': job = self.scheduler.get_job(str(task.id)) if job: job.remove()
class FreezerScheduler(object): def __init__(self, apiclient, interval, job_path): # config_manager self.client = apiclient self.freezerc_executable = spawn.find_executable('freezer-agent') if self.freezerc_executable is None: # Needed in the case of a non-activated virtualenv self.freezerc_executable = spawn.find_executable('freezer-agent', path=':'.join( sys.path)) LOG.debug('Freezer-agent found at {0}'.format( self.freezerc_executable)) self.job_path = job_path self._client = None self.lock = threading.Lock() self.execution_lock = threading.Lock() job_defaults = {'coalesce': True, 'max_instances': 2} executors = { 'default': { 'type': 'threadpool', 'max_workers': 1 }, 'threadpool': { 'type': 'threadpool', 'max_workers': 10 } } self.scheduler = BackgroundScheduler(job_defaults=job_defaults, executors=executors) if self.client: self.scheduler.add_job(self.poll, 'interval', seconds=interval, id='api_poll', executor='default') self.add_job = self.scheduler.add_job self.remove_job = self.scheduler.remove_job self.jobs = {} def get_jobs(self): if self.client: job_doc_list = utils.get_active_jobs_from_api(self.client) try: utils.save_jobs_to_disk(job_doc_list, self.job_path) except Exception as e: LOG.error('Unable to save jobs to {0}. ' '{1}'.format(self.job_path, e)) return job_doc_list else: return utils.get_jobs_from_disk(self.job_path) def start_session(self, session_id, job_id, session_tag): if self.client: return self.client.sessions.start_session(session_id, job_id, session_tag) else: raise Exception("Unable to start session: api not in use.") def end_session(self, session_id, job_id, session_tag, result): if self.client: return self.client.sessions.end_session(session_id, job_id, session_tag, result) else: raise Exception("Unable to end session: api not in use.") def upload_metadata(self, metadata_doc): if self.client: self.client.backups.create(metadata_doc) def start(self): utils.do_register(self.client) self.poll() self.scheduler.start() try: while True: # Due to the new Background scheduler nature, we need to keep # the main thread alive. time.sleep(1) except (KeyboardInterrupt, SystemExit): # Not strictly necessary if daemonic mode is enabled but # should be done if possible self.scheduler.shutdown(wait=False) def update_job(self, job_id, job_doc): if self.client: try: return self.client.jobs.update(job_id, job_doc) except Exception as e: LOG.error("Job update error: {0}".format(e)) def update_job_schedule(self, job_id, job_schedule): """ Pushes to the API the updates the job_schedule information of the job_doc :param job_id: id of the job to modify :param job_schedule: dict containing the job_scheduler information :return: None """ doc = {'job_schedule': job_schedule} self.update_job(job_id, doc) def update_job_status(self, job_id, status): doc = {'job_schedule': {'status': status}} self.update_job(job_id, doc) def is_scheduled(self, job_id): return self.scheduler.get_job(job_id) is not None def create_job(self, job_doc): job = scheduler_job.Job.create(self, self.freezerc_executable, job_doc) if job: self.jobs[job.id] = job LOG.info("Created job {0}".format(job.id)) return job def poll(self): try: work_job_doc_list = self.get_jobs() except Exception as e: LOG.error("Unable to get jobs: {0}".format(e)) return work_job_id_list = [] # create job if necessary, then let it process its events for job_doc in work_job_doc_list: job_id = job_doc['job_id'] work_job_id_list.append(job_id) job = self.jobs.get(job_id, None) or self.create_job(job_doc) if job: # check for abort status if job_doc['job_schedule']['event'] == 'abort': pid = int(job_doc['job_schedule']['current_pid']) utils.terminate_subprocess(pid, 'freezer-agent') job.process_event(job_doc) # request removal of any job that has been removed in the api for job_id, job in six.iteritems(self.jobs): if job_id not in work_job_id_list: job.remove() remove_list = [ job_id for job_id, job in self.jobs.items() if job.can_be_removed() ] for k in remove_list: self.jobs.pop(k) def stop(self): sys.exit() def reload(self): LOG.warning("reload not supported")
class BileanScheduler(object): """Billing scheduler based on apscheduler""" job_types = ( NOTIFY, DAILY, FREEZE, ) = ( 'notify', 'daily', 'freeze', ) trigger_types = (DATE, CRON) = ('date', 'cron') def __init__(self, **kwargs): super(BileanScheduler, self).__init__() self._scheduler = BackgroundScheduler() self.notifier = notifier.Notifier() self.engine_id = kwargs.get('engine_id', None) self.context = kwargs.get('context', None) if not self.context: self.context = bilean_context.get_admin_context() if cfg.CONF.bilean_task.store_ap_job: self._scheduler.add_jobstore(cfg.CONF.bilean_task.backend, url=cfg.CONF.bilean_task.connection) def init_scheduler(self): """Init all jobs related to the engine from db.""" jobs = db_api.job_get_all(self.context, engine_id=self.engine_id) if not jobs: LOG.info(_LI("No job found from db")) return True for job in jobs: if self.bilean_scheduler.is_exist(job.id): continue task_name = "_%s_task" % (job.job_type) task = getattr(self, task_name) self.bilean_task.add_job(task, job.id, job_type=job.job_type, params=job.parameters) def add_job(self, task, job_id, trigger_type='date', **kwargs): """Add a job to scheduler by given data. :param str|unicode user_id: used as job_id :param datetime alarm_time: when to first run the job """ mg_time = cfg.CONF.bilean_task.misfire_grace_time job_time_zone = cfg.CONF.bilean_task.time_zone user_id = job_id.split('-')[1] if trigger_type == 'date': run_date = kwargs.get('run_date') if run_date is None: msg = "Param run_date cannot be None for trigger type 'date'." raise exception.InvalidInput(reason=msg) self._scheduler.add_job(task, 'date', timezone=job_time_zone, run_date=run_date, args=[user_id], id=job_id, misfire_grace_time=mg_time) return True # Add a cron type job hour = kwargs.get('hour', None) minute = kwargs.get('minute', None) if not hour or not minute: hour, minute = self._generate_timer() self._scheduler.add_job(task, 'cron', timezone=job_time_zone, hour=hour, minute=minute, args=[user_id], id=job_id, misfire_grace_time=mg_time) return True def modify_job(self, job_id, **changes): """Modifies the properties of a single job. Modifications are passed to this method as extra keyword arguments. :param str|unicode job_id: the identifier of the job """ self._scheduler.modify_job(job_id, **changes) def remove_job(self, job_id): """Removes a job, preventing it from being run any more. :param str|unicode job_id: the identifier of the job """ self._scheduler.remove_job(job_id) def start(self): LOG.info(_('Starting Billing scheduler')) self._scheduler.start() def stop(self): LOG.info(_('Stopping Billing scheduler')) self._scheduler.shutdown() def is_exist(self, job_id): """Returns if the Job exists that matches the given ``job_id``. :param str|unicode job_id: the identifier of the job :return: True|False """ job = self._scheduler.get_job(job_id) return job is not None def _notify_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) msg = {'user': user.id, 'notification': 'The balance is almost use up'} self.notifier.info('billing.notify', msg) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'notify')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) self._add_freeze_job(user) def _daily_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'daily')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def _freeze_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'freeze')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def _add_notify_job(self, user): if not user.rate: return False total_seconds = user['balance'] / user['rate'] prior_notify_time = cfg.CONF.bilean_task.prior_notify_time * 3600 notify_seconds = total_seconds - prior_notify_time notify_seconds = notify_seconds if notify_seconds > 0 else 0 run_date = timeutils.utcnow() + timedelta(seconds=notify_seconds) job_params = {'run_date': run_date} job_id = self._generate_job_id(user['id'], self.NOTIFY) self.add_job(self._notify_task, job_id, params=job_params) # Save job to database job = {'id': job_id, 'job_type': self.NOTIFY, 'engine_id': self.engine_id, 'parameters': {'run_date': run_date}} db_api.job_create(self.context, job) def _add_freeze_job(self, user): if not user.rate: return False total_seconds = user.balance / user.rate run_date = timeutils.utcnow() + timedelta(seconds=total_seconds) job_params = {'run_date': run_date} job_id = self._generate_job_id(user.id, self.FREEZE) self.add_job(self._freeze_task, job_id, params=job_params) # Save job to database job = {'id': job_id, 'job_type': self.FREEZE, 'engine_id': self.engine_id, 'parameters': {'run_date': run_date}} db_api.job_create(self.context, job) return True def _add_daily_job(self, user): job_id = self._generate_job_id(user.id, self.DAILY) params = {'hour': random.randint(0, 23), 'minute': random.randint(0, 59)} self.add_job(self._daily_task, job_id, trigger_type='cron', params=params) # Save job to database job = {'id': job_id, 'job_type': self.DAILY, 'engine_id': self.engine_id, 'parameters': params} db_api.job_create(self.context, job) return True def _delete_all_job(self, user): for job_type in self.job_types: job_id = self._generate_job_id(user.id, job_type) if self.is_exist(job_id): self.remove_job(job_id) try: db_api.job_delete(self.context, job_id) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def update_user_job(self, user): """Update user's billing job""" if user.status not in [user.ACTIVE, user.WARNING]: self._delete_all_job(user.id) return for job_type in self.NOTIFY, self.FREEZE: job_id = self._generate_job_id(user.id, job_type) if self.is_exist(job_id): self.remove_job(job_id) try: db_api.job_delete(self.context, job_id) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) daily_job_id = self._generate_job_id(user.id, self.DAILY) if not self.is_exist(daily_job_id): self._add_daily_job(user) if user.status == user.ACTIVE: self._add_notify_job(user) else: self._add_freeze_job(user) def _generate_timer(self): """Generate a random timer include hour and minute.""" hour = random.randint(0, 23) minute = random.randint(0, 59) return hour, minute def _generate_job_id(self, user_id, job_type): """Generate job id by given user_id and job type""" return "%s-%s" % (job_type, user_id)
class ScheduleCommand(Command): schedule_sender = None def __init__(self, logger, message_sender): super().__init__(logger, message_sender) # Message scheduler configuration self.scheduler = BackgroundScheduler() configuration = self.message_sender.get_configuration() sdb_url = configuration.get('Message Scheduler', 'db_path') self.scheduler.add_jobstore('sqlalchemy', url=sdb_url) self.scheduler.start() ScheduleCommand.schedule_sender = message_sender def process(self, chat_id, user_id, username, arguments): if len(arguments) < 1: return self.help() operation = arguments[0] command_arguments = arguments[1:] if operation == "add": return self.add_message(username, chat_id, command_arguments) elif operation == "time": return self.get_local_time() elif operation == "remove": return self.remove_message(username, command_arguments) else: return self.help() def get_local_time(self): date_str = str(datetime.today().strftime('%Y-%m-%d %H:%M:%S')) return "My time is {}".format(date_str) def add_message(self, username, chat_id, arguments): if len(arguments) <2: return self.help() date = None message = None if arguments[0] == 'relative': if len(arguments)<3: return self.help() date = self.on_delta_parse(arguments[1]) message = arguments[2] else: date = self.on_date_parse(arguments[0]) message = arguments[1] if message.startswith('"') and message.endswith('"'): message = message[1:-1] reference = self.user_reference(username) if date is None: return "{}Date format not recognized".format(reference) current_date = datetime.today() if date < current_date: current_str = self.get_human_string_date(current_date) return "{}Sorry, I can't travel to the past, my current date is: {}".format(reference, current_str) message_id = str(random.randrange(0, 999999999)) self.scheduler.add_job(ScheduleCommand.send_programmed_message, \ 'date', run_date=date, args=[username, \ chat_id, message], id = message_id) date_str = self.get_human_string_date(date) return "{}The message [{}] has been successfully scheduled on {}"\ .format(reference, message_id, date_str) def send_programmed_message(username, chat_id, message): reference = ScheduleCommand.user_reference(username) text = "{}{}".format(reference, message) ScheduleCommand.schedule_sender.send_message(chat_id, text) send_programmed_message = staticmethod(send_programmed_message) def remove_message(self, username, arguments): if len(arguments) < 1: return self.help() founds = 0 ids = self.get_comma_arguments(arguments[0]) for message_id in ids: job = self.scheduler.get_job(message_id) if job == None: continue if job.args[0] != username: continue self.scheduler.remove_job(message_id) founds += 1 reference = self.user_reference(username) if founds == len(ids): if len(ids) ==1: return "{}The scheduled message was canceled.".format(reference) else: return "{}The scheduled messages were canceled.".format(reference) elif founds > 0: return "{}Some scheduled messages were canceled, but some others were not found or you aren't the owner.".format(reference) else: if len(ids) == 1: return "{}The scheduled message was not found or you aren't the owner.".format(reference) else: return "{}The scheduled message were not found or you aren't the owner.".format(reference) def get_human_string_date(self,datetime): return str(datetime.strftime('%Y-%m-%d %H:%M:%S')) def on_delta_parse(self, text_date): regex = re.compile(r'^((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?$') parts = regex.match(text_date) if not parts: return None parts = parts.groupdict() time_params = {} valid_params = 0 for (name, param) in parts.items(): if param: time_params[name] = int(param) valid_params = valid_params + 1 if valid_params == 0: return None return datetime.now() + timedelta(**time_params) def on_date_parse(self, text_date): try: return parser.parse(text_date, dayfirst = True, yearfirst = True) except ValueError: return None def user_reference(username): if username is None: return "" else: return "@{}: ".format(username) user_reference = staticmethod(user_reference) def help(self): self.logger.info("Printing help for schedule message command.") return self.get_file_help(__file__, "schedule_message.man") def name(self): return "schedule_message" def description(self): return "Schedule a message to be sent in a specified date."
class Trigger: def __init__(self, app): self.scheduler = None self.app = app def setup(self): self.scheduler = BackgroundScheduler({ 'apscheduler.jobstores.default': { 'type': 'sqlalchemy', 'url': self.app.config["TRIGGER_DATABASE_URL"] #os.environ.get('TRIGGER_DATABASE_URL') }, 'apscheduler.executors.processpool': { 'type': 'processpool', 'max_workers': '30' }, 'apscheduler.job_defaults.coalesce': 'false', 'apscheduler.job_defaults.max_instances': '20', 'apscheduler.timezone': 'UTC', }) def start(self): self.scheduler.start() def is_running(self): return self.scheduler.running() def shutdown(self): self.scheduler.shutdown() def load_job_list(self): with self.app.app_context(): projects = AutoProject.query.all() # key_list = ("minute", "hour", "day", "month", "day_of_week") for p in projects: if p.enable and self.scheduler.get_job(p.id) is None: cron = p.cron.replace("\n", "").strip().split(" ") #print(cron) if len(cron) < 5: continue j = self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, replace_existing=True, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % p.id, args=(p.id,)) else: self.update_job(p.id) def add_job(self, func, name, id, cron): if self.scheduler.get_job(id) is None: self.scheduler.add_job(func=func, trigger='cron', name=name, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % id) def update_job(self, id): with self.app.app_context(): p = AutoProject.query.filter_by(id=id).first() if p.enable: cron = p.cron.replace("\n", "").strip().split(" ") if len(cron) < 5: return False print(self.scheduler.get_job(id)) if self.scheduler.get_job(id) is None: self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % id, args=(id,)) else: self.remove_job(id) self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % id, args=(id,)) return True def remove_job(self, id): if self.scheduler.get_job(id) is not None: self.scheduler.remove_job(id) def pause_job(self, id): pass def resume_job(self, id): pass def get_jobs(self): to_zone = tz.gettz("CST") #jobs = self.scheduler.get_jobs() urls = { "pass": "******", "fail": "fail.png", "running": "run.gif", "none": "project.png" } projects = AutoProject.query.order_by(AutoProject.id.desc()).all() data = {"total": len(projects), "rows": []} for p in projects: next_run_time = "调度未启动" status = "pass" job = self.scheduler.get_job(p.id) if job is not None: next_run_time = job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S") # 获取该job下最后一次运行状态 task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first() if task is not None: output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no) if os.path.exists(output_dir + "/report.html"): tree = ET.parse(output_dir + "/output.xml") root = tree.getroot() # passed = root.find("./statistics/suite/stat").attrib["pass"] fail = root.find("./statistics/suite/stat").attrib["fail"] if int(fail) != 0: status = 'fail' else: status = 'pass' else: status = "none" data["rows"].append({"id": "%s" % p.id, "name": p.name, "enable": p.enable, "status": status, "url": url_for('static', filename='images/%s' % urls[status]), "cron": p.cron, "next_run_time": next_run_time }) """ for job in jobs: status = "running" task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first() if task is None: continue output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no) if os.path.exists(output_dir + "/report.html"): tree = ET.parse(output_dir + "/output.xml") root = tree.getroot() #passed = root.find("./statistics/suite/stat").attrib["pass"] fail = root.find("./statistics/suite/stat").attrib["fail"] if int(fail) != 0: status = 'fail' else: status = 'pass' data["rows"].append({"id": "%s" % job.id, "name": job.name, "status": status, "url": url_for('static', filename='images/%s' % urls[status]), "cron": AutoProject.query.filter_by(id=job.id).first().cron, "next_run_time": job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S") }) """ return data def print_jobs(self): pass
class Scheduler: timezone = settings.TIME_ZONE def __init__(self): self.scheduler = BackgroundScheduler(timezone=self.timezone) self.scheduler.add_listener( self._handle_event, EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES | EVENT_JOB_EXECUTED) def _record_alarm(self, obj, status): duration = seconds_to_human(time.time() - obj.latest_fault_time) Alarm.objects.create( name=obj.name, type=obj.get_type_display(), status=status, duration=duration, notify_grp=obj.notify_grp, notify_mode=obj.notify_mode) def _do_notify(self, event, obj): grp = json.loads(obj.notify_grp) for mode in json.loads(obj.notify_mode): if mode == '1': spug.notify_by_wx(event, obj.name, grp) elif mode == '3': spug.notify_by_dd(event, obj.name, grp) elif mode == '4': spug.notify_by_email(event, obj.name, grp) def _handle_notify(self, obj, old_status): if obj.latest_status == 0: if old_status == 1: self._record_alarm(obj, '2') logger.info(f'{human_datetime()} recover job_id: {obj.id}') self._do_notify('2', obj) else: if obj.fault_times >= obj.threshold: if time.time() - obj.latest_notify_time >= obj.quiet * 60: obj.latest_notify_time = int(time.time()) obj.save() self._record_alarm(obj, '1') logger.info(f'{human_datetime()} notify job_id: {obj.id}') self._do_notify('1', obj) def _handle_event(self, event): close_old_connections() obj = SimpleLazyObject(lambda: Detection.objects.filter(pk=event.job_id).first()) if event.code == EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('monitor', '1', '调度器已关闭', '调度器意外关闭,你可以在github上提交issue', False) elif event.code == EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('monitor', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时') elif event.code == EVENT_JOB_ERROR: logger.info(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}') Notify.make_notify('monitor', '1', f'{obj.name} - 执行异常', f'{event.exception}') elif event.code == EVENT_JOB_EXECUTED: obj = Detection.objects.filter(pk=event.job_id).first() old_status = obj.latest_status obj.latest_status = 0 if event.retval else 1 obj.latest_run_time = human_datetime(event.scheduled_run_time) if old_status in [0, None] and event.retval is False: obj.latest_fault_time = int(time.time()) if obj.latest_status == 0: obj.latest_notify_time = 0 obj.fault_times = 0 else: obj.fault_times += 1 obj.save() self._handle_notify(obj, old_status) def _init(self): self.scheduler.start() for item in Detection.objects.filter(is_active=True): trigger = IntervalTrigger(minutes=int(item.rate), timezone=self.timezone) self.scheduler.add_job( dispatch, trigger, id=str(item.id), args=(item.type, item.addr, item.extra), ) def run(self): rds_cli = get_redis_connection() self._init() rds_cli.delete(settings.MONITOR_KEY) logger.info('Running monitor') while True: _, data = rds_cli.blpop(settings.MONITOR_KEY) task = AttrDict(json.loads(data)) if task.action in ('add', 'modify'): trigger = IntervalTrigger(minutes=int(task.rate), timezone=self.timezone) self.scheduler.add_job( dispatch, trigger, id=str(task.id), args=(task.type, task.addr, task.extra), replace_existing=True ) elif task.action == 'remove': job = self.scheduler.get_job(str(task.id)) if job: job.remove()
class VolatileDictionary(dict): def __init__(self, *args, **kwargs): super().__init__(self, *args, **kwargs) self._scheduler = BackgroundScheduler() self._scheduler.start() self._evaporation_jobs = {} def is_set_volatile(self, key): return key in self._evaporation_jobs def __setitem__(self, key, value, t=None): if is_key_time_tuple(key): key, t = key super().__setitem__(key, value) if t is not None: self._schedule_evaporation(key, t) def __delitem__(self, key): if key in self._evaporation_jobs: self.cancel_volatility(key) super().__delitem__(key) def _schedule_evaporation(self, key, t): date = datetime.now() + timedelta(seconds=t) job = self._scheduler.add_job(self._evaporate, 'date', run_date=date, args=(key, )) self._evaporation_jobs[key] = job.id def _evaporate(self, key): super().__delitem__(key) del self._evaporation_jobs[key] def cancel_volatility(self, key): if not self.is_set_volatile(key): raise NonvolatileTypeError(key) self._scheduler.remove_job(self._evaporation_jobs[key]) del self._evaporation_jobs[key] def get_set_lifetime(self, key): if not self.is_set_volatile(key): raise NonvolatileTypeError(key) job_id = self._evaporation_jobs[key] job = self._scheduler.get_job(job_id) job_date = job.trigger.run_date return (job_date - datetime.now(job_date.tzinfo)).total_seconds() def volatile_keys(self): return [key for key in self if self.is_set_volatile(key)] def nonvolatile_keys(self): return [key for key in self if not self.is_set_volatile(key)] def volatile_values(self): return [self[key] for key in self.volatile_keys()] def nonvolatile_values(self): return [self[key] for key in self.nonvolatile_keys()] def volatile_items(self): return [(key, self[key]) for key in self.volatile_keys()] def nonvolatile_items(self): return [(key, self[key]) for key in self.nonvolatile_keys()] def __str__(self): volatile_sets = self.volatile_items() nonvolatile_sets = self.nonvolatile_items() string = '' if len(volatile_sets) > 0: string += 'Volatile sets:\n' for key, value in volatile_sets: t = self.get_set_lifetime(key) string += '\t{}: {} [{}s]\n'.format(key, value, t) string += '\n' if len(nonvolatile_sets) > 0: string += 'Nonvolatile sets:\n' for key, value in nonvolatile_sets: string += '\t{}: {}\n'.format(key, value) return string
class JobManager(object): def __init__(self): self.scheduler = BackgroundScheduler(executors=EXECUTORS, job_defaults=JOB_DEFAULTS, timezone='Asia/Shanghai') self.jobs = {} self.scheduler.start() def add_job_store(self): pass def add_job(self, method, jobtype, trigger, jobid, args=None, kwargs=None): job = None if jobtype == 'interval': job = self.scheduler.add_job(method, 'interval', seconds=trigger, id=jobid, args=args, kwargs=kwargs) if jobtype == 'cron': Trigger = CronTrigger(**trigger) job = self.scheduler.add_job(method, Trigger, id=jobid, args=args, kwargs=kwargs) if jobtype == 'once': job = self.scheduler.add_job(method, 'date', run_date=trigger, id=jobid, args=args, kwargs=kwargs) LOG.debug("add job: %s", jobid) if job is not None: self.jobs[jobid] = job def delete_job(self, jobid): if jobid in self.jobs: self.scheduler.remove_job(jobid) del self.jobs[jobid] def disable_job(self, jobid): if jobid in self.jobs: self.scheduler.pause_job(jobid) def enable_job(self, jobid): if jobid in self.jobs: self.scheduler.resume_job(jobid) def get_job_next_run_time(self, jobid): job = self.scheduler.get_job(jobid) if not job: return None next_run_time = job.next_run_time return next_run_time.strftime("%Y-%m-%d %H:%M:%S")
class Trigger: def __init__(self, app): self.scheduler = None self.app = app def setup(self): self.scheduler = BackgroundScheduler({ 'apscheduler.jobstores.default': { 'type': 'sqlalchemy', 'url': self.app.config["TRIGGER_DATABASE_URL"] #os.environ.get('TRIGGER_DATABASE_URL') }, 'apscheduler.executors.processpool': { 'type': 'processpool', 'max_workers': '30' }, 'apscheduler.job_defaults.coalesce': 'false', 'apscheduler.job_defaults.max_instances': '20', 'apscheduler.timezone': 'UTC', }) def start(self): self.scheduler.start() def is_running(self): return self.scheduler.running() def shutdown(self): self.scheduler.shutdown() def load_job_list(self): with self.app.app_context(): projects = AutoProject.query.all() # key_list = ("minute", "hour", "day", "month", "day_of_week") for p in projects: if p.enable and self.scheduler.get_job(p.id) is None: cron = p.cron.replace("\n", "").strip().split(" ") #print(cron) if len(cron) < 5: continue j = self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, replace_existing=True, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % p.id, args=(p.id,)) else: self.update_job(p.id) def update_job(self, id): with self.app.app_context(): p = AutoProject.query.filter_by(id=id).first() if p.enable: cron = p.cron.replace("\n", "").strip().split(" ") if len(cron) < 5: return False print(self.scheduler.get_job(id)) if self.scheduler.get_job(id) is None: self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % id, args=(id,)) else: self.remove_job(id) self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % id, args=(id,)) return True def remove_job(self, id): if self.scheduler.get_job(id) is not None: self.scheduler.remove_job(id) def pause_job(self, id): pass def resume_job(self, id): pass def get_jobs(self): to_zone = tz.gettz("CST") #jobs = self.scheduler.get_jobs() urls = { "pass": "******", "fail": "fail.png", "running": "run.gif", "none": "project.png" } projects = AutoProject.query.order_by(AutoProject.id.desc()).all() data = {"total": len(projects), "rows": []} for p in projects: next_run_time = "调度未启动" job = self.scheduler.get_job(p.id) if job is not None: next_run_time = job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S") # 获取该job下最后一次运行状态 task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first() if task is not None: output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no) if os.path.exists(output_dir + "/report.html"): tree = ET.parse(output_dir + "/output.xml") root = tree.getroot() # passed = root.find("./statistics/suite/stat").attrib["pass"] fail = root.find("./statistics/suite/stat").attrib["fail"] if int(fail) != 0: status = 'fail' else: status = 'pass' else: status = "none" data["rows"].append({"id": "%s" % p.id, "name": p.name, "enable": p.enable, "status": status, "url": url_for('static', filename='images/%s' % urls[status]), "cron": p.cron, "next_run_time": next_run_time }) """ for job in jobs: status = "running" task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first() if task is None: continue output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no) if os.path.exists(output_dir + "/report.html"): tree = ET.parse(output_dir + "/output.xml") root = tree.getroot() #passed = root.find("./statistics/suite/stat").attrib["pass"] fail = root.find("./statistics/suite/stat").attrib["fail"] if int(fail) != 0: status = 'fail' else: status = 'pass' data["rows"].append({"id": "%s" % job.id, "name": job.name, "status": status, "url": url_for('static', filename='images/%s' % urls[status]), "cron": AutoProject.query.filter_by(id=job.id).first().cron, "next_run_time": job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S") }) """ return data def print_jobs(self): pass
class OnionooManager(): def __init__(self, proxy: Proxy = None): self.log = logging.getLogger('theonionbox') self.proxy = proxy self.documents = {} executors = {'default': ThreadPoolExecutor(50)} job_defaults = {'coalesce': True, 'max_instances': 10} self.scheduler = BackgroundScheduler(logger=self.log, executors=executors, job_defaults=job_defaults) self.scheduler.start() def shutdown(self): self.scheduler.shutdown() def query(self, update: Document, document: OnionooDocument, fingerprint: str): # https://trac.torproject.org/projects/tor/ticket/6320 hash = hashlib.sha1(a2b_hex(fingerprint)).hexdigest() payload = {'lookup': hash} headers = {'accept-encoding': 'gzip'} if len(update.ifModSince) > 0: headers['if-modified-since'] = update.ifModSince proxy_address = self.proxy.address( ) if self.proxy is not None else None if proxy_address is None: proxies = {} query_base = ONIONOO_OPEN else: proxies = { 'http': 'socks5h://' + proxy_address, 'https': 'socks5h://' + proxy_address } query_base = ONIONOO_HIDDEN[randint(0, len(ONIONOO_HIDDEN) - 1)] query_address = '{}/{}'.format(query_base, document.value) r = None self.log.debug( f"OoM|{fingerprint[:6]}: Launching query @ '{query_address}'.") try: r = requests.get(query_address, params=payload, headers=headers, proxies=proxies, timeout=10) except requests.exceptions.ConnectTimeout: self.log.info( f"OoM|{fingerprint[:6]}/{document.value}: Connection timeout @ '{query_base}'." ) # We'll try again next time... except Exception as exc: self.log.warning( f"OoM|{fingerprint[:6]}: Failed querying '{query_address}' -> {exc}." ) else: self.log.debug( f"OoM|{fingerprint[:6]}/{document.value}: {r.status_code}; {len(r.text)} chars received." ) if r is None: return update.ifModSince = r.headers['last-modified'] if r.status_code != requests.codes.ok: return try: data = r.json() except Exception as exc: self.log.debug( f"OoM|{fingerprint[:6]}/{document.value}: Failed to un-json network data -> {exc}." ) return update.update(data) def register(self, fingerprint: str) -> OnionooData: self.log.debug(f'Registering OoM|{fingerprint[:6]}.') documents = {} for d in OnionooDocument: hash = hashlib.sha256(f'{fingerprint.lower()}|{d.value}'.encode( 'UTF-8')).hexdigest() if hash not in self.documents: self.documents[hash] = Document() self.scheduler.add_job( self.query, trigger='interval', hours=2, jitter=3600, next_run_time=datetime.now( ), # to trigger once immediately! kwargs={ 'update': self.documents[hash], 'document': d, 'fingerprint': fingerprint }, id=hash) documents[d] = self.documents[hash] return OnionooData(documents) def trigger(self, fingerprint: str): for d in OnionooDocument: hash = hashlib.sha256(f'{fingerprint.lower()}|{d.value}'.encode( 'UTF-8')).hexdigest() with contextlib.suppress(Exception): self.scheduler.get_job(job_id=hash).modify( next_run_time=datetime.now())
class Scheduler(object): def __init__(self, tasks, settings): self.settings = settings self.logger = get_logger('scheduler', self.settings) self.intervals = self.settings['scheduler']['intervals'] self.apscheduler = { 'apscheduler': self.settings['scheduler']['apscheduler'] } if not isinstance(tasks, dict): self.logger.error('tasks is not a dictionary') return if not isinstance(self.intervals, dict): self.logger.error('intervals is not a dictionary') return self.tasks = self._flatten_dict(tasks, '') self.logger.debug('Tasks found:') self.logger.debug(LINE_SPLITTER) for key in self.tasks: self.logger.debug('%45s %30s' % (key, self.tasks[key].task_type)) self.logger.debug(LINE_SPLITTER) #self.logger.debug('Checking tasks paths!') # TODO: Check if paths are valid def init(self): """ Initializes the queue, and adds the tasks """ self.logger.info('Initilizing APScheduler...') apsched_kwargs = self._flatten_dict(self.apscheduler, '') apsched_kwargs['apscheduler.logger'] = get_logger('apscheduler', self.settings) self.sched = BackgroundScheduler(apsched_kwargs) for (id, task) in self.tasks.items(): task_type = task.task_type self.logger.debug('Adding task "%s" [%s]' % (id, task_type)) if not task_type in self.intervals: self.logger.info('Interval not defined for "%s" class. Assuming it is an once-time task' % task_type) self.add_task(id, task) continue self.add_task(id, task, self.intervals[task_type]) self.logger.info('APScheduler initialized!') def clear(self): """ Removes all jobs from scheduler """ if not isinstance(self.sched, BaseScheduler): self.logger.error('Scheduler is not initialized') return for job in self.sched.get_jobs(): job.remove() def start(self): """ Start the scheduler by starting the instance of APScheduler """ if not isinstance(self.sched, BaseScheduler): self.logger.error('Scheduler is not initialized') return try: self.sched.start() except SchedulerAlreadyRunningError as e: self.logger.warning(e) def stop(self, wait=True): """ Stop the scheduler. If wait=True, then it will be stopped after all jobs that are currently executed will finish """ if not isinstance(self.sched, BaseScheduler): self.logger.warning('Scheduler is not initialized') return try: self.sched.shutdown(wait=wait) except SchedulerNotRunningError as e: self.logger.warning(e) def add_task(self, id, func, interval=None): """ Adds a new task into the queue. If interval is None then the task will be executed once. """ if not isinstance(id, basestring): self.logger.error('"id" argument is not an instance of basestring') return if not hasattr(func, '__call__'): self.logger.error('"func" is not callable') return try: if isinstance(interval, dict): self.sched.add_job(func, trigger='interval', id=id, **interval) elif interval is None: # Run once (ommit trigger) self.sched.add_job(func, id=id) else: self.logger.error('"interval" is not an instance of [time|None]') return except ConflictingIdError as e: self.logger.warning(e) def remove_task(self, id): """ Remove a job from the queue """ if not isinstance(id, basestring): self.logger.error('"id" argument is not an instance of basestring') return try: self.sched.remove_job(id) except JobLookupError as e: self.logger.warning(e) def force_update(self, job_id=None): """ Updates a job with id == job_id, or all jobs if no id is given """ if not isinstance(self.sched, BaseScheduler): self.logger.warning('Scheduler is not initialized') return if not job_id: self.logger.info("Forcing update of all jobs") for job in self.sched.get_jobs(): self.__run_job(job) else: self.logger.info("Forcing update of job %s" % job_id) job = self.sched.get_job(job_id) if not job: self.logger.warn("Job %s not found" % job_id) else: self.__run_job(job) def __run_job(self, job): if job.func: # Add the job to the scheduler and run it just once self.sched.add_job(job.func) # If we explicity call job.func() then we block the thread and we get multiple # missed executions from apscheduler # job.func() else: self.logger.warn("Job %s has a None type callable func" % job.id) def _flatten_dict(self, d, path): new_dict = { } for key in d: if isinstance(d[key], dict): new_path = '%s.%s' % (path, key) if path else key x = self._flatten_dict(d[key],new_path).copy() new_dict.update(x) else: new_key = '%s.%s' % (path, key) new_dict[new_key] = d[key] return new_dict
class Core(object): def __init__(self): self.started = False self.daemon = None self.io_loop = IOLoop().instance() self.pid = os.getpid() self.tz = tz.tzlocal() self.config_file = None self.data_dir = None self.cache_dir = None self.quite = None self.no_launch = None self.web_port = None self.developer = None self.debug = None self.newest_version = None self.newest_version_string = None self.user_agent = 'SiCKRAGE.CE.1/({};{};{})'.format(platform.system(), platform.release(), str(uuid.uuid1())) self.sys_encoding = get_sys_encoding() self.languages = [language for language in os.listdir(sickrage.LOCALE_DIR) if '_' in language] self.showlist = [] self.api = None self.adba_connection = None self.notifier_providers = None self.metadata_providers = None self.search_providers = None self.log = None self.config = None self.alerts = None self.main_db = None self.cache_db = None self.failed_db = None self.scheduler = None self.wserver = None self.wsession = None self.google_auth = None self.name_cache = None self.show_queue = None self.search_queue = None self.postprocessor_queue = None self.version_updater = None self.show_updater = None self.daily_searcher = None self.backlog_searcher = None self.proper_searcher = None self.trakt_searcher = None self.subtitle_searcher = None self.auto_postprocessor = None # patch modules with encoding kludge patch_modules() def start(self): self.started = True # thread name threading.currentThread().setName('CORE') # init core classes self.notifier_providers = NotifierProviders() self.metadata_providers = MetadataProviders() self.search_providers = SearchProviders() self.log = Logger() self.config = Config() self.api = API() self.alerts = Notifications() self.main_db = MainDB() self.cache_db = CacheDB() self.failed_db = FailedDB() self.scheduler = BackgroundScheduler() self.wserver = WebServer() self.wsession = WebSession() self.google_auth = GoogleAuth() self.name_cache = NameCache() self.show_queue = ShowQueue() self.search_queue = SearchQueue() self.postprocessor_queue = PostProcessorQueue() self.version_updater = VersionUpdater() self.show_updater = ShowUpdater() self.daily_searcher = DailySearcher() self.backlog_searcher = BacklogSearcher() self.proper_searcher = ProperSearcher() self.trakt_searcher = TraktSearcher() self.subtitle_searcher = SubtitleSearcher() self.auto_postprocessor = AutoPostProcessor() # Check if we need to perform a restore first if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))): success = restoreSR(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir) print("Restoring SiCKRAGE backup: %s!\n" % ("FAILED", "SUCCESSFUL")[success]) if success: shutil.rmtree(os.path.abspath(os.path.join(self.data_dir, 'restore')), ignore_errors=True) # migrate old database file names to new ones if os.path.isfile(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))): if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')): helpers.moveFile(os.path.join(self.data_dir, 'sickrage.db'), os.path.join(self.data_dir, '{}.bak-{}' .format('sickrage.db', datetime.datetime.now().strftime( '%Y%m%d_%H%M%S')))) helpers.moveFile(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')), os.path.abspath(os.path.join(self.data_dir, 'sickrage.db'))) # load config self.config.load() # set language self.config.change_gui_lang(self.config.gui_lang) # set socket timeout socket.setdefaulttimeout(self.config.socket_timeout) # setup logger settings self.log.logSize = self.config.log_size self.log.logNr = self.config.log_nr self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log') self.log.debugLogging = self.config.debug self.log.consoleLogging = not self.quite # start logger self.log.start() # user agent if self.config.random_user_agent: self.user_agent = UserAgent().random urlparse.uses_netloc.append('scgi') urllib.FancyURLopener.version = self.user_agent # Check available space try: total_space, available_space = getFreeSpace(self.data_dir) if available_space < 100: self.log.error('Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data ' 'otherwise. Only %sMB left', available_space) return except Exception: self.log.error('Failed getting diskspace: %s', traceback.format_exc()) # perform database startup actions for db in [self.main_db, self.cache_db, self.failed_db]: # initialize database db.initialize() # check integrity of database db.check_integrity() # migrate database db.migrate() # misc database cleanups db.cleanup() # compact main database if not sickrage.app.developer and self.config.last_db_compact < time.time() - 604800: # 7 days self.main_db.compact() self.config.last_db_compact = int(time.time()) # load name cache self.name_cache.load() # load data for shows from database self.load_shows() if self.config.default_page not in ('home', 'schedule', 'history', 'news', 'IRC'): self.config.default_page = 'home' # cleanup cache folder for folder in ['mako', 'sessions', 'indexers']: try: shutil.rmtree(os.path.join(sickrage.app.cache_dir, folder), ignore_errors=True) except Exception: continue # init anidb connection if self.config.use_anidb: def anidb_logger(msg): return self.log.debug("AniDB: {} ".format(msg)) try: self.adba_connection = adba.Connection(keepAlive=True, log=anidb_logger) self.adba_connection.auth(self.config.anidb_username, self.config.anidb_password) except Exception as e: self.log.warning("AniDB exception msg: %r " % repr(e)) if self.config.web_port < 21 or self.config.web_port > 65535: self.config.web_port = 8081 if not self.config.web_cookie_secret: self.config.web_cookie_secret = generateCookieSecret() # attempt to help prevent users from breaking links by using a bad url if not self.config.anon_redirect.endswith('?'): self.config.anon_redirect = '' if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs): self.config.root_dirs = '' self.config.naming_force_folders = check_force_season_folders() if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'): self.config.nzb_method = 'blackhole' if self.config.torrent_method not in ('blackhole', 'utorrent', 'transmission', 'deluge', 'deluged', 'download_station', 'rtorrent', 'qbittorrent', 'mlnet', 'putio'): self.config.torrent_method = 'blackhole' if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq: self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq: self.config.daily_searcher_freq = self.config.min_daily_searcher_freq self.config.min_backlog_searcher_freq = self.backlog_searcher.get_backlog_cycle_time() if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq: self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq if self.config.version_updater_freq < self.config.min_version_updater_freq: self.config.version_updater_freq = self.config.min_version_updater_freq if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq: self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq if self.config.proper_searcher_interval not in ('15m', '45m', '90m', '4h', 'daily'): self.config.proper_searcher_interval = 'daily' if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23: self.config.showupdate_hour = 0 if self.config.subtitles_languages[0] == '': self.config.subtitles_languages = [] # add version checker job self.scheduler.add_job( self.version_updater.run, IntervalTrigger( hours=self.config.version_updater_freq ), name="VERSIONUPDATER", id="VERSIONUPDATER" ) # add network timezones updater job self.scheduler.add_job( update_network_dict, IntervalTrigger( days=1 ), name="TZUPDATER", id="TZUPDATER" ) # add show updater job self.scheduler.add_job( self.show_updater.run, IntervalTrigger( days=1, start_date=datetime.datetime.now().replace(hour=self.config.showupdate_hour) ), name="SHOWUPDATER", id="SHOWUPDATER" ) # add daily search job self.scheduler.add_job( self.daily_searcher.run, IntervalTrigger( minutes=self.config.daily_searcher_freq, start_date=datetime.datetime.now() + datetime.timedelta(minutes=4) ), name="DAILYSEARCHER", id="DAILYSEARCHER" ) # add backlog search job self.scheduler.add_job( self.backlog_searcher.run, IntervalTrigger( minutes=self.config.backlog_searcher_freq, start_date=datetime.datetime.now() + datetime.timedelta(minutes=30) ), name="BACKLOG", id="BACKLOG" ) # add auto-postprocessing job self.scheduler.add_job( self.auto_postprocessor.run, IntervalTrigger( minutes=self.config.autopostprocessor_freq ), name="POSTPROCESSOR", id="POSTPROCESSOR" ) # add find proper job self.scheduler.add_job( self.proper_searcher.run, IntervalTrigger( minutes={'15m': 15, '45m': 45, '90m': 90, '4h': 4 * 60, 'daily': 24 * 60}[ self.config.proper_searcher_interval] ), name="PROPERSEARCHER", id="PROPERSEARCHER" ) # add trakt.tv checker job self.scheduler.add_job( self.trakt_searcher.run, IntervalTrigger( hours=1 ), name="TRAKTSEARCHER", id="TRAKTSEARCHER" ) # add subtitles finder job self.scheduler.add_job( self.subtitle_searcher.run, IntervalTrigger( hours=self.config.subtitle_searcher_freq ), name="SUBTITLESEARCHER", id="SUBTITLESEARCHER" ) # start scheduler service self.scheduler.start() # Pause/Resume PROPERSEARCHER job (self.scheduler.get_job('PROPERSEARCHER').pause, self.scheduler.get_job('PROPERSEARCHER').resume )[self.config.download_propers]() # Pause/Resume TRAKTSEARCHER job (self.scheduler.get_job('TRAKTSEARCHER').pause, self.scheduler.get_job('TRAKTSEARCHER').resume )[self.config.use_trakt]() # Pause/Resume SUBTITLESEARCHER job (self.scheduler.get_job('SUBTITLESEARCHER').pause, self.scheduler.get_job('SUBTITLESEARCHER').resume )[self.config.use_subtitles]() # Pause/Resume POSTPROCESS job (self.scheduler.get_job('POSTPROCESSOR').pause, self.scheduler.get_job('POSTPROCESSOR').resume )[self.config.process_automatically]() # start queue's self.search_queue.start() self.show_queue.start() self.postprocessor_queue.start() # start webserver self.wserver.start() def shutdown(self, restart=False): if self.started: self.log.info('SiCKRAGE IS SHUTTING DOWN!!!') # shutdown webserver self.wserver.shutdown() # shutdown show queue if self.show_queue: self.log.debug("Shutting down show queue") self.show_queue.shutdown() del self.show_queue # shutdown search queue if self.search_queue: self.log.debug("Shutting down search queue") self.search_queue.shutdown() del self.search_queue # shutdown post-processor queue if self.postprocessor_queue: self.log.debug("Shutting down post-processor queue") self.postprocessor_queue.shutdown() del self.postprocessor_queue # log out of ADBA if self.adba_connection: self.log.debug("Shutting down ANIDB connection") self.adba_connection.stop() # save all show and config settings self.save_all() # close databases for db in [self.main_db, self.cache_db, self.failed_db]: if db.opened: self.log.debug("Shutting down {} database connection".format(db.name)) db.close() # shutdown logging self.log.close() if restart: os.execl(sys.executable, sys.executable, *sys.argv) if sickrage.app.daemon: sickrage.app.daemon.stop() self.started = False def save_all(self): # write all shows self.log.info("Saving all shows to the database") for show in self.showlist: try: show.saveToDB() except Exception: continue # save config self.config.save() def load_shows(self): """ Populates the showlist with shows from the database """ for dbData in [x['doc'] for x in self.main_db.db.all('tv_shows', with_doc=True)]: try: self.log.debug("Loading data for show: [{}]".format(dbData['show_name'])) show = TVShow(int(dbData['indexer']), int(dbData['indexer_id'])) show.nextEpisode() self.showlist += [show] except Exception as e: self.log.error("Show error in [%s]: %s" % (dbData['location'], e.message))
class Scheduler: timezone = settings.TIME_ZONE week_map = { '*': '*', '7': '6', '0': '6', '1': '0', '2': '1', '3': '2', '4': '3', '5': '4', '6': '5', } def __init__(self): self.scheduler = BackgroundScheduler(timezone=self.timezone) self.scheduler.add_listener( self._handle_event, EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES | EVENT_JOB_EXECUTED) @classmethod def parse_trigger(cls, trigger, trigger_args): if trigger == 'interval': return IntervalTrigger(seconds=int(trigger_args), timezone=cls.timezone) elif trigger == 'date': return DateTrigger(run_date=trigger_args, timezone=cls.timezone) elif trigger == 'cron': args = json.loads(trigger_args) if not isinstance( trigger_args, dict) else trigger_args minute, hour, day, month, week = args['rule'].split() week = cls.week_map[week] return CronTrigger(minute=minute, hour=hour, day=day, month=month, day_of_week=week, start_date=args['start'], end_date=args['stop']) else: raise TypeError(f'unknown schedule policy: {trigger!r}') def _handle_event(self, event): close_old_connections() obj = SimpleLazyObject( lambda: Task.objects.filter(pk=event.job_id).first()) if event.code == EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('schedule', '1', '调度器已关闭', '调度器意外关闭,你可以在github上提交issue') elif event.code == EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('schedule', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时') elif event.code == EVENT_JOB_ERROR: logger.info( f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}' ) Notify.make_notify('schedule', '1', f'{obj.name} - 执行异常', f'{event.exception}') elif event.code == EVENT_JOB_EXECUTED: if event.retval: score = 0 for item in event.retval: score += 1 if item[1] else 0 history = History.objects.create( task_id=event.job_id, status=2 if score == len(event.retval) else 1 if score else 0, run_time=human_datetime(event.scheduled_run_time), output=json.dumps(event.retval)) Task.objects.filter(pk=event.job_id).update(latest=history) if score != 0 and time.time() - counter.get(event.job_id, 0) > 3600: counter[event.job_id] = time.time() Notify.make_notify('schedule', '1', f'{obj.name} - 执行失败', '请在任务计划中查看失败详情') def _init_builtin_jobs(self): self.scheduler.add_job(auto_clean_records, 'cron', hour=0, minute=0) self.scheduler.add_job(auto_clean_schedule_history, 'cron', hour=0, minute=0) def _init(self): self.scheduler.start() self._init_builtin_jobs() for task in Task.objects.filter(is_active=True): trigger = self.parse_trigger(task.trigger, task.trigger_args) self.scheduler.add_job( dispatch, trigger, id=str(task.id), args=(task.command, json.loads(task.targets)), ) def run(self): rds_cli = get_redis_connection() self._init() rds_cli.delete(settings.SCHEDULE_KEY) logger.info('Running scheduler') while True: _, data = rds_cli.brpop(settings.SCHEDULE_KEY) task = AttrDict(json.loads(data)) if task.action in ('add', 'modify'): trigger = self.parse_trigger(task.trigger, task.trigger_args) self.scheduler.add_job(dispatch, trigger, id=str(task.id), args=(task.command, task.targets), replace_existing=True) elif task.action == 'remove': job = self.scheduler.get_job(str(task.id)) if job: job.remove()
import os, time from apscheduler.schedulers.blocking import BlockingScheduler from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.executors.pool import ProcessPoolExecutor from datetime import datetime from datetime import timedelta import sqlite3 delta_2min = timedelta(minutes=2) def build_stock_day_data(): os.system("python 3_build_stock_day_data.py") sched = BackgroundScheduler() sched.add_job(build_stock_day_data, 'interval', minutes=3, id='001', next_run_time=datetime.now(), max_instances=1) sched.start() while 1: print(sched.get_job('001')) time.sleep(60)