def task(self): """ !!!!this function is useless don't run it!!!! Parameters: year (int|str) – 4-digit year month (int|str) – month (1-12) day (int|str) – day of the (1-31) week (int|str) – ISO week (1-53) day_of_week (int|str) – number or name of weekday (0-6 or mon,tue,wed,thu,fri,sat,sun) hour (int|str) – hour (0-23) minute (int|str) – minute (0-59) second (int|str) – second (0-59) start_date (datetime|str) – earliest possible date/time to trigger on (inclusive) end_date (datetime|str) – latest possible date/time to trigger on (inclusive) timezone (datetime.tzinfo|str) – time zone to use for the date/time calculations (defaults to scheduler timezone) :return: """ scheduler = BlockingScheduler() #scheduler.add_job(self.task_func, trigger='cron', day='*/1', hour='1') scheduler.add_job(self.task_func, trigger='cron', minute='*/5') #scheduler.add_job(func, 'date', run_date='2016-10-25 13:51:30') try: scheduler.start() except Exception as e: # TODO 执行错误的处理方案 Global.logger.error('定时任务错误:%s' % e) scheduler.shutdown()
class ScheduledCheck(Check, metaclass=ABCMeta): ''' An abstract base class for a check that runs based on the Scheduler from apscheduler Child classes need to implement the check method ''' def __init__(self, queue=None, notify_on_exception=True, name=None, **kwargs): ''' Create a new instance of this Check The kwargs are handed over to apscheduler.blocking.BlockingScheduler.add_job and decide when the checks are run. For example `trigger='cron', hour=8` will run this check every day at 8 o'clock ''' super().__init__(queue=queue, notify_on_exception=notify_on_exception, name=name) self.scheduler = BlockingScheduler( job_defaults={'misfire_grace_time': 5 * 60}) self.scheduler.add_job(self.wrapped_check, **kwargs) def run(self): self.scheduler.start() def stop(self): self.scheduler.shutdown() self.log.info('Check %s stopped', self.__class__.__name__)
class ScheduledCheck(Check, metaclass=ABCMeta): ''' An abstract base class for a check that runs based on the Scheduler from apscheduler Child classes need to implement the check method ''' def __init__(self, queue=None, notify_on_exception=True, name=None, **kwargs): ''' Create a new instance of this Check The kwargs are handed over to apscheduler.blocking.BlockingScheduler.add_job and decide when the checks are run. For example `trigger='cron', hour=8` will run this check every day at 8 o'clock ''' super().__init__(queue=queue, notify_on_exception=notify_on_exception, name=name) self.scheduler = BlockingScheduler( job_defaults={'misfire_grace_time': 5*60} ) self.scheduler.add_job(self.wrapped_check, **kwargs) def run(self): self.scheduler.start() def stop(self): self.scheduler.shutdown() self.log.info('Check %s stopped', self.__class__.__name__)
def _run(self): """ run your fun""" scheduler = BlockingScheduler() scheduler.add_job(PrintText().start, 'interval', seconds=3) # scheduler.add_job(PrintText().start, 'cron', hour=start_hour, minute=start_minute,second='0') try: scheduler.start() except KeyboardInterrupt, SystemExit: scheduler.shutdown() logger.error('Exit The Job!')
def log(self, container_id, is_watch=False, default_loop_time=20): if is_watch: scheduler = BlockingScheduler() scheduler.add_job( self.output_log_single(container_id=container_id), 'interval', seconds=default_loop_time) try: scheduler.start() except (KeyboardInterrupt, SystemExit): scheduler.shutdown() else: self.output_log_single(container_id=container_id)
def schedule(ctx, hour): email = ctx.parent.params['email'] username = ctx.parent.params['username'] email_to = ctx.parent.params['email_to'] password = ctx.obj.get('password', None) gsheet = ctx.parent.params['gsheet'] doc_key = ctx.parent.params['doc_key'] schedule = BlockingScheduler() schedule.add_job(run, kwargs={"email": email, "gsheet": gsheet, "doc_key": doc_key, "username": username, "email_to": email_to, "password": password}, trigger='cron', hour=hour) try: schedule.start() except (KeyboardInterrupt, SystemExit): schedule.shutdown()
def main(): log = logging.getLogger("main") cameras, light, project = parse_configs() storage_dir = Path(project["storage_dir"]) if not storage_dir.is_dir(): storage_dir.mkdir(parents=True) # We will use queue to connect scheduler thread with I2C communication thread connection_queue = Queue() # Create separate thread for I2C communication log.info("Starting I2C thread") i2c_driver = I2CDriver(0x04) i2c_thread = Thread(target=i2c_thread_function, args=(i2c_driver, connection_queue)) i2c_thread.start() log.info("Running pipeline for the first time") pipeline_executor = PipelineExecutor(storage_dir, cameras, light, connection_queue, project["pipeline"]) # For the first time, execute pipeline manually, then schedule it pipeline_executor.execute() # Create a scheduler and add job to it log.info("Scheduling the pipeline") scheduler = BlockingScheduler() scheduler.add_job( func=(lambda executor=pipeline_executor: executor.execute()), trigger="interval", seconds=project['run_interval_seconds']) atexit.register(lambda: scheduler.shutdown()) scheduler.start() # Blocks thread
def engage(): banner.print_banner() oneness_scheduler = BlockingScheduler({ 'apscheduler.executors.processpool': { 'class': 'apscheduler.executors.pool:ProcessPoolExecutor', 'max_workers': '20' }, 'job_defaults': { 'coalesce': False, 'executor': 'processpool' } }) oneness_scheduler.add_executor('processpool') t_retweet = oneness_scheduler.add_job(twitter.retweet.start, 'interval', minutes=60, id='twitter_retweet_bot') t_follow = oneness_scheduler.add_job(twitter.follow.start, 'interval', minutes=10, id='twitter_follow_bot') # quoted_im_generator = oneness_scheduler.add_job( # image_generator.quoted_image.start, # 'interval', minutes=300, # id='quoted_im_generator', # kwargs={'overlay_flag': True} # ) im_with_quote_generator = oneness_scheduler.add_job( image_generator.quoted_image.start, 'interval', minutes=120, id='image_with_quote_generator', kwargs={'overlay_flag': False}) try: # oneness_scheduler.start() for job in oneness_scheduler.get_jobs(): job.modify(next_run_time=datetime.now()) oneness_scheduler.start() except (KeyboardInterrupt, SystemExit): oneness_scheduler.shutdown()
def watch(self): docker_log_watch = BackgroundScheduler() docker_log_watch.add_job(self.watch_upload_docker_log, 'interval', seconds=10) try: docker_log_watch.start() except (KeyboardInterrupt, SystemExit): docker_log_watch.shutdown() scheduler = BlockingScheduler() scheduler.add_job(self.upload_machine_information, 'interval', seconds=self.time, max_instances=2) try: scheduler.start() except (KeyboardInterrupt, SystemExit): scheduler.shutdown()
def main(): logging.basicConfig(filename='test.log', level=logging.ERROR, \ format='%(asctime)s:%(levelname)s:%(name)s:%(message)s') #Kill gphoto2 killgphoto() #Declaring/Calculating variables needed scheduler = BlockingScheduler(timezone='US/Central') hourbound = tl.getConfig('START_HOUR') + '-' + tl.getConfig('FINISH_HOUR') #Ensure photo_local_root exists if not os.path.exists(tl.getConfig('photo_local_root')): os.makedirs(tl.getConfig('photo_local_root')) #GP2 Log and Camera Setup gp.check_result(gp.use_python_logging()) context = gp.gp_context_new() camera = gp.check_result(gp.gp_camera_new()) gp.check_result(gp.gp_camera_init(camera, context)) #Adding job to scheduler scheduler.add_job(captureandsave, 'cron', args=[camera,context], \ day_of_week='mon-sun', second='*/'+str(tl.getConfig('INTERVAL')), \ hour=hourbound) print('Press Ctrl+{0} to exit'.format( \ 'Break' if os.name == 'nt' else 'C')) try: scheduler.start() except (KeyboardInterrupt, SystemExit): scheduler.shutdown() pass #Close Camera gp.check_result(gp.gp_camera_exit(camera, context)) return 0
def daily_task(self): def func(): day = datetime.datetime.now().strftime('%Y-%m-%d') # 活期 self.da.init_balance(day, 1) self.logger.info(day, '活期每日余额计算完成') # 定期 self.da.init_balance(day, 2) self.logger.info(day, '定期每日余额计算完成') # 理财 self.da.init_balance(day, 3) self.logger.info(day, '理财每日余额计算完成') scheduler = BlockingScheduler() scheduler.add_job(func,'cron',day='*',hour='1') # 每天凌晨1点运行 try: scheduler.start() except Exception as e: # TODO 执行错误的处理方案 self.logger.error('每日AUM计算出错:',e) scheduler.shutdown()
class TaskMgr: def __init__(self): self.scheduler = BlockingScheduler() self.update_time = Moment(hour=0, minute=0, second=0) self.list_task = [] def add_task(self, task): self.list_task.append(task) def set_moment(self, hour): self.update_time.hour = hour def _at_the_moment(self): now = datetime.now() option_moment = datetime(year=now.year, month=now.month, day=now.day, hour=0, minute=0, second=0) option_moment = option_moment.replace(hour=self.update_time.hour) return self._time_equal(t1=now.timestamp(), t2=option_moment.timestamp(), delta=30) def _time_equal(self, t1, t2, delta): if t1 > t2 - delta and t1 < t2 + delta: return True return False def _cron(self): for task in self.list_task: if self._at_the_moment() or not task.flag: task.run() def start(self, run_immedately=True): try: if run_immedately: for task in self.list_task: task.run() self.scheduler.add_job(func=self._cron, trigger='cron', minute=0, second=0) self.scheduler.start() except (KeyboardInterrupt, SystemExit): self.scheduler.shutdown()
class Scheduler: def __init__(self): conf = configparser.ConfigParser() conf.read("../agent.ini") ip = conf.get("redis", "ip") port = conf.getint("redis", "port") timeout = conf.getint("redis", "timeout") self.invoker_id = self._get_invoker_id() self.max_tasks = conf.getint("invoker", "max_tasks") self.live_seconds = conf.getint("invoker", "live_seconds") self.db = SchedulerDb(ip, port, timeout) logging.config.fileConfig("../logger.ini") self.logger = logging.getLogger("main") executors = { 'default': {'type': 'processpool', 'max_workers': self.max_tasks + 1} } self.blockScheduler = BlockingScheduler() self.jobs = {} self.lock = threading.Lock() @staticmethod def _get_invoker_id(): hostname = socket.gethostname() pid = os.getpid() return hostname + "-" + str(pid) def task_invoke(self, task_instance, task_param): if task_param.cmd.startswith('http'): executor = HttpExecutor(self.db, task_instance, task_param) executor.execute() else: pass def break_heart(self): """ invoker每隔一段时间就心跳一下,看看是否有新任务,是否有任务需要更新 :param bs: :return: """ # 先看看参数是否有变化的把调度重启或者关闭 try: self.lock.acquire() self.refresh_local_invoker() self.refresh_other_invokers() if len(self.jobs) >= self.max_tasks: return task_instances, task_params = self.db.query_waiting_run_tasks(self.invoker_id, self.max_tasks - len(self.jobs), True) if len(task_instances) == 0: return for i in range(len(task_instances)): task_instance = task_instances[i] task_param = task_params[i] if task_instance.id not in self.jobs.keys(): self.logger.info("分配了新任务%s", task_instance.id) job = self.blockScheduler.add_job(self.task_invoke, next_run_time=( datetime.datetime.now() + datetime.timedelta(seconds=2)), args=[task_instance, task_param], id=task_instance.id) self.jobs[job.id] = job self.db.lock_invoker_instance(self.invoker_id, task_instance.id, self.live_seconds) else: self.logger.error("%s任务已经在运行", task_instance.id) finally: self.lock.release() def refresh_local_invoker(self): """ 调度的参数是否发生变化,如有需要重启调度 :param bs: :return: """ self.db.update_invoker_time(self.invoker_id, self.jobs.keys(), self.live_seconds) self.logger.info("%s心跳更新成功!", self.invoker_id) # 看看是否有需要停止的任务再自己这里,释放掉 stop_tasks = self.db.query_need_stop_tasks(self.invoker_id) for stop_task in stop_tasks: if stop_task in self.jobs.keys(): try: job = self.jobs[stop_task] task_instance = job.args[0] task_instance.status = 'off' job.pause() job.remove() except Exception as e: self.logger.error(e) self.jobs.pop(stop_task) try: self.blockScheduler.remove_job(stop_task) except Exception as e1: self.logger.error(e1) self.logger.info("人工停止了任务%s", stop_task) self.db.unlock_invoker_instance(self.invoker_id, stop_task, self.live_seconds) # 是否有参数变化的任务需要重启 c_jobs = copy.copy(self.jobs) for key in c_jobs.keys(): if key not in self.jobs.keys(): continue job = self.jobs[key] task_instance = job.args[0] old_task_param = job.args[1] # 判断参数是否发生变化,如果有变化重新执行任务 new_task_param = self.db.query_task_param(task_instance.task_param_id) # if new_task_param if not new_task_param.has_diff(old_task_param): continue try: task_instance.status = 'off' job.pause() job.remove() except Exception as e: self.logger.error(e) self.jobs.pop(key) try: self.blockScheduler.remove_job(key) except Exception as e1: self.logger.error(e1) self.logger.info("参数变化停止了任务%s", task_instance.id) self.db.unlock_invoker_instance(self.invoker_id, task_instance.id, self.live_seconds) self.db.add_task_waiting_run(task_instance.id) def refresh_other_invokers(self): """ 遍历所有的invoker,判断invoker是否超过存活期 :return: """ invokers = self.db.query_all_invokers() for invoker_id in invokers.keys(): if not self.db.invoker_is_live(self.invoker_id): task_instance_list = self.db.query_invoker_tasks(self.invoker_id) for task_instance_id in task_instance_list: self.db.add_task_waiting_run(task_instance_id) def main(self): try: self.db.register_invoker(self.invoker_id, self.max_tasks, self.live_seconds); self.blockScheduler.add_listener(self._job_listener, events.EVENT_JOB_ERROR | events.EVENT_JOB_MISSED) self.blockScheduler.add_job(self.break_heart, "interval", seconds=self.live_seconds / 2, id="break_heart") self.logger.info("开始启动调度...") self.blockScheduler.start() self.logger.info("启动调度成功!") except KeyboardInterrupt as e: self.logger.info(e) self.blockScheduler.shutdown() def _job_listener(self, ev): """ 监听job的事件,job完成后再发起下次调用,对于异常也要处理 :param ev: :return: """ if ev.code == events.EVENT_JOB_ERROR: self.logger.error(ev.exception) self.logger.error(ev.traceback) else: pass
class ActionScheduler: def __init__(self): self.scheduler = BlockingScheduler(jobstores=APSCHEDULER_SETTINGS['jobstores'], executors=APSCHEDULER_SETTINGS['executors'], job_defaults=APSCHEDULER_SETTINGS['job_defaults'], timezone=TIMEZONE_PST8PDT) pass def start(self): self._add_event_listener() # self._add_example_jobs() self._add_jobs() self.scheduler.start() def shutdown(self): # self.scheduler.remove_all_jobs() # save all jobs into sqlite, do not remove them self.scheduler.shutdown() def _add_event_listener(self): self.scheduler.add_listener(ActionScheduler.listener_jobs_status, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) self.scheduler.add_listener(ActionScheduler.listener_all_jobs_finished, EVENT_ALL_JOBS_REMOVED) # examples def _add_example_jobs(self): import datetime self.scheduler.add_job(func=ActionScheduler.job_example, args=["cron", ], trigger='cron', second='*/5', misfire_grace_time=DEFAULT_MISFIRE_GRACE_TIME, replace_existing=True, id="cron") self.scheduler.add_job(func=ActionScheduler.job_example, args=["interval", ], trigger='interval', seconds=60, misfire_grace_time=DEFAULT_MISFIRE_GRACE_TIME, replace_existing=True, id="interval") self.scheduler.add_job(func=ActionScheduler.job_example, args=["date", ], trigger='date', run_date=get_cur_time()+datetime.timedelta(seconds=12), id="date") # examples @staticmethod def job_example(job_type): print("job_example: {}".format(job_type)) def _add_jobs(self): # add reap alerts immediate job TODO test # self.scheduler.add_job(id="reap_alerts_immediate", func=ActionScheduler.job_reap_alerts_and_start_action_tasks, args=[], # misfire_grace_time=DEFAULT_MISFIRE_GRACE_TIME, replace_existing=True, ) # add reap alerts interval job # self.scheduler.add_job(id="reap_alerts", func=ActionScheduler.job_reap_alerts_and_start_action_tasks, # args=[], trigger='interval', seconds=REAP_INTERVAL_SECONDS, # misfire_grace_time=DEFAULT_MISFIRE_GRACE_TIME, replace_existing=True, ) # add gather & retry failed action tasks immediate job TODO test # self.scheduler.add_job(id="check_tasks_immediate", func=ActionScheduler.job_gather_and_retry_failed_action_tasks, args=[], # misfire_grace_time=DEFAULT_MISFIRE_GRACE_TIME, replace_existing=True, ) # add gather & retry failed action tasks interval job # self.scheduler.add_job(id="check_tasks", func=ActionScheduler.job_gather_and_retry_failed_action_tasks, # args=[], trigger='interval', seconds=GATHER_FAILED_TASKS_INTERVAL_SECONDS, # misfire_grace_time=DEFAULT_MISFIRE_GRACE_TIME, replace_existing=True, ) pass @staticmethod def listener_all_jobs_finished(event): # this would hardly be invoked logger_.info('All jobs are done.') @staticmethod def listener_jobs_status(event): if event.exception: logger_.warn('Job {} crashed.'.format(event.job_id)) else: logger_.info('Job {} executed.'.format(event.job_id))
class AutoCopy(object): config_class = Config default_config = ImmutableDict({ 'DEBUG': False, 'DRIVE_LIST': [], 'DST_PATH': 'C:\\autocopy', 'EXTENSION': ['.txt'], 'FILE_SIZE': "10 MB", 'SECONDS': 2, 'UNIT': { "KB": 1024, "MB": 1024 * 1024, "GB": 1024 * 1024 * 1024 } }) def __init__(self, import_name, *args, **kwargs): self.import_name = import_name self.get_logica_drives = win32file.GetLogicalDrives self.config = self.make_config() self.scheduler = BlockingScheduler() def init(self): self.drive_list = self.config.get('DRIVE_LIST') self.dst_path = self.config.get('DST_PATH') self.unit = self.config.get('UNIT') self.size = self.config.get('FILE_SIZE') self.extension = self.config.get('EXTENSION') self.seconds = self.config.get('SECONDS') self.is_dst_path() def make_config(self): return self.config_class(self.default_config) def is_u_disk(self, drive, drive_num=2): return win32file.GetDriveType(drive) == drive_num def get_u_disk(self, drive_list=None): sign = self.get_logica_drives() drive_list = drive_list or self.drive_list drives = (drive_list[i] for i in range(len(drive_list) - 1) if (sign & 1 << i and self.is_u_disk(drive_list[i]))) return drives def is_dst_path(self, dst_path=None): dst_path = dst_path or self.dst_path if not os.path.exists(dst_path) or not os.path.isdir(dst_path): try: os.mkdir(dst_path) except Exception: os.remove(dst_path) os.mkdir(dst_path) return None def _get_size(self, size, unit='KB'): units = self.unit if isinstance(size, integer_types): return size * units.get(unit, 1024) try: size, unit = size.split(' ') except Exception: pass else: return int(size) * units.get(unit, 1024) return 1024 def is_file_size(self, file, size=None): if not isinstance(file, basestring): raise TypeError("This is not a string.") size = size or self.size return os.path.getsize(file) < self._get_size(size=size) def _copyfile(self, path, dst_path=None, extension=None): for path, _, file_list in os.walk(path): for file in file_list: _, ext = os.path.splitext(file) file = os.path.join(path, file) if ext in extension and self.is_file_size(file): try: shutil.copy(file, dst_path) except Exception: self.is_dst_path() def copyfile(self, dst_path=None, extension=None): extension = extension or self.extension dst_path = dst_path or self.dst_path drives = self.get_u_disk() for drive in drives: self._copyfile(drive, dst_path, extension) def _run(self, *args, **kwargs): self.copyfile(*args, **kwargs) def run(self, timer=True, seconds=None, *args, **kwargs): self.init() if not timer: return self._run trigger = IntervalTrigger(seconds=seconds or self.seconds) self.scheduler.add_job(self._run, trigger) # self.scheduler.add_job() try: self.scheduler.start() except (KeyboardInterrupt, SystemExit): self.scheduler.shutdown()
class St2Timer(object): """ A timer interface that uses APScheduler 3.0. """ def __init__(self, local_timezone=None): self._timezone = local_timezone self._scheduler = BlockingScheduler(timezone=self._timezone) self._jobs = {} self._trigger_types = TIMER_TRIGGER_TYPES.keys() self._trigger_watcher = TriggerWatcher( create_handler=self._handle_create_trigger, update_handler=self._handle_update_trigger, delete_handler=self._handle_delete_trigger, trigger_types=self._trigger_types, queue_suffix='timers') self._trigger_dispatcher = TriggerDispatcher(LOG) def start(self): self._register_timer_trigger_types() self._trigger_watcher.start() self._scheduler.start() def cleanup(self): self._scheduler.shutdown(wait=True) def add_trigger(self, trigger): self._add_job_to_scheduler(trigger) def update_trigger(self, trigger): self.remove_trigger(trigger) self.add_trigger(trigger) def remove_trigger(self, trigger): id = trigger['id'] try: job_id = self._jobs[id] except KeyError: LOG.info('Job not found: %s', id) return self._scheduler.remove_job(job_id) def _add_job_to_scheduler(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref] try: jsonschema.validate(trigger['parameters'], trigger_type['parameters_schema']) except jsonschema.ValidationError as e: LOG.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) raise # Or should we just return? time_spec = trigger['parameters'] time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone')) time_type = None if trigger_type['name'] == 'st2.IntervalTimer': unit = time_spec.get('unit', None) value = time_spec.get('delta', None) time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone}) elif trigger_type['name'] == 'st2.DateTimer': # Raises an exception if date string isn't a valid one. dat = date_parser.parse(time_spec.get('date', None)) time_type = DateTrigger(dat, timezone=time_zone) elif trigger_type['name'] == 'st2.CronTimer': cron = time_spec.copy() cron['timezone'] = time_zone time_type = CronTrigger(**cron) if hasattr(time_type, 'run_date') and datetime.now(tzutc()) > time_type.run_date: LOG.warning('Not scheduling expired timer: %s : %s', trigger['parameters'], time_type.run_date) else: self._add_job(trigger, time_type) def _add_job(self, trigger, time_type, replace=True): try: job = self._scheduler.add_job(self._emit_trigger_instance, trigger=time_type, args=[trigger], replace_existing=replace) LOG.info('Job %s scheduled.', job.id) self._jobs[trigger['id']] = job.id except Exception as e: LOG.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) def _emit_trigger_instance(self, trigger): LOG.info('Timer fired at: %s. Trigger: %s', str(datetime.utcnow()), trigger) payload = { 'executed_at': str(datetime.utcnow()), 'schedule': trigger['parameters'].get('time') } self._trigger_dispatcher.dispatch(trigger, payload) def _register_timer_trigger_types(self): return container_utils.add_trigger_models(TIMER_TRIGGER_TYPES.values()) ############################################## # Event handler methods for the trigger events ############################################## def _handle_create_trigger(self, trigger): LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.add_trigger(trigger=trigger) def _handle_update_trigger(self, trigger): LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.update_trigger(trigger=trigger) def _handle_delete_trigger(self, trigger): LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.remove_trigger(trigger=trigger) def _sanitize_trigger(self, trigger): sanitized = trigger._data if 'id' in sanitized: # Friendly objectid rather than the MongoEngine representation. sanitized['id'] = str(sanitized['id']) return sanitized
class JobLauncher(object): def __init__(self, background=False, deamon=True, **kwargs): logging.basicConfig(format="[%(asctime)s] %(message)s", atefmt="%Y-%m-%d %H:%M:%S") logging.getLogger('apscheduler').setLevel(logging.DEBUG) if background: self.sched = BackgroundScheduler(deamon=deamon) # background else: self.sched = BlockingScheduler(deamon=deamon) # foreground # TODO: Read from configuration file. self.sched.configure( jobstores={ # "sqlite": SQLAlchemyJobStore(url='sqlite:///app/database/example.db'), # "default": MemoryJobStore() "default": SQLAlchemyJobStore(url='sqlite:///app/database/example.db') }, executors={ 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) }, job_defaults={ 'coalesce': False, 'max_instances': 3 }, timezone=get_localzone() # Asia/Seoul ) self.retried = 0 self.logger = logging.getLogger('apscheduler') super(JobLauncher, self).__init__() def start(self): try: if self.sched.state != STATE_RUNNING: self.printJobs(jobstore='default') started = self.sched.start() except ConflictingIdError as e: traceback.print_exc() except KeyboardInterrupt as e: traceback.print_exc() finally: pass # Remove all remained store. # self.sched.remove_all_jobs() # for job in self.getJobs(): # if job.pending: # job.pause() self.logger.info('Finished') self.logger.info(self.getJobs()) self.printJobs() def stop(self, wait=False): if self.sched.state == STATE_RUNNING: self.sched.shutdown(wait=wait) def resume(self): if self.sched.state == STATE_RUNNING: self.sched.resume() def pause(self): if self.sched.state == STATE_RUNNING: self.sched.pause() def addListener(self, listener, types): self.sched.add_listener(listener, types) def addJob(self, job, **kwargs): execute, trigger, options = job.build(**kwargs) added_job = self.sched.add_job(execute, trigger, **options) self.printJobs() return added_job def getJob(self, job_id): return self.sched.get_job(job_id) def getJobs(self, jobstore=None): return self.sched.get_jobs(jobstore=jobstore) def removeJob(self, job_id, jobstore=None): return self.sched.remove_job(job_id, jobstore=jobstore) def removeAllJob(self, jobstore=None): return self.sched.remove_all_jobs(jobstore=jobstore) def printJobs(self, jobstore=None, out=None): return self.sched.print_jobs(jobstore=jobstore, out=None) def getJobState(self, job_id=None, jobstore=None): state = list() if job_id is not None: job = self.sched.get_job(job_id, jobstore=jobstore) if job is not None: temp = dict() temp[job.id] = { "next_run_time": job.next_run_time, "state": job.pending, } state.append(temp) else: for job in self.sched.get_jobs(jobstore=jobstore): temp = dict() temp[job.id] = { "next_run_time": job.next_run_time, "state": job.pending, } state.append(temp) return state
# sched.add_job(build_add, 'date', run_date='2018-03-08 18:01:50',misfire_grace_time=10000,max_instances=3) # # # sched = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) # # print "scheduled job from store",sched.get_jobs('default') # date='2018-03-08 18:05:58' # @sched.scheduled_job('date', run_date=date) # def build_scheduled(): # # supplying the values for kwargs # # server_c = jenkins.Jenkins('http://10.144.169.116:8080', username='******', password='******') # # print request.POST['CARD/CPS/Authorization'] # # url=request.POST['url2'] # # print url # try: # # build=server_c.build_job('POC/ET_Trail'); # print "job build initiated successfully" # # return HttpResponse(simplejson.dumps(build), content_type='application/json') # except Exception as e: # print e # with open("test.txt", "a") as myfile: # myfile.write("2017-03-07 13:37:40' printed") print "scheduled job from store", sched.get_jobs('default') # print "scheduled jobs are ",sched.get_jobs(pending='True') sched.start() sched.wakeup() sched.shutdown(wait=True)
class TaskExecutor: def __init__(self, db, task_instance, task_param): self.task_instance = task_instance self.task_param = task_param self.db = db # invoke log self.invoke_log_map = {} self.jobs = {} logging.config.fileConfig("../logger.ini") self.logger = logging.getLogger("taskExecutor") invoke_count = int(self.task_param.get_invoke_args()['invoke_count']) executors = { 'default': { 'type': 'threadpool', 'max_workers': invoke_count + 1 } } self.scheduler = BlockingScheduler(executors=executors) def execute(self): self.scheduler.add_listener( self._job_listener, events.EVENT_JOB_EXECUTED | events.EVENT_JOB_ERROR | events.EVENT_JOB_ADDED | events.EVENT_JOB_MISSED) # invoke_log_map up server self.scheduler.add_job(self._invoke_break_heart, "interval", seconds=2) try: self.scheduler.start() except Exception as e: print(e) self.scheduler.shutdown(wait=True) def _job_listener(self, ev): """ 监听job的事件,job完成后再发起下次调用,对于异常也要处理 :param ev: :return: """ if self.task_instance.status == 'off': return if ev.code == events.EVENT_JOB_ADDED: self.jobs[ev.job_id] = self.scheduler.get_job(ev.job_id) elif ev.code == events.EVENT_JOB_EXECUTED or ev.code == events.EVENT_JOB_ERROR: if ev.code == events.EVENT_JOB_ERROR: self.logger.error(ev.exception) self.logger.error(ev.traceback) job = self.jobs[ev.job_id] self.scheduler.add_job( job.func, next_run_time=(datetime.datetime.now() + datetime.timedelta(seconds=1)), id=ev.job_id, args=job.args) else: pass def _invoke_break_heart(self): if self.task_instance.status == 'off': jobs = self.scheduler.get_jobs() for job in jobs: try: job.pause() job.remove() except Exception as e: self.logger.error(e) self.db.save_task_logs(self.invoke_log_map)
class Feeds(Thread): WINDOW = 1000 def __init__(self, sources, feeds, sleep, logger): Thread.__init__(self) self.sleep = sleep self.logger = logger self.coords = deque([(source.strip(), feed.strip()) for source, feed in zip(sources, feeds)]) self.entries = [] self.last_45 = {feed: [] for _, feed in self.coords} socket.setdefaulttimeout(3) def run(self): job_defaults = {'coalesce': True, 'max_instances': 1} self.blocker = BlockingScheduler(job_defaults=job_defaults) self.blocker.add_job(self.parse_feed, 'cron', second=f'*/{self.sleep}', id='parse_feed') self.blocker.start() def on_close(self): self.blocker.shutdown() self.join() def parse_feed(self): self.coords.rotate() self.source, self.feed = self.coords[0] try: response = feedparser.parse(self.feed) except Exception as e: self.logger.warning(f"Status,{self.source},{self.feed},{e}") return status = response.get('status', None) if not status: self.logger.warning(f"Status,{self.source},{self.feed},None") return if status != 200: self.logger.warning(f"Status,{self.source},{self.feed},{status}") return entries = response.get('entries', None) if not entries: self.logger.warning(f"Entries,{self.source},{self.feed},None") return for entry in entries: entry_str = json.dumps(entry).encode() entry_hash = sha256(entry_str).hexdigest() if entry_hash in self.last_45[self.feed]: break self.last_45[self.feed].append(entry_hash) self.last_45[self.feed] = self.last_45[self.feed][-self.WINDOW:] utc_now = datetime.now( tz=timezone.utc).strftime("%Y-%d-%m %H:%M:%S.%f") entry['oscrap_acquisition_datetime'] = utc_now entry['oscrap_source'] = self.source print(self.source) self.entries.append(entry) if len(self.entries) > 0: with open(f"{DIR}/news_data/{str(uuid.uuid4())}.txt", "w") as file: file.write(json.dumps(self.entries)) self.entries = []
class St2Timer(object): """ A timer interface that uses APScheduler 3.0. """ def __init__(self, local_timezone=None): self._timezone = local_timezone self._scheduler = BlockingScheduler(timezone=self._timezone) self._jobs = {} self._trigger_types = list(TIMER_TRIGGER_TYPES.keys()) self._trigger_watcher = TriggerWatcher( create_handler=self._handle_create_trigger, update_handler=self._handle_update_trigger, delete_handler=self._handle_delete_trigger, trigger_types=self._trigger_types, queue_suffix=self.__class__.__name__, exclusive=True) self._trigger_dispatcher = TriggerDispatcher(LOG) def start(self): self._register_timer_trigger_types() self._trigger_watcher.start() self._scheduler.start() def cleanup(self): self._scheduler.shutdown(wait=True) def add_trigger(self, trigger): self._add_job_to_scheduler(trigger) def update_trigger(self, trigger): self.remove_trigger(trigger) self.add_trigger(trigger) def remove_trigger(self, trigger): trigger_id = trigger['id'] try: job_id = self._jobs[trigger_id] except KeyError: LOG.info('Job not found: %s', trigger_id) return self._scheduler.remove_job(job_id) del self._jobs[trigger_id] def _add_job_to_scheduler(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref] try: util_schema.validate(instance=trigger['parameters'], schema=trigger_type['parameters_schema'], cls=util_schema.CustomValidator, use_default=True, allow_default_none=True) except jsonschema.ValidationError as e: LOG.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) raise # Or should we just return? time_spec = trigger['parameters'] time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone')) time_type = None if trigger_type['name'] == 'st2.IntervalTimer': unit = time_spec.get('unit', None) value = time_spec.get('delta', None) time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone}) elif trigger_type['name'] == 'st2.DateTimer': # Raises an exception if date string isn't a valid one. dat = date_parser.parse(time_spec.get('date', None)) time_type = DateTrigger(dat, timezone=time_zone) elif trigger_type['name'] == 'st2.CronTimer': cron = time_spec.copy() cron['timezone'] = time_zone time_type = CronTrigger(**cron) utc_now = date_utils.get_datetime_utc_now() if hasattr(time_type, 'run_date') and utc_now > time_type.run_date: LOG.warning('Not scheduling expired timer: %s : %s', trigger['parameters'], time_type.run_date) else: self._add_job(trigger, time_type) return time_type def _add_job(self, trigger, time_type, replace=True): try: job = self._scheduler.add_job(self._emit_trigger_instance, trigger=time_type, args=[trigger], replace_existing=replace) LOG.info('Job %s scheduled.', job.id) self._jobs[trigger['id']] = job.id except Exception as e: LOG.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) def _emit_trigger_instance(self, trigger): utc_now = date_utils.get_datetime_utc_now() # debug logging is reasonable for this one. A high resolution timer will end up # trashing standard logs. LOG.debug('Timer fired at: %s. Trigger: %s', str(utc_now), trigger) payload = { 'executed_at': str(utc_now), 'schedule': trigger['parameters'].get('time') } trace_context = TraceContext(trace_tag='%s-%s' % (self._get_trigger_type_name(trigger), trigger.get('name', uuid.uuid4().hex))) self._trigger_dispatcher.dispatch(trigger, payload, trace_context=trace_context) def _get_trigger_type_name(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref] return trigger_type['name'] def _register_timer_trigger_types(self): return trigger_services.add_trigger_models( list(TIMER_TRIGGER_TYPES.values())) ############################################## # Event handler methods for the trigger events ############################################## def _handle_create_trigger(self, trigger): LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.add_trigger(trigger=trigger) def _handle_update_trigger(self, trigger): LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.update_trigger(trigger=trigger) def _handle_delete_trigger(self, trigger): LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.remove_trigger(trigger=trigger) def _sanitize_trigger(self, trigger): sanitized = TriggerAPI.from_model(trigger).to_dict() return sanitized
# ch19/example1.py from datetime import datetime from apscheduler.schedulers.background import BlockingScheduler def tick(): print(f'Tick! The time is: {datetime.now()}') if __name__ == '__main__': scheduler = BlockingScheduler() scheduler.add_job(tick, 'interval', seconds=3) try: scheduler.start() print('Printing in the main thread.') except KeyboardInterrupt: pass scheduler.shutdown()
class St2TimerSensor(Sensor): ''' A timer sensor that uses APScheduler 3.0. ''' def __init__(self, sensor_service=None): self._timezone = 'America/Los_Angeles' # Whatever TZ local box runs in. self._sensor_service = sensor_service self._log = self._sensor_service.get_logger(self.__class__.__name__) self._scheduler = BlockingScheduler(timezone=self._timezone) self._jobs = {} def setup(self): pass def run(self): self._scheduler.start() def cleanup(self): self._scheduler.shutdown(wait=True) def add_trigger(self, trigger): self._add_job_to_scheduler(trigger) def update_trigger(self, trigger): self.remove_trigger(trigger) self.add_trigger(trigger) def remove_trigger(self, trigger): id = trigger['id'] try: job_id = self._jobs[id] except KeyError: self._log.info('Job not found: %s', id) return self._scheduler.remove_job(job_id) def _get_trigger_type(self, ref): pass def _add_job_to_scheduler(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TRIGGER_TYPES[trigger_type_ref] try: jsonschema.validate(trigger['parameters'], trigger_type['parameters_schema']) except jsonschema.ValidationError as e: self._log.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) raise # Or should we just return? time_spec = trigger['parameters'] time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone')) time_type = None if trigger_type['name'] == 'st2.IntervalTimer': unit = time_spec.get('unit', None) value = time_spec.get('delta', None) time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone}) elif trigger_type['name'] == 'st2.DateTimer': # Raises an exception if date string isn't a valid one. dat = date_parser.parse(time_spec.get('date', None)) time_type = DateTrigger(dat, timezone=time_zone) elif trigger_type['name'] == 'st2.CronTimer': cron = time_spec.copy() cron['timezone'] = time_zone time_type = CronTrigger(**cron) if hasattr(time_type, 'run_date') and datetime.now(tzutc()) > time_type.run_date: self._log.warning('Not scheduling expired timer: %s : %s', trigger['parameters'], time_type.run_date) else: self._add_job(trigger, time_type) def _add_job(self, trigger, time_type, replace=True): try: job = self._scheduler.add_job(self._emit_trigger_instance, trigger=time_type, args=[trigger], replace_existing=replace) self._log.info('Job %s scheduled.', job.id) self._jobs[trigger['id']] = job.id except Exception as e: self._log.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) def _emit_trigger_instance(self, trigger): self._log.info('Timer fired at: %s. Trigger: %s', str(datetime.utcnow()), trigger) payload = { 'executed_at': str(datetime.utcnow()), 'schedule': trigger['parameters'].get('time') } self._sensor_service.dispatch(trigger, payload)
class Feeds(Thread): WINDOW = 10_000 def __init__(self, sources, feeds, sleep, logger): Thread.__init__(self) self.sleep = sleep self.logger = logger self.coords = deque([(source.strip(), feed.strip()) for source, feed in zip(sources, feeds)]) self.entries = [] self.last = {feed: [] for _, feed in self.coords} socket.setdefaulttimeout(3) def run(self): job_defaults = {'coalesce': True, 'max_instances': 1} self.blocker = BlockingScheduler(job_defaults=job_defaults) self.blocker.add_job(self.parse_feed, 'cron', second=f'*/{self.sleep}', id='parse_feed') self.blocker.start() def on_close(self): self.blocker.shutdown() self.join() def parse_feed(self): self.coords.rotate() self.source, self.feed = self.coords[0] try: response = feedparser.parse(self.feed) except Exception as e: self.logger.warning(f"Status,{self.source},{self.feed},{e}") return status = response.get('status', None) if not status: self.logger.warning(f"Status,{self.source},{self.feed},None") return if status != 200: self.logger.warning(f"Status,{self.source},{self.feed},{status}") return entries = response.get('entries', None) if not entries: self.logger.warning(f"Entries,{self.source},{self.feed},None") return for entry in entries: _id = entry['id'] if self.source == 'Google' else get_id( entry.copy()) if _id in self.last[self.feed]: continue self.last[self.feed].append(_id) self.last[self.feed] = self.last[self.feed][-self.WINDOW:] entry['acquisition_datetime'] = datetime.utcnow().isoformat()[:19] entry['feed_source'] = self.source entry['_source'] = 'rss' entry['_id'] = _id print(self.source) self.entries.append(entry) if len(self.entries) > 0: with open(f"{DIR}/news_data/{str(uuid.uuid4())}.json", "w") as file: file.write(json.dumps(self.entries)) self.entries = []
class St2Timer(object): """ A timer interface that uses APScheduler 3.0. """ def __init__(self, local_timezone=None): self._timezone = local_timezone self._scheduler = BlockingScheduler(timezone=self._timezone) self._jobs = {} self._trigger_types = TIMER_TRIGGER_TYPES.keys() self._trigger_watcher = TriggerWatcher(create_handler=self._handle_create_trigger, update_handler=self._handle_update_trigger, delete_handler=self._handle_delete_trigger, trigger_types=self._trigger_types, queue_suffix=self.__class__.__name__, exclusive=True) self._trigger_dispatcher = TriggerDispatcher(LOG) def start(self): self._register_timer_trigger_types() self._trigger_watcher.start() self._scheduler.start() def cleanup(self): self._scheduler.shutdown(wait=True) def add_trigger(self, trigger): self._add_job_to_scheduler(trigger) def update_trigger(self, trigger): self.remove_trigger(trigger) self.add_trigger(trigger) def remove_trigger(self, trigger): trigger_id = trigger['id'] try: job_id = self._jobs[trigger_id] except KeyError: LOG.info('Job not found: %s', trigger_id) return self._scheduler.remove_job(job_id) del self._jobs[trigger_id] def _add_job_to_scheduler(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref] try: jsonschema.validate(trigger['parameters'], trigger_type['parameters_schema']) except jsonschema.ValidationError as e: LOG.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) raise # Or should we just return? time_spec = trigger['parameters'] time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone')) time_type = None if trigger_type['name'] == 'st2.IntervalTimer': unit = time_spec.get('unit', None) value = time_spec.get('delta', None) time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone}) elif trigger_type['name'] == 'st2.DateTimer': # Raises an exception if date string isn't a valid one. dat = date_parser.parse(time_spec.get('date', None)) time_type = DateTrigger(dat, timezone=time_zone) elif trigger_type['name'] == 'st2.CronTimer': cron = time_spec.copy() cron['timezone'] = time_zone time_type = CronTrigger(**cron) utc_now = date_utils.get_datetime_utc_now() if hasattr(time_type, 'run_date') and utc_now > time_type.run_date: LOG.warning('Not scheduling expired timer: %s : %s', trigger['parameters'], time_type.run_date) else: self._add_job(trigger, time_type) return time_type def _add_job(self, trigger, time_type, replace=True): try: job = self._scheduler.add_job(self._emit_trigger_instance, trigger=time_type, args=[trigger], replace_existing=replace) LOG.info('Job %s scheduled.', job.id) self._jobs[trigger['id']] = job.id except Exception as e: LOG.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) def _emit_trigger_instance(self, trigger): utc_now = date_utils.get_datetime_utc_now() # debug logging is reasonable for this one. A high resolution timer will end up # trashing standard logs. LOG.debug('Timer fired at: %s. Trigger: %s', str(utc_now), trigger) payload = { 'executed_at': str(utc_now), 'schedule': trigger['parameters'].get('time') } trace_context = TraceContext(trace_tag='%s-%s' % (self._get_trigger_type_name(trigger), trigger.get('name', uuid.uuid4().hex))) self._trigger_dispatcher.dispatch(trigger, payload, trace_context=trace_context) def _get_trigger_type_name(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TIMER_TRIGGER_TYPES[trigger_type_ref] return trigger_type['name'] def _register_timer_trigger_types(self): return trigger_services.add_trigger_models(TIMER_TRIGGER_TYPES.values()) ############################################## # Event handler methods for the trigger events ############################################## def _handle_create_trigger(self, trigger): LOG.debug('Calling "add_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.add_trigger(trigger=trigger) def _handle_update_trigger(self, trigger): LOG.debug('Calling "update_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.update_trigger(trigger=trigger) def _handle_delete_trigger(self, trigger): LOG.debug('Calling "remove_trigger" method (trigger.type=%s)' % (trigger.type)) trigger = self._sanitize_trigger(trigger=trigger) self.remove_trigger(trigger=trigger) def _sanitize_trigger(self, trigger): sanitized = trigger._data if 'id' in sanitized: # Friendly objectid rather than the MongoEngine representation. sanitized['id'] = str(sanitized['id']) return sanitized
class St2TimerSensor(object): ''' A timer sensor that uses APScheduler 3.0. ''' def __init__(self, container_service): self._timezone = 'America/Los_Angeles' # Whatever TZ local box runs in. self._container_service = container_service self._log = self._container_service.get_logger(self.__class__.__name__) self._scheduler = BlockingScheduler(timezone=self._timezone) self._jobs = {} def setup(self): pass def start(self): self._scheduler.start() def stop(self): self._scheduler.shutdown(wait=True) def add_trigger(self, trigger): self._add_job_to_scheduler(trigger) def update_trigger(self, trigger): self.remove_trigger(trigger) self.add_trigger(trigger) def remove_trigger(self, trigger): id = trigger['id'] try: job_id = self._jobs[id] except KeyError: self._log.info('Job not found: %s', id) return self._scheduler.remove_job(job_id) def get_trigger_types(self): return [trigger_type for trigger_type in six.itervalues(TRIGGER_TYPES)] def _get_trigger_type(self, ref): pass def _add_job_to_scheduler(self, trigger): trigger_type_ref = trigger['type'] trigger_type = TRIGGER_TYPES[trigger_type_ref] try: jsonschema.validate(trigger['parameters'], trigger_type['parameters_schema']) except jsonschema.ValidationError as e: self._log.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) raise # Or should we just return? time_spec = trigger['parameters'] time_zone = aps_utils.astimezone(trigger['parameters'].get('timezone')) time_type = None if trigger_type['name'] == 'st2.IntervalTimer': unit = time_spec.get('unit', None) value = time_spec.get('delta', None) time_type = IntervalTrigger(**{unit: value, 'timezone': time_zone}) elif trigger_type['name'] == 'st2.DateTimer': # Raises an exception if date string isn't a valid one. dat = date_parser.parse(time_spec.get('date', None)) time_type = DateTrigger(dat, timezone=time_zone) elif trigger_type['name'] == 'st2.CronTimer': cron = time_spec.copy() cron['timezone'] = time_zone time_type = CronTrigger(**cron) if hasattr(time_type, 'run_date') and datetime.now(tzutc()) > time_type.run_date: self._log.warning('Not scheduling expired timer: %s : %s', trigger['parameters'], time_type.run_date) else: self._add_job(trigger, time_type) def _add_job(self, trigger, time_type, replace=True): try: job = self._scheduler.add_job(self._emit_trigger_instance, trigger=time_type, args=[trigger], replace_existing=replace) self._log.info('Job %s scheduled.', job.id) self._jobs[trigger['id']] = job.id except Exception as e: self._log.error('Exception scheduling timer: %s, %s', trigger['parameters'], e, exc_info=True) def _emit_trigger_instance(self, trigger): self._log.info('Timer fired at: %s. Trigger: %s', str(datetime.utcnow()), trigger) payload = { 'executed_at': str(datetime.utcnow()), 'schedule': trigger['parameters'].get('time') } self._container_service.dispatch(trigger, payload)
class TimerWork(): def __init__(self): self.jobs = [] self.sched = BlockingScheduler() self.sched.add_listener(self.listener, EVENT_JOB_ERROR | EVENT_JOB_EXECUTED) def listener(self, event): if event.exception: print 'JOB :<' else: print 'JOB :>' pass def exit(self): print 'shutdown now' self.sched.shutdown(wait=True) print 'shutdown ok' pass def pause(self): BaseScheduler.pause_job() pass def resume(self): BaseScheduler.remove_job() pass def add_job(self, func, gap_time=0, first_run_time=None, args=None, kwargs=None): ''' 添加定时任务 :param func: 执行函数 :param gap_time: 间隔时间,如果设置小于等于0的值忽略 单位s :param first_run_time: 第一次执行时间 datetime.datetime 如果不设置就按正常处理 :param args: 参数 :param kwargs: 参数 :return: ''' self.jobs.append(func) if type(first_run_time) == types.IntType: first_run_time = datetime.datetime.now() + datetime.timedelta(seconds=first_run_time) if gap_time > 0: if first_run_time: self.sched.add_job(func, 'interval', seconds=gap_time, next_run_time=first_run_time, args=args, kwargs=kwargs) else: self.sched.add_job(func, 'interval', seconds=gap_time, args=args, kwargs=kwargs) else: if first_run_time: self.sched.add_job(func, next_run_time=first_run_time, args=args, kwargs=kwargs) pass else: self.sched.add_job(func, args=args, kwargs=kwargs) pass pass def run(self): if len(self.jobs) > 0: self.sched.start() else: print 'no add jobs , so check'
class Monitor (): def __init__ (self, bay = 2, temperature_ctrl = None, wait_email = 20, wait_T_readout = 30): self._bay = bay self._name = 'bay'+str(bay) self._notifications = True self._wait_email = wait_email self._wait_T_readout = wait_T_readout self._offset = 100 self._pwd = None self._temperature_ctrl = temperature_ctrl if temperature_ctrl: self._Tctrl = temperature_ctrl else: self._Tctrl = None print ("No temperature controller!") self._max_T = 10 self._scheduler = BlockingScheduler() self._scheduler.configure(timezone='UTC') self._scheduler.add_job(self._check_email, 'interval', seconds=self._wait_email) self._scheduler.add_job(self._get_temperature, 'interval', seconds=self._wait_T_readout) def login (self): try: print ("Enter password...") self._pwd = getpass.getpass() self._email = QPLemail.QPLmail(bay=self._bay, password=self._pwd) except: print ("Login failed!") def set_max_temperature (self, T=10): self._max_T = T def set_channel (self, channel): self._channel = channel def _check_email (self): msg_dict = self._email.fetch_unread() for msg in msg_dict: body = msg['body'][0].as_string() #print (msg) sender = msg['mail_from'][0] sender_addr = msg['mail_from'][1] #print (sender) #print (sender_addr) if (body.find ('notifications-off')>0): self._deactivate(sender_addr) elif (body.find ('notifications-on')>0): self._activate(sender_addr) elif (body.find('get-temperature')>0): T = self._get_temperature() # here I need to extract the sender email address, not the name self._email.send (to=[sender_addr], subject='Temperature readout', message='Current temperature: '+str(self._curr_T)+'K') elif (body.find ('send-report')>0): self._send_report() else: print ("None") def _send_alarm_email (self): email_to = ['*****@*****.**', '*****@*****.**'] #email_to = ['*****@*****.**'] self._email.send (to=email_to, subject='Help!', message='Current temperature: '+str(self._curr_T)+'K') print ("ALARM: temperature = "+str(self._curr_T)+ "K. Email sent to: ") print (email_to) def _activate(self, sender): self._notifications = True print ("Notifications activated, as requested by: "+sender) self._email.send (to=['*****@*****.**', sender], subject='Settings change', message='Notifications activated, as requested by '+sender) def _deactivate(self, sender): self._notifications = False print ("Notifications de-activated, as requested by: "+sender) self._email.send (to=['*****@*****.**', sender], subject='Settings change', message='Notifications de-activated, as requested by '+sender) def _get_temperature (self, overrule_notifications=False): self._curr_T = self._temperature_ctrl.get_kelvin(channel = self._channel) #print ("Read temperature: ", self._curr_T) if (self._curr_T>self._max_T): if (self._notifications): self._send_alarm_email() return self._curr_T def _send_report (self): pass def start (self): print('Press Ctrl+C to exit') try: self._scheduler.start() while True: time.sleep(1) except (KeyboardInterrupt, SystemExit): # Not strictly necessary if daemonic mode is enabled but should be done if possible self._scheduler.shutdown()
get_coverage = openweather.run if __name__ == '__main__': cba = timezone('America/Argentina/Cordoba') log.info("Starting Domus core...") jobstores = { 'mysql': SQLAlchemyJobStore(url=JOB_STORAGE) } executors = { 'default': ThreadPoolExecutor(20), } job_defaults = { 'coalesce': False, 'max_instances': 5 } log.info("Starting core...") log.debug("Connecting to job store...") scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=cba) log.debug("Creating Jobs...") scheduler.add_job(cache_data, 'interval', minutes=20, id='data_from_wunderground') scheduler.add_job(get_coverage, 'interval', minutes=5, id='data_from_openwrt') scheduler.add_job(do_forecast, trigger='cron', minute='30', hour='8,13', id='twitting forecast') try: scheduler.start() except (KeyboardInterrupt, SystemExit): print "quitting" scheduler.shutdown(wait=False) pass
class Scheduler: def __init__(self, main_config): self.main_config = main_config self.create_dirs() self.logger = get_logger(main_config['project_name'], file=main_config['logs_dir'], level=main_config['log_level']) self.board = None self.scheduler = BlockingScheduler() self.setup() atexit.register(self._exit) def create_dirs(self): try: Path(self.main_config['logs_dir']).mkdir(parents=True, exist_ok=True) Path(self.main_config['data_dir']).mkdir(parents=True, exist_ok=True) Path(self.main_config['data_dir']).joinpath('sensors/').mkdir( parents=True, exist_ok=True) cameras_config = read_json(self.main_config['cameras_config']) for camera in cameras_config: Path(self.main_config['data_dir'])\ .joinpath('images/' + str(camera['type'] + '_' + str(camera['id'])))\ .mkdir(parents=True, exist_ok=True) except Exception as e: self.logger.error('Error creating file structure!') def setup(self): try: board_scheme = read_json(self.main_config['board']) sensors = read_json(self.main_config['sensors_config']) board = Board(board_scheme, sensors, self.logger) self.board = board except Exception as e: self.logger.warning( 'No board specified in config or some error in Board init') self.logger.warning(str(e)) raise UserWarning(str(e)) for p in self.main_config['pipelines']: pipeline = read_json(p) pipeline_executor = PipelineExecutor( logger=get_logger(self.main_config['project_name'] + '.' + pipeline['name'], file=self.main_config['logs_dir'], level=self.main_config['log_level']), pipeline=pipeline['pipeline'], main_config=self.main_config, pipeline_name=pipeline['name'], board=self.board) self.scheduler.add_job( func=(lambda executor=pipeline_executor: executor.execute()), **pipeline['run_interval']) def start(self): try: self.logger.info(self.main_config['project_name'] + ' started') self.scheduler.start() except Exception as e: self.logger.error('Error starting scheduler!') self.logger.error(str(e)) def _exit(self): self.board.exit() print('EXITING!!!') self.logger.info('System exited normally') self.scheduler.shutdown()
class TaskerDo: ''' schedule template to file generation based on redis ''' ip_list =[] def __init__(self, conf_file): self.get_configuration(conf_file) self.client = redis.Redis(**self.config['settings']['redis']) self.set_logging(self.config['settings']['logfile']) self.sched = BlockingScheduler() self.set_scheduler(self.run, self.config['generate']['schedule']) def set_scheduler(self,job,schedule,args=None): ''' set schedual jobs :param schedule: dict, contain 'name', 'trigger' =[cron|interval|more apscheduler on web] , and params minuets,second, and more on apscheduler :return: None ''' self.sched.add_job(job, args=args,trigger=schedule['trigger'], name=schedule['name'],**schedule['params']) def start(self): self.sched.start() def stop(self): self.sched.shutdown() def run(self): ''' This is where the magic starts and all the jobs been executed. optional load ip_list to redis, get ips from redis -> generate file -> run reload job :param ip_list: optional, list of ip to loan into redis :return: nothing yet ''' if len(self.ip_list) > 0: self.redis_push_ip_list(self.ip_list, expiration=self.config['bad_ip_expiration']) self.ip_list = [] ip_list = self.redis_get_ip_list() logging.info("%s items in database: %s" % (len(ip_list), ",".join(ip_list))) if len(ip_list) > 0: generate = self.config['generate'] status = self.file_generate(template_file=generate['template'],generated_file=generate['file'], ip_list=ip_list) if status: logging.info("configuration file: %s was created"%(generate['file'])) for command in self.config['generate']['reload_commands']: logging.info("executing command: %s" % (command)) try: if self.reload_services(command): logging.info("Success") except CommandFail as e: self.logger.error(e) break else: logging.info('reload not needed') def is_diff(self, content_str, current_file): ''' Compare new version to existing one :param content_str: string, new version content string :param current_file: file, existing file :return: bool, true is different ''' try: with open(current_file) as f: current_str = f.read() except FileNotFoundError: return True if content_str == current_str: return False else: return True def get_configuration(self, conf_file): ''' :param conf_file: yaml configuration file with report data :return: configuration data ''' with open(conf_file) as f: yaml_data = f.read() self.config = yaml.load(yaml_data) def set_logging(self, log_file): self.logger = logging.getLogger() hdlr = logging.FileHandler(log_file) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) self.logger.addHandler(hdlr) self.logger.setLevel(logging.INFO) def file_generate(self, template_file, generated_file, **kwargs): ''' generate application config file using template :param template_file: file, the template file to generate using Jinja2 template engine :param generated_file: file, the file to generate :param kwargs: the argument to send to templates :return: bool status ''' with open(template_file) as t: template_str = t.read() t = Template(template_str) content_str = t.render(**kwargs) is_diff = self.is_diff(content_str, generated_file) if is_diff: return self.write_content(content_str, generated_file) logging.info("New version exist") else: logging.info("No changes in current file, generation not needed") return False def redis_push_ip_list(self, ip_list, expiration): ''' push list of ip's to redis and set expiration :param ip_list: list of strings :param expiration: int second :return: bool status ''' ip_list = [ip for ip in ip_list if not ipaddress.ip_address(ip).is_private] if len(ip_list) == 0: raise ValueError('Empty list supplied') for key in ip_list: _ = self.client.set(key, None, ex=expiration) return True def redis_get_ip_list(self): ''' get the list of ips from redis :return: list ''' ips = self.client.keys('*') return [i.decode() for i in ips] def write_content(self, content_str, generated_file, backup=True): ''' helper to generate_file function, write the data to file and create backup file :param content_str: string, the generated content :param generated_file: :param backup: bool :return: bool, status ''' if backup and os.path.exists(generated_file): date = datetime.now().strftime('%y%m%d%H%M%S') os.rename(generated_file, generated_file + '.' + date) with open(generated_file, 'w') as conf_file: _ = conf_file.write(content_str) return True def reload_services(self, commands_list): ''' list of command_str to execute, stop if one failed :param commands_list: list of shell command_str :return: bool, status ''' for command_str in commands_list: exit_status,output = subprocess.getstatusoutput(command_str) if exit_status > 0: raise CommandFail("Command %s failed with status %s output: %s" % (command_str, exit_status, output)) return True