class SchedRegistry(): def __init__(self): self._sched = BlockingScheduler() monitor_class = map( lambda module: "monitor.sched.sched_collect.%sCollectScheduler" % module.capitalize(), [item for item in GLOBAL_CONFIG.MONITOR_ITEM]) jobs = ModuleLoader.load_modules(monitor_class) for job in jobs: self.add_job(job()) def list_jobs(self): return self._sched.get_jobs() def add_job(self, sched_job): LOG.info("sched_job[%s] add the registry" % sched_job.get_name()) self._sched.add_job(sched_job.run, trigger=sched_job.get_trigger_type(), **sched_job.get_trigger_args()) def sched_start(self): LOG.debug('The scheduler will be start...') self._sched.start() LOG.debug('The scheduler started successfully.') def sched_stop(self): self._sched.shutdown(wait=30)
def run_cronjob(master: bool, crawl: bool, vaildator: bool): """ 执行数据抓取及校验定时任务 :param master: 是否是主节点 :param crawl: 是否运行爬虫 :param vaildator: 是否运行校验器 """ sched = BlockingScheduler() if vaildator and master: sched.add_job(run_validate_pub, 'interval', seconds=PROXY_VALIDATE_TIME) if crawl: sched.add_job( run_crawl, args=[master], trigger='interval', seconds=PROXY_UPDATE_TIME, next_run_time=datetime.datetime.now(), # 立刻执行 max_instances=5) try: if sched.get_jobs(): sched.start() except (KeyboardInterrupt, SystemExit): pass
def create_scheduler(): scheduler = BlockingScheduler(executors={"default": ThreadPoolExecutor()}) scheduler.add_job(todoist_assigned_issues, "interval", minutes=15) scheduler.add_job(todoist_repo_prs, "interval", minutes=15) for job in scheduler.get_jobs(): if isinstance(job.trigger, IntervalTrigger): scheduler.add_job(job.func) return scheduler
def main(): ws = WeiboSche('./account/a.json') sche = BlockingScheduler() sche.add_job(ws.get_forword, 'cron', hour='0-23/2', minute=1) # 获取微博 sche.add_job(ws.forword, 'cron', hour='0-23/1', minute=2) # 转发微博 sche.add_job(ws.send, 'cron', args=('one_word', ), hour=8, minute=1) # 一言 sche.add_job(ws.send, 'cron', args=('weather0', ), hour=6, minute=1) # 早安 sche.add_job(ws.send, 'cron', args=('weather1', ), hour=23, minute=1) # 晚安 sche.add_job(ws.send, 'cron', args=('news', ), hour=9, minute=1) # 每日国际视野 sche.add_job(ws.send, 'cron', args=('history_of_today', ), hour=10, minute=1) # 历史上的今天 for job in sche.get_jobs(): ws.ow.logger.info('add job {} {}'.format(job.name, job.args)) sche.start()
def startCronTask(task, **interval_config): # 定义全局变量scheduler,用于控制定时任务的启动和停止 global scheduler scheduler = BlockingScheduler() scheduler.add_listener(CronTask_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) scheduler._logger = logger logger.info( '==================================== 新的日志分段 ==============================================' ) scheduler.add_job(func=task, trigger='interval', **interval_config, id='push_to_github') logger.info('当前所有定时任务job1:%s', scheduler.get_jobs()) logger.info('定时任务调度器状态1:%s', scheduler.state) scheduler.start()
class SchedulerManager: def __init__(self): if config.ENVIRONMENT == "test": self.scheduler = BackgroundScheduler() else: self.scheduler = BlockingScheduler() def init(self, websites, func): self.scheduler.configure(timezone="utc") for website in websites: self.scheduler.add_job( func, "interval", seconds=website.get("interval"), id=website.get("url"), kwargs={"url": website.get("url"), "regexp_rules": website.get("regexp_rules")}, ) self.scheduler.start() def get_jobs(self): return self.scheduler.get_jobs()
dump() time.sleep(1) keep_files() time.sleep(1) sync() except Exception as e: logging.error(e) pass if __name__ == '__main__': if run_once_immediately: main() job = scheduler.add_job( func=main, trigger=trigger, name='rclone to oss', id='1', replace_existing=True ) if run_immediately: job.modify(next_run_time=datetime.now(tz=scheduler.timezone) + timedelta(seconds=5)) for job in scheduler.get_jobs(): logging.info(f"{job.name}: {job.trigger}") def handler_stop_signals(signum, frame): logging.info(f'shutdown from stop signum: {signum}') scheduler.shutdown() signal.signal(signal.SIGINT, handler_stop_signals) signal.signal(signal.SIGTERM, handler_stop_signals) scheduler.start()
class StandaloneScheduler(BaseScheduler): """Main workers refreshing data from AWS """ name = 'Standalone Scheduler' ns = 'scheduler_standalone' pool = None scheduler = None options = ( ConfigOption('worker_threads', 20, 'int', 'Number of worker threads to spawn'), ConfigOption('worker_interval', 30, 'int', 'Delay between each worker thread being spawned, in seconds'), ) def __init__(self): super().__init__() self.collectors = {} self.auditors = [] self.region_workers = [] self.pool = ProcessPoolExecutor(self.dbconfig.get('worker_threads', self.ns, 20)) self.scheduler = APScheduler( threadpool=self.pool, job_defaults={ 'coalesce': True, 'misfire_grace_time': 30 } ) self.load_plugins() def execute_scheduler(self): # Schedule a daily job to cleanup stuff thats been left around (eip's with no instances etc) self.scheduler.add_job( self.cleanup, trigger='cron', name='cleanup', hour=3, minute=0, second=0 ) # Schedule periodic scheduling of jobs self.scheduler.add_job( self.schedule_jobs, trigger='interval', name='schedule_jobs', seconds=60, start_date=datetime.now() + timedelta(seconds=1) ) # Periodically reload the dbconfiguration self.scheduler.add_job( self.dbconfig.reload_data, trigger='interval', name='reload_dbconfig', minutes=5, start_date=datetime.now() + timedelta(seconds=3) ) self.scheduler.start() def execute_worker(self): """This method is not used for the standalone scheduler.""" print('The standalone scheduler does not have a separate worker model. ' 'Executing the scheduler will also execute the workers') def schedule_jobs(self): current_jobs = { x.name: x for x in self.scheduler.get_jobs() if x.name not in ( 'cleanup', 'schedule_jobs', 'reload_dbconfig' ) } new_jobs = [] start = datetime.now() + timedelta(seconds=1) _, accounts = BaseAccount.search(include_disabled=False) # region Global collectors (non-aws) if CollectorType.GLOBAL in self.collectors: for wkr in self.collectors[CollectorType.GLOBAL]: job_name = 'global_{}'.format(wkr.name) new_jobs.append(job_name) if job_name in current_jobs: continue self.scheduler.add_job( self.execute_global_worker, trigger='interval', name=job_name, minutes=wkr.interval, start_date=start, args=[wkr], kwargs={} ) start += timedelta(seconds=30) # endregion # region AWS collectors aws_accounts = list(filter(lambda x: x.account_type == AWSAccount.account_type, accounts)) for acct in aws_accounts: if CollectorType.AWS_ACCOUNT in self.collectors: for wkr in self.collectors[CollectorType.AWS_ACCOUNT]: job_name = '{}_{}'.format(acct.account_name, wkr.name) new_jobs.append(job_name) if job_name in current_jobs: continue self.scheduler.add_job( self.execute_aws_account_worker, trigger='interval', name=job_name, minutes=wkr.interval, start_date=start, args=[wkr], kwargs={'account': acct.account_name} ) if CollectorType.AWS_REGION in self.collectors: for wkr in self.collectors[CollectorType.AWS_REGION]: for region in AWS_REGIONS: job_name = '{}_{}_{}'.format(acct.account_name, region, wkr.name) new_jobs.append(job_name) if job_name in current_jobs: continue self.scheduler.add_job( self.execute_aws_region_worker, trigger='interval', name=job_name, minutes=wkr.interval, start_date=start, args=[wkr], kwargs={'account': acct.account_name, 'region': region} ) db.session.commit() start += timedelta(seconds=self.dbconfig.get('worker_interval', self.ns, 30)) # endregion # region Auditors start = datetime.now() + timedelta(seconds=1) for wkr in self.auditors: job_name = 'auditor_{}'.format(wkr.name) new_jobs.append(job_name) if job_name in current_jobs: continue if app_config.log_level == 'DEBUG': audit_start = start + timedelta(seconds=5) else: audit_start = start + timedelta(minutes=5) self.scheduler.add_job( self.execute_auditor_worker, trigger='interval', name=job_name, minutes=wkr.interval, start_date=audit_start, args=[wkr], kwargs={} ) start += timedelta(seconds=self.dbconfig.get('worker_interval', self.ns, 30)) # endregion extra_jobs = list(set(current_jobs) - set(new_jobs)) for job in extra_jobs: self.log.warning('Removing job {} as it is no longer needed'.format(job)) current_jobs[job].remove() def execute_global_worker(self, data, **kwargs): try: cls = self.get_class_from_ep(data.entry_point) worker = cls(**kwargs) self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S"))) self.log.info('Starting global {} worker'.format(data.name)) worker.run() except Exception as ex: self.log.exception('Global Worker {}: {}'.format(data.name, ex)) finally: db.session.rollback() self.log.info('Completed run for global {} worker'.format(data.name)) def execute_aws_account_worker(self, data, **kwargs): try: cls = self.get_class_from_ep(data.entry_point) worker = cls(**kwargs) self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S"))) worker.run() except Exception as ex: self.log.exception('AWS Account Worker {}/{}: {}'.format(data.name, kwargs['account'], ex)) finally: db.session.rollback() self.log.info('Completed run for {} worker on {}'.format(data.name, kwargs['account'])) def execute_aws_region_worker(self, data, **kwargs): try: cls = self.get_class_from_ep(data.entry_point) worker = cls(**kwargs) self.log.info('RUN_INFO: {} starting at {} for account {} / region {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), kwargs['account'], kwargs['region'], (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S"))) worker.run() except Exception as ex: self.log.exception('AWS Region Worker {}/{}/{}: {}'.format( data.name, kwargs['account'], kwargs['region'], ex )) finally: db.session.rollback() self.log.info('Completed run for {} worker on {}/{}'.format( data.name, kwargs['account'], kwargs['region'] )) def execute_auditor_worker(self, data, **kwargs): try: cls = self.get_class_from_ep(data.entry_point) worker = cls(**kwargs) self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S"))) worker.run() except Exception as ex: self.log.exception('Auditor Worker {}: {}'.format(data.name, ex)) finally: db.session.rollback() self.log.info('Completed run for auditor {}'.format(data.name)) def cleanup(self): try: self.log.info('Running cleanup tasks') log_purge_date = datetime.now() - timedelta(days=self.dbconfig.get('log_keep_days', 'log', default=31)) db.LogEvent.find(LogEvent.timestamp < log_purge_date) db.session.commit() finally: db.session.rollback()
class SQSScheduler(BaseScheduler): name = 'SQS Scheduler' ns = NS_SCHEDULER_SQS options = ( ConfigOption('queue_region', 'us-west-2', 'string', 'Region of the SQS Queues'), ConfigOption('job_queue_url', '', 'string', 'URL of the SQS Queue for pending jobs'), ConfigOption('status_queue_url', '', 'string', 'URL of the SQS Queue for worker reports'), ConfigOption('job_delay', 2, 'float', 'Time between each scheduled job, in seconds. Can be used to ' 'avoid spiky load during execution of tasks'), ) def __init__(self): """Initialize the SQSScheduler, setting up the process pools, scheduler and connecting to the required SQS Queues""" super().__init__() self.pool = ProcessPoolExecutor(1) self.scheduler = APScheduler( threadpool=self.pool, job_defaults={ 'coalesce': True, 'misfire_grace_time': 30 } ) session = get_local_aws_session() sqs = session.resource('sqs', self.dbconfig.get('queue_region', self.ns)) self.job_queue = sqs.Queue(self.dbconfig.get('job_queue_url', self.ns)) self.status_queue = sqs.Queue(self.dbconfig.get('status_queue_url', self.ns)) def execute_scheduler(self): """Main entry point for the scheduler. This method will start two scheduled jobs, `schedule_jobs` which takes care of scheduling the actual SQS messaging and `process_status_queue` which will track the current status of the jobs as workers are executing them Returns: `None` """ try: # Schedule periodic scheduling of jobs self.scheduler.add_job( self.schedule_jobs, trigger='interval', name='schedule_jobs', minutes=15, start_date=datetime.now() + timedelta(seconds=1) ) self.scheduler.add_job( self.process_status_queue, trigger='interval', name='process_status_queue', seconds=30, start_date=datetime.now() + timedelta(seconds=5), max_instances=1 ) self.scheduler.start() except KeyboardInterrupt: self.scheduler.shutdown() def list_current_jobs(self): """Return a list of the currently scheduled jobs in APScheduler Returns: `dict` of `str`: :obj:`apscheduler/job:Job` """ jobs = {} for job in self.scheduler.get_jobs(): if job.name not in ('schedule_jobs', 'process_status_queue'): jobs[job.name] = job return jobs def schedule_jobs(self): """Schedule or remove jobs as needed. Checks to see if there are any jobs that needs to be scheduled, after refreshing the database configuration as well as the list of collectors and auditors. Returns: `None` """ self.dbconfig.reload_data() self.collectors = {} self.auditors = [] self.load_plugins() _, accounts = BaseAccount.search(include_disabled=False) current_jobs = self.list_current_jobs() new_jobs = [] batch_id = str(uuid4()) batch = SchedulerBatch() batch.batch_id = batch_id batch.status = SchedulerStatus.PENDING db.session.add(batch) db.session.commit() start = datetime.now() + timedelta(seconds=1) job_delay = dbconfig.get('job_delay', self.ns, 0.5) # region Global collectors (non-aws) if CollectorType.GLOBAL in self.collectors: for worker in self.collectors[CollectorType.GLOBAL]: job_name = get_hash(worker) if job_name in current_jobs: continue self.scheduler.add_job( self.send_worker_queue_message, trigger='interval', name=job_name, minutes=worker.interval, start_date=start, kwargs={ 'batch_id': batch_id, 'job_name': job_name, 'entry_point': worker.entry_point, 'worker_args': {} } ) start += timedelta(seconds=job_delay) # endregion # region AWS collectors aws_accounts = list(filter(lambda x: x.account_type == AWSAccount.account_type, accounts)) if CollectorType.AWS_ACCOUNT in self.collectors: for worker in self.collectors[CollectorType.AWS_ACCOUNT]: for account in aws_accounts: job_name = get_hash((account.account_name, worker)) if job_name in current_jobs: continue new_jobs.append(job_name) self.scheduler.add_job( self.send_worker_queue_message, trigger='interval', name=job_name, minutes=worker.interval, start_date=start, kwargs={ 'batch_id': batch_id, 'job_name': job_name, 'entry_point': worker.entry_point, 'worker_args': { 'account': account.account_name } } ) start += timedelta(seconds=job_delay) if CollectorType.AWS_REGION in self.collectors: for worker in self.collectors[CollectorType.AWS_REGION]: for region in AWS_REGIONS: for account in aws_accounts: job_name = get_hash((account.account_name, region, worker)) if job_name in current_jobs: continue new_jobs.append(job_name) self.scheduler.add_job( self.send_worker_queue_message, trigger='interval', name=job_name, minutes=worker.interval, start_date=start, kwargs={ 'batch_id': batch_id, 'job_name': job_name, 'entry_point': worker.entry_point, 'worker_args': { 'account': account.account_name, 'region': region } } ) start += timedelta(seconds=job_delay) # endregion # region Auditors if app_config.log_level == 'DEBUG': audit_start = start + timedelta(seconds=5) else: audit_start = start + timedelta(minutes=5) for worker in self.auditors: job_name = get_hash((worker,)) if job_name in current_jobs: continue new_jobs.append(job_name) self.scheduler.add_job( self.send_worker_queue_message, trigger='interval', name=job_name, minutes=worker.interval, start_date=audit_start, kwargs={ 'batch_id': batch_id, 'job_name': job_name, 'entry_point': worker.entry_point, 'worker_args': {} } ) audit_start += timedelta(seconds=job_delay) # endregion def send_worker_queue_message(self, *, batch_id, job_name, entry_point, worker_args, retry_count=0): """Send a message to the `worker_queue` for a worker to execute the requests job Args: batch_id (`str`): Unique ID of the batch the job belongs to job_name (`str`): Non-unique ID of the job. This is used to ensure that the same job is only scheduled a single time per batch entry_point (`dict`): A dictionary providing the entry point information for the worker to load the class worker_args (`dict`): A dictionary with the arguments required by the worker class (if any, can be an empty dictionary) retry_count (`int`): The number of times this one job has been attempted to be executed. If a job fails to execute after 3 retries it will be marked as failed Returns: `None` """ try: job_id = str(uuid4()) self.job_queue.send_message( MessageBody=json.dumps({ 'batch_id': batch_id, 'job_id': job_id, 'job_name': job_name, 'entry_point': entry_point, 'worker_args': worker_args, }), MessageDeduplicationId=job_id, MessageGroupId=batch_id, MessageAttributes={ 'RetryCount': { 'StringValue': str(retry_count), 'DataType': 'Number' } } ) if retry_count == 0: job = SchedulerJob() job.job_id = job_id job.batch_id = batch_id job.status = SchedulerStatus.PENDING job.data = worker_args db.session.add(job) db.session.commit() except: self.log.exception('Error when processing worker task') def execute_worker(self): """Retrieve a message from the `worker_queue` and process the request. This function will read a single message from the `worker_queue` and load the specified `EntryPoint` and execute the worker with the provided arguments. Upon completion (failure or otherwise) a message is sent to the `status_queue` information the scheduler about the return status (success/failure) of the worker Returns: `None` """ try: try: messages = self.job_queue.receive_messages( MaxNumberOfMessages=1, MessageAttributeNames=('RetryCount',) ) except ClientError: self.log.exception('Failed fetching messages from SQS queue') return if not messages: self.log.debug('No pending jobs') return for message in messages: try: retry_count = int(message.message_attributes['RetryCount']['StringValue']) data = json.loads(message.body) try: # SQS FIFO queues will not allow another thread to get any new messages until the messages # in-flight are returned to the queue or deleted, so we remove the message from the queue as # soon as we've loaded the data self.send_status_message(data['job_id'], SchedulerStatus.STARTED) message.delete() cls = self.get_class_from_ep(data['entry_point']) worker = cls(**data['worker_args']) if hasattr(worker, 'type'): if worker.type == CollectorType.GLOBAL: self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S"))) elif worker.type == CollectorType.AWS_REGION: self.log.info('RUN_INFO: {} starting at {} for account {} / region {}, next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), data['worker_args']['account'], data['worker_args']['region'], (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S"))) elif worker.type == CollectorType.AWS_ACCOUNT: self.log.info('RUN_INFO: {} starting at {} for account {} next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), data['worker_args']['account'], (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S"))) else: self.log.info('RUN_INFO: {} starting at {} next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S"))) worker.run() self.send_status_message(data['job_id'], SchedulerStatus.COMPLETED) except InquisitorError: # If the job failed for some reason, reschedule it unless it has already been retried 3 times if retry_count >= 3: self.send_status_message(data['job_id'], SchedulerStatus.FAILED) else: self.send_worker_queue_message( batch_id=data['batch_id'], job_name=data['job_name'], entry_point=data['entry_point'], worker_args=data['worker_args'], retry_count=retry_count + 1 ) except: self.log.exception('Failed processing scheduler job: {}'.format(message.body)) except KeyboardInterrupt: self.log.info('Shutting down worker thread') @retry def send_status_message(self, object_id, status): """Send a message to the `status_queue` to update a job's status. Returns `True` if the message was sent, else `False` Args: object_id (`str`): ID of the job that was executed status (:obj:`SchedulerStatus`): Status of the job Returns: `bool` """ try: body = json.dumps({ 'id': object_id, 'status': status }) self.status_queue.send_message( MessageBody=body, MessageGroupId='job_status', MessageDeduplicationId=get_hash((object_id, status)) ) return True except Exception as ex: print(ex) return False @retry def process_status_queue(self): """Process all messages in the `status_queue` and check for any batches that needs to change status Returns: `None` """ self.log.debug('Start processing status queue') while True: messages = self.status_queue.receive_messages(MaxNumberOfMessages=10) if not messages: break for message in messages: data = json.loads(message.body) job = SchedulerJob.get(data['id']) try: if job and job.update_status(data['status']): db.session.commit() except SchedulerError as ex: if hasattr(ex, 'message') and ex.message == 'Attempting to update already completed job': pass message.delete() # Close any batch that is now complete open_batches = db.SchedulerBatch.find(SchedulerBatch.status < SchedulerStatus.COMPLETED) for batch in open_batches: open_jobs = list(filter(lambda x: x.status < SchedulerStatus.COMPLETED, batch.jobs)) if not open_jobs: open_batches.remove(batch) batch.update_status(SchedulerStatus.COMPLETED) self.log.debug('Closed completed batch {}'.format(batch.batch_id)) else: started_jobs = list(filter(lambda x: x.status > SchedulerStatus.PENDING, open_jobs)) if batch.status == SchedulerStatus.PENDING and len(started_jobs) > 0: batch.update_status(SchedulerStatus.STARTED) self.log.debug('Started batch manually {}'.format(batch.batch_id)) # Check for stale batches / jobs for batch in open_batches: if batch.started < datetime.now() - timedelta(hours=2): self.log.warning('Closing a stale scheduler batch: {}'.format(batch.batch_id)) for job in batch.jobs: if job.status < SchedulerStatus.COMPLETED: job.update_status(SchedulerStatus.ABORTED) batch.update_status(SchedulerStatus.ABORTED) db.session.commit()
# 精确时间执行 # scheduler.add_job(job1, 'date', run_date=datetime(2020, 8, 8, 16, 30, 5), id='job1') # scheduler.add_job(job1, 'date', run_date=‘2020-8-8 16:30:5’, id='job1') # 每5秒执行一次 scheduler.add_job(job1, 'interval', seconds=5, id='job1') scheduler.add_job(job1, 'interval', seconds=5, id='job2') # 移除任务,得在start()之前 # scheduler.remove_job('job1') # 暂停作业 # scheduler.pause_job('job1') # 恢复作业 # scheduler.resume_job('job1') # 获取作业列表 jobs = scheduler.get_jobs() print(jobs) # 根据id获取某个作业 job = scheduler.get_job('job1') print(job) scheduler.start() # 关闭调度器,加上false表示为不想等待 scheduler.shutdown() scheduler.shutdown(wait=False)
class SchedUtility(object, metaclass=Singleton): def __init__(self): try: self.Global = Global() self.Utility = Utility() self.InfraUtil = InfraUtility() self.db = DBMySql('Scheduler') self.myModulePyFile = os.path.abspath(__file__) self.myClass = self.__class__.__name__ #Setting the infrastructure self.Infra = self.InfraUtil.setInfra(self.Global.SchedulerInfraKey) if not self.Infra: raise InfraInitializationError('Could not initialize {cls}'.format(cls=(self.myModulePyFile,self.myClass))) # we need to get the proper logger for a given module self.logger = self.Infra.getInfraLogger(self.Global.SchedulerInfraKey) # loading Schduler config and starting scheduler self.__startScheduler__() except Exception as err: raise err def __startScheduler__(self): try: mySchedulerType = self.Global.DefaultSchedulerType mySchedulerMode = self.Global.DefaultSchedulerMode if mySchedulerMode == 'Run': myArgPaused = False else: myArgPaused = True #fi mySchedulerConfig = self.Utility.getACopy(self.Infra.schedulerConfigData) if mySchedulerType == 'Background': self.Scheduler = BackgroundScheduler(mySchedulerConfig) else: self.Scheduler = BlockingScheduler(mySchedulerConfig) #fi if not self.Scheduler.running: self.Scheduler.start(paused = myArgPaused) except Exception as err: raise err def getAllJobDetail(self): ''' Description: Returns all jobs as stored in scheduler ''' myJobDetail = [] for job in self.Scheduler.get_jobs(): myJobDetail.append(self.getAJobDetail(job.id)) return myJobDetail def getAJobDetail(self, jobIdArg): ''' Description: Print all jobs as stored in scheduler ''' myJobId = jobIdArg job = self.Scheduler.get_job(myJobId) myJobDetail = job.__getstate__() return myJobDetail def suspendJob(self, jobIdArg): myJobId = jobIdArg job = self.Scheduler.get_job(myJobId) job.pause() def resumeJob(self, jobIdArg): myJobId = jobIdArg job = self.Scheduler.get_job(myJobId) job.resume() def getCurrentlyExecutingJob(self): return len(self.Scheduler.get_jobs()) def removeJob(self, jobId): try: self.Scheduler.remove_job(jobId) except JobLookupError as err: print('Invalid Job !!') def removeAllJobs(self): try: self.Scheduler.remove_all_jobs() except Exception as err: raise err def getAllJobsFromRep(self): for job in self.Scheduler.get_jobs(): myJobDetail = self.Scheduler.get_job(job.id) print(job,myJobDetail) def getNewJob(self,prefixArg): # random number between 10 and 99 to ensure we always get 2 digit if isinstance(prefixArg,str) and prefixArg is not None: return prefixArg + '_' + str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99))) else: return datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99)) def getJobInfoFromDb(self, jobIdArg): try: myResponse = self.Utility.getResponseTemplate() myJobId = self.Utility.getACopy(jobIdArg) self.logger.debug('arg [{arg}] received'.format(arg = myJobId)) myJobCriteria = 'JobId = %s ' %repr(myJobId) return self.db.processDbRequest(operation = self.Global.fetch, container = 'ScheduledJobs', contents = ['*'], criteria = myJobCriteria) except Exception as err: myErrorMsg, myTraceback = self.Utility.getErrorTraceback() self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg) self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback) self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg) return myResponse def getNextSeqForJob(self, jobIdArg): try: myResponse = self.Utility.getResponseTemplate() myJobId = self.Utility.getACopy(jobIdArg) self.logger.debug('arg [{arg}] received'.format(arg = myJobId)) myJobCriteria = 'JobId = %s ' %repr(myJobId) return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) + 1 except Exception as err: myErrorMsg, myTraceback = self.Utility.getErrorTraceback() self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg) self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback) return myErrorMsg def getCurrentSeqForJob(self, jobIdArg): try: myResponse = self.Utility.getResponseTemplate() myJobId = self.Utility.getACopy(jobIdArg) self.logger.debug('arg [{arg}] received'.format(arg = myJobId)) myJobCriteria = 'JobId = %s ' %repr(myJobId) return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) except Exception as err: myErrorMsg, myTraceback = self.Utility.getErrorTraceback() self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg) self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback) return myErrorMsg def getElapsedStatsForJob(self, jobIdArg): try: myResponse = self.Utility.getResponseTemplate() myJobId = self.Utility.getACopy(jobIdArg) self.logger.debug('arg [{arg}] received'.format(arg = myJobId)) myJobCriteria = 'JobId = %s ' %repr(myJobId) return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) except Exception as err: myErrorMsg, myTraceback = self.Utility.getErrorTraceback() self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg) self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback) return myErrorMsg def processJobStartEvent(self, jobIdArg): ''' 1. Mark job started in ScheduledJobs 2. Create new entry for this job in ScheduledJobsRunLog ''' try: # initializing myResponse = self.Utility.getResponseTemplate() myJobId = self.Utility.getACopy(jobIdArg) self.logger.debug('arg [{arg}] received'.format(arg=myJobId)) myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data'] if myJobDetailsFromDb: # building data for SchedulerJobsRunLog myJobCriteria = ' JobId = %s' %repr(myJobId) myNextSeqForJob = self.getNextSeqForJob(myJobId) # will mark the job started and creat the run log for this run self.db.processDbRequest(operation='change', container='ScheduledJobs', \ dataDict={'Status': 'Executing'}, criteria = myJobCriteria, commitWork=True ) # creating run information self.db.processDbRequest(operation='create', container='ScheduledJobsRunLog', \ dataDict={'JobId':myJobId, 'Seq' : myNextSeqForJob, 'ExecutionStarted': self.Utility.getCurrentTime()}, commitWork=True ) self.Utility.buildResponse(myResponse, self.Global.Success, self.Global.Success, {'Seq':myNextSeqForJob}) else: self.Utility.buildResponse(myResponse, self.Global.UnSuccess, 'Cound not find job details for job {job}'.format(job = myJobId)) return myResponse except Exception as err: myErrorMsg, myTraceback = self.Utility.getErrorTraceback() self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg) self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback) self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg) #raise err # will raise the error so this can be logged by scheduler as an error occurred in processing job return myResponse def processJobFinishEvent(self, jobIdArg, execDetailsArg): ''' 1. Mark job completed (update failure cnt and total count and consc fail count, lastrunstatus) in ScheduledJobs 2. Update ScheduledJobsRunlog container ''' try: # initializing myResponse = self.Utility.getResponseTemplate() myJobId = self.Utility.getACopy(jobIdArg) myExecDetails = execDetailsArg myJobStatus = self.Global.NextJobRun self.logger.debug('arg [{arg}] received'.format(arg=myJobId)) myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data'] if myJobDetailsFromDb: self.logger.debug('Job details found, proceeding with finish event') myJobCriteria = 'JobId = %s' %repr(myJobId) myCurrentSeqForJob = self.getCurrentSeqForJob(myJobId) myJobRunCriteria = ' JobId = %s and Seq = %s ' %(repr(myJobId), myCurrentSeqForJob) self.logger.debug('Job criteria {criteria}'.format(criteria = myJobCriteria)) self.logger.debug('Job criteria with seq {criteria}'.format(criteria = myJobRunCriteria)) myJobDetailsFromSched = self.getAJobDetail(myJobId) # Updating execution details in ScheduledJobsRunLog self.logger.debug('udating statistics of this run') myDbResult = self.db.processDbRequest(operation = 'change', container = 'ScheduledJobsRunLog', \ dataDict={ 'Status': myExecDetails['Status'], 'ElapsedSeconds':myExecDetails['Data']['ElapsedSecs'], 'ExecutionCompleted': self.Utility.getCurrentTime(), 'ExecutionDetail': json.dumps(myExecDetails['Data']) }, criteria = myJobRunCriteria, commitWork=True ) self.logger.debug('ScheduledJobsRunLog: db results >> {results}'.format(results = myDbResult)) # Updating execution details in ScheduledJobs #if myExecDetails['Status'] == self.Global.Success: # if success, reset consecfailcnt to 0, increment totalrun by 1 and update next run myElapsedStats = self.db.executeDynamicSql(\ operation = 'fetch', \ sql_text = 'select min(ElapsedSeconds) "Min", max(ElapsedSeconds) "Max", avg(ElapsedSeconds) "Avg" from ScheduledJobsRunLog') self.logger.debug('Elapsed Stats: {stats}'.format(stats = myElapsedStats)) myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \ dataDict={ 'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1, 'NextRun' : myJobDetailsFromSched['next_run_time'].strftime('%Y-%m-%d% %H:%M:%S'), 'LatConsecFailCnt' : 0, 'MinElapsedSecs' : myElapsedStats['Data'][0]['Min'], 'MaxElapsedSecs' : myElapsedStats['Data'][0]['Min'] , 'AvgElapsedSecs' : myElapsedStats['Data'][0]['Avg'] }, criteria = myJobCriteria, commitWork=True ) self.logger.debug('ScheduledJobs: last stats update >> {result}'.format(result = myDbResult)) #self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success) ''' else: # process job was unsuccessful if myJobDetailsFromDb[0]['LatConsecFailCnt'] >= self.Global.SchedConsecFailCntThreshold: myJobStatus = self.Global.SuspendMode self.logger.info('suspending job {job}'.format(job=myJobId)) self.suspendJob(myJobId) myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \ dataDict={ 'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1, 'next_run' : myJobDetailsFromSched['next_run_time'], 'LatConsecFailCnt' : myJobDetailsFromDb[0]['LatConsecFailCnt'] + 1, 'TotalFailure' : myJobDetailsFromDb[0]['TotalFailure' + 1] }, criteria = myJobCriteria, commitWork=True ) # will suspend the job if total failure count has been reached beyond Total consecutive failure threshold self.Utility.buildResponse(myResponse, self.Global.UnSuccess,self.Global.UnSuccess) raise processJobError(myExecDetails['Message']) ''' self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success) return myResponse except Exception as err: myErrorMsg, myTraceback = self.Utility.getErrorTraceback() self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg) self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback) self.Utility.buildResponse(myResponse, self.Global.UnSuccess, myErrorMsg) return myResponse
class AlgoTrade(): def __init__(self, event_engine): self.event_engine = event_engine self._expire_at = datetime.now() self._trade_status = {'status':'close'} self.sched = Scheduler() def daily_jobs(self, data): logger.info('判断当前时候交易日') if data.market_status == 'close': logger.info('当前不是交易日,耐心等待') logger.info(self.sched.get_jobs()) return logger.warning('当前为交易日,准备执行各类交易策略') on_open_event = Event('on_open', data) on_tick_event = Event('on_tick', data) before_trade_event = Event('before_trade', data) on_ipo_event = Event('on_ipo', data) pre_close_event = Event('pre_close', data) on_close_event = Event('on_close', data) self.sched.add_job(self.event_engine.trigger,'date', run_date=data.market_am_open + timedelta(minutes=1), args=[on_open_event], id='on_open' ) self.sched.add_job(self.event_engine.trigger, 'date', run_date=data.market_am_close + timedelta(minutes=randint(30, 90)), args=[on_ipo_event], id='on_ipo' ) self.sched.add_job(self.event_engine.trigger, 'date', run_date=data.market_fm_close - timedelta(minutes=10), args=[pre_close_event], id='pre_close' ) self.sched.add_job(self.event_engine.trigger, 'date', run_date=data.market_fm_close - timedelta(minutes=5), args=[on_close_event], id='on_close' ) self.sched.add_job(self.event_engine.trigger, 'interval', seconds=15, start_date=data.market_am_open + timedelta(minutes=1), end_date=data.market_am_close, args=[on_tick_event], id='on_tick_am' ) self.sched.add_job(self.event_engine.trigger, 'interval', seconds=15, start_date=data.market_fm_open + timedelta(minutes=1), end_date=data.market_fm_close - timedelta(minutes=5), args=[on_tick_event], id='on_tick_fm' ) self.event_engine.trigger(before_trade_event) logger.info(self.sched.get_jobs()) def run(self, data): try: self.event_engine.start() self.sched.add_job(self.daily_jobs, 'cron', day_of_week='0-4', hour=9, minute=27, args=[data]) self.daily_jobs(data) logger.warning('策略启动执行') self.sched.start() except: self.event_engine.stop() self.sched.shutdown(wait=True) logger.warning('策略停止执行成功')
id='test_job1') #定时任务示例 scheduler.add_job(job1, 'cron', second='*/4', args=('定时', ), id='test_job2') #一次性任务示例 scheduler.add_job(job1, next_run_time=(datetime.datetime.now() + datetime.timedelta(seconds=5)), args=('一次', ), id='test_job3') ''' 传递参数的方式有元组(tuple)、列表(list)、字典(dict) 注意:不过需要注意采用元组传递参数时后边需要多加一个逗号 ''' # #基于list # scheduler.add_job(job2, 'interval', seconds=5, args=['a','b','list'], id='test_job4') # #基于tuple # scheduler.add_job(job2, 'interval', seconds=5, args=('a','b','tuple',), id='test_job5') # #基于dict # scheduler.add_job(job3, 'interval', seconds=5, kwargs={'f':'dict', 'a':1,'b':2}, id='test_job6') #带有参数的示例 # scheduler.add_job(job2, 'interval', seconds=5, args=['a','b'], id='test_job7') # scheduler.add_job(job2, 'interval', seconds=5, args=('a','b',), id='test_job8') # scheduler.add_job(job3, 'interval', seconds=5, kwargs={'a':1,'b':2}, id='test_job9') print(scheduler.get_jobs()) scheduler.start()
class Host: def __init__(self, event_queue_id, event_queue_name, policy_storage, log_group=None, metrics=None, output_dir=None): logging.basicConfig(level=logging.INFO, format='%(message)s') log.info("Running Azure Cloud Custodian Self-Host") load_resources() self.session = local_session(Session) # Load configuration self.options = Host.build_options(output_dir, log_group, metrics) self.policy_storage_uri = policy_storage self.event_queue_name = event_queue_name self.event_queue_id = event_queue_id # Prepare storage bits self.policy_blob_client = None self.blob_cache = {} self.queue_storage_account = self.prepare_queue_storage( self.event_queue_id, self.event_queue_name) self.queue_service = None # Track required event subscription updates self.require_event_update = False # Policy cache and dictionary self.policy_cache = tempfile.mkdtemp() self.policies = {} # Configure scheduler self.scheduler = BlockingScheduler() logging.getLogger('apscheduler.executors.default').setLevel( logging.ERROR) # Schedule recurring policy updates self.scheduler.add_job(self.update_policies, 'interval', seconds=policy_update_seconds, id="update_policies", next_run_time=datetime.now()) # Schedule recurring queue polling self.scheduler.add_job(self.poll_queue, 'interval', seconds=queue_poll_seconds, id="poll_queue") self.scheduler.start() def update_policies(self): """ Enumerate all policies from storage. Use the MD5 hashes in the enumerated policies and a local dictionary to decide if we should bother downloading/updating each blob. We maintain an on-disk policy cache for future features. """ if not self.policy_blob_client: self.policy_blob_client = Storage.get_blob_client_by_uri( self.policy_storage_uri, self.session) (client, container, prefix) = self.policy_blob_client try: # All blobs with YAML extension blobs = [ b for b in client.list_blobs(container) if Host.has_yaml_ext(b.name) ] except AzureHttpError as e: # If blob methods are failing don't keep # a cached client self.policy_blob_client = None raise e # Filter to hashes we have not seen before new_blobs = self._get_new_blobs(blobs) # Get all YAML files on disk that are no longer in blob storage cached_policy_files = [ f for f in os.listdir(self.policy_cache) if Host.has_yaml_ext(f) ] removed_files = [ f for f in cached_policy_files if f not in [b.name for b in blobs] ] if not (removed_files or new_blobs): return # Update a copy so we don't interfere with # iterations on other threads policies_copy = self.policies.copy() for f in removed_files: path = os.path.join(self.policy_cache, f) self.unload_policy_file(path, policies_copy) # Get updated YML files for blob in new_blobs: policy_path = os.path.join(self.policy_cache, blob.name) if os.path.exists(policy_path): self.unload_policy_file(policy_path, policies_copy) client.get_blob_to_path(container, blob.name, policy_path) self.load_policy(policy_path, policies_copy) self.blob_cache.update( {blob.name: blob.properties.content_settings.content_md5}) # Assign our copy back over the original self.policies = policies_copy if self.require_event_update: self.update_event_subscriptions() def _get_new_blobs(self, blobs): new_blobs = [] for blob in blobs: md5_hash = blob.properties.content_settings.content_md5 if not md5_hash: blob, md5_hash = self._try_create_md5_content_hash(blob) if blob and md5_hash and md5_hash != self.blob_cache.get( blob.name): new_blobs.append(blob) return new_blobs def _try_create_md5_content_hash(self, blob): # Not all storage clients provide the md5 hash when uploading a file # so, we need to make sure that hash exists. (client, container, _) = self.policy_blob_client log.info("Applying md5 content hash to policy {}".format(blob.name)) try: # Get the blob contents blob_bytes = client.get_blob_to_bytes(container, blob.name) # Re-upload the blob. validate_content ensures that the md5 hash is created client.create_blob_from_bytes(container, blob.name, blob_bytes.content, validate_content=True) # Re-fetch the blob with the new hash hashed_blob = client.get_blob_properties(container, blob.name) return hashed_blob, hashed_blob.properties.content_settings.content_md5 except AzureHttpError as e: log.warning("Failed to apply a md5 content hash to policy {}. " "This policy will be skipped.".format(blob.name)) log.error(e) return None, None def load_policy(self, path, policies): """ Loads a YAML file and prompts scheduling updates :param path: Path to YAML file on disk :param policies: Dictionary of policies to update """ with open(path, "r") as stream: try: policy_config = yaml.safe_load(stream) new_policies = PolicyCollection.from_data( policy_config, self.options) if new_policies: for p in new_policies: log.info("Loading Policy %s from %s" % (p.name, path)) p.validate() policies.update({p.name: {'policy': p}}) # Update periodic and set event update flag policy_mode = p.data.get('mode', {}).get('type') if policy_mode == CONTAINER_TIME_TRIGGER_MODE: self.update_periodic(p) elif policy_mode == CONTAINER_EVENT_TRIGGER_MODE: self.require_event_update = True else: log.warning( "Unsupported policy mode for Azure Container Host: {}. " "{} will not be run. " "Supported policy modes include \"{}\" and \"{}\"." .format(policy_mode, p.data['name'], CONTAINER_EVENT_TRIGGER_MODE, CONTAINER_TIME_TRIGGER_MODE)) except Exception as exc: log.error('Invalid policy file %s %s' % (path, exc)) def unload_policy_file(self, path, policies): """ Unload a policy file that has changed or been removed. Take the copy from disk and pop all policies from dictionary and update scheduled jobs and event registrations. """ with open(path, "r") as stream: try: policy_config = yaml.safe_load(stream) except yaml.YAMLError as exc: log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc)) os.unlink(path) return removed = [ policies.pop(p['name']) for p in policy_config.get('policies', []) ] log.info('Removing policies %s' % removed) # update periodic periodic_names = \ [p['name'] for p in policy_config['policies'] if p.get('mode', {}).get('schedule')] periodic_to_remove = \ [p for p in periodic_names if p in [j.id for j in self.scheduler.get_jobs()]] for name in periodic_to_remove: self.scheduler.remove_job(job_id=name) # update event event_names = \ [p['name'] for p in policy_config['policies'] if p.get('mode', {}).get('events')] if event_names: self.require_event_update = True os.unlink(path) return path def update_periodic(self, policy): """ Update scheduled policies using cron type periodic scheduling. """ trigger = CronTrigger.from_crontab(policy.data['mode']['schedule']) trigger.jitter = jitter_seconds self.scheduler.add_job(self.run_policy, trigger, id=policy.name, name=policy.name, args=[policy, None, None], coalesce=True, max_instances=1, replace_existing=True, misfire_grace_time=20) def update_event_subscriptions(self): """ Find unique list of all subscribed events and update a single event subscription to channel them to an Azure Queue. """ log.info('Updating event grid subscriptions') destination = \ StorageQueueEventSubscriptionDestination(resource_id=self.queue_storage_account.id, queue_name=self.event_queue_name) # Get total unique event list to use in event subscription policy_items = self.policies.items() events_lists = [ v['policy'].data.get('mode', {}).get('events') for n, v in policy_items ] flat_events = [e for l in events_lists if l for e in l if e] resolved_events = AzureEvents.get_event_operations(flat_events) unique_events = set(resolved_events) # Build event filter strings advance_filter = StringInAdvancedFilter(key='Data.OperationName', values=unique_events) event_filter = EventSubscriptionFilter( advanced_filters=[advance_filter]) # Update event subscription AzureEventSubscription.create(destination, self.event_queue_name, self.session.get_subscription_id(), self.session, event_filter) self.require_event_update = False def poll_queue(self): """ Poll the Azure queue and loop until there are no visible messages remaining. """ # Exit if we don't have any policies if not self.policies: return if not self.queue_service: self.queue_service = Storage.get_queue_client_by_storage_account( self.queue_storage_account, self.session) while True: try: messages = Storage.get_queue_messages( self.queue_service, self.event_queue_name, num_messages=queue_message_count, visibility_timeout=queue_timeout_seconds) except AzureHttpError: self.queue_service = None raise if len(messages) == 0: break log.info('Pulled %s events to process while polling queue.' % len(messages)) for message in messages: if message.dequeue_count > max_dequeue_count: Storage.delete_queue_message(self.queue_service, self.event_queue_name, message=message) log.warning( "Event deleted due to reaching maximum retry count.") else: # Run matching policies self.run_policies_for_event(message) # We delete events regardless of policy result Storage.delete_queue_message(self.queue_service, self.event_queue_name, message=message) def run_policies_for_event(self, message): """ Find all policies subscribed to this event type and schedule them for immediate execution. """ # Load up the event event = json.loads(base64.b64decode(message.content).decode('utf-8')) operation_name = event['data']['operationName'] # Execute all policies matching the event type for k, v in self.policies.items(): events = v['policy'].data.get('mode', {}).get('events') if not events: continue events = AzureEvents.get_event_operations(events) if operation_name in events: self.scheduler.add_job(self.run_policy, id=k + event['id'], name=k, args=[v['policy'], event, None], misfire_grace_time=60 * 3) def run_policy(self, policy, event, context): try: policy.push(event, context) except Exception as e: log.error("Exception running policy: %s error: %s", policy.name, e) def prepare_queue_storage(self, queue_resource_id, queue_name): """ Create a storage client using unusual ID/group reference as this is what we require for event subscriptions """ # Use a different session object if the queue is in a different subscription queue_subscription_id = ResourceIdParser.get_subscription_id( queue_resource_id) if queue_subscription_id != self.session.subscription_id: session = Session(queue_subscription_id) else: session = self.session storage_client = session.client( 'azure.mgmt.storage.StorageManagementClient') account = storage_client.storage_accounts.get_properties( ResourceIdParser.get_resource_group(queue_resource_id), ResourceIdParser.get_resource_name(queue_resource_id)) Storage.create_queue_from_storage_account(account, queue_name, self.session) return account @staticmethod def build_options(output_dir=None, log_group=None, metrics=None): """ Initialize the Azure provider to apply global config across all policy executions. """ if not output_dir: output_dir = tempfile.mkdtemp() log.warning( 'Output directory not specified. Using directory: %s' % output_dir) config = Config.empty(**{ 'log_group': log_group, 'metrics': metrics, 'output_dir': output_dir }) return Azure().initialize(config) @staticmethod def has_yaml_ext(filename): return filename.lower().endswith(('.yml', '.yaml'))
class VpsScheduler(object): def __init__(self): self.log = logging.getLogger("Main.VpsScheduler") self.interface = salt_runner.VpsHerder() self.sched = BlockingScheduler({ 'apscheduler.job_defaults.coalesce': 'true', 'apscheduler.timezone': 'UTC', }) self.sched.add_job(self.ensure_active_workers, 'interval', seconds=60) self.install_destroyer_jobs() def create_vm(self, vm_name): vm_idx = int(vm_name.split("-")[-1]) - 1 self.log.info("Creating VM named: %s, index: %s", vm_name, vm_idx) try: self.interface.make_client(vm_name) self.interface.configure_client(vm_name, vm_idx) self.log.info("VM %s created.", vm_name) except marshaller_exceptions.VmCreateFailed: self.log.info("Failure instantiating VM %s.", vm_name) self.destroy_vm(vm_name) def destroy_vm(self, vm_name): self.log.info("Destroying VM named: %s", vm_name) self.interface.destroy_client(vm_name) self.log.info("VM %s destroyed.", vm_name) def build_target_vm_list(self): workers = [] for x in range(settings.VPS_ACTIVE_WORKERS): # start VPS numbers at 1 # Mostly for nicer printing workers.append(VPS_NAME_FORMAT.format(number=x + 1)) assert len(set(workers)) == len(workers), "Duplicate VPS target names!" return set(workers) def get_active_vms(self): # workers = ['scrape-worker-1', 'scrape-worker-2', 'scrape-worker-a', 'scrape-worker-5', 'utility'] workers = self.interface.list_nodes() ret = [ worker for worker in workers if worker.startswith('scrape-worker') ] assert len(set(ret)) == len(ret), "VPS instances with duplicate names!" return set(ret) def ensure_active_workers(self): self.log.info("Validating active VPSes") active = self.get_active_vms() target = self.build_target_vm_list() self.log.info("Active managed VPSes: %s", active) self.log.info("Target VPS set: %s", target) missing = target - active extra = active - target self.log.info("Need to create VMs: %s", missing) self.log.info("Need to destroy VMs: %s", extra) for vm_name in extra: self.destroy_vm(vm_name) for vm_name in missing: self.create_vm(vm_name) existing = self.sched.get_jobs() tznow = datetime.datetime.now(tz=pytz.utc) for job in existing: self.log.info(" %s, %s", job, job.args) def install_destroyer_jobs(self): # vms = self.get_active_vms() vms = self.build_target_vm_list() hours = time.time() / (60 * 60) restart_interval = settings.VPS_LIFETIME_HOURS / settings.VPS_ACTIVE_WORKERS basetime = time.time() basetime = basetime - (basetime % hrs_to_sec(settings.VPS_LIFETIME_HOURS)) print(hours % settings.VPS_LIFETIME_HOURS, restart_interval, basetime) for vm in vms: vm_num = int(vm.split("-")[-1]) start_offset = vm_num * restart_interval nextrun = basetime + hrs_to_sec(start_offset) # Don't schedule a destruction before we start the scheduler. if nextrun + 120 < time.time(): nextrun += hrs_to_sec(settings.VPS_LIFETIME_HOURS) self.sched.add_job(self.destroy_vm, trigger='interval', args=(vm, ), seconds=hrs_to_sec(settings.VPS_LIFETIME_HOURS), next_run_time=datetime.datetime.fromtimestamp( nextrun, tz=pytz.utc)) print("Item nextrun: ", nextrun, nextrun - time.time()) def run(self): self.sched.start()
}, { "delete": { "_id": "system-log" } }, { "delete": { "_id": "SSFirewall-log" } }, { "delete": { "_id": "All-log" } }] print printjson( es.bulk(index=".kibana", doc_type="select-buffter", body=actions))[1] @sched.scheduled_job('cron', day="*", hour=2, minute=0, second=0) def main(): insert_kibana('.kibana') if __name__ == '__main__': #print printjson(es.info())[1] #print es.ping() #insert_kibana('.kibana') #main() #delete_kibana() print sched.get_jobs() sched.start()
acticle_links = self.parser.parse_list(wechat_url, html_cont) if acticle_links == None: break for link in acticle_links: html = self.downloader.download_articles_ph(link) data = self.parser.parse_article(html) #解析出文本 if data == None: continue (title, date, content, readNum, praise_num, discuss_content, discuss_praise) = data # self.urls.add_new_urls(new_urls) # self.outputer.collect_data(data) self.outputer.output_file(full_path, data) if __name__ == "__main__": logging.basicConfig() sched = BlockingScheduler() sched.add_job(job_period, 'cron', start_date='2016-09-01', hour=0, minute=0, second=1, end_date='2016-11-30') a = sched.get_jobs() print(a) sched.start() # job_period()
start_date='2020-03-30 00:00:00') scheduler.add_job(func=invmbBoxSizeWork, name='INVMBBoxSize', id='INVMBBoxSize', trigger='interval', weeks=1, hours=2, start_date='2020-03-30 00:00:00') scheduler.add_job(func=getBomListWork, name='GetBomList', id='GetBomList', trigger='interval', weeks=1, hours=3, start_date='2020-03-30 00:00:00') try: loggerMain.logger.warning('Main_定时任务开始') loggerMain.logger.warning('Main_现有任务:{}'.format(scheduler.get_jobs())) loggerMain.logger.warning('Main_执行需优先处理任务') autoPlanWork() scheduler.start() except (KeyboardInterrupt, SystemExit): scheduler.shutdown() loggerMain.logger.warning('Main_定时任务关闭') except Exception as e: loggerMain.logger.error('Main:{}'.format(str(e)))
def tick(): page(1) time.sleep(30) Run_Algebra.run_algebra() print( ''' ************************************************************************************ ************************************************************************************ ***********************************定时任务运行成功*********************************** ************************************************************************************ ************************************************************************************ ''', "***************************************** 现在的时间是: %s" % datetime.now()) if __name__ == '__main__': scheduler = BlockingScheduler() scheduler.add_job(tick, 'interval', seconds=300) print('按 Ctrl+{0} 退出'.format('Break' if os.name == 'nt' else 'C ')) try: scheduler.start() print(scheduler.get_jobs()) print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) except (KeyboardInterrupt, SystemExit, BaseException) as e: scheduler.shutdown(wait=False) Send_mail.send_mail(subject="警报:‘Timed_Task’ 模块错误,请以 nohup 重新运行程序", content_text=e + scheduler.get_jobs(), attachments=None) print('退出工作!')
def run_job(): scheduler = BlockingScheduler() scheduler.add_job(hello_job, "interval", seconds=5) print scheduler.get_jobs() scheduler.start()
# some convenience variables one = timedelta(seconds=1) three = timedelta(seconds=3) five = timedelta(seconds=5) cam = Camera() cron = BlockingScheduler() #schedule_alerts(eclipse) with open('mtjefferson.csv', 'r') as csvfile: reader = csv.DictReader(csvfile, delimiter=',') for row in reader: #print(row) cron.add_job(take_picture, 'date', next_run_time=row['start'], kwargs=row) for job in cron.get_jobs(): print(job) try: print("begin at: %s" % (datetime.now())) cron.start() except (KeyboardInterrupt, SystemExit): GPIO.cleanup() cam.exit() pass
def get_jobs(): scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) jobs_array = scheduler.get_jobs() for job_info in jobs_array: print job_info.id, job_info.name, job_info.next_run_time
class JobManage(): def __init__(self): jobstores = {'default': MemoryJobStore()} executors = { 'default': ThreadPoolExecutor(50) # 'processpool': ProcessPoolExecutor(3) } job_defaults = {'coalesce': False, 'max_instances': 50} self.sched = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) self.addError() self.addJobExecuted() def addJob(self, func, jobId=None, cron=None, args=[], kwargs={}): ''' 只支持cron的形式 * * * * * command 分 时 日 月 周 命令 第1列表示分钟1~59 每分钟用*或者 */1表示 第2列表示小时1~23(0表示0点) 第3列表示日期1~31 第4列表示月份1~12 第5列标识号星期0~6(0表示星期天) 第6列要运行的命令 ''' if cron is None: raise Exception("cron cannot be Null") (minute, hour, day, month, week) = cron.split(" ") self.sched.add_job(func, trigger='cron', id=jobId, hour=hour, minute=minute, day=day, month=month, week=week, args=args, kwargs=kwargs) def removeJob(self, jobId): self.sched.remove_job(jobId) def start(self): self.sched.start() def shutdown(self): self.sched.shutdown() def printJobs(self): self.sched.print_jobs() def getJobs(self): return self.sched.get_jobs() def addError(self, func=None): if func is None: func = self.listener self.sched.add_listener(func, EVENT_JOB_ERROR) def addJobExecuted(self, func=None): if func is None: func = self.listener self.sched.add_listener(func, EVENT_JOB_EXECUTED) def listener(self, event): if event.exception: log.error("任务【%s】 任务出错 : %s" % (event.job_id, event.traceback)) else: log.debug("任务【%s】已经跑完,结束时间 : %s " % (event.job_id, getNow())) # jobMange = JobManage()
class SpiderSchedule(object): """ 爬虫调度,不同的数据源,采取不同的爬取策略 """ def __init__(self): self.log = logging.getLogger("proxy.spider") self.sched = BlockingScheduler() self.client = RedisClient(host=REDIS['host'], port=REDIS['port'], db=REDIS['db'], password=REDIS['password'], max_conns=REDIS['max_conns']) self._config_schedule() def _config_schedule(self): """ 配置任务 :return: """ for crawl in crawl_list: # 是否可用 if not crawl["enable"]: continue self.log.info("添加job:{}".format(crawl["name"])) #执行方式,是间隔时间,还是定时任务 if "interval" in crawl: d = crawl["interval"] self.sched.add_job(self._spider, "interval", [crawl["name"]], **d) elif "cron" in crawl: d = crawl["cron"] self.sched.add_job(self._spider, "cron", [crawl["name"]], **d) def _spider(self, name): """ 爬虫实现 :param name: :return: """ self.log.info("爬取源:{}".format(name)) crawl_conf = get_crawl_by_name(name) for url in crawl_conf["urls"]: # 延时下载 time.sleep(crawl_conf.get("delay", None) or DOWNLOAD_DELAY) content = Downloader.download(url, timeout=config.DOWNLOAD_TIMEOUT, retries=config.DOWNLOAD_RETRIES) if content is None: self.log.error("download失败,url:" + url) continue #解析页面 proxy_list = HtmlParser().parse(url, content, crawl_conf) #保存proxy self._save(proxy_list, crawl_conf) def _save(self, proxy_list, crawl_conf): self.client.lpushlist(QUEUE_NAME, proxy_list) def run(self): try: # 判断是否有job jobs = self.sched.get_jobs() if len(jobs) == 0: self.log.error("当前jobs为0") return self.sched.start() except Exception: self.log.error("执行调度任务失败")
def test_EndpointStatuses(mocker): """Tests the main workflow of endpointStatuses. We mock away at two places, gordo_components.watchman.endpoints_status.watch_for_model_server_service which listens for kubernetes events is mocked away, and we construct the events manually. And the job-scheduler is never actually started, so we dont proceed to do any of the jobs, we just check that they are added/removed as desired. """ # We will never call start on the scheduler, so we wont actually do any of the # scheduled jobs| scheduler = BlockingScheduler() project_name = "super_project" project_version = "101" namespace = "somenamespace" host = "localhost" target_names = ["target 1", "target 2"] mocked_watch = mocker.patch( "gordo_components.watchman.endpoints_status.watch_for_model_server_service" ) eps = EndpointStatuses( scheduler=scheduler, project_name=project_name, ambassador_host=host, model_names=target_names, project_version=project_version, namespace=namespace, ) assert namespace == mocked_watch.call_args[1]["namespace"] assert project_name == mocked_watch.call_args[1]["project_name"] assert project_version == mocked_watch.call_args[1]["project_version"] event_handler = mocked_watch.call_args[1]["event_handler"] # Before receiving any events we only have the targets in `target_names` cur_status = eps.statuses() assert set([ep["target"] for ep in cur_status]) == set(target_names) # And none of them are healthy assert all([ep["healthy"] is False for ep in cur_status]) # Lets start adding some events. # Target 1 is online! # We make a fake event mock_event_obj = MagicMock() mock_event_obj.metadata = MagicMock() mock_event_obj.metadata.labels = { "applications.gordo.equinor.com/model-name": "target 1" } # And we let the caller know about it event_handler({"type": "ADDED", "object": mock_event_obj}) # The job to update target 1 is added to the joblist jobs = scheduler.get_jobs() assert len(jobs) == 1 assert scheduler.get_job("update_model_metadata_target 1") is not None # Target 2 is up as well mock_event_obj.metadata.labels[ "applications.gordo.equinor.com/model-name"] = "target 2" event_handler({"type": "ADDED", "object": mock_event_obj}) jobs = scheduler.get_jobs() assert len(jobs) == 2 assert scheduler.get_job("update_model_metadata_target 2") is not None # Oida, target 1 seems to be removed! mock_event_obj.metadata.labels = { "applications.gordo.equinor.com/model-name": "target 1" } event_handler({"type": "DELETED", "object": mock_event_obj}) jobs = scheduler.get_jobs() assert len(jobs) == 1 assert scheduler.get_job("update_model_metadata_target 1") is None assert scheduler.get_job("update_model_metadata_target 2") is not None
class VpsScheduler(object): def __init__(self): self.log = logging.getLogger("Main.VpsScheduler") self.interface = salt_runner.VpsHerder() self.sched = BlockingScheduler({ 'apscheduler.job_defaults.coalesce': 'true', 'apscheduler.timezone': 'UTC', }) self.stats_con = statsd.StatsClient( host=settings.GRAPHITE_DB_IP, port=8125, prefix='ReadableWebProxy.VpsHerder', ) self.sched.add_job(poke_statsd, 'interval', seconds=60) self.sched.add_job(self.ensure_active_workers, 'interval', seconds=60 * 5) self.install_destroyer_jobs() def create_vm(self, vm_name): vm_idx = int(vm_name.split("-")[-1]) - 1 provider = "unknown" self.log.info("Creating VM named: %s, index: %s", vm_name, vm_idx) try: # VM Create time is 30 minutes, max with stopit.ThreadingTimeout(60 * 30, swallow_exc=False): # This is slightly horrible. provider, kwargs = self.interface.generate_conf() with self.stats_con.timer("VM-Creation-{}".format(provider)): client_make = self.interface.make_client( vm_name, provider, kwargs) self.log.info("Client make result: %s", client_make) client_conf = self.interface.configure_client( vm_name, vm_idx) self.log.info("Client conf result: %s", client_conf) self.log.info("VM %s created.", vm_name) CREATE_WATCHDOG.value += 1 self.stats_con.incr( "vm-create.{provider}.ok".format(provider=provider)) except stopit.TimeoutException: self.log.error("Timeout instantiating VM %s.", vm_name) self.stats_con.incr( "vm-create.{provider}.fail.timeout".format(provider=provider)) for line in traceback.format_exc().split("\n"): self.log.error(line) for _ in range(5): self.destroy_vm(vm_name) time.sleep(2.5) return except marshaller_exceptions.LocationNotAvailableResponse: self.log.warning("Failure instantiating VM %s.", vm_name) self.stats_con.incr( "vm-create.{provider}.fail.locationnotavilable".format( provider=provider)) for line in traceback.format_exc().split("\n"): self.log.warning(line) for _ in range(5): self.destroy_vm(vm_name) time.sleep(2.5) return except marshaller_exceptions.InvalidDeployResponse: self.log.warning("Failure instantiating VM %s.", vm_name) self.stats_con.incr( "vm-create.{provider}.fail.invaliddeployresponse".format( provider=provider)) for line in traceback.format_exc().split("\n"): self.log.warning(line) for _ in range(5): self.destroy_vm(vm_name) time.sleep(2.5) return except marshaller_exceptions.InvalidExpectParameter: self.log.warning("Failure instantiating VM %s.", vm_name) self.stats_con.incr( "vm-create.{provider}.fail.invalidexpectparameter".format( provider=provider)) for line in traceback.format_exc().split("\n"): self.log.warning(line) for _ in range(5): self.destroy_vm(vm_name) time.sleep(2.5) return except marshaller_exceptions.VmCreateFailed: self.log.warning("Failure instantiating VM %s.", vm_name) self.stats_con.incr( "vm-create.{provider}.fail.vmcreatefailed".format( provider=provider)) for line in traceback.format_exc().split("\n"): self.log.warning(line) for _ in range(5): self.destroy_vm(vm_name) time.sleep(2.5) return except Exception as e: self.stats_con.incr( "vm-create.{provider}.fail.unknown-error".format( provider=provider)) self.log.error("Unknown failure instantiating VM %s!", vm_name) for line in traceback.format_exc().split("\n"): self.log.error(line) for _ in range(5): self.destroy_vm(vm_name) time.sleep(2.5) return self.log.info("VM Creation complete.") def destroy_vm(self, vm_name): self.interface.list_nodes() self.log.info( "Destroying VM named: %s, current workers: %s", vm_name, [worker for provider, worker in self.interface.list_nodes()]) dest_cnt = 0 while vm_name in [ worker for provider, worker in self.interface.list_nodes() ]: self.interface.destroy_client(vm_name) dest_cnt += 1 self.log.info("VM %s destroyed (%s calls).", vm_name, dest_cnt) self.interface.list_nodes() def build_target_vm_list(self): workers = [] for x in range(settings.VPS_ACTIVE_WORKERS): # start VPS numbers at 1 # Mostly for nicer printing workers.append(VPS_NAME_FORMAT.format(number=x + 1)) assert len(set(workers)) == len( workers), "Duplicate VPS target names: %s!" % workers return set(workers) def get_active_vms(self): # workers = ['scrape-worker-1', 'scrape-worker-2', 'scrape-worker-a', 'scrape-worker-5', 'utility'] workers = self.interface.list_nodes() ret = [] active_each = {} for provider, worker in workers: if worker.startswith('scrape-worker'): ret.append(worker) active_each.setdefault(provider, []).append(worker) workers = [ worker for provider, worker in workers if worker.startswith('scrape-worker') ] if len(set(workers)) != len(workers): self.log.error("Duplicate VPS target names in active workers: %s!", workers) for worker in set(workers): if workers.count(worker) > 1: self.log.info("Destroying VM: ") self.destroy_vm(worker) return set(workers) def worker_lister(self): ''' Maximally dumb function to kick over the stats system. ''' self.get_active_vms() def ensure_active_workers(self): self.log.info("Validating active VPSes") active = self.get_active_vms() self.log.info("Active nodes:") for node_tmp in active: self.log.info(" %s", node_tmp) target = self.build_target_vm_list() # Whoo set math! missing = target - active extra = active - target self.log.info("Active managed VPSes: %s", active) self.log.info("Target VPS set : %s", target) self.log.info("Need to create VMs : %s", missing) self.log.info("Need to destroy VMs : %s", extra) for vm_name in extra: self.destroy_vm(vm_name) self.interface.list_nodes() for vm_name in missing: self.create_vm(vm_name) self.interface.list_nodes() existing = self.sched.get_jobs() for job in existing: self.log.info(" %s, %s", job, job.args) def install_destroyer_jobs(self): # vms = self.get_active_vms() vms = self.build_target_vm_list() hours = time.time() / (60 * 60) # The lifetime value needs to be a float. settings.VPS_LIFETIME_HOURS = float(settings.VPS_LIFETIME_HOURS) restart_interval = settings.VPS_LIFETIME_HOURS / settings.VPS_ACTIVE_WORKERS basetime = time.time() basetime = basetime - (basetime % hrs_to_sec(settings.VPS_LIFETIME_HOURS)) self.log.info( "VPS Lifetime (hours): %s. Step interval: %s. Modulo start-time: %s", hours % settings.VPS_LIFETIME_HOURS, restart_interval, basetime) for vm in vms: vm_num = int(vm.split("-")[-1]) start_offset = vm_num * restart_interval nextrun = basetime + hrs_to_sec(start_offset) # Don't schedule a destruction before we start the scheduler. if nextrun + 120 < time.time(): nextrun += hrs_to_sec(settings.VPS_LIFETIME_HOURS) self.sched.add_job(self.destroy_vm, trigger='interval', args=(vm, ), seconds=hrs_to_sec(settings.VPS_LIFETIME_HOURS), next_run_time=datetime.datetime.fromtimestamp( nextrun, tz=pytz.utc)) self.log.info("VM %s next run: %s (in %s seconds): ", vm, nextrun, nextrun - time.time()) def run(self): self.sched.start()
# Run ping to a specific server url to keep it alive (awake) def ping(): pingurl = os.environ.get('DOMAIN_URL') + '/ping' # TODO: Add error handling try: r = requests.get(pingurl) except: print("Unexpected error:", sys.exc_info()) else: print('Ping {} at {}: {}'.format( pingurl, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), r.status_code)) sched = BlockingScheduler() if os.environ.get('KEEP_ALIVE', '0') == "1": try: every = int(os.environ.get('PING_EVERY_X_MINUTES', '15')) except: every = 15 print('Job: Add keepalive, running every {} minutes'.format(every)) sched.add_job(ping, 'interval', minutes=every) if len(sched.get_jobs()) > 0: sched.start()
class abstract_schedule(metaclass=ABCMeta): def __init__(self): self.sched = BlockingScheduler() def lstCronJob(self, job_id=None): result = {} if not job_id: jobs = self.sched.get_jobs() for j in jobs: result[j.id] = j else: jobs = self.sched.get_job(job_id) result[job_id] = jobs return result def delCronJob(self, job_id): jobs = self.lstCronJob(job_id) if not jobs: sys.stdout.write("Job %s not found" %job_id) else: self.sched.remove_job(job_id) sys.stdout.write("Job %s 删除成功!"%job_id) return True def addCronJob(self, job_id, func, policy, args): cron = CronTrigger(**policy) self.sched.add_job(func, cron, args=args, id=job_id) def start(self): print("123123") self.sched.add_job(self.autoAddJob, IntervalTrigger(seconds=5), id="autoAddJob") self.sched.start() def autoAddJob(self): history_jobs = self.lstCronJob() print(history_jobs, 'history_jobs') current_jobs = self.getBackupPolicy() print(current_jobs, 'current_jobs') only_current_jobs = set(current_jobs.keys()).difference(set(history_jobs.keys())) print(only_current_jobs, 'only_current_jobs') # 当前任务调度列表中有的 历史任务列表中没有的 only_history_jobs = set(history_jobs.keys()).difference(set(current_jobs.keys())) print(only_history_jobs, 'only_history_jobs') #历史任务中有的当前任务列表中没有的任务 # for j in only_history_jobs: if j == 'autoAddJob': continue self.delCronJob(job_id=j) for j in only_current_jobs: func = current_jobs[j].pop('func') args = current_jobs[j].pop('args') policy = current_jobs[j] self.addCronJob(job_id=j, func=func, policy=policy, args=args) @abstractmethod def getBackupPolicy(self): pass
def Create_Scheduler(x): global i i = 0 for u in x: temp = datetime.strptime(u['schedule_time'], '%H:%M:%S') #format = '%H:%M:%S' sched.add_job(task, 'cron', hour=temp.hour, minute=temp.minute, id=str(i), kwargs={ "a": u['Tag'], "b": u['food_amount'] }) i += 1 def delete_Create_Scheduler(x): for j in i: sched.remove_job(str(j)) if __name__ == "__main__": x = Get_Scheduler() Create_Scheduler(x) print(sched.get_jobs()) sched.start()
class Host: def __init__(self, storage_id, queue_name, policy_uri, log_group=None, metrics=None, output_dir=None): logging.basicConfig(level=logging.INFO, format='%(message)s') log.info("Running Azure Cloud Custodian Self-Host") resources.load_available() self.session = local_session(Session) self.storage_session = self.session storage_subscription_id = ResourceIdParser.get_subscription_id(storage_id) if storage_subscription_id != self.session.subscription_id: self.storage_session = Session(subscription_id=storage_subscription_id) # Load configuration self.options = Host.build_options(output_dir, log_group, metrics) self.policy_storage_uri = policy_uri self.event_queue_id = storage_id self.event_queue_name = queue_name # Default event queue name is the subscription ID if not self.event_queue_name: self.event_queue_name = self.session.subscription_id # Prepare storage bits self.policy_blob_client = None self.blob_cache = {} self.queue_storage_account = self.prepare_queue_storage( self.event_queue_id, self.event_queue_name) self.queue_service = None # Register event subscription self.update_event_subscription() # Policy cache and dictionary self.policy_cache = tempfile.mkdtemp() self.policies = {} # Configure scheduler self.scheduler = BlockingScheduler(Host.get_scheduler_config()) logging.getLogger('apscheduler.executors.default').setLevel(logging.ERROR) logging.getLogger('apscheduler').setLevel(logging.ERROR) # Schedule recurring policy updates self.scheduler.add_job(self.update_policies, 'interval', seconds=policy_update_seconds, id="update_policies", next_run_time=datetime.now(), executor='threadpool') # Schedule recurring queue polling self.scheduler.add_job(self.poll_queue, 'interval', seconds=queue_poll_seconds, id="poll_queue", executor='threadpool') self.scheduler.start() def update_policies(self): """ Enumerate all policies from storage. Use the MD5 hashes in the enumerated policies and a local dictionary to decide if we should bother downloading/updating each blob. We maintain an on-disk policy cache for future features. """ if not self.policy_blob_client: self.policy_blob_client = Storage.get_blob_client_by_uri(self.policy_storage_uri, self.storage_session) (client, container, prefix) = self.policy_blob_client try: # All blobs with YAML extension blobs = [b for b in client.list_blobs(container) if Host.has_yaml_ext(b.name)] except AzureHttpError as e: # If blob methods are failing don't keep # a cached client self.policy_blob_client = None raise e # Filter to hashes we have not seen before new_blobs = self._get_new_blobs(blobs) # Get all YAML files on disk that are no longer in blob storage cached_policy_files = [f for f in os.listdir(self.policy_cache) if Host.has_yaml_ext(f)] removed_files = [f for f in cached_policy_files if f not in [b.name for b in blobs]] if not (removed_files or new_blobs): return # Update a copy so we don't interfere with # iterations on other threads policies_copy = self.policies.copy() for f in removed_files: path = os.path.join(self.policy_cache, f) self.unload_policy_file(path, policies_copy) # Get updated YML files for blob in new_blobs: policy_path = os.path.join(self.policy_cache, blob.name) if os.path.exists(policy_path): self.unload_policy_file(policy_path, policies_copy) elif not os.path.isdir(os.path.dirname(policy_path)): os.makedirs(os.path.dirname(policy_path)) client.get_blob_to_path(container, blob.name, policy_path) self.load_policy(policy_path, policies_copy) self.blob_cache.update({blob.name: blob.properties.content_settings.content_md5}) # Assign our copy back over the original self.policies = policies_copy def _get_new_blobs(self, blobs): new_blobs = [] for blob in blobs: md5_hash = blob.properties.content_settings.content_md5 if not md5_hash: blob, md5_hash = self._try_create_md5_content_hash(blob) if blob and md5_hash and md5_hash != self.blob_cache.get(blob.name): new_blobs.append(blob) return new_blobs def _try_create_md5_content_hash(self, blob): # Not all storage clients provide the md5 hash when uploading a file # so, we need to make sure that hash exists. (client, container, _) = self.policy_blob_client log.info("Applying md5 content hash to policy {}".format(blob.name)) try: # Get the blob contents blob_bytes = client.get_blob_to_bytes(container, blob.name) # Re-upload the blob. validate_content ensures that the md5 hash is created client.create_blob_from_bytes(container, blob.name, blob_bytes.content, validate_content=True) # Re-fetch the blob with the new hash hashed_blob = client.get_blob_properties(container, blob.name) return hashed_blob, hashed_blob.properties.content_settings.content_md5 except AzureHttpError as e: log.warning("Failed to apply a md5 content hash to policy {}. " "This policy will be skipped.".format(blob.name)) log.error(e) return None, None def load_policy(self, path, policies): """ Loads a YAML file and prompts scheduling updates :param path: Path to YAML file on disk :param policies: Dictionary of policies to update """ with open(path, "r") as stream: try: policy_config = yaml.safe_load(stream) new_policies = PolicyCollection.from_data(policy_config, self.options) if new_policies: for p in new_policies: log.info("Loading Policy %s from %s" % (p.name, path)) p.validate() policies.update({p.name: {'policy': p}}) # Update periodic policy_mode = p.data.get('mode', {}).get('type') if policy_mode == CONTAINER_TIME_TRIGGER_MODE: self.update_periodic(p) elif policy_mode != CONTAINER_EVENT_TRIGGER_MODE: log.warning( "Unsupported policy mode for Azure Container Host: {}. " "{} will not be run. " "Supported policy modes include \"{}\" and \"{}\"." .format( policy_mode, p.data['name'], CONTAINER_EVENT_TRIGGER_MODE, CONTAINER_TIME_TRIGGER_MODE ) ) except Exception as exc: log.error('Invalid policy file %s %s' % (path, exc)) def unload_policy_file(self, path, policies): """ Unload a policy file that has changed or been removed. Take the copy from disk and pop all policies from dictionary and update scheduled jobs. """ with open(path, "r") as stream: try: policy_config = yaml.safe_load(stream) except yaml.YAMLError as exc: log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc)) os.unlink(path) return path try: # Some policies might have bad format, so they have never been loaded removed = [policies.pop(p['name']) for p in policy_config.get('policies', []) if p['name'] in policies] log.info('Removing policies %s' % removed) # update periodic periodic_names = \ [p['name'] for p in policy_config.get('policies', []) if p.get('mode', {}).get('schedule')] periodic_to_remove = \ [p for p in periodic_names if p in [j.id for j in self.scheduler.get_jobs()]] for name in periodic_to_remove: self.scheduler.remove_job(job_id=name) except (AttributeError, KeyError) as exc: log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc)) os.unlink(path) return path def update_periodic(self, policy): """ Update scheduled policies using cron type periodic scheduling. """ trigger = CronTrigger.from_crontab(policy.data['mode']['schedule']) self.scheduler.add_job(Host.run_policy, trigger, id=policy.name, name=policy.name, args=[policy, None, None], coalesce=True, max_instances=1, replace_existing=True, misfire_grace_time=60) def update_event_subscription(self): """ Create a single event subscription to channel all events to an Azure Queue. """ log.info('Updating event grid subscriptions') destination = StorageQueueEventSubscriptionDestination( resource_id=self.queue_storage_account.id, queue_name=self.event_queue_name) # Build event filter event_filter = EventSubscriptionFilter( included_event_types=['Microsoft.Resources.ResourceWriteSuccess']) # Update event subscription AzureEventSubscription.create(destination, self.event_queue_name, self.session.get_subscription_id(), self.session, event_filter) def poll_queue(self): """ Poll the Azure queue and loop until there are no visible messages remaining. """ # Exit if we don't have any policies if not self.policies: return if not self.queue_service: self.queue_service = Storage.get_queue_client_by_storage_account( self.queue_storage_account, self.storage_session) while True: try: messages = Storage.get_queue_messages( self.queue_service, self.event_queue_name, num_messages=queue_message_count, visibility_timeout=queue_timeout_seconds) except AzureHttpError: self.queue_service = None raise if len(messages) == 0: break log.info('Pulled %s events to process while polling queue.' % len(messages)) for message in messages: if message.dequeue_count > max_dequeue_count: Storage.delete_queue_message(self.queue_service, self.event_queue_name, message=message) log.warning("Event deleted due to reaching maximum retry count.") else: # Run matching policies self.run_policies_for_event(message) # We delete events regardless of policy result Storage.delete_queue_message( self.queue_service, self.event_queue_name, message=message) def run_policies_for_event(self, message): """ Find all policies subscribed to this event type and schedule them for immediate execution. """ # Load up the event event = json.loads(base64.b64decode(message.content).decode('utf-8')) operation_name = event['data']['operationName'] # Execute all policies matching the event type for k, v in self.policies.items(): events = v['policy'].data.get('mode', {}).get('events') if not events: continue events = AzureEvents.get_event_operations(events) if operation_name.upper() in (event.upper() for event in events): self.scheduler.add_job(Host.run_policy, id=k + event['id'], name=k, args=[v['policy'], event, None], misfire_grace_time=60 * 3) def prepare_queue_storage(self, queue_resource_id, queue_name): """ Create a storage client using unusual ID/group reference as this is what we require for event subscriptions """ storage_client = self.storage_session \ .client('azure.mgmt.storage.StorageManagementClient') account = storage_client.storage_accounts.get_properties( ResourceIdParser.get_resource_group(queue_resource_id), ResourceIdParser.get_resource_name(queue_resource_id)) Storage.create_queue_from_storage_account(account, queue_name, self.session) return account @staticmethod def run_policy(policy, event, context): try: policy.push(event, context) except Exception: log.exception("Policy Failed: %s", policy.name) @staticmethod def build_options(output_dir=None, log_group=None, metrics=None): """ Initialize the Azure provider to apply global config across all policy executions. """ if not output_dir: output_dir = tempfile.mkdtemp() log.warning('Output directory not specified. Using directory: %s' % output_dir) config = Config.empty( **{ 'log_group': log_group, 'metrics': metrics, 'output_dir': output_dir } ) return Azure().initialize(config) @staticmethod def get_scheduler_config(): if os.name == 'nt': executor = "apscheduler.executors.pool:ThreadPoolExecutor" else: executor = "apscheduler.executors.pool:ProcessPoolExecutor" return { 'apscheduler.jobstores.default': { 'type': 'memory' }, 'apscheduler.executors.default': { 'class': executor, 'max_workers': '4' }, 'apscheduler.executors.threadpool': { 'type': 'threadpool', 'max_workers': '20' }, 'apscheduler.job_defaults.coalesce': 'true', 'apscheduler.job_defaults.max_instances': '1', 'apscheduler.timezone': 'UTC', } @staticmethod def has_yaml_ext(filename): return filename.lower().endswith(('.yml', '.yaml')) @staticmethod @click.command(help="Periodically run a set of policies from an Azure storage container " "against a single subscription. The host will update itself with new " "policies and event subscriptions as they are added.") @click.option("--storage-id", "-q", envvar=ENV_CONTAINER_STORAGE_RESOURCE_ID, required=True, help="The resource id of the storage account to create the event queue in") @click.option("--queue-name", "-n", envvar=ENV_CONTAINER_QUEUE_NAME, help="The name of the event queue to create") @click.option("--policy-uri", "-p", envvar=ENV_CONTAINER_POLICY_URI, required=True, help="The URI to the Azure storage container that holds the policies") @click.option("--log-group", "-l", envvar=ENV_CONTAINER_OPTION_LOG_GROUP, help="Location to send policy logs") @click.option("--metrics", "-m", envvar=ENV_CONTAINER_OPTION_METRICS, help="The resource name or instrumentation key for uploading metrics") @click.option("--output-dir", "-d", envvar=ENV_CONTAINER_OPTION_OUTPUT_DIR, help="The directory for policy output") def cli(**kwargs): Host(**kwargs)
class JobScheduler(metaclass=Singleton): __function_dict__ = {} def __init__(self): self.logger = Log().logger self.scheduler = BlockingScheduler(logger=self.logger) self.parser = DateTimeParser() def get_lives(self): return {job.name: job for job in self.scheduler.get_jobs()} def add_job(self, name, trigger, trigger_args, func_args=None, func_kwargs=None): if name not in self.get_lives(): func = self.get_functions(name) self.scheduler.add_job(func, trigger=trigger, args=func_args, kwargs=func_kwargs, id=name, name=name, **trigger_args) self.logger.info('[%s][%s]作业已添加,并已启动' % (trigger, name)) def add_date_job(self, name, time, func_args=None, func_kwargs=None): time = self.parser.set_date(time).set_time(time).datetime self.add_job(name, 'date', {'run_date': time}, func_args, func_kwargs) def add_interval_job(self, name, weeks=0, days=0, hours=0, minutes=0, seconds=0, func_args=None, func_kwargs=None): self.add_job(name, 'interval', {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': seconds}, func_args, func_kwargs) def add_cron_job(self, name, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None, second=None, func_args=None, func_kwargs=None): self.add_job(name, 'cron', {'year': year, 'month': month, 'day': day, 'week': week, 'day_of_week': day_of_week, 'hour': hour, 'minute': minute, 'second': second}, func_args, func_kwargs) def add_started_job(self, name, after_seconds=1, func_args=None, func_kwargs=None): time = datetime.now() + timedelta(seconds=after_seconds) self.add_date_job(name, time, func_args, func_kwargs) def delete_job(self, name): jobs = self.get_lives() if name in jobs: self.scheduler.remove_job(jobs[name].id) self.logger.info('作业[%s]已移除' % name) @classmethod def register(cls, name): def add_method(f): cls.__function_dict__[name.strip()] = f return f return add_method def get_functions(self, name): return self.__function_dict__.get(name.strip()) def get_function_doc(self, name): return self.get_functions(name).__doc__ def get_function_names(self): return sorted(self.__function_dict__.keys())