class SchedRegistry():
def __init__(self):
self._sched = BlockingScheduler()
monitor_class = map(
lambda module: "monitor.sched.sched_collect.%sCollectScheduler" %
module.capitalize(), [item for item in GLOBAL_CONFIG.MONITOR_ITEM])
jobs = ModuleLoader.load_modules(monitor_class)
for job in jobs:
self.add_job(job())
def list_jobs(self):
return self._sched.get_jobs()
def add_job(self, sched_job):
LOG.info("sched_job[%s] add the registry" % sched_job.get_name())
self._sched.add_job(sched_job.run,
trigger=sched_job.get_trigger_type(),
**sched_job.get_trigger_args())
def sched_start(self):
LOG.debug('The scheduler will be start...')
self._sched.start()
LOG.debug('The scheduler started successfully.')
def sched_stop(self):
self._sched.shutdown(wait=30)
def run_cronjob(master: bool, crawl: bool, vaildator: bool):
"""
执行数据抓取及校验定时任务
:param master: 是否是主节点
:param crawl: 是否运行爬虫
:param vaildator: 是否运行校验器
"""
sched = BlockingScheduler()
if vaildator and master:
sched.add_job(run_validate_pub,
'interval',
seconds=PROXY_VALIDATE_TIME)
if crawl:
sched.add_job(
run_crawl,
args=[master],
trigger='interval',
seconds=PROXY_UPDATE_TIME,
next_run_time=datetime.datetime.now(), # 立刻执行
max_instances=5)
try:
if sched.get_jobs():
sched.start()
except (KeyboardInterrupt, SystemExit):
pass
def create_scheduler():
scheduler = BlockingScheduler(executors={"default": ThreadPoolExecutor()})
scheduler.add_job(todoist_assigned_issues, "interval", minutes=15)
scheduler.add_job(todoist_repo_prs, "interval", minutes=15)
for job in scheduler.get_jobs():
if isinstance(job.trigger, IntervalTrigger):
scheduler.add_job(job.func)
return scheduler
def main():
ws = WeiboSche('./account/a.json')
sche = BlockingScheduler()
sche.add_job(ws.get_forword, 'cron', hour='0-23/2', minute=1) # 获取微博
sche.add_job(ws.forword, 'cron', hour='0-23/1', minute=2) # 转发微博
sche.add_job(ws.send, 'cron', args=('one_word', ), hour=8, minute=1) # 一言
sche.add_job(ws.send, 'cron', args=('weather0', ), hour=6, minute=1) # 早安
sche.add_job(ws.send, 'cron', args=('weather1', ), hour=23, minute=1) # 晚安
sche.add_job(ws.send, 'cron', args=('news', ), hour=9, minute=1) # 每日国际视野
sche.add_job(ws.send,
'cron',
args=('history_of_today', ),
hour=10,
minute=1) # 历史上的今天
for job in sche.get_jobs():
ws.ow.logger.info('add job {} {}'.format(job.name, job.args))
sche.start()
def startCronTask(task, **interval_config):
# 定义全局变量scheduler，用于控制定时任务的启动和停止
global scheduler
scheduler = BlockingScheduler()
scheduler.add_listener(CronTask_listener,
EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
scheduler._logger = logger
logger.info(
'==================================== 新的日志分段 =============================================='
)
scheduler.add_job(func=task,
trigger='interval',
**interval_config,
id='push_to_github')
logger.info('当前所有定时任务job1：%s', scheduler.get_jobs())
logger.info('定时任务调度器状态1：%s', scheduler.state)
scheduler.start()
class SchedulerManager:
def __init__(self):
if config.ENVIRONMENT == "test":
self.scheduler = BackgroundScheduler()
else:
self.scheduler = BlockingScheduler()
def init(self, websites, func):
self.scheduler.configure(timezone="utc")
for website in websites:
self.scheduler.add_job(
func,
"interval",
seconds=website.get("interval"),
id=website.get("url"),
kwargs={"url": website.get("url"), "regexp_rules": website.get("regexp_rules")},
)
self.scheduler.start()
def get_jobs(self):
return self.scheduler.get_jobs()
dump()
time.sleep(1)
keep_files()
time.sleep(1)
sync()
except Exception as e:
logging.error(e)
pass
if __name__ == '__main__':
if run_once_immediately:
main()
job = scheduler.add_job(
func=main, trigger=trigger, name='rclone to oss', id='1', replace_existing=True
)
if run_immediately:
job.modify(next_run_time=datetime.now(tz=scheduler.timezone) + timedelta(seconds=5))
for job in scheduler.get_jobs():
logging.info(f"{job.name}: {job.trigger}")
def handler_stop_signals(signum, frame):
logging.info(f'shutdown from stop signum: {signum}')
scheduler.shutdown()
signal.signal(signal.SIGINT, handler_stop_signals)
signal.signal(signal.SIGTERM, handler_stop_signals)
scheduler.start()
class StandaloneScheduler(BaseScheduler):
"""Main workers refreshing data from AWS
"""
name = 'Standalone Scheduler'
ns = 'scheduler_standalone'
pool = None
scheduler = None
options = (
ConfigOption('worker_threads', 20, 'int', 'Number of worker threads to spawn'),
ConfigOption('worker_interval', 30, 'int', 'Delay between each worker thread being spawned, in seconds'),
)
def __init__(self):
super().__init__()
self.collectors = {}
self.auditors = []
self.region_workers = []
self.pool = ProcessPoolExecutor(self.dbconfig.get('worker_threads', self.ns, 20))
self.scheduler = APScheduler(
threadpool=self.pool,
job_defaults={
'coalesce': True,
'misfire_grace_time': 30
}
)
self.load_plugins()
def execute_scheduler(self):
# Schedule a daily job to cleanup stuff thats been left around (eip's with no instances etc)
self.scheduler.add_job(
self.cleanup,
trigger='cron',
name='cleanup',
hour=3,
minute=0,
second=0
)
# Schedule periodic scheduling of jobs
self.scheduler.add_job(
self.schedule_jobs,
trigger='interval',
name='schedule_jobs',
seconds=60,
start_date=datetime.now() + timedelta(seconds=1)
)
# Periodically reload the dbconfiguration
self.scheduler.add_job(
self.dbconfig.reload_data,
trigger='interval',
name='reload_dbconfig',
minutes=5,
start_date=datetime.now() + timedelta(seconds=3)
)
self.scheduler.start()
def execute_worker(self):
"""This method is not used for the standalone scheduler."""
print('The standalone scheduler does not have a separate worker model. '
'Executing the scheduler will also execute the workers')
def schedule_jobs(self):
current_jobs = {
x.name: x for x in self.scheduler.get_jobs() if x.name not in (
'cleanup',
'schedule_jobs',
'reload_dbconfig'
)
}
new_jobs = []
start = datetime.now() + timedelta(seconds=1)
_, accounts = BaseAccount.search(include_disabled=False)
# region Global collectors (non-aws)
if CollectorType.GLOBAL in self.collectors:
for wkr in self.collectors[CollectorType.GLOBAL]:
job_name = 'global_{}'.format(wkr.name)
new_jobs.append(job_name)
if job_name in current_jobs:
continue
self.scheduler.add_job(
self.execute_global_worker,
trigger='interval',
name=job_name,
minutes=wkr.interval,
start_date=start,
args=[wkr],
kwargs={}
)
start += timedelta(seconds=30)
# endregion
# region AWS collectors
aws_accounts = list(filter(lambda x: x.account_type == AWSAccount.account_type, accounts))
for acct in aws_accounts:
if CollectorType.AWS_ACCOUNT in self.collectors:
for wkr in self.collectors[CollectorType.AWS_ACCOUNT]:
job_name = '{}_{}'.format(acct.account_name, wkr.name)
new_jobs.append(job_name)
if job_name in current_jobs:
continue
self.scheduler.add_job(
self.execute_aws_account_worker,
trigger='interval',
name=job_name,
minutes=wkr.interval,
start_date=start,
args=[wkr],
kwargs={'account': acct.account_name}
)
if CollectorType.AWS_REGION in self.collectors:
for wkr in self.collectors[CollectorType.AWS_REGION]:
for region in AWS_REGIONS:
job_name = '{}_{}_{}'.format(acct.account_name, region, wkr.name)
new_jobs.append(job_name)
if job_name in current_jobs:
continue
self.scheduler.add_job(
self.execute_aws_region_worker,
trigger='interval',
name=job_name,
minutes=wkr.interval,
start_date=start,
args=[wkr],
kwargs={'account': acct.account_name, 'region': region}
)
db.session.commit()
start += timedelta(seconds=self.dbconfig.get('worker_interval', self.ns, 30))
# endregion
# region Auditors
start = datetime.now() + timedelta(seconds=1)
for wkr in self.auditors:
job_name = 'auditor_{}'.format(wkr.name)
new_jobs.append(job_name)
if job_name in current_jobs:
continue
if app_config.log_level == 'DEBUG':
audit_start = start + timedelta(seconds=5)
else:
audit_start = start + timedelta(minutes=5)
self.scheduler.add_job(
self.execute_auditor_worker,
trigger='interval',
name=job_name,
minutes=wkr.interval,
start_date=audit_start,
args=[wkr],
kwargs={}
)
start += timedelta(seconds=self.dbconfig.get('worker_interval', self.ns, 30))
# endregion
extra_jobs = list(set(current_jobs) - set(new_jobs))
for job in extra_jobs:
self.log.warning('Removing job {} as it is no longer needed'.format(job))
current_jobs[job].remove()
def execute_global_worker(self, data, **kwargs):
try:
cls = self.get_class_from_ep(data.entry_point)
worker = cls(**kwargs)
self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S")))
self.log.info('Starting global {} worker'.format(data.name))
worker.run()
except Exception as ex:
self.log.exception('Global Worker {}: {}'.format(data.name, ex))
finally:
db.session.rollback()
self.log.info('Completed run for global {} worker'.format(data.name))
def execute_aws_account_worker(self, data, **kwargs):
try:
cls = self.get_class_from_ep(data.entry_point)
worker = cls(**kwargs)
self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S")))
worker.run()
except Exception as ex:
self.log.exception('AWS Account Worker {}/{}: {}'.format(data.name, kwargs['account'], ex))
finally:
db.session.rollback()
self.log.info('Completed run for {} worker on {}'.format(data.name, kwargs['account']))
def execute_aws_region_worker(self, data, **kwargs):
try:
cls = self.get_class_from_ep(data.entry_point)
worker = cls(**kwargs)
self.log.info('RUN_INFO: {} starting at {} for account {} / region {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), kwargs['account'], kwargs['region'], (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S")))
worker.run()
except Exception as ex:
self.log.exception('AWS Region Worker {}/{}/{}: {}'.format(
data.name,
kwargs['account'],
kwargs['region'],
ex
))
finally:
db.session.rollback()
self.log.info('Completed run for {} worker on {}/{}'.format(
data.name,
kwargs['account'],
kwargs['region']
))
def execute_auditor_worker(self, data, **kwargs):
try:
cls = self.get_class_from_ep(data.entry_point)
worker = cls(**kwargs)
self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data.entry_point['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=data.interval)).strftime("%Y-%m-%d %H:%M:%S")))
worker.run()
except Exception as ex:
self.log.exception('Auditor Worker {}: {}'.format(data.name, ex))
finally:
db.session.rollback()
self.log.info('Completed run for auditor {}'.format(data.name))
def cleanup(self):
try:
self.log.info('Running cleanup tasks')
log_purge_date = datetime.now() - timedelta(days=self.dbconfig.get('log_keep_days', 'log', default=31))
db.LogEvent.find(LogEvent.timestamp < log_purge_date)
db.session.commit()
finally:
db.session.rollback()
class SQSScheduler(BaseScheduler):
name = 'SQS Scheduler'
ns = NS_SCHEDULER_SQS
options = (
ConfigOption('queue_region', 'us-west-2', 'string', 'Region of the SQS Queues'),
ConfigOption('job_queue_url', '', 'string', 'URL of the SQS Queue for pending jobs'),
ConfigOption('status_queue_url', '', 'string', 'URL of the SQS Queue for worker reports'),
ConfigOption('job_delay', 2, 'float', 'Time between each scheduled job, in seconds. Can be used to '
'avoid spiky load during execution of tasks'),
)
def __init__(self):
"""Initialize the SQSScheduler, setting up the process pools, scheduler and connecting to the required
SQS Queues"""
super().__init__()
self.pool = ProcessPoolExecutor(1)
self.scheduler = APScheduler(
threadpool=self.pool,
job_defaults={
'coalesce': True,
'misfire_grace_time': 30
}
)
session = get_local_aws_session()
sqs = session.resource('sqs', self.dbconfig.get('queue_region', self.ns))
self.job_queue = sqs.Queue(self.dbconfig.get('job_queue_url', self.ns))
self.status_queue = sqs.Queue(self.dbconfig.get('status_queue_url', self.ns))
def execute_scheduler(self):
"""Main entry point for the scheduler. This method will start two scheduled jobs, `schedule_jobs` which takes
care of scheduling the actual SQS messaging and `process_status_queue` which will track the current status
of the jobs as workers are executing them
Returns:
`None`
"""
try:
# Schedule periodic scheduling of jobs
self.scheduler.add_job(
self.schedule_jobs,
trigger='interval',
name='schedule_jobs',
minutes=15,
start_date=datetime.now() + timedelta(seconds=1)
)
self.scheduler.add_job(
self.process_status_queue,
trigger='interval',
name='process_status_queue',
seconds=30,
start_date=datetime.now() + timedelta(seconds=5),
max_instances=1
)
self.scheduler.start()
except KeyboardInterrupt:
self.scheduler.shutdown()
def list_current_jobs(self):
"""Return a list of the currently scheduled jobs in APScheduler
Returns:
`dict` of `str`: :obj:`apscheduler/job:Job`
"""
jobs = {}
for job in self.scheduler.get_jobs():
if job.name not in ('schedule_jobs', 'process_status_queue'):
jobs[job.name] = job
return jobs
def schedule_jobs(self):
"""Schedule or remove jobs as needed.
Checks to see if there are any jobs that needs to be scheduled, after refreshing the database configuration
as well as the list of collectors and auditors.
Returns:
`None`
"""
self.dbconfig.reload_data()
self.collectors = {}
self.auditors = []
self.load_plugins()
_, accounts = BaseAccount.search(include_disabled=False)
current_jobs = self.list_current_jobs()
new_jobs = []
batch_id = str(uuid4())
batch = SchedulerBatch()
batch.batch_id = batch_id
batch.status = SchedulerStatus.PENDING
db.session.add(batch)
db.session.commit()
start = datetime.now() + timedelta(seconds=1)
job_delay = dbconfig.get('job_delay', self.ns, 0.5)
# region Global collectors (non-aws)
if CollectorType.GLOBAL in self.collectors:
for worker in self.collectors[CollectorType.GLOBAL]:
job_name = get_hash(worker)
if job_name in current_jobs:
continue
self.scheduler.add_job(
self.send_worker_queue_message,
trigger='interval',
name=job_name,
minutes=worker.interval,
start_date=start,
kwargs={
'batch_id': batch_id,
'job_name': job_name,
'entry_point': worker.entry_point,
'worker_args': {}
}
)
start += timedelta(seconds=job_delay)
# endregion
# region AWS collectors
aws_accounts = list(filter(lambda x: x.account_type == AWSAccount.account_type, accounts))
if CollectorType.AWS_ACCOUNT in self.collectors:
for worker in self.collectors[CollectorType.AWS_ACCOUNT]:
for account in aws_accounts:
job_name = get_hash((account.account_name, worker))
if job_name in current_jobs:
continue
new_jobs.append(job_name)
self.scheduler.add_job(
self.send_worker_queue_message,
trigger='interval',
name=job_name,
minutes=worker.interval,
start_date=start,
kwargs={
'batch_id': batch_id,
'job_name': job_name,
'entry_point': worker.entry_point,
'worker_args': {
'account': account.account_name
}
}
)
start += timedelta(seconds=job_delay)
if CollectorType.AWS_REGION in self.collectors:
for worker in self.collectors[CollectorType.AWS_REGION]:
for region in AWS_REGIONS:
for account in aws_accounts:
job_name = get_hash((account.account_name, region, worker))
if job_name in current_jobs:
continue
new_jobs.append(job_name)
self.scheduler.add_job(
self.send_worker_queue_message,
trigger='interval',
name=job_name,
minutes=worker.interval,
start_date=start,
kwargs={
'batch_id': batch_id,
'job_name': job_name,
'entry_point': worker.entry_point,
'worker_args': {
'account': account.account_name,
'region': region
}
}
)
start += timedelta(seconds=job_delay)
# endregion
# region Auditors
if app_config.log_level == 'DEBUG':
audit_start = start + timedelta(seconds=5)
else:
audit_start = start + timedelta(minutes=5)
for worker in self.auditors:
job_name = get_hash((worker,))
if job_name in current_jobs:
continue
new_jobs.append(job_name)
self.scheduler.add_job(
self.send_worker_queue_message,
trigger='interval',
name=job_name,
minutes=worker.interval,
start_date=audit_start,
kwargs={
'batch_id': batch_id,
'job_name': job_name,
'entry_point': worker.entry_point,
'worker_args': {}
}
)
audit_start += timedelta(seconds=job_delay)
# endregion
def send_worker_queue_message(self, *, batch_id, job_name, entry_point, worker_args, retry_count=0):
"""Send a message to the `worker_queue` for a worker to execute the requests job
Args:
batch_id (`str`): Unique ID of the batch the job belongs to
job_name (`str`): Non-unique ID of the job. This is used to ensure that the same job is only scheduled
a single time per batch
entry_point (`dict`): A dictionary providing the entry point information for the worker to load the class
worker_args (`dict`): A dictionary with the arguments required by the worker class (if any, can be an
empty dictionary)
retry_count (`int`): The number of times this one job has been attempted to be executed. If a job fails to
execute after 3 retries it will be marked as failed
Returns:
`None`
"""
try:
job_id = str(uuid4())
self.job_queue.send_message(
MessageBody=json.dumps({
'batch_id': batch_id,
'job_id': job_id,
'job_name': job_name,
'entry_point': entry_point,
'worker_args': worker_args,
}),
MessageDeduplicationId=job_id,
MessageGroupId=batch_id,
MessageAttributes={
'RetryCount': {
'StringValue': str(retry_count),
'DataType': 'Number'
}
}
)
if retry_count == 0:
job = SchedulerJob()
job.job_id = job_id
job.batch_id = batch_id
job.status = SchedulerStatus.PENDING
job.data = worker_args
db.session.add(job)
db.session.commit()
except:
self.log.exception('Error when processing worker task')
def execute_worker(self):
"""Retrieve a message from the `worker_queue` and process the request.
This function will read a single message from the `worker_queue` and load the specified `EntryPoint`
and execute the worker with the provided arguments. Upon completion (failure or otherwise) a message is sent
to the `status_queue` information the scheduler about the return status (success/failure) of the worker
Returns:
`None`
"""
try:
try:
messages = self.job_queue.receive_messages(
MaxNumberOfMessages=1,
MessageAttributeNames=('RetryCount',)
)
except ClientError:
self.log.exception('Failed fetching messages from SQS queue')
return
if not messages:
self.log.debug('No pending jobs')
return
for message in messages:
try:
retry_count = int(message.message_attributes['RetryCount']['StringValue'])
data = json.loads(message.body)
try:
# SQS FIFO queues will not allow another thread to get any new messages until the messages
# in-flight are returned to the queue or deleted, so we remove the message from the queue as
# soon as we've loaded the data
self.send_status_message(data['job_id'], SchedulerStatus.STARTED)
message.delete()
cls = self.get_class_from_ep(data['entry_point'])
worker = cls(**data['worker_args'])
if hasattr(worker, 'type'):
if worker.type == CollectorType.GLOBAL:
self.log.info('RUN_INFO: {} starting at {}, next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S")))
elif worker.type == CollectorType.AWS_REGION:
self.log.info('RUN_INFO: {} starting at {} for account {} / region {}, next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), data['worker_args']['account'], data['worker_args']['region'], (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S")))
elif worker.type == CollectorType.AWS_ACCOUNT:
self.log.info('RUN_INFO: {} starting at {} for account {} next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), data['worker_args']['account'], (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S")))
else:
self.log.info('RUN_INFO: {} starting at {} next run will be at approximately {}'.format(data['entry_point']['module_name'], datetime.now().strftime("%Y-%m-%d %H:%M:%S"), (datetime.now() + timedelta(minutes=worker.interval)).strftime("%Y-%m-%d %H:%M:%S")))
worker.run()
self.send_status_message(data['job_id'], SchedulerStatus.COMPLETED)
except InquisitorError:
# If the job failed for some reason, reschedule it unless it has already been retried 3 times
if retry_count >= 3:
self.send_status_message(data['job_id'], SchedulerStatus.FAILED)
else:
self.send_worker_queue_message(
batch_id=data['batch_id'],
job_name=data['job_name'],
entry_point=data['entry_point'],
worker_args=data['worker_args'],
retry_count=retry_count + 1
)
except:
self.log.exception('Failed processing scheduler job: {}'.format(message.body))
except KeyboardInterrupt:
self.log.info('Shutting down worker thread')
@retry
def send_status_message(self, object_id, status):
"""Send a message to the `status_queue` to update a job's status.
Returns `True` if the message was sent, else `False`
Args:
object_id (`str`): ID of the job that was executed
status (:obj:`SchedulerStatus`): Status of the job
Returns:
`bool`
"""
try:
body = json.dumps({
'id': object_id,
'status': status
})
self.status_queue.send_message(
MessageBody=body,
MessageGroupId='job_status',
MessageDeduplicationId=get_hash((object_id, status))
)
return True
except Exception as ex:
print(ex)
return False
@retry
def process_status_queue(self):
"""Process all messages in the `status_queue` and check for any batches that needs to change status
Returns:
`None`
"""
self.log.debug('Start processing status queue')
while True:
messages = self.status_queue.receive_messages(MaxNumberOfMessages=10)
if not messages:
break
for message in messages:
data = json.loads(message.body)
job = SchedulerJob.get(data['id'])
try:
if job and job.update_status(data['status']):
db.session.commit()
except SchedulerError as ex:
if hasattr(ex, 'message') and ex.message == 'Attempting to update already completed job':
pass
message.delete()
# Close any batch that is now complete
open_batches = db.SchedulerBatch.find(SchedulerBatch.status < SchedulerStatus.COMPLETED)
for batch in open_batches:
open_jobs = list(filter(lambda x: x.status < SchedulerStatus.COMPLETED, batch.jobs))
if not open_jobs:
open_batches.remove(batch)
batch.update_status(SchedulerStatus.COMPLETED)
self.log.debug('Closed completed batch {}'.format(batch.batch_id))
else:
started_jobs = list(filter(lambda x: x.status > SchedulerStatus.PENDING, open_jobs))
if batch.status == SchedulerStatus.PENDING and len(started_jobs) > 0:
batch.update_status(SchedulerStatus.STARTED)
self.log.debug('Started batch manually {}'.format(batch.batch_id))
# Check for stale batches / jobs
for batch in open_batches:
if batch.started < datetime.now() - timedelta(hours=2):
self.log.warning('Closing a stale scheduler batch: {}'.format(batch.batch_id))
for job in batch.jobs:
if job.status < SchedulerStatus.COMPLETED:
job.update_status(SchedulerStatus.ABORTED)
batch.update_status(SchedulerStatus.ABORTED)
db.session.commit()
# 精确时间执行
# scheduler.add_job(job1, 'date', run_date=datetime(2020, 8, 8, 16, 30, 5), id='job1')
# scheduler.add_job(job1, 'date', run_date=‘2020-8-8 16:30:5’, id='job1')
# 每5秒执行一次
scheduler.add_job(job1, 'interval', seconds=5, id='job1')
scheduler.add_job(job1, 'interval', seconds=5, id='job2')
# 移除任务，得在start()之前
# scheduler.remove_job('job1')
# 暂停作业
# scheduler.pause_job('job1')
# 恢复作业
# scheduler.resume_job('job1')
# 获取作业列表
jobs = scheduler.get_jobs()
print(jobs)
# 根据id获取某个作业
job = scheduler.get_job('job1')
print(job)
scheduler.start()
# 关闭调度器,加上false表示为不想等待
scheduler.shutdown()
scheduler.shutdown(wait=False)
class SchedUtility(object, metaclass=Singleton):
def __init__(self):
try:
self.Global = Global()
self.Utility = Utility()
self.InfraUtil = InfraUtility()
self.db = DBMySql('Scheduler')
self.myModulePyFile = os.path.abspath(__file__)
self.myClass = self.__class__.__name__
#Setting the infrastructure
self.Infra = self.InfraUtil.setInfra(self.Global.SchedulerInfraKey)
if not self.Infra:
raise InfraInitializationError('Could not initialize {cls}'.format(cls=(self.myModulePyFile,self.myClass)))
# we need to get the proper logger for a given module
self.logger = self.Infra.getInfraLogger(self.Global.SchedulerInfraKey)
# loading Schduler config and starting scheduler
self.__startScheduler__()
except Exception as err:
raise err
def __startScheduler__(self):
try:
mySchedulerType = self.Global.DefaultSchedulerType
mySchedulerMode = self.Global.DefaultSchedulerMode
if mySchedulerMode == 'Run':
myArgPaused = False
else:
myArgPaused = True
#fi
mySchedulerConfig = self.Utility.getACopy(self.Infra.schedulerConfigData)
if mySchedulerType == 'Background':
self.Scheduler = BackgroundScheduler(mySchedulerConfig)
else:
self.Scheduler = BlockingScheduler(mySchedulerConfig)
#fi
if not self.Scheduler.running:
self.Scheduler.start(paused = myArgPaused)
except Exception as err:
raise err
def getAllJobDetail(self):
'''
Description: Returns all jobs as stored in scheduler
'''
myJobDetail = []
for job in self.Scheduler.get_jobs():
myJobDetail.append(self.getAJobDetail(job.id))
return myJobDetail
def getAJobDetail(self, jobIdArg):
'''
Description: Print all jobs as stored in scheduler
'''
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
myJobDetail = job.__getstate__()
return myJobDetail
def suspendJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.pause()
def resumeJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.resume()
def getCurrentlyExecutingJob(self):
return len(self.Scheduler.get_jobs())
def removeJob(self, jobId):
try:
self.Scheduler.remove_job(jobId)
except JobLookupError as err:
print('Invalid Job !!')
def removeAllJobs(self):
try:
self.Scheduler.remove_all_jobs()
except Exception as err:
raise err
def getAllJobsFromRep(self):
for job in self.Scheduler.get_jobs():
myJobDetail = self.Scheduler.get_job(job.id)
print(job,myJobDetail)
def getNewJob(self,prefixArg):
# random number between 10 and 99 to ensure we always get 2 digit
if isinstance(prefixArg,str) and prefixArg is not None:
return prefixArg + '_' + str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99)))
else:
return datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99))
def getJobInfoFromDb(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.processDbRequest(operation = self.Global.fetch, container = 'ScheduledJobs', contents = ['*'], criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
return myResponse
def getNextSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) + 1
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getCurrentSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getElapsedStatsForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def processJobStartEvent(self, jobIdArg):
'''
1. Mark job started in ScheduledJobs
2. Create new entry for this job in ScheduledJobsRunLog
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
# building data for SchedulerJobsRunLog
myJobCriteria = ' JobId = %s' %repr(myJobId)
myNextSeqForJob = self.getNextSeqForJob(myJobId)
# will mark the job started and creat the run log for this run
self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={'Status': 'Executing'}, criteria = myJobCriteria, commitWork=True )
# creating run information
self.db.processDbRequest(operation='create', container='ScheduledJobsRunLog', \
dataDict={'JobId':myJobId, 'Seq' : myNextSeqForJob, 'ExecutionStarted': self.Utility.getCurrentTime()}, commitWork=True )
self.Utility.buildResponse(myResponse, self.Global.Success, self.Global.Success, {'Seq':myNextSeqForJob})
else:
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, 'Cound not find job details for job {job}'.format(job = myJobId))
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
#raise err # will raise the error so this can be logged by scheduler as an error occurred in processing job
return myResponse
def processJobFinishEvent(self, jobIdArg, execDetailsArg):
'''
1. Mark job completed (update failure cnt and total count and consc fail count, lastrunstatus) in ScheduledJobs
2. Update ScheduledJobsRunlog container
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
myExecDetails = execDetailsArg
myJobStatus = self.Global.NextJobRun
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
self.logger.debug('Job details found, proceeding with finish event')
myJobCriteria = 'JobId = %s' %repr(myJobId)
myCurrentSeqForJob = self.getCurrentSeqForJob(myJobId)
myJobRunCriteria = ' JobId = %s and Seq = %s ' %(repr(myJobId), myCurrentSeqForJob)
self.logger.debug('Job criteria {criteria}'.format(criteria = myJobCriteria))
self.logger.debug('Job criteria with seq {criteria}'.format(criteria = myJobRunCriteria))
myJobDetailsFromSched = self.getAJobDetail(myJobId)
# Updating execution details in ScheduledJobsRunLog
self.logger.debug('udating statistics of this run')
myDbResult = self.db.processDbRequest(operation = 'change', container = 'ScheduledJobsRunLog', \
dataDict={
'Status': myExecDetails['Status'], 'ElapsedSeconds':myExecDetails['Data']['ElapsedSecs'],
'ExecutionCompleted': self.Utility.getCurrentTime(), 'ExecutionDetail': json.dumps(myExecDetails['Data'])
}, criteria = myJobRunCriteria, commitWork=True )
self.logger.debug('ScheduledJobsRunLog: db results >> {results}'.format(results = myDbResult))
# Updating execution details in ScheduledJobs
#if myExecDetails['Status'] == self.Global.Success:
# if success, reset consecfailcnt to 0, increment totalrun by 1 and update next run
myElapsedStats = self.db.executeDynamicSql(\
operation = 'fetch', \
sql_text = 'select min(ElapsedSeconds) "Min", max(ElapsedSeconds) "Max", avg(ElapsedSeconds) "Avg" from ScheduledJobsRunLog')
self.logger.debug('Elapsed Stats: {stats}'.format(stats = myElapsedStats))
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'NextRun' : myJobDetailsFromSched['next_run_time'].strftime('%Y-%m-%d% %H:%M:%S'), 'LatConsecFailCnt' : 0,
'MinElapsedSecs' : myElapsedStats['Data'][0]['Min'], 'MaxElapsedSecs' : myElapsedStats['Data'][0]['Min'] ,
'AvgElapsedSecs' : myElapsedStats['Data'][0]['Avg']
}, criteria = myJobCriteria, commitWork=True )
self.logger.debug('ScheduledJobs: last stats update >> {result}'.format(result = myDbResult))
#self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
'''
else:
# process job was unsuccessful
if myJobDetailsFromDb[0]['LatConsecFailCnt'] >= self.Global.SchedConsecFailCntThreshold:
myJobStatus = self.Global.SuspendMode
self.logger.info('suspending job {job}'.format(job=myJobId))
self.suspendJob(myJobId)
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'next_run' : myJobDetailsFromSched['next_run_time'], 'LatConsecFailCnt' : myJobDetailsFromDb[0]['LatConsecFailCnt'] + 1,
'TotalFailure' : myJobDetailsFromDb[0]['TotalFailure' + 1]
}, criteria = myJobCriteria, commitWork=True )
# will suspend the job if total failure count has been reached beyond Total consecutive failure threshold
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,self.Global.UnSuccess)
raise processJobError(myExecDetails['Message'])
'''
self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, myErrorMsg)
return myResponse
class AlgoTrade():
def __init__(self, event_engine):
self.event_engine = event_engine
self._expire_at = datetime.now()
self._trade_status = {'status':'close'}
self.sched = Scheduler()
def daily_jobs(self, data):
logger.info('判断当前时候交易日')
if data.market_status == 'close':
logger.info('当前不是交易日，耐心等待')
logger.info(self.sched.get_jobs())
return
logger.warning('当前为交易日，准备执行各类交易策略')
on_open_event = Event('on_open', data)
on_tick_event = Event('on_tick', data)
before_trade_event = Event('before_trade', data)
on_ipo_event = Event('on_ipo', data)
pre_close_event = Event('pre_close', data)
on_close_event = Event('on_close', data)
self.sched.add_job(self.event_engine.trigger,'date',
run_date=data.market_am_open + timedelta(minutes=1),
args=[on_open_event],
id='on_open'
)
self.sched.add_job(self.event_engine.trigger, 'date',
run_date=data.market_am_close + timedelta(minutes=randint(30, 90)),
args=[on_ipo_event],
id='on_ipo'
)
self.sched.add_job(self.event_engine.trigger, 'date',
run_date=data.market_fm_close - timedelta(minutes=10),
args=[pre_close_event],
id='pre_close'
)
self.sched.add_job(self.event_engine.trigger, 'date',
run_date=data.market_fm_close - timedelta(minutes=5),
args=[on_close_event],
id='on_close'
)
self.sched.add_job(self.event_engine.trigger, 'interval', seconds=15,
start_date=data.market_am_open + timedelta(minutes=1),
end_date=data.market_am_close,
args=[on_tick_event],
id='on_tick_am'
)
self.sched.add_job(self.event_engine.trigger, 'interval', seconds=15,
start_date=data.market_fm_open + timedelta(minutes=1),
end_date=data.market_fm_close - timedelta(minutes=5),
args=[on_tick_event],
id='on_tick_fm'
)
self.event_engine.trigger(before_trade_event)
logger.info(self.sched.get_jobs())
def run(self, data):
try:
self.event_engine.start()
self.sched.add_job(self.daily_jobs, 'cron', day_of_week='0-4', hour=9, minute=27, args=[data])
self.daily_jobs(data)
logger.warning('策略启动执行')
self.sched.start()
except:
self.event_engine.stop()
self.sched.shutdown(wait=True)
logger.warning('策略停止执行成功')
id='test_job1')
#定时任务示例
scheduler.add_job(job1,
'cron',
second='*/4',
args=('定时', ),
id='test_job2')
#一次性任务示例
scheduler.add_job(job1,
next_run_time=(datetime.datetime.now() +
datetime.timedelta(seconds=5)),
args=('一次', ),
id='test_job3')
'''
传递参数的方式有元组(tuple)、列表(list)、字典(dict)
注意：不过需要注意采用元组传递参数时后边需要多加一个逗号
'''
# #基于list
# scheduler.add_job(job2, 'interval', seconds=5, args=['a','b','list'], id='test_job4')
# #基于tuple
# scheduler.add_job(job2, 'interval', seconds=5, args=('a','b','tuple',), id='test_job5')
# #基于dict
# scheduler.add_job(job3, 'interval', seconds=5, kwargs={'f':'dict', 'a':1,'b':2}, id='test_job6')
#带有参数的示例
# scheduler.add_job(job2, 'interval', seconds=5, args=['a','b'], id='test_job7')
# scheduler.add_job(job2, 'interval', seconds=5, args=('a','b',), id='test_job8')
# scheduler.add_job(job3, 'interval', seconds=5, kwargs={'a':1,'b':2}, id='test_job9')
print(scheduler.get_jobs())
scheduler.start()
class Host:
def __init__(self,
event_queue_id,
event_queue_name,
policy_storage,
log_group=None,
metrics=None,
output_dir=None):
logging.basicConfig(level=logging.INFO, format='%(message)s')
log.info("Running Azure Cloud Custodian Self-Host")
load_resources()
self.session = local_session(Session)
# Load configuration
self.options = Host.build_options(output_dir, log_group, metrics)
self.policy_storage_uri = policy_storage
self.event_queue_name = event_queue_name
self.event_queue_id = event_queue_id
# Prepare storage bits
self.policy_blob_client = None
self.blob_cache = {}
self.queue_storage_account = self.prepare_queue_storage(
self.event_queue_id, self.event_queue_name)
self.queue_service = None
# Track required event subscription updates
self.require_event_update = False
# Policy cache and dictionary
self.policy_cache = tempfile.mkdtemp()
self.policies = {}
# Configure scheduler
self.scheduler = BlockingScheduler()
logging.getLogger('apscheduler.executors.default').setLevel(
logging.ERROR)
# Schedule recurring policy updates
self.scheduler.add_job(self.update_policies,
'interval',
seconds=policy_update_seconds,
id="update_policies",
next_run_time=datetime.now())
# Schedule recurring queue polling
self.scheduler.add_job(self.poll_queue,
'interval',
seconds=queue_poll_seconds,
id="poll_queue")
self.scheduler.start()
def update_policies(self):
"""
Enumerate all policies from storage.
Use the MD5 hashes in the enumerated policies
and a local dictionary to decide if we should
bother downloading/updating each blob.
We maintain an on-disk policy cache for future
features.
"""
if not self.policy_blob_client:
self.policy_blob_client = Storage.get_blob_client_by_uri(
self.policy_storage_uri, self.session)
(client, container, prefix) = self.policy_blob_client
try:
# All blobs with YAML extension
blobs = [
b for b in client.list_blobs(container)
if Host.has_yaml_ext(b.name)
]
except AzureHttpError as e:
# If blob methods are failing don't keep
# a cached client
self.policy_blob_client = None
raise e
# Filter to hashes we have not seen before
new_blobs = self._get_new_blobs(blobs)
# Get all YAML files on disk that are no longer in blob storage
cached_policy_files = [
f for f in os.listdir(self.policy_cache) if Host.has_yaml_ext(f)
]
removed_files = [
f for f in cached_policy_files if f not in [b.name for b in blobs]
]
if not (removed_files or new_blobs):
return
# Update a copy so we don't interfere with
# iterations on other threads
policies_copy = self.policies.copy()
for f in removed_files:
path = os.path.join(self.policy_cache, f)
self.unload_policy_file(path, policies_copy)
# Get updated YML files
for blob in new_blobs:
policy_path = os.path.join(self.policy_cache, blob.name)
if os.path.exists(policy_path):
self.unload_policy_file(policy_path, policies_copy)
client.get_blob_to_path(container, blob.name, policy_path)
self.load_policy(policy_path, policies_copy)
self.blob_cache.update(
{blob.name: blob.properties.content_settings.content_md5})
# Assign our copy back over the original
self.policies = policies_copy
if self.require_event_update:
self.update_event_subscriptions()
def _get_new_blobs(self, blobs):
new_blobs = []
for blob in blobs:
md5_hash = blob.properties.content_settings.content_md5
if not md5_hash:
blob, md5_hash = self._try_create_md5_content_hash(blob)
if blob and md5_hash and md5_hash != self.blob_cache.get(
blob.name):
new_blobs.append(blob)
return new_blobs
def _try_create_md5_content_hash(self, blob):
# Not all storage clients provide the md5 hash when uploading a file
# so, we need to make sure that hash exists.
(client, container, _) = self.policy_blob_client
log.info("Applying md5 content hash to policy {}".format(blob.name))
try:
# Get the blob contents
blob_bytes = client.get_blob_to_bytes(container, blob.name)
# Re-upload the blob. validate_content ensures that the md5 hash is created
client.create_blob_from_bytes(container,
blob.name,
blob_bytes.content,
validate_content=True)
# Re-fetch the blob with the new hash
hashed_blob = client.get_blob_properties(container, blob.name)
return hashed_blob, hashed_blob.properties.content_settings.content_md5
except AzureHttpError as e:
log.warning("Failed to apply a md5 content hash to policy {}. "
"This policy will be skipped.".format(blob.name))
log.error(e)
return None, None
def load_policy(self, path, policies):
"""
Loads a YAML file and prompts scheduling updates
:param path: Path to YAML file on disk
:param policies: Dictionary of policies to update
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
new_policies = PolicyCollection.from_data(
policy_config, self.options)
if new_policies:
for p in new_policies:
log.info("Loading Policy %s from %s" % (p.name, path))
p.validate()
policies.update({p.name: {'policy': p}})
# Update periodic and set event update flag
policy_mode = p.data.get('mode', {}).get('type')
if policy_mode == CONTAINER_TIME_TRIGGER_MODE:
self.update_periodic(p)
elif policy_mode == CONTAINER_EVENT_TRIGGER_MODE:
self.require_event_update = True
else:
log.warning(
"Unsupported policy mode for Azure Container Host: {}. "
"{} will not be run. "
"Supported policy modes include \"{}\" and \"{}\"."
.format(policy_mode, p.data['name'],
CONTAINER_EVENT_TRIGGER_MODE,
CONTAINER_TIME_TRIGGER_MODE))
except Exception as exc:
log.error('Invalid policy file %s %s' % (path, exc))
def unload_policy_file(self, path, policies):
"""
Unload a policy file that has changed or been removed.
Take the copy from disk and pop all policies from dictionary
and update scheduled jobs and event registrations.
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
except yaml.YAMLError as exc:
log.warning('Failure loading cached policy for cleanup %s %s' %
(path, exc))
os.unlink(path)
return
removed = [
policies.pop(p['name']) for p in policy_config.get('policies', [])
]
log.info('Removing policies %s' % removed)
# update periodic
periodic_names = \
[p['name'] for p in policy_config['policies'] if p.get('mode', {}).get('schedule')]
periodic_to_remove = \
[p for p in periodic_names if p in [j.id for j in self.scheduler.get_jobs()]]
for name in periodic_to_remove:
self.scheduler.remove_job(job_id=name)
# update event
event_names = \
[p['name'] for p in policy_config['policies'] if p.get('mode', {}).get('events')]
if event_names:
self.require_event_update = True
os.unlink(path)
return path
def update_periodic(self, policy):
"""
Update scheduled policies using cron type
periodic scheduling.
"""
trigger = CronTrigger.from_crontab(policy.data['mode']['schedule'])
trigger.jitter = jitter_seconds
self.scheduler.add_job(self.run_policy,
trigger,
id=policy.name,
name=policy.name,
args=[policy, None, None],
coalesce=True,
max_instances=1,
replace_existing=True,
misfire_grace_time=20)
def update_event_subscriptions(self):
"""
Find unique list of all subscribed events and
update a single event subscription to channel
them to an Azure Queue.
"""
log.info('Updating event grid subscriptions')
destination = \
StorageQueueEventSubscriptionDestination(resource_id=self.queue_storage_account.id,
queue_name=self.event_queue_name)
# Get total unique event list to use in event subscription
policy_items = self.policies.items()
events_lists = [
v['policy'].data.get('mode', {}).get('events')
for n, v in policy_items
]
flat_events = [e for l in events_lists if l for e in l if e]
resolved_events = AzureEvents.get_event_operations(flat_events)
unique_events = set(resolved_events)
# Build event filter strings
advance_filter = StringInAdvancedFilter(key='Data.OperationName',
values=unique_events)
event_filter = EventSubscriptionFilter(
advanced_filters=[advance_filter])
# Update event subscription
AzureEventSubscription.create(destination, self.event_queue_name,
self.session.get_subscription_id(),
self.session, event_filter)
self.require_event_update = False
def poll_queue(self):
"""
Poll the Azure queue and loop until
there are no visible messages remaining.
"""
# Exit if we don't have any policies
if not self.policies:
return
if not self.queue_service:
self.queue_service = Storage.get_queue_client_by_storage_account(
self.queue_storage_account, self.session)
while True:
try:
messages = Storage.get_queue_messages(
self.queue_service,
self.event_queue_name,
num_messages=queue_message_count,
visibility_timeout=queue_timeout_seconds)
except AzureHttpError:
self.queue_service = None
raise
if len(messages) == 0:
break
log.info('Pulled %s events to process while polling queue.' %
len(messages))
for message in messages:
if message.dequeue_count > max_dequeue_count:
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
log.warning(
"Event deleted due to reaching maximum retry count.")
else:
# Run matching policies
self.run_policies_for_event(message)
# We delete events regardless of policy result
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
def run_policies_for_event(self, message):
"""
Find all policies subscribed to this event type
and schedule them for immediate execution.
"""
# Load up the event
event = json.loads(base64.b64decode(message.content).decode('utf-8'))
operation_name = event['data']['operationName']
# Execute all policies matching the event type
for k, v in self.policies.items():
events = v['policy'].data.get('mode', {}).get('events')
if not events:
continue
events = AzureEvents.get_event_operations(events)
if operation_name in events:
self.scheduler.add_job(self.run_policy,
id=k + event['id'],
name=k,
args=[v['policy'], event, None],
misfire_grace_time=60 * 3)
def run_policy(self, policy, event, context):
try:
policy.push(event, context)
except Exception as e:
log.error("Exception running policy: %s error: %s", policy.name, e)
def prepare_queue_storage(self, queue_resource_id, queue_name):
"""
Create a storage client using unusual ID/group reference
as this is what we require for event subscriptions
"""
# Use a different session object if the queue is in a different subscription
queue_subscription_id = ResourceIdParser.get_subscription_id(
queue_resource_id)
if queue_subscription_id != self.session.subscription_id:
session = Session(queue_subscription_id)
else:
session = self.session
storage_client = session.client(
'azure.mgmt.storage.StorageManagementClient')
account = storage_client.storage_accounts.get_properties(
ResourceIdParser.get_resource_group(queue_resource_id),
ResourceIdParser.get_resource_name(queue_resource_id))
Storage.create_queue_from_storage_account(account, queue_name,
self.session)
return account
@staticmethod
def build_options(output_dir=None, log_group=None, metrics=None):
"""
Initialize the Azure provider to apply global config across all policy executions.
"""
if not output_dir:
output_dir = tempfile.mkdtemp()
log.warning(
'Output directory not specified. Using directory: %s' %
output_dir)
config = Config.empty(**{
'log_group': log_group,
'metrics': metrics,
'output_dir': output_dir
})
return Azure().initialize(config)
@staticmethod
def has_yaml_ext(filename):
return filename.lower().endswith(('.yml', '.yaml'))
class VpsScheduler(object):
def __init__(self):
self.log = logging.getLogger("Main.VpsScheduler")
self.interface = salt_runner.VpsHerder()
self.sched = BlockingScheduler({
'apscheduler.job_defaults.coalesce': 'true',
'apscheduler.timezone': 'UTC',
})
self.sched.add_job(self.ensure_active_workers, 'interval', seconds=60)
self.install_destroyer_jobs()
def create_vm(self, vm_name):
vm_idx = int(vm_name.split("-")[-1]) - 1
self.log.info("Creating VM named: %s, index: %s", vm_name, vm_idx)
try:
self.interface.make_client(vm_name)
self.interface.configure_client(vm_name, vm_idx)
self.log.info("VM %s created.", vm_name)
except marshaller_exceptions.VmCreateFailed:
self.log.info("Failure instantiating VM %s.", vm_name)
self.destroy_vm(vm_name)
def destroy_vm(self, vm_name):
self.log.info("Destroying VM named: %s", vm_name)
self.interface.destroy_client(vm_name)
self.log.info("VM %s destroyed.", vm_name)
def build_target_vm_list(self):
workers = []
for x in range(settings.VPS_ACTIVE_WORKERS):
# start VPS numbers at 1
# Mostly for nicer printing
workers.append(VPS_NAME_FORMAT.format(number=x + 1))
assert len(set(workers)) == len(workers), "Duplicate VPS target names!"
return set(workers)
def get_active_vms(self):
# workers = ['scrape-worker-1', 'scrape-worker-2', 'scrape-worker-a', 'scrape-worker-5', 'utility']
workers = self.interface.list_nodes()
ret = [
worker for worker in workers if worker.startswith('scrape-worker')
]
assert len(set(ret)) == len(ret), "VPS instances with duplicate names!"
return set(ret)
def ensure_active_workers(self):
self.log.info("Validating active VPSes")
active = self.get_active_vms()
target = self.build_target_vm_list()
self.log.info("Active managed VPSes: %s", active)
self.log.info("Target VPS set: %s", target)
missing = target - active
extra = active - target
self.log.info("Need to create VMs: %s", missing)
self.log.info("Need to destroy VMs: %s", extra)
for vm_name in extra:
self.destroy_vm(vm_name)
for vm_name in missing:
self.create_vm(vm_name)
existing = self.sched.get_jobs()
tznow = datetime.datetime.now(tz=pytz.utc)
for job in existing:
self.log.info(" %s, %s", job, job.args)
def install_destroyer_jobs(self):
# vms = self.get_active_vms()
vms = self.build_target_vm_list()
hours = time.time() / (60 * 60)
restart_interval = settings.VPS_LIFETIME_HOURS / settings.VPS_ACTIVE_WORKERS
basetime = time.time()
basetime = basetime - (basetime %
hrs_to_sec(settings.VPS_LIFETIME_HOURS))
print(hours % settings.VPS_LIFETIME_HOURS, restart_interval, basetime)
for vm in vms:
vm_num = int(vm.split("-")[-1])
start_offset = vm_num * restart_interval
nextrun = basetime + hrs_to_sec(start_offset)
# Don't schedule a destruction before we start the scheduler.
if nextrun + 120 < time.time():
nextrun += hrs_to_sec(settings.VPS_LIFETIME_HOURS)
self.sched.add_job(self.destroy_vm,
trigger='interval',
args=(vm, ),
seconds=hrs_to_sec(settings.VPS_LIFETIME_HOURS),
next_run_time=datetime.datetime.fromtimestamp(
nextrun, tz=pytz.utc))
print("Item nextrun: ", nextrun, nextrun - time.time())
def run(self):
self.sched.start()
}, {
"delete": {
"_id": "system-log"
}
}, {
"delete": {
"_id": "SSFirewall-log"
}
}, {
"delete": {
"_id": "All-log"
}
}]
print printjson(
es.bulk(index=".kibana", doc_type="select-buffter", body=actions))[1]
@sched.scheduled_job('cron', day="*", hour=2, minute=0, second=0)
def main():
insert_kibana('.kibana')
if __name__ == '__main__':
#print printjson(es.info())[1]
#print es.ping()
#insert_kibana('.kibana')
#main()
#delete_kibana()
print sched.get_jobs()
sched.start()
acticle_links = self.parser.parse_list(wechat_url, html_cont)
if acticle_links == None:
break
for link in acticle_links:
html = self.downloader.download_articles_ph(link)
data = self.parser.parse_article(html) #解析出文本
if data == None:
continue
(title, date, content, readNum, praise_num, discuss_content,
discuss_praise) = data
# self.urls.add_new_urls(new_urls)
# self.outputer.collect_data(data)
self.outputer.output_file(full_path, data)
if __name__ == "__main__":
logging.basicConfig()
sched = BlockingScheduler()
sched.add_job(job_period,
'cron',
start_date='2016-09-01',
hour=0,
minute=0,
second=1,
end_date='2016-11-30')
a = sched.get_jobs()
print(a)
sched.start()
# job_period()
start_date='2020-03-30 00:00:00')
scheduler.add_job(func=invmbBoxSizeWork,
name='INVMBBoxSize',
id='INVMBBoxSize',
trigger='interval',
weeks=1,
hours=2,
start_date='2020-03-30 00:00:00')
scheduler.add_job(func=getBomListWork,
name='GetBomList',
id='GetBomList',
trigger='interval',
weeks=1,
hours=3,
start_date='2020-03-30 00:00:00')
try:
loggerMain.logger.warning('Main_定时任务开始')
loggerMain.logger.warning('Main_现有任务:{}'.format(scheduler.get_jobs()))
loggerMain.logger.warning('Main_执行需优先处理任务')
autoPlanWork()
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
loggerMain.logger.warning('Main_定时任务关闭')
except Exception as e:
loggerMain.logger.error('Main:{}'.format(str(e)))
def tick():
page(1)
time.sleep(30)
Run_Algebra.run_algebra()
print(
'''
************************************************************************************
************************************************************************************
***********************************定时任务运行成功***********************************
************************************************************************************
************************************************************************************
''', "***************************************** 现在的时间是: %s" %
datetime.now())
if __name__ == '__main__':
scheduler = BlockingScheduler()
scheduler.add_job(tick, 'interval', seconds=300)
print('按 Ctrl+{0} 退出'.format('Break' if os.name == 'nt' else 'C '))
try:
scheduler.start()
print(scheduler.get_jobs())
print('Press Ctrl+{0} to exit'.format('Break' if os.name ==
'nt' else 'C'))
except (KeyboardInterrupt, SystemExit, BaseException) as e:
scheduler.shutdown(wait=False)
Send_mail.send_mail(subject="警报：‘Timed_Task’ 模块错误，请以 nohup 重新运行程序",
content_text=e + scheduler.get_jobs(),
attachments=None)
print('退出工作!')
def run_job():
scheduler = BlockingScheduler()
scheduler.add_job(hello_job, "interval", seconds=5)
print scheduler.get_jobs()
scheduler.start()
# some convenience variables
one = timedelta(seconds=1)
three = timedelta(seconds=3)
five = timedelta(seconds=5)
cam = Camera()
cron = BlockingScheduler()
#schedule_alerts(eclipse)
with open('mtjefferson.csv', 'r') as csvfile:
reader = csv.DictReader(csvfile, delimiter=',')
for row in reader:
#print(row)
cron.add_job(take_picture,
'date',
next_run_time=row['start'],
kwargs=row)
for job in cron.get_jobs():
print(job)
try:
print("begin at: %s" % (datetime.now()))
cron.start()
except (KeyboardInterrupt, SystemExit):
GPIO.cleanup()
cam.exit()
pass
def get_jobs():
scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
jobs_array = scheduler.get_jobs()
for job_info in jobs_array:
print job_info.id, job_info.name, job_info.next_run_time
class JobManage():
def __init__(self):
jobstores = {'default': MemoryJobStore()}
executors = {
'default': ThreadPoolExecutor(50)
# 'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 50}
self.sched = BlockingScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
self.addError()
self.addJobExecuted()
def addJob(self, func, jobId=None, cron=None, args=[], kwargs={}):
'''
只支持cron的形式
* * * * * command
分 时 日 月 周 命令
第1列表示分钟1～59 每分钟用*或者 */1表示
第2列表示小时1～23（0表示0点）
第3列表示日期1～31
第4列表示月份1～12
第5列标识号星期0～6（0表示星期天）
第6列要运行的命令
'''
if cron is None:
raise Exception("cron cannot be Null")
(minute, hour, day, month, week) = cron.split(" ")
self.sched.add_job(func,
trigger='cron',
id=jobId,
hour=hour,
minute=minute,
day=day,
month=month,
week=week,
args=args,
kwargs=kwargs)
def removeJob(self, jobId):
self.sched.remove_job(jobId)
def start(self):
self.sched.start()
def shutdown(self):
self.sched.shutdown()
def printJobs(self):
self.sched.print_jobs()
def getJobs(self):
return self.sched.get_jobs()
def addError(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_ERROR)
def addJobExecuted(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_EXECUTED)
def listener(self, event):
if event.exception:
log.error("任务【%s】 任务出错 : %s" % (event.job_id, event.traceback))
else:
log.debug("任务【%s】已经跑完，结束时间 : %s " % (event.job_id, getNow()))
# jobMange = JobManage()
class SpiderSchedule(object):
"""
爬虫调度，不同的数据源，采取不同的爬取策略
"""
def __init__(self):
self.log = logging.getLogger("proxy.spider")
self.sched = BlockingScheduler()
self.client = RedisClient(host=REDIS['host'],
port=REDIS['port'],
db=REDIS['db'],
password=REDIS['password'],
max_conns=REDIS['max_conns'])
self._config_schedule()
def _config_schedule(self):
"""
配置任务
:return:
"""
for crawl in crawl_list:
# 是否可用
if not crawl["enable"]:
continue
self.log.info("添加job：{}".format(crawl["name"]))
#执行方式，是间隔时间，还是定时任务
if "interval" in crawl:
d = crawl["interval"]
self.sched.add_job(self._spider, "interval", [crawl["name"]],
**d)
elif "cron" in crawl:
d = crawl["cron"]
self.sched.add_job(self._spider, "cron", [crawl["name"]], **d)
def _spider(self, name):
"""
爬虫实现
:param name:
:return:
"""
self.log.info("爬取源：{}".format(name))
crawl_conf = get_crawl_by_name(name)
for url in crawl_conf["urls"]:
# 延时下载
time.sleep(crawl_conf.get("delay", None) or DOWNLOAD_DELAY)
content = Downloader.download(url,
timeout=config.DOWNLOAD_TIMEOUT,
retries=config.DOWNLOAD_RETRIES)
if content is None:
self.log.error("download失败，url：" + url)
continue
#解析页面
proxy_list = HtmlParser().parse(url, content, crawl_conf)
#保存proxy
self._save(proxy_list, crawl_conf)
def _save(self, proxy_list, crawl_conf):
self.client.lpushlist(QUEUE_NAME, proxy_list)
def run(self):
try:
# 判断是否有job
jobs = self.sched.get_jobs()
if len(jobs) == 0:
self.log.error("当前jobs为0")
return
self.sched.start()
except Exception:
self.log.error("执行调度任务失败")
def test_EndpointStatuses(mocker):
"""Tests the main workflow of endpointStatuses.
We mock away at two places,
gordo_components.watchman.endpoints_status.watch_for_model_server_service
which listens for kubernetes events is mocked away, and we construct the events
manually.
And the job-scheduler is never actually started, so we dont proceed to do any of
the jobs, we just check that they are added/removed as desired.
"""
# We will never call start on the scheduler, so we wont actually do any of the
# scheduled jobs|
scheduler = BlockingScheduler()
project_name = "super_project"
project_version = "101"
namespace = "somenamespace"
host = "localhost"
target_names = ["target 1", "target 2"]
mocked_watch = mocker.patch(
"gordo_components.watchman.endpoints_status.watch_for_model_server_service"
)
eps = EndpointStatuses(
scheduler=scheduler,
project_name=project_name,
ambassador_host=host,
model_names=target_names,
project_version=project_version,
namespace=namespace,
)
assert namespace == mocked_watch.call_args[1]["namespace"]
assert project_name == mocked_watch.call_args[1]["project_name"]
assert project_version == mocked_watch.call_args[1]["project_version"]
event_handler = mocked_watch.call_args[1]["event_handler"]
# Before receiving any events we only have the targets in `target_names`
cur_status = eps.statuses()
assert set([ep["target"] for ep in cur_status]) == set(target_names)
# And none of them are healthy
assert all([ep["healthy"] is False for ep in cur_status])
# Lets start adding some events.
# Target 1 is online!
# We make a fake event
mock_event_obj = MagicMock()
mock_event_obj.metadata = MagicMock()
mock_event_obj.metadata.labels = {
"applications.gordo.equinor.com/model-name": "target 1"
}
# And we let the caller know about it
event_handler({"type": "ADDED", "object": mock_event_obj})
# The job to update target 1 is added to the joblist
jobs = scheduler.get_jobs()
assert len(jobs) == 1
assert scheduler.get_job("update_model_metadata_target 1") is not None
# Target 2 is up as well
mock_event_obj.metadata.labels[
"applications.gordo.equinor.com/model-name"] = "target 2"
event_handler({"type": "ADDED", "object": mock_event_obj})
jobs = scheduler.get_jobs()
assert len(jobs) == 2
assert scheduler.get_job("update_model_metadata_target 2") is not None
# Oida, target 1 seems to be removed!
mock_event_obj.metadata.labels = {
"applications.gordo.equinor.com/model-name": "target 1"
}
event_handler({"type": "DELETED", "object": mock_event_obj})
jobs = scheduler.get_jobs()
assert len(jobs) == 1
assert scheduler.get_job("update_model_metadata_target 1") is None
assert scheduler.get_job("update_model_metadata_target 2") is not None
class VpsScheduler(object):
def __init__(self):
self.log = logging.getLogger("Main.VpsScheduler")
self.interface = salt_runner.VpsHerder()
self.sched = BlockingScheduler({
'apscheduler.job_defaults.coalesce': 'true',
'apscheduler.timezone': 'UTC',
})
self.stats_con = statsd.StatsClient(
host=settings.GRAPHITE_DB_IP,
port=8125,
prefix='ReadableWebProxy.VpsHerder',
)
self.sched.add_job(poke_statsd, 'interval', seconds=60)
self.sched.add_job(self.ensure_active_workers,
'interval',
seconds=60 * 5)
self.install_destroyer_jobs()
def create_vm(self, vm_name):
vm_idx = int(vm_name.split("-")[-1]) - 1
provider = "unknown"
self.log.info("Creating VM named: %s, index: %s", vm_name, vm_idx)
try:
# VM Create time is 30 minutes, max
with stopit.ThreadingTimeout(60 * 30, swallow_exc=False):
# This is slightly horrible.
provider, kwargs = self.interface.generate_conf()
with self.stats_con.timer("VM-Creation-{}".format(provider)):
client_make = self.interface.make_client(
vm_name, provider, kwargs)
self.log.info("Client make result: %s", client_make)
client_conf = self.interface.configure_client(
vm_name, vm_idx)
self.log.info("Client conf result: %s", client_conf)
self.log.info("VM %s created.", vm_name)
CREATE_WATCHDOG.value += 1
self.stats_con.incr(
"vm-create.{provider}.ok".format(provider=provider))
except stopit.TimeoutException:
self.log.error("Timeout instantiating VM %s.", vm_name)
self.stats_con.incr(
"vm-create.{provider}.fail.timeout".format(provider=provider))
for line in traceback.format_exc().split("\n"):
self.log.error(line)
for _ in range(5):
self.destroy_vm(vm_name)
time.sleep(2.5)
return
except marshaller_exceptions.LocationNotAvailableResponse:
self.log.warning("Failure instantiating VM %s.", vm_name)
self.stats_con.incr(
"vm-create.{provider}.fail.locationnotavilable".format(
provider=provider))
for line in traceback.format_exc().split("\n"):
self.log.warning(line)
for _ in range(5):
self.destroy_vm(vm_name)
time.sleep(2.5)
return
except marshaller_exceptions.InvalidDeployResponse:
self.log.warning("Failure instantiating VM %s.", vm_name)
self.stats_con.incr(
"vm-create.{provider}.fail.invaliddeployresponse".format(
provider=provider))
for line in traceback.format_exc().split("\n"):
self.log.warning(line)
for _ in range(5):
self.destroy_vm(vm_name)
time.sleep(2.5)
return
except marshaller_exceptions.InvalidExpectParameter:
self.log.warning("Failure instantiating VM %s.", vm_name)
self.stats_con.incr(
"vm-create.{provider}.fail.invalidexpectparameter".format(
provider=provider))
for line in traceback.format_exc().split("\n"):
self.log.warning(line)
for _ in range(5):
self.destroy_vm(vm_name)
time.sleep(2.5)
return
except marshaller_exceptions.VmCreateFailed:
self.log.warning("Failure instantiating VM %s.", vm_name)
self.stats_con.incr(
"vm-create.{provider}.fail.vmcreatefailed".format(
provider=provider))
for line in traceback.format_exc().split("\n"):
self.log.warning(line)
for _ in range(5):
self.destroy_vm(vm_name)
time.sleep(2.5)
return
except Exception as e:
self.stats_con.incr(
"vm-create.{provider}.fail.unknown-error".format(
provider=provider))
self.log.error("Unknown failure instantiating VM %s!", vm_name)
for line in traceback.format_exc().split("\n"):
self.log.error(line)
for _ in range(5):
self.destroy_vm(vm_name)
time.sleep(2.5)
return
self.log.info("VM Creation complete.")
def destroy_vm(self, vm_name):
self.interface.list_nodes()
self.log.info(
"Destroying VM named: %s, current workers: %s", vm_name,
[worker for provider, worker in self.interface.list_nodes()])
dest_cnt = 0
while vm_name in [
worker for provider, worker in self.interface.list_nodes()
]:
self.interface.destroy_client(vm_name)
dest_cnt += 1
self.log.info("VM %s destroyed (%s calls).", vm_name, dest_cnt)
self.interface.list_nodes()
def build_target_vm_list(self):
workers = []
for x in range(settings.VPS_ACTIVE_WORKERS):
# start VPS numbers at 1
# Mostly for nicer printing
workers.append(VPS_NAME_FORMAT.format(number=x + 1))
assert len(set(workers)) == len(
workers), "Duplicate VPS target names: %s!" % workers
return set(workers)
def get_active_vms(self):
# workers = ['scrape-worker-1', 'scrape-worker-2', 'scrape-worker-a', 'scrape-worker-5', 'utility']
workers = self.interface.list_nodes()
ret = []
active_each = {}
for provider, worker in workers:
if worker.startswith('scrape-worker'):
ret.append(worker)
active_each.setdefault(provider, []).append(worker)
workers = [
worker for provider, worker in workers
if worker.startswith('scrape-worker')
]
if len(set(workers)) != len(workers):
self.log.error("Duplicate VPS target names in active workers: %s!",
workers)
for worker in set(workers):
if workers.count(worker) > 1:
self.log.info("Destroying VM: ")
self.destroy_vm(worker)
return set(workers)
def worker_lister(self):
'''
Maximally dumb function to kick over the stats system.
'''
self.get_active_vms()
def ensure_active_workers(self):
self.log.info("Validating active VPSes")
active = self.get_active_vms()
self.log.info("Active nodes:")
for node_tmp in active:
self.log.info(" %s", node_tmp)
target = self.build_target_vm_list()
# Whoo set math!
missing = target - active
extra = active - target
self.log.info("Active managed VPSes: %s", active)
self.log.info("Target VPS set : %s", target)
self.log.info("Need to create VMs : %s", missing)
self.log.info("Need to destroy VMs : %s", extra)
for vm_name in extra:
self.destroy_vm(vm_name)
self.interface.list_nodes()
for vm_name in missing:
self.create_vm(vm_name)
self.interface.list_nodes()
existing = self.sched.get_jobs()
for job in existing:
self.log.info(" %s, %s", job, job.args)
def install_destroyer_jobs(self):
# vms = self.get_active_vms()
vms = self.build_target_vm_list()
hours = time.time() / (60 * 60)
# The lifetime value needs to be a float.
settings.VPS_LIFETIME_HOURS = float(settings.VPS_LIFETIME_HOURS)
restart_interval = settings.VPS_LIFETIME_HOURS / settings.VPS_ACTIVE_WORKERS
basetime = time.time()
basetime = basetime - (basetime %
hrs_to_sec(settings.VPS_LIFETIME_HOURS))
self.log.info(
"VPS Lifetime (hours): %s. Step interval: %s. Modulo start-time: %s",
hours % settings.VPS_LIFETIME_HOURS, restart_interval, basetime)
for vm in vms:
vm_num = int(vm.split("-")[-1])
start_offset = vm_num * restart_interval
nextrun = basetime + hrs_to_sec(start_offset)
# Don't schedule a destruction before we start the scheduler.
if nextrun + 120 < time.time():
nextrun += hrs_to_sec(settings.VPS_LIFETIME_HOURS)
self.sched.add_job(self.destroy_vm,
trigger='interval',
args=(vm, ),
seconds=hrs_to_sec(settings.VPS_LIFETIME_HOURS),
next_run_time=datetime.datetime.fromtimestamp(
nextrun, tz=pytz.utc))
self.log.info("VM %s next run: %s (in %s seconds): ", vm, nextrun,
nextrun - time.time())
def run(self):
self.sched.start()
# Run ping to a specific server url to keep it alive (awake)
def ping():
pingurl = os.environ.get('DOMAIN_URL') + '/ping'
# TODO: Add error handling
try:
r = requests.get(pingurl)
except:
print("Unexpected error:", sys.exc_info())
else:
print('Ping {} at {}: {}'.format(
pingurl,
datetime.now().strftime("%Y-%m-%d %H:%M:%S"), r.status_code))
sched = BlockingScheduler()
if os.environ.get('KEEP_ALIVE', '0') == "1":
try:
every = int(os.environ.get('PING_EVERY_X_MINUTES', '15'))
except:
every = 15
print('Job: Add keepalive, running every {} minutes'.format(every))
sched.add_job(ping, 'interval', minutes=every)
if len(sched.get_jobs()) > 0:
sched.start()
class abstract_schedule(metaclass=ABCMeta):
def __init__(self):
self.sched = BlockingScheduler()
def lstCronJob(self, job_id=None):
result = {}
if not job_id:
jobs = self.sched.get_jobs()
for j in jobs:
result[j.id] = j
else:
jobs = self.sched.get_job(job_id)
result[job_id] = jobs
return result
def delCronJob(self, job_id):
jobs = self.lstCronJob(job_id)
if not jobs:
sys.stdout.write("Job %s not found" %job_id)
else:
self.sched.remove_job(job_id)
sys.stdout.write("Job %s 删除成功!"%job_id)
return True
def addCronJob(self, job_id, func, policy, args):
cron = CronTrigger(**policy)
self.sched.add_job(func, cron, args=args, id=job_id)
def start(self):
print("123123")
self.sched.add_job(self.autoAddJob, IntervalTrigger(seconds=5), id="autoAddJob")
self.sched.start()
def autoAddJob(self):
history_jobs = self.lstCronJob()
print(history_jobs, 'history_jobs')
current_jobs = self.getBackupPolicy()
print(current_jobs, 'current_jobs')
only_current_jobs = set(current_jobs.keys()).difference(set(history_jobs.keys()))
print(only_current_jobs, 'only_current_jobs')
# 当前任务调度列表中有的 历史任务列表中没有的
only_history_jobs = set(history_jobs.keys()).difference(set(current_jobs.keys()))
print(only_history_jobs, 'only_history_jobs')
#历史任务中有的当前任务列表中没有的任务
#
for j in only_history_jobs:
if j == 'autoAddJob':
continue
self.delCronJob(job_id=j)
for j in only_current_jobs:
func = current_jobs[j].pop('func')
args = current_jobs[j].pop('args')
policy = current_jobs[j]
self.addCronJob(job_id=j, func=func, policy=policy, args=args)
@abstractmethod
def getBackupPolicy(self):
pass
def Create_Scheduler(x):
global i
i = 0
for u in x:
temp = datetime.strptime(u['schedule_time'], '%H:%M:%S')
#format = '%H:%M:%S'
sched.add_job(task,
'cron',
hour=temp.hour,
minute=temp.minute,
id=str(i),
kwargs={
"a": u['Tag'],
"b": u['food_amount']
})
i += 1
def delete_Create_Scheduler(x):
for j in i:
sched.remove_job(str(j))
if __name__ == "__main__":
x = Get_Scheduler()
Create_Scheduler(x)
print(sched.get_jobs())
sched.start()
class Host:
def __init__(self, storage_id, queue_name, policy_uri,
log_group=None, metrics=None, output_dir=None):
logging.basicConfig(level=logging.INFO, format='%(message)s')
log.info("Running Azure Cloud Custodian Self-Host")
resources.load_available()
self.session = local_session(Session)
self.storage_session = self.session
storage_subscription_id = ResourceIdParser.get_subscription_id(storage_id)
if storage_subscription_id != self.session.subscription_id:
self.storage_session = Session(subscription_id=storage_subscription_id)
# Load configuration
self.options = Host.build_options(output_dir, log_group, metrics)
self.policy_storage_uri = policy_uri
self.event_queue_id = storage_id
self.event_queue_name = queue_name
# Default event queue name is the subscription ID
if not self.event_queue_name:
self.event_queue_name = self.session.subscription_id
# Prepare storage bits
self.policy_blob_client = None
self.blob_cache = {}
self.queue_storage_account = self.prepare_queue_storage(
self.event_queue_id,
self.event_queue_name)
self.queue_service = None
# Register event subscription
self.update_event_subscription()
# Policy cache and dictionary
self.policy_cache = tempfile.mkdtemp()
self.policies = {}
# Configure scheduler
self.scheduler = BlockingScheduler(Host.get_scheduler_config())
logging.getLogger('apscheduler.executors.default').setLevel(logging.ERROR)
logging.getLogger('apscheduler').setLevel(logging.ERROR)
# Schedule recurring policy updates
self.scheduler.add_job(self.update_policies,
'interval',
seconds=policy_update_seconds,
id="update_policies",
next_run_time=datetime.now(),
executor='threadpool')
# Schedule recurring queue polling
self.scheduler.add_job(self.poll_queue,
'interval',
seconds=queue_poll_seconds,
id="poll_queue",
executor='threadpool')
self.scheduler.start()
def update_policies(self):
"""
Enumerate all policies from storage.
Use the MD5 hashes in the enumerated policies
and a local dictionary to decide if we should
bother downloading/updating each blob.
We maintain an on-disk policy cache for future
features.
"""
if not self.policy_blob_client:
self.policy_blob_client = Storage.get_blob_client_by_uri(self.policy_storage_uri,
self.storage_session)
(client, container, prefix) = self.policy_blob_client
try:
# All blobs with YAML extension
blobs = [b for b in client.list_blobs(container) if Host.has_yaml_ext(b.name)]
except AzureHttpError as e:
# If blob methods are failing don't keep
# a cached client
self.policy_blob_client = None
raise e
# Filter to hashes we have not seen before
new_blobs = self._get_new_blobs(blobs)
# Get all YAML files on disk that are no longer in blob storage
cached_policy_files = [f for f in os.listdir(self.policy_cache)
if Host.has_yaml_ext(f)]
removed_files = [f for f in cached_policy_files if f not in [b.name for b in blobs]]
if not (removed_files or new_blobs):
return
# Update a copy so we don't interfere with
# iterations on other threads
policies_copy = self.policies.copy()
for f in removed_files:
path = os.path.join(self.policy_cache, f)
self.unload_policy_file(path, policies_copy)
# Get updated YML files
for blob in new_blobs:
policy_path = os.path.join(self.policy_cache, blob.name)
if os.path.exists(policy_path):
self.unload_policy_file(policy_path, policies_copy)
elif not os.path.isdir(os.path.dirname(policy_path)):
os.makedirs(os.path.dirname(policy_path))
client.get_blob_to_path(container, blob.name, policy_path)
self.load_policy(policy_path, policies_copy)
self.blob_cache.update({blob.name: blob.properties.content_settings.content_md5})
# Assign our copy back over the original
self.policies = policies_copy
def _get_new_blobs(self, blobs):
new_blobs = []
for blob in blobs:
md5_hash = blob.properties.content_settings.content_md5
if not md5_hash:
blob, md5_hash = self._try_create_md5_content_hash(blob)
if blob and md5_hash and md5_hash != self.blob_cache.get(blob.name):
new_blobs.append(blob)
return new_blobs
def _try_create_md5_content_hash(self, blob):
# Not all storage clients provide the md5 hash when uploading a file
# so, we need to make sure that hash exists.
(client, container, _) = self.policy_blob_client
log.info("Applying md5 content hash to policy {}".format(blob.name))
try:
# Get the blob contents
blob_bytes = client.get_blob_to_bytes(container, blob.name)
# Re-upload the blob. validate_content ensures that the md5 hash is created
client.create_blob_from_bytes(container, blob.name, blob_bytes.content,
validate_content=True)
# Re-fetch the blob with the new hash
hashed_blob = client.get_blob_properties(container, blob.name)
return hashed_blob, hashed_blob.properties.content_settings.content_md5
except AzureHttpError as e:
log.warning("Failed to apply a md5 content hash to policy {}. "
"This policy will be skipped.".format(blob.name))
log.error(e)
return None, None
def load_policy(self, path, policies):
"""
Loads a YAML file and prompts scheduling updates
:param path: Path to YAML file on disk
:param policies: Dictionary of policies to update
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
new_policies = PolicyCollection.from_data(policy_config, self.options)
if new_policies:
for p in new_policies:
log.info("Loading Policy %s from %s" % (p.name, path))
p.validate()
policies.update({p.name: {'policy': p}})
# Update periodic
policy_mode = p.data.get('mode', {}).get('type')
if policy_mode == CONTAINER_TIME_TRIGGER_MODE:
self.update_periodic(p)
elif policy_mode != CONTAINER_EVENT_TRIGGER_MODE:
log.warning(
"Unsupported policy mode for Azure Container Host: {}. "
"{} will not be run. "
"Supported policy modes include \"{}\" and \"{}\"."
.format(
policy_mode,
p.data['name'],
CONTAINER_EVENT_TRIGGER_MODE,
CONTAINER_TIME_TRIGGER_MODE
)
)
except Exception as exc:
log.error('Invalid policy file %s %s' % (path, exc))
def unload_policy_file(self, path, policies):
"""
Unload a policy file that has changed or been removed.
Take the copy from disk and pop all policies from dictionary
and update scheduled jobs.
"""
with open(path, "r") as stream:
try:
policy_config = yaml.safe_load(stream)
except yaml.YAMLError as exc:
log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc))
os.unlink(path)
return path
try:
# Some policies might have bad format, so they have never been loaded
removed = [policies.pop(p['name'])
for p in policy_config.get('policies', [])
if p['name'] in policies]
log.info('Removing policies %s' % removed)
# update periodic
periodic_names = \
[p['name'] for p in policy_config.get('policies', [])
if p.get('mode', {}).get('schedule')]
periodic_to_remove = \
[p for p in periodic_names if p in [j.id for j in self.scheduler.get_jobs()]]
for name in periodic_to_remove:
self.scheduler.remove_job(job_id=name)
except (AttributeError, KeyError) as exc:
log.warning('Failure loading cached policy for cleanup %s %s' % (path, exc))
os.unlink(path)
return path
def update_periodic(self, policy):
"""
Update scheduled policies using cron type
periodic scheduling.
"""
trigger = CronTrigger.from_crontab(policy.data['mode']['schedule'])
self.scheduler.add_job(Host.run_policy,
trigger,
id=policy.name,
name=policy.name,
args=[policy, None, None],
coalesce=True,
max_instances=1,
replace_existing=True,
misfire_grace_time=60)
def update_event_subscription(self):
"""
Create a single event subscription to channel
all events to an Azure Queue.
"""
log.info('Updating event grid subscriptions')
destination = StorageQueueEventSubscriptionDestination(
resource_id=self.queue_storage_account.id, queue_name=self.event_queue_name)
# Build event filter
event_filter = EventSubscriptionFilter(
included_event_types=['Microsoft.Resources.ResourceWriteSuccess'])
# Update event subscription
AzureEventSubscription.create(destination,
self.event_queue_name,
self.session.get_subscription_id(),
self.session, event_filter)
def poll_queue(self):
"""
Poll the Azure queue and loop until
there are no visible messages remaining.
"""
# Exit if we don't have any policies
if not self.policies:
return
if not self.queue_service:
self.queue_service = Storage.get_queue_client_by_storage_account(
self.queue_storage_account,
self.storage_session)
while True:
try:
messages = Storage.get_queue_messages(
self.queue_service,
self.event_queue_name,
num_messages=queue_message_count,
visibility_timeout=queue_timeout_seconds)
except AzureHttpError:
self.queue_service = None
raise
if len(messages) == 0:
break
log.info('Pulled %s events to process while polling queue.' % len(messages))
for message in messages:
if message.dequeue_count > max_dequeue_count:
Storage.delete_queue_message(self.queue_service,
self.event_queue_name,
message=message)
log.warning("Event deleted due to reaching maximum retry count.")
else:
# Run matching policies
self.run_policies_for_event(message)
# We delete events regardless of policy result
Storage.delete_queue_message(
self.queue_service,
self.event_queue_name,
message=message)
def run_policies_for_event(self, message):
"""
Find all policies subscribed to this event type
and schedule them for immediate execution.
"""
# Load up the event
event = json.loads(base64.b64decode(message.content).decode('utf-8'))
operation_name = event['data']['operationName']
# Execute all policies matching the event type
for k, v in self.policies.items():
events = v['policy'].data.get('mode', {}).get('events')
if not events:
continue
events = AzureEvents.get_event_operations(events)
if operation_name.upper() in (event.upper() for event in events):
self.scheduler.add_job(Host.run_policy,
id=k + event['id'],
name=k,
args=[v['policy'],
event,
None],
misfire_grace_time=60 * 3)
def prepare_queue_storage(self, queue_resource_id, queue_name):
"""
Create a storage client using unusual ID/group reference
as this is what we require for event subscriptions
"""
storage_client = self.storage_session \
.client('azure.mgmt.storage.StorageManagementClient')
account = storage_client.storage_accounts.get_properties(
ResourceIdParser.get_resource_group(queue_resource_id),
ResourceIdParser.get_resource_name(queue_resource_id))
Storage.create_queue_from_storage_account(account,
queue_name,
self.session)
return account
@staticmethod
def run_policy(policy, event, context):
try:
policy.push(event, context)
except Exception:
log.exception("Policy Failed: %s", policy.name)
@staticmethod
def build_options(output_dir=None, log_group=None, metrics=None):
"""
Initialize the Azure provider to apply global config across all policy executions.
"""
if not output_dir:
output_dir = tempfile.mkdtemp()
log.warning('Output directory not specified. Using directory: %s' % output_dir)
config = Config.empty(
**{
'log_group': log_group,
'metrics': metrics,
'output_dir': output_dir
}
)
return Azure().initialize(config)
@staticmethod
def get_scheduler_config():
if os.name == 'nt':
executor = "apscheduler.executors.pool:ThreadPoolExecutor"
else:
executor = "apscheduler.executors.pool:ProcessPoolExecutor"
return {
'apscheduler.jobstores.default': {
'type': 'memory'
},
'apscheduler.executors.default': {
'class': executor,
'max_workers': '4'
},
'apscheduler.executors.threadpool': {
'type': 'threadpool',
'max_workers': '20'
},
'apscheduler.job_defaults.coalesce': 'true',
'apscheduler.job_defaults.max_instances': '1',
'apscheduler.timezone': 'UTC',
}
@staticmethod
def has_yaml_ext(filename):
return filename.lower().endswith(('.yml', '.yaml'))
@staticmethod
@click.command(help="Periodically run a set of policies from an Azure storage container "
"against a single subscription. The host will update itself with new "
"policies and event subscriptions as they are added.")
@click.option("--storage-id", "-q", envvar=ENV_CONTAINER_STORAGE_RESOURCE_ID, required=True,
help="The resource id of the storage account to create the event queue in")
@click.option("--queue-name", "-n", envvar=ENV_CONTAINER_QUEUE_NAME,
help="The name of the event queue to create")
@click.option("--policy-uri", "-p", envvar=ENV_CONTAINER_POLICY_URI, required=True,
help="The URI to the Azure storage container that holds the policies")
@click.option("--log-group", "-l", envvar=ENV_CONTAINER_OPTION_LOG_GROUP,
help="Location to send policy logs")
@click.option("--metrics", "-m", envvar=ENV_CONTAINER_OPTION_METRICS,
help="The resource name or instrumentation key for uploading metrics")
@click.option("--output-dir", "-d", envvar=ENV_CONTAINER_OPTION_OUTPUT_DIR,
help="The directory for policy output")
def cli(**kwargs):
Host(**kwargs)
class JobScheduler(metaclass=Singleton):
__function_dict__ = {}
def __init__(self):
self.logger = Log().logger
self.scheduler = BlockingScheduler(logger=self.logger)
self.parser = DateTimeParser()
def get_lives(self):
return {job.name: job for job in self.scheduler.get_jobs()}
def add_job(self, name, trigger, trigger_args, func_args=None, func_kwargs=None):
if name not in self.get_lives():
func = self.get_functions(name)
self.scheduler.add_job(func, trigger=trigger, args=func_args, kwargs=func_kwargs,
id=name, name=name, **trigger_args)
self.logger.info('[%s][%s]作业已添加，并已启动' % (trigger, name))
def add_date_job(self, name, time, func_args=None, func_kwargs=None):
time = self.parser.set_date(time).set_time(time).datetime
self.add_job(name, 'date', {'run_date': time}, func_args, func_kwargs)
def add_interval_job(self, name, weeks=0, days=0, hours=0, minutes=0, seconds=0,
func_args=None, func_kwargs=None):
self.add_job(name, 'interval',
{'weeks': weeks, 'days': days,
'hours': hours, 'minutes': minutes, 'seconds': seconds},
func_args, func_kwargs)
def add_cron_job(self, name, year=None, month=None, day=None, week=None,
day_of_week=None, hour=None, minute=None, second=None,
func_args=None, func_kwargs=None):
self.add_job(name, 'cron',
{'year': year, 'month': month, 'day': day, 'week': week,
'day_of_week': day_of_week,
'hour': hour, 'minute': minute, 'second': second},
func_args, func_kwargs)
def add_started_job(self, name, after_seconds=1, func_args=None, func_kwargs=None):
time = datetime.now() + timedelta(seconds=after_seconds)
self.add_date_job(name, time, func_args, func_kwargs)
def delete_job(self, name):
jobs = self.get_lives()
if name in jobs:
self.scheduler.remove_job(jobs[name].id)
self.logger.info('作业[%s]已移除' % name)
@classmethod
def register(cls, name):
def add_method(f):
cls.__function_dict__[name.strip()] = f
return f
return add_method
def get_functions(self, name):
return self.__function_dict__.get(name.strip())
def get_function_doc(self, name):
return self.get_functions(name).__doc__
def get_function_names(self):
return sorted(self.__function_dict__.keys())
