class SchedulerBackgroud(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.scheduler = BackgroundScheduler()
self.executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
self.job_defaults = {'coalesce': False, 'max_instances': 1}
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'),
job_defaults=self.job_defaults,
executors=self.executors)
def Run(self):
self.scheduler.add_job(Task.count_es_logs,
'cron',
second='0',
minute='*',
id=Task.count_es_logs.__name__,
replace_existing=True)
self.scheduler.add_job(Task.server_per,
'cron',
second='0',
minute='*/5',
id=Task.server_per.__name__,
replace_existing=True)
self.scheduler.start()
loging.write("Scheduler backgroud start on %s ......" % HOST)
def get_scheduler(store_path=None, log_file=None):
if store_path is None:
store_path = r'jobstore.sqlite'
if log_file is None:
log_file = r'logger.log'
scheduler = BackgroundScheduler({'apscheduler.timezone': 'Asia/Shanghai'})
jobstores = {
'default': SQLAlchemyJobStore(url='sqlite:///{0}'.format(store_path))
}
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 1
}
scheduler.configure(jobstores=jobstores, executors=executors)
# 事件记录
scheduler.add_listener(
lambda event: event_listener(event, scheduler),
EVENT_JOB_EXECUTED | EVENT_JOB_ERROR | EVENT_JOB_ADDED | EVENT_JOB_SUBMITTED | EVENT_JOB_REMOVED
)
# 日志定制
scheduler._logger = modify_logger(scheduler._logger, log_file=log_file)
return scheduler
def scheduler(event):
scheduler = BackgroundScheduler()
settings = event.app.registry.settings
jobstores = {'default': SQLAlchemyJobStore(url=settings['scheduler.url'])}
executors = {
'default': {
'type': settings['scheduler.executors.type'],
'max_workers': settings['scheduler.executors.max_workers']
},
'processpool': ProcessPoolExecutor(
max_workers=settings['scheduler.executors.processpool.max_workers']
)
}
job_defaults = {
'coalesce': False,
'max_instances': settings['scheduler.job_defaults.max_instances']
}
scheduler.configure(
jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=timezone('UTC')
)
if settings['scheduler.autostart'] == 'true':
scheduler.start()
event.app.registry.registerUtility(scheduler, IScheduler)
def runDownScheduler():
"""
down shecdule random(1-5)hour
:return:
"""
timezone = ConfigHandler().timezone
scheduler_log = LogHandler("schedule")
scheduler = BackgroundScheduler(logger=scheduler_log, timezone=timezone)
intlTime = random.randint(1, 5)
scheduler.add_job(execDown(random.randint(5, 30)),
'interval',
hour=intlTime,
id="down_url",
name="url下载")
executors = {
'default': {
'type': 'threadpool',
'max_workers': 20
},
'processpool': ProcessPoolExecutor(max_workers=5)
}
job_defaults = {'coalesce': False, 'max_instances': 10}
scheduler.configure(executors=executors,
job_defaults=job_defaults,
timezone=timezone)
scheduler.start()
class Scheduler(Singleton):
scheduler: BackgroundScheduler
def __init__(self, maxWorkers: int = 20, maxExecuters:int = 5):
jobstores = {
'default': MemoryJobStore()
}
executors = {
'default': {'type': 'threadpool', 'max_workers': maxWorkers}
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
self.scheduler = BackgroundScheduler()
self.scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
self.scheduler.add_listener(self.onEvent)
def onEvent(self, event: JobEvent):
print(event)
def addJob(self, crontabStr: str, func: FunctionType) -> Job:
trigger: CronTrigger = None
if crontabStr:
trigger = CronTrigger.from_crontab(crontabStr)
return self.scheduler.add_job(func, trigger)
def parseCronTab(self, crontabStr: str) -> CronTrigger:
return CronTrigger.from_crontab(crontabStr)
class cScheduler():
def __init__(self):
jobstores = {
'default': SQLAlchemyJobStore(url='sqlite:///__secret//jobs.db')
}
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
job_defaults = {
'coalesce': False,
'max_instances': 1
}
self.the_sched = BackgroundScheduler()
self.the_sched.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc)
self.telegram_server = None
self.job_queue = None
def start(self):
self.the_sched.start()
def set_telegram_server(self, telegram_server):
self.telegram_server = telegram_server
self.telegram_job_queue = telegram_server.get_job_queue()
def start_main_schedule(self):
self.telegram_job_queue.put(self.telegram_server.extjob_send_all, 5, repeat=True)
def start():
jobstores = {
# 'mongo': {'type': 'mongodb'},
'default': SQLAlchemyJobStore(url='sqlite:///db.sqlite3')
}
executors = {
'default': {
'type': 'threadpool',
'max_workers': 20
},
'processpool': ProcessPoolExecutor(max_workers=5)
}
job_defaults = {'coalesce': False, 'max_instances': 3}
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone='Asia/kolkata')
scheduler.add_job(
background_task,
'cron',
day_of_week='*',
hour=18,
minute=00,
id="scheduler_id",
max_instances=1,
replace_existing=True,
)
scheduler.start()
class YtsmScheduler(object):
def __init__(self):
self._apscheduler = BackgroundScheduler()
def initialize(self):
# set state of existing jobs as "interrupted"
JobExecution.objects\
.filter(status=JOB_STATES_MAP['running'])\
.update(status=JOB_STATES_MAP['interrupted'])
self._configure_scheduler()
self._apscheduler.start()
def _configure_scheduler(self):
logger = logging.getLogger('scheduler')
executors = {
'default': {
'type': 'threadpool',
'max_workers': appconfig.concurrency
}
}
job_defaults = {
'misfire_grace_time': 60 * 60 * 24 * 365 # 1 year
}
self._apscheduler.configure(logger=logger, executors=executors, job_defaults=job_defaults)
def _run_job(self, job_class: Type[Job], user: Optional[User], args: Union[tuple, list]):
job_execution = JobExecution(user=user, status=JOB_STATES_MAP['running'])
job_execution.save()
job_instance = job_class(job_execution, *args)
# update description
job_execution.description = job_instance.get_description()
job_execution.save()
try:
job_instance.run()
job_execution.status = JOB_STATES_MAP['finished']
except Exception as ex:
job_instance.log.critical("Job failed with exception: %s", traceback.format_exc())
job_instance.usr_err(job_instance.name + " operation failed: " + str(ex))
job_execution.status = JOB_STATES_MAP['failed']
finally:
job_execution.end_date = datetime.datetime.now(tz=pytz.UTC)
job_execution.save()
def add_job(self, job_class: Type[Job], trigger: Union[str, BaseTrigger] = None,
args: Union[list, tuple] = None,
user: Optional[User] = None,
**kwargs):
if args is None:
args = []
return self._apscheduler.add_job(YtsmScheduler._run_job, trigger=trigger, args=[self, job_class, user, args],
**kwargs)
def get_scheduler(host, port, password=None):
""" Returns an APScheduler instance with Redis store """
scheduler = BackgroundScheduler()
scheduler.configure(
jobstores={
'default': RedisJobStore(host=host, port=port, password=password)
})
return scheduler
class Scheduler_publish(object):
def __init__(self):
self.tm = time.strftime('%M',time.localtime())
self.scheduler = BackgroundScheduler({'apscheduler.job_defaults.max_instances': '5'})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Scheduler_mem(self,func,publish_key = 'None',taskKey = 'None',):
self.scheduler.add_job(func,'cron', second='*/2',minute = self.tm,args=[publish_key,taskKey],id='%s_%s'%(taskKey,self.tm))
return self.scheduler
class Scheduler_publish(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.tm = time.strftime('%Y%m%d%H%M',time.localtime())
self.scheduler = BackgroundScheduler({'apscheduler.job_defaults.max_instances': '5'})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Scheduler_mem(self,func,publish_key = 'None',taskKey = 'None'):
self.scheduler.add_job(func,'date', run_date=self.run_date,args=[publish_key,taskKey],id='%s_%s'%(taskKey,self.tm),replace_existing=False)
return self.scheduler
def init():
sched = BackgroundScheduler(daemon=True,
timezone=utc,
misfire_grace_time=None)
atexit.register(lambda: sched.shutdown(wait=False))
sched.configure(misfire_grace_time=None)
sched.start()
# jjj = sched.get_job('barn2_snapshot_1')
return sched
class MiningScheduler(object):
def __init__(self, mining_server, conf):
self.logger = logging.getLogger("root")
self.conn = get_conn(conf.get("db", "host"), conf.getint("db", "port"), conf.get("db", "database"))
self.mining_server = mining_server
# self.init_seed()
self.init_job()
def init_job(self):
self.scheduler = BackgroundScheduler()
executors = {"default": {"type": "threadpool", "max_workers": 5}}
self.scheduler.configure(logger=logging.getLogger("apscheduler"), executors=executors)
self.scheduler.add_listener(
self.err_handle, apscheduler.events.EVENT_JOB_ERROR | apscheduler.events.EVENT_JOB_MISSED
)
job_co = self.conn.mining_seed_job
total = job_co.count()
interval = 60 * 60
time_delta = interval * 1.0 / total
all_job = list(job_co.find())
for i, job in enumerate(all_job):
next_run_time = datetime.now() + timedelta(milliseconds=int(time_delta * i * 1000), seconds=10)
self.scheduler.add_job(
self.put_to_queue,
"interval",
args=(job,),
seconds=interval,
next_run_time=next_run_time,
id=str(job.get("_id")),
misfire_grace_time=10,
)
def init_seed(self):
data_list = []
with open("../data/url_filter.txt") as fin:
for line in fin:
line = line.strip()
_, url, block = line.split("\t")
data_list.append(dict(url=url, block=block))
self.conn.mining_seed_job.remove({})
self.conn.mining_seed_job.insert_many(data_list)
def err_handle(self, ev):
self.logger.error("apscheduler error:" + str(ev))
def start(self):
self.logger.info("scheduler started")
self.scheduler.start()
def put_to_queue(self, job):
url = job.get("url")
rs = self.conn.mining_seed_task.insert_one(dict(job_id=job.get("_id"), time=datetime.now()))
self.mining_server.process_task(url, rs.inserted_id)
class Slave(Arctic):
def __init__(self, connection = None):
if not connection:
connection = 'localhost'
self.__mongo_client = connection
# tick db
Arctic.__init__(self, self.__mongo_client)
# job system
self.__job_db = self.__mongo_client.jobs
self.__job_collection = self.__job_db.queue
# scheduler
self.__scheduler = BackgroundScheduler()
self.__scheduler.configure(executors = executors, job_defaults= job_defaults, timezone=utc)
self.__scheduler.start()
def __del__(self):
self.__scheduler.shutdown(wait=False)
# TODO to have different functors query from various data sources
def work(self, job):
"""
:param job: a json job object contains symbol and other stuff
:return:
"""
pass
def get_job(self):
"""
Periodically query new job(s) from the master
if running jobs is less than MAXIMUM
@param desc : a json file of jobs
"""
doc_jobs = self.__job_collection.find({"status" : {"$ne" : "removed" }})
doc_jobs = doc_jobs.sort('add_time', pymongo.DESCENDING)
return doc_jobs[0]
def update_jobs(self):
"""
Periodically check jobs' status, if job(s) from the master
to be paused, or removed
"""
pass
def list_jobs(self):
doc_jobs = self.__job_collection.find({"status" : {"$ne" : "removed"}})
for job in doc_jobs:
pprint.pprint(job)
def init_scheduler():
jobstores = {}
executors = {'default': {'type': 'threadpool', 'max_workers': 20}}
job_defaults = {'coalesce': True, 'max_instances': 3}
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone="utc")
return scheduler
def schdeuler(log):
scheduler = BackgroundScheduler()
job_defaults = {
'coalesce': False,
'max_instances': 50,
'misfire_grace_time': 100,
}
scheduler.configure(job_defaults=job_defaults)
scheduler._logger = log
return scheduler
class Scheduler_publish(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.tm = time.strftime('%Y%m%d%H%M%S',time.localtime())
self.scheduler = BackgroundScheduler()
self.executors = {'default': ThreadPoolExecutor(10), 'processpool': ProcessPoolExecutor(5)}
self.job_defaults = {'coalesce': False, 'max_instances': 1}
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'),job_defaults=self.job_defaults,executors=self.executors)
def Scheduler_mem(self,func,args = []):
self.scheduler.add_job(func,'date', run_date=self.run_date,args=[args],id=self.tm,replace_existing=True)
return self.scheduler
def config_Scheduler(self, timezone):
jobstores = {
'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')
}
executors = {'processpool': ProcessPoolExecutor(max_workers=5)}
job_defaults = {'coalesce': False, 'max_instances': 3}
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=timezone)
sched.add_job(job_function, 'interval', seconds=1)
class APSchedulerBundle(object):
def __init__(self):
self.config_mapping = {
"apscheduler": {
"jobstores": "",
"executors": "",
"coalesce": False,
"max_instances": 3,
"timezone": "UTC"
}
}
self.scheduler = BackgroundScheduler()
self.injection_bindings = {
Scheduler: self.scheduler
}
self.event_listeners = [
(KernelReadyEvent, self.kernel_ready),
(KernelShutdownEvent, self.kernel_shutdown)
]
self.bundle_lock = Lock()
self.bundle_lock.acquire()
def kernel_shutdown(self, event):
self.bundle_lock.release()
@inject.params(config=Configuration)
@inject.params(kernel=Kernel)
def kernel_ready(self, event, config, kernel):
self.scheduler.configure(self.build_config(config.apscheduler))
logging.info("APScheduler ready")
kernel.run_service(self.run_service)
def run_service(self):
self.scheduler.start()
self.bundle_lock.acquire()
self.scheduler.shutdown()
def build_config(self, config):
apsconfig = {}
for prop in ["jobstores", "executors"]:
if isinstance(getattr(config, prop), dict) > 0:
for prop_name, prop_config in getattr(config, prop).items():
apsconfig[".".join(['apscheduler', prop, prop_name])] = prop_config
apsconfig["apscheduler.job_defaults.coalesce"] = config.coalesce
apsconfig["apscheduler.job_defaults.max_instances"] = config.max_instances
apsconfig["apscheduler.timezone"] = config.timezone
return apsconfig
def schedule_jobs():
# https://apscheduler.readthedocs.io/en/stable/userguide.html?highlight=max_instances#limiting-the-number-of-concurrently-executing-instances-of-a-job
scheduler = BackgroundScheduler(
{'apscheduler.job_defaults.max_instances': '1'})
scheduler.configure(timezone=utc)
scheduler.start()
scheduler.add_job(notification_scraper,
args=['icinga'],
trigger='interval',
seconds=settings.NOTIFICATIONS_SCRAPE_INTERVAL_SECONDS)
scheduler.add_job(notification_scraper,
args=['zabbix'],
trigger='interval',
seconds=settings.NOTIFICATIONS_SCRAPE_INTERVAL_SECONDS)
def create_app(self):
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'
app.config['ROOT_USER'] = '******'
app.config['ROOT_PASS'] = '******'
app.config['SECRET_KEY'] = 'secret'
app.debug = False
self.app = app
self.db = db
# Initializing test scheduler
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc)
self.scheduler = scheduler
self.scheduler.start()
return app
class TaskScheduler:
def __init__(self):
jobs_database_name = 'jobs.sqlite'
jobstores = {
# 'default': SQLAlchemyJobStore(url=F'sqlite:///{jobs_database_name}')
}
executors = {
'default': {
'type': 'threadpool',
'max_workers': 20
},
'processpool': ProcessPoolExecutor(max_workers=5)
}
job_defaults = {'coalesce': False, 'max_instances': 3}
self.scheduler = BackgroundScheduler()
self.scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
def get_task(self, task_id):
return self.scheduler.get_job(task_id)
def get_tasks_ids(self):
task_ids = []
jobs = self.scheduler.get_jobs()
for j in jobs:
task_ids.append(j.id)
return task_ids
def start_scheduler(self):
self.scheduler.start()
def stop_scheduler(self):
self.scheduler.shutdown()
def add_task(self, task_func, interval_minutes, args, task_id):
print('Adding an interval task')
self.scheduler.add_job(task_func,
IntervalTrigger(minutes=interval_minutes),
args=args,
id=str(task_id))
print('Adding the interval task finished')
def remove_task(self, id):
print(F'Removing task (Id: {id})')
self.scheduler.remove_job(id)
print('The task removed.')
class Scheduler_backgroud(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.scheduler = BackgroundScheduler(
{'apscheduler.job_defaults.max_instances': '15'})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Run(self):
if HOST in task_servers:
self.scheduler.add_job(Task.analytics_internet_logs,
'date',
run_date=self.run_date,
id='analytics_internet_logs',
replace_existing=True)
self.scheduler.add_job(Task.analytics_internet2_logs,
'date',
run_date=self.run_date,
id='analytics_internet2_logs',
replace_existing=True)
self.scheduler.add_job(Task.analytics_internet3_logs,
'date',
run_date=self.run_date,
id='analytics_internet3_logs',
replace_existing=True)
self.scheduler.add_job(Task.analytics_intranet_logs,
'date',
run_date=self.run_date,
id='analytics_intranet_logs',
replace_existing=True)
self.scheduler.add_job(Task.analytics_intranet2_logs,
'date',
run_date=self.run_date,
id='analytics_intranet2_logs',
replace_existing=True)
self.scheduler.add_job(Task.WAF_logs,
'date',
run_date=self.run_date,
id='WAF_logs',
replace_existing=True)
self.scheduler.add_job(Task.httpry_logs,
'date',
run_date=self.run_date,
id='httpry_logs',
replace_existing=True)
self.scheduler.start()
loging.write('Scheduler_run start......')
else:
loging.write('%s not in task server list!' % HOST)
def configure_scheduler():
global scheduler
jobstores = {
'default': SQLAlchemyJobStore(url=DATABASE_URL)
}
executors = {
'default': {'type': 'threadpool', 'max_workers': 20}
}
job_defaults = {
'coalesce': True,
'max_instances': 3
}
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone='America/New_York')
def init_app(app):
app.register_blueprint(ads, url_prefix='/ads')
# let SQLAlchemy knows the current app
db.init_app(app)
with app.app_context():
# initial database check
logic_scheduler.initial_update()
# register and start weekly update job, perform update on every sunday
scheduler = BackgroundScheduler()
scheduler.configure(timezone=utc)
scheduler.add_job(
func=logic_scheduler.weekly_update, trigger='cron', args=[app], day_of_week=6, hour=12, minute=0, second=0,
id='ads_weekly_update', replace_existing=True
)
scheduler.start()
def create_app(self):
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'
app.config['ROOT_USER'] = '******'
app.config['ROOT_PASS'] = '******'
app.config['SECRET_KEY'] = 'secret'
app.debug = False
self.app = app
self.db = db
# Initializing test scheduler
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
self.scheduler = scheduler
self.scheduler.start()
return app
class Scheduler_publish(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.tm = time.strftime('%Y%m%d%H%M', time.localtime())
self.scheduler = BackgroundScheduler(
{'apscheduler.job_defaults.max_instances': '5'})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Scheduler_mem(self, func, publish_key='None', taskKey='None'):
self.scheduler.add_job(func,
'date',
run_date=self.run_date,
args=[publish_key, taskKey],
id='%s_%s' % (taskKey, self.tm),
replace_existing=False)
return self.scheduler
class Scheduler_backgroud(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.scheduler = BackgroundScheduler({'apscheduler.job_defaults.max_instances': '15'})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Run(self):
if HOST in task_servers:
self.scheduler.add_job(Task.analytics_internet_logs, 'date', run_date=self.run_date,id='analytics_internet_logs', replace_existing=True)
self.scheduler.add_job(Task.analytics_internet2_logs, 'date', run_date=self.run_date,id='analytics_internet2_logs', replace_existing=True)
self.scheduler.add_job(Task.analytics_internet3_logs, 'date', run_date=self.run_date,id='analytics_internet3_logs', replace_existing=True)
self.scheduler.add_job(Task.analytics_intranet_logs, 'date', run_date=self.run_date,id='analytics_intranet_logs',replace_existing=True)
self.scheduler.add_job(Task.analytics_intranet2_logs, 'date', run_date=self.run_date,id='analytics_intranet2_logs', replace_existing=True)
self.scheduler.add_job(Task.WAF_logs, 'date', run_date=self.run_date,id='WAF_logs',replace_existing=True)
self.scheduler.add_job(Task.httpry_logs,'date', run_date=self.run_date,id='httpry_logs', replace_existing=True)
self.scheduler.start()
loging.write('Scheduler_run start......')
else:
loging.write('%s not in task server list!' %HOST)
def init_scheduler():
"""初始化调度器"""
logger.debug('init scheduler')
global scheduler
# SQLAlchemyJobStore指定存储链接
jobstores = {'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')}
# 最大工作线程数20
executors = {
'default': {
'type': 'threadpool',
'max_workers': 20
},
}
scheduler = BackgroundScheduler()
# scheduler.configure(executors=executors)
# 持久化存储不支持回调函数
scheduler.configure(jobstores=jobstores, executors=executors)
scheduler.start()
class NidoDaemon(Daemon):
def run(self):
self._l.debug('Starting run loop for Nido daemon')
self.controller = Controller()
config = Config().get_config()
poll_interval = config['schedule']['poll_interval']
db_path = config['schedule']['db']
rpc_port = config['schedule']['rpc_port']
self.scheduler = BackgroundScheduler()
jobstores = {
'default': {'type': 'memory'},
'schedule': SQLAlchemyJobStore(
url='sqlite:///{}'.format(db_path)
)
}
job_defaults = {'coalesce': True, 'misfire_grace_time': 10}
self.scheduler.configure(jobstores=jobstores,
job_defaults=job_defaults)
self.scheduler.add_job(
NidoSchedulerService.wakeup, trigger='interval',
seconds=poll_interval, name='Poll'
)
self.scheduler.add_job(NidoSchedulerService.wakeup, name='Poll')
self.scheduler.start()
RPCserver = ThreadedServer(
NidoSchedulerService(self.scheduler),
port=rpc_port,
protocol_config={
'allow_public_attrs': True,
'allow_pickle': True
}
)
RPCserver.start()
def quit(self):
self.scheduler.shutdown()
self.controller.shutdown()
self._l.info('Nido daemon shutdown')
self._l.info('********************')
return
class Scheduler_publish(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.tm = time.strftime('%Y%m%d%H%M%S', time.localtime())
self.scheduler = BackgroundScheduler({
'apscheduler.job_defaults.max_instances':
'50',
'apscheduler.job_defaults.coalesce':
'false'
})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Scheduler_mem(self, func, args=[]):
self.scheduler.add_job(func,
'date',
run_date=self.run_date,
args=[args],
id=self.tm,
replace_existing=True)
return self.scheduler
class SeedScheduler(object):
def __init__(self, scheduler, handler, conf):
'''
scheduler: blocking or background
'''
self.logger = logging.getLogger("root")
self.handler = handler
self.init_job(scheduler, conf)
def init_job(self, scheduler, conf):
base = SeedBase(conf)
if scheduler == 'blocking':
self.scheduler = BlockingScheduler()
elif scheduler == 'background':
self.scheduler = BackgroundScheduler()
executors = {
'default': {'type': 'threadpool', 'max_workers': 5},
'processpool': ProcessPoolExecutor(max_workers=5)
}
self.scheduler.configure(logger=logging.getLogger('apscheduler'), executors=executors)
self.scheduler.add_listener(self.err_handle,
apscheduler.events.EVENT_JOB_ERROR | apscheduler.events.EVENT_JOB_MISSED)
for seed in base.get_all_seed():
print seed
start_date=datetime.datetime.now()
self.scheduler.add_job(self.handler.handle, 'interval',
args=(seed, self.logger),
seconds=seed['interval'],
id=str(seed['jobid'])
)
def err_handle(self, ev):
self.logger.error(str(ev))
def start(self):
self.logger.info('scheduler started')
self.scheduler.start()
class Scheduler_backgroud(object):
def __init__(self):
self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3)
self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S')
self.scheduler = BackgroundScheduler({
'apscheduler.job_defaults.max_instances':
'50',
'apscheduler.job_defaults.coalesce':
'false'
})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
def Run(self):
if HOST in TASK_BACKGROUD:
self.scheduler.add_job(Task.count_es_logs,
'cron',
second='0',
minute='*',
id=Task.count_es_logs.__name__,
replace_existing=True)
self.scheduler.start()
loging.write("Scheduler backgroud start on %s ......" % HOST)
def initScheduler(self):
jobstores = {'default': MemoryJobStore()}
executors = {
'default': {
'type': 'threadpool',
'max_workers': 20
},
}
job_defaults = {'coalesce': False, 'max_instances': 3}
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone="Europe/Athens")
#scheduler.add_listener(my_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
scheduler.start()
#logging.basicConfig()
return scheduler
class Scheduler_publish(object):
def __init__(self):
self.tm = time.strftime('%M',time.localtime())
self.scheduler = BackgroundScheduler({'apscheduler.job_defaults.max_instances': '5'})
self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'))
self.path = app.root_path
def job_update_java(self):
path = '%s/../Script/java_update_script.py' % self.path
os.system('/usr/bin/python %s' % path)
def job_publish_java(self):
path = '%s/../Script/java_publish_script.py' % self.path
os.system('/usr/bin/python %s' % path)
def job_update_php(self):
path = '%s/../Script/php_update_script.py' % self.path
os.system('/usr/bin/python %s' % path)
def job_publish_php(self):
path = '%s/../Script/php_publish_script.py' % self.path
os.system('/usr/bin/python %s' % path)
def Scheduler_mem(self,func):
self.scheduler.add_job(func,'cron', second='*/2',minute = self.tm )
return self.scheduler
from apscheduler.schedulers.background import BackgroundScheduler from pytz import utc from slack.bot import start_bot from django.conf import settings scheduler = BackgroundScheduler() scheduler.configure(timezone=settings.TIME_ZONE) scheduler.add_job(start_bot) scheduler.start()
class JobQueue:
"""This class allows you to periodically perform tasks with the bot. It is a convenience
wrapper for the APScheduler library.
Attributes:
scheduler (:class:`apscheduler.schedulers.background.BackgroundScheduler`): The APScheduler
bot (:class:`telegram.Bot`): The bot instance that should be passed to the jobs.
DEPRECATED: Use :attr:`set_dispatcher` instead.
"""
def __init__(self) -> None:
self._dispatcher: 'Dispatcher' = None # type: ignore[assignment]
self.logger = logging.getLogger(self.__class__.__name__)
self.scheduler = BackgroundScheduler(timezone=pytz.utc)
self.scheduler.add_listener(
self._update_persistence, mask=EVENT_JOB_EXECUTED | EVENT_JOB_ERROR
)
# Dispatch errors and don't log them in the APS logger
def aps_log_filter(record): # type: ignore
return 'raised an exception' not in record.msg
logging.getLogger('apscheduler.executors.default').addFilter(aps_log_filter)
self.scheduler.add_listener(self._dispatch_error, EVENT_JOB_ERROR)
def _build_args(self, job: 'Job') -> List[Union[CallbackContext, 'Bot', 'Job']]:
if self._dispatcher.use_context:
return [CallbackContext.from_job(job, self._dispatcher)]
return [self._dispatcher.bot, job]
def _tz_now(self) -> datetime.datetime:
return datetime.datetime.now(self.scheduler.timezone)
def _update_persistence(self, event: JobEvent) -> None: # pylint: disable=W0613
self._dispatcher.update_persistence()
def _dispatch_error(self, event: JobEvent) -> None:
try:
self._dispatcher.dispatch_error(None, event.exception)
# Errors should not stop the thread.
except Exception:
self.logger.exception(
'An error was raised while processing the job and an '
'uncaught error was raised while handling the error '
'with an error_handler.'
)
@overload
def _parse_time_input(self, time: None, shift_day: bool = False) -> None:
...
@overload
def _parse_time_input(
self,
time: Union[float, int, datetime.timedelta, datetime.datetime, datetime.time],
shift_day: bool = False,
) -> datetime.datetime:
...
def _parse_time_input(
self,
time: Union[float, int, datetime.timedelta, datetime.datetime, datetime.time, None],
shift_day: bool = False,
) -> Optional[datetime.datetime]:
if time is None:
return None
if isinstance(time, (int, float)):
return self._tz_now() + datetime.timedelta(seconds=time)
if isinstance(time, datetime.timedelta):
return self._tz_now() + time
if isinstance(time, datetime.time):
date_time = datetime.datetime.combine(
datetime.datetime.now(tz=time.tzinfo or self.scheduler.timezone).date(), time
)
if date_time.tzinfo is None:
date_time = self.scheduler.timezone.localize(date_time)
if shift_day and date_time <= datetime.datetime.now(pytz.utc):
date_time += datetime.timedelta(days=1)
return date_time
# isinstance(time, datetime.datetime):
return time
def set_dispatcher(self, dispatcher: 'Dispatcher') -> None:
"""Set the dispatcher to be used by this JobQueue. Use this instead of passing a
:class:`telegram.Bot` to the JobQueue, which is deprecated.
Args:
dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher.
"""
self._dispatcher = dispatcher
if dispatcher.bot.defaults:
if dispatcher.bot.defaults:
self.scheduler.configure(timezone=dispatcher.bot.defaults.tzinfo or pytz.utc)
def run_once(
self,
callback: Callable[['CallbackContext'], None],
when: Union[float, datetime.timedelta, datetime.datetime, datetime.time],
context: object = None,
name: str = None,
job_kwargs: JSONDict = None,
) -> 'Job':
"""Creates a new ``Job`` that runs once and adds it to the queue.
Args:
callback (:obj:`callable`): The callback function that should be executed by the new
job. Callback signature for context based API:
``def callback(CallbackContext)``
``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
its ``job.context`` or change it to a repeating job.
when (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \
:obj:`datetime.datetime` | :obj:`datetime.time`):
Time in or at which the job should run. This parameter will be interpreted
depending on its type.
* :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the
job should run.
* :obj:`datetime.timedelta` will be interpreted as "time from now" in which the
job should run.
* :obj:`datetime.datetime` will be interpreted as a specific date and time at
which the job should run. If the timezone (``datetime.tzinfo``) is :obj:`None`,
the default timezone of the bot will be used.
* :obj:`datetime.time` will be interpreted as a specific time of day at which the
job should run. This could be either today or, if the time has already passed,
tomorrow. If the timezone (``time.tzinfo``) is :obj:`None`, the
default timezone of the bot will be used.
context (:obj:`object`, optional): Additional data needed for the callback function.
Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`.
name (:obj:`str`, optional): The name of the new job. Defaults to
``callback.__name__``.
job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the
``scheduler.add_job()``.
Returns:
:class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
queue.
"""
if not job_kwargs:
job_kwargs = {}
name = name or callback.__name__
job = Job(callback, context, name, self)
date_time = self._parse_time_input(when, shift_day=True)
j = self.scheduler.add_job(
callback,
name=name,
trigger='date',
run_date=date_time,
args=self._build_args(job),
timezone=date_time.tzinfo or self.scheduler.timezone,
**job_kwargs,
)
job.job = j
return job
def run_repeating(
self,
callback: Callable[['CallbackContext'], None],
interval: Union[float, datetime.timedelta],
first: Union[float, datetime.timedelta, datetime.datetime, datetime.time] = None,
last: Union[float, datetime.timedelta, datetime.datetime, datetime.time] = None,
context: object = None,
name: str = None,
job_kwargs: JSONDict = None,
) -> 'Job':
"""Creates a new ``Job`` that runs at specified intervals and adds it to the queue.
Args:
callback (:obj:`callable`): The callback function that should be executed by the new
job. Callback signature for context based API:
``def callback(CallbackContext)``
``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
its ``job.context`` or change it to a repeating job.
interval (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta`): The interval in which
the job will run. If it is an :obj:`int` or a :obj:`float`, it will be interpreted
as seconds.
first (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \
:obj:`datetime.datetime` | :obj:`datetime.time`, optional):
Time in or at which the job should run. This parameter will be interpreted
depending on its type.
* :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the
job should run.
* :obj:`datetime.timedelta` will be interpreted as "time from now" in which the
job should run.
* :obj:`datetime.datetime` will be interpreted as a specific date and time at
which the job should run. If the timezone (``datetime.tzinfo``) is :obj:`None`,
the default timezone of the bot will be used.
* :obj:`datetime.time` will be interpreted as a specific time of day at which the
job should run. This could be either today or, if the time has already passed,
tomorrow. If the timezone (``time.tzinfo``) is :obj:`None`, the
default timezone of the bot will be used.
Defaults to ``interval``
last (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \
:obj:`datetime.datetime` | :obj:`datetime.time`, optional):
Latest possible time for the job to run. This parameter will be interpreted
depending on its type. See ``first`` for details.
If ``last`` is :obj:`datetime.datetime` or :obj:`datetime.time` type
and ``last.tzinfo`` is :obj:`None`, the default timezone of the bot will be
assumed.
Defaults to :obj:`None`.
context (:obj:`object`, optional): Additional data needed for the callback function.
Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`.
name (:obj:`str`, optional): The name of the new job. Defaults to
``callback.__name__``.
job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the
``scheduler.add_job()``.
Returns:
:class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
queue.
Note:
`interval` is always respected "as-is". That means that if DST changes during that
interval, the job might not run at the time one would expect. It is always recommended
to pin servers to UTC time, then time related behaviour can always be expected.
"""
if not job_kwargs:
job_kwargs = {}
name = name or callback.__name__
job = Job(callback, context, name, self)
dt_first = self._parse_time_input(first)
dt_last = self._parse_time_input(last)
if dt_last and dt_first and dt_last < dt_first:
raise ValueError("'last' must not be before 'first'!")
if isinstance(interval, datetime.timedelta):
interval = interval.total_seconds()
j = self.scheduler.add_job(
callback,
trigger='interval',
args=self._build_args(job),
start_date=dt_first,
end_date=dt_last,
seconds=interval,
name=name,
**job_kwargs,
)
job.job = j
return job
def run_monthly(
self,
callback: Callable[['CallbackContext'], None],
when: datetime.time,
day: int,
context: object = None,
name: str = None,
day_is_strict: bool = True,
job_kwargs: JSONDict = None,
) -> 'Job':
"""Creates a new ``Job`` that runs on a monthly basis and adds it to the queue.
Args:
callback (:obj:`callable`): The callback function that should be executed by the new
job. Callback signature for context based API:
``def callback(CallbackContext)``
``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
its ``job.context`` or change it to a repeating job.
when (:obj:`datetime.time`): Time of day at which the job should run. If the timezone
(``when.tzinfo``) is :obj:`None`, the default timezone of the bot will be used.
day (:obj:`int`): Defines the day of the month whereby the job would run. It should
be within the range of 1 and 31, inclusive.
context (:obj:`object`, optional): Additional data needed for the callback function.
Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`.
name (:obj:`str`, optional): The name of the new job. Defaults to
``callback.__name__``.
day_is_strict (:obj:`bool`, optional): If :obj:`False` and day > month.days, will pick
the last day in the month. Defaults to :obj:`True`.
job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the
``scheduler.add_job()``.
Returns:
:class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
queue.
"""
if not job_kwargs:
job_kwargs = {}
name = name or callback.__name__
job = Job(callback, context, name, self)
if day_is_strict:
j = self.scheduler.add_job(
callback,
trigger='cron',
args=self._build_args(job),
name=name,
day=day,
hour=when.hour,
minute=when.minute,
second=when.second,
timezone=when.tzinfo or self.scheduler.timezone,
**job_kwargs,
)
else:
trigger = OrTrigger(
[
CronTrigger(
day=day,
hour=when.hour,
minute=when.minute,
second=when.second,
timezone=when.tzinfo,
**job_kwargs,
),
CronTrigger(
day='last',
hour=when.hour,
minute=when.minute,
second=when.second,
timezone=when.tzinfo or self.scheduler.timezone,
**job_kwargs,
),
]
)
j = self.scheduler.add_job(
callback, trigger=trigger, args=self._build_args(job), name=name, **job_kwargs
)
job.job = j
return job
def run_daily(
self,
callback: Callable[['CallbackContext'], None],
time: datetime.time,
days: Tuple[int, ...] = Days.EVERY_DAY,
context: object = None,
name: str = None,
job_kwargs: JSONDict = None,
) -> 'Job':
"""Creates a new ``Job`` that runs on a daily basis and adds it to the queue.
Args:
callback (:obj:`callable`): The callback function that should be executed by the new
job. Callback signature for context based API:
``def callback(CallbackContext)``
``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
its ``job.context`` or change it to a repeating job.
time (:obj:`datetime.time`): Time of day at which the job should run. If the timezone
(``time.tzinfo``) is :obj:`None`, the default timezone of the bot will be used.
days (Tuple[:obj:`int`], optional): Defines on which days of the week the job should
run. Defaults to ``EVERY_DAY``
context (:obj:`object`, optional): Additional data needed for the callback function.
Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`.
name (:obj:`str`, optional): The name of the new job. Defaults to
``callback.__name__``.
job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the
``scheduler.add_job()``.
Returns:
:class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
queue.
Note:
For a note about DST, please see the documentation of `APScheduler`_.
.. _`APScheduler`: https://apscheduler.readthedocs.io/en/stable/modules/triggers/cron.html
#daylight-saving-time-behavior
"""
if not job_kwargs:
job_kwargs = {}
name = name or callback.__name__
job = Job(callback, context, name, self)
j = self.scheduler.add_job(
callback,
name=name,
args=self._build_args(job),
trigger='cron',
day_of_week=','.join([str(d) for d in days]),
hour=time.hour,
minute=time.minute,
second=time.second,
timezone=time.tzinfo or self.scheduler.timezone,
**job_kwargs,
)
job.job = j
return job
def run_custom(
self,
callback: Callable[['CallbackContext'], None],
job_kwargs: JSONDict,
context: object = None,
name: str = None,
) -> 'Job':
"""Creates a new customly defined ``Job``.
Args:
callback (:obj:`callable`): The callback function that should be executed by the new
job. Callback signature for context based API:
``def callback(CallbackContext)``
``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access
its ``job.context`` or change it to a repeating job.
job_kwargs (:obj:`dict`): Arbitrary keyword arguments. Used as arguments for
``scheduler.add_job``.
context (:obj:`object`, optional): Additional data needed for the callback function.
Can be accessed through ``job.context`` in the callback. Defaults to ``None``.
name (:obj:`str`, optional): The name of the new job. Defaults to
``callback.__name__``.
Returns:
:class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job
queue.
"""
name = name or callback.__name__
job = Job(callback, context, name, self)
j = self.scheduler.add_job(callback, args=self._build_args(job), name=name, **job_kwargs)
job.job = j
return job
def start(self) -> None:
"""Starts the job_queue thread."""
if not self.scheduler.running:
self.scheduler.start()
def stop(self) -> None:
"""Stops the thread."""
if self.scheduler.running:
self.scheduler.shutdown()
def jobs(self) -> Tuple['Job', ...]:
"""Returns a tuple of all jobs that are currently in the ``JobQueue``."""
return tuple(Job.from_aps_job(job, self) for job in self.scheduler.get_jobs())
def get_jobs_by_name(self, name: str) -> Tuple['Job', ...]:
"""Returns a tuple of jobs with the given name that are currently in the ``JobQueue``"""
return tuple(job for job in self.jobs() if job.name == name)
class CronManager:
def __init__(self, use_mongo_db=True):
self.scheduler = BackgroundScheduler(timezone=shanghai_tz)
self.scheduler.configure()
if use_mongo_db:
self.job_store = MongoDBJobStore(database='apscheduler',
collection='cronTab',
client=db)
self.scheduler.add_jobstore(self.job_store)
self.is_replace_existing = True
else:
self.is_replace_existing = False
def add_cron(self, cron_instance):
if not isinstance(cron_instance, Cron):
raise TypeError('please add correct cron！')
if cron_instance.trigger_type == 'interval':
seconds = cron_instance.trigger_args.get('seconds')
if not isinstance(seconds,
int) and not common.can_convert_to_int(seconds):
raise TypeError('请输入合法的时间间隔！')
seconds = int(seconds)
if seconds <= 0:
raise TypeError('请输入大于0的时间间隔！')
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
seconds=seconds,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_id(),
max_instances=5,
jitter=0) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'date':
run_date = cron_instance.trigger_args.get('run_date')
# TODO 判断run_date类型
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
run_date=run_date,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_id()) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'cron':
raise TypeError('暂时不支持 trigger_type 等于 \'cron\'')
return cron_instance.get_id()
def start(self, paused=False):
self.scheduler.start(paused=paused)
def pause_cron(self, cron_id=None, pause_all=False):
if pause_all:
self.scheduler.pause()
elif cron_id:
self.scheduler.pause_job(job_id=cron_id)
def resume_cron(self, cron_id=None, resume_all=False):
if resume_all:
self.scheduler.resume()
elif cron_id:
self.scheduler.resume_job(job_id=cron_id)
def del_cron(self, cron_id=None, del_all=False):
if del_all:
self.scheduler.remove_all_jobs()
elif cron_id:
self.scheduler.remove_job(job_id=cron_id)
def update_cron(self, cron_id, cron_info):
if not isinstance(cron_id, str):
raise TypeError('cron_id must be str')
if not isinstance(cron_info, dict):
raise TypeError('cron_info must be dict')
trigger_type = cron_info.get('triggerType')
interval = cron_info.get('interval')
run_date = cron_info.get('runDate')
test_case_suite_id_list = cron_info.get('testCaseSuiteIdList')
is_execute_forbiddened_case = cron_info.get('isExecuteForbiddenedCase')
test_case_id_list = cron_info.get('testCaseIdList')
test_domain = cron_info.get('testDomain')
global_vars_id = cron_info.get('globalVarsId')
alarm_mail_list = cron_info.get('alarmMailList')
is_ding_ding_notify = cron_info.get('isDingDingNotify')
ding_ding_access_token = cron_info.get('dingdingAccessToken')
ding_ding_notify_strategy = cron_info.get('dingdingNotifyStrategy')
is_enterprise_wechat_notify = cron_info.get('isEnterpriseWechatNotify')
enterprise_wechat_access_token = cron_info.get(
'enterpriseWechatAccessToken')
enterprise_wechat_notify_strategy = cron_info.get(
'enterpriseWechatNotifyStrategy')
cron_name = cron_info.get('name')
try:
if trigger_type == 'interval' and int(interval) > 0:
self.scheduler.modify_job(
job_id=cron_id, trigger=IntervalTrigger(seconds=interval))
elif trigger_type == 'date':
# TODO 判断run_date类型
self.scheduler.modify_job(
job_id=cron_id, trigger=DateTrigger(run_date=run_date))
else:
raise TypeError('更新定时任务触发器失败！')
if run_date:
cron = Cron(
test_case_suite_id_list=test_case_suite_id_list,
is_execute_forbiddened_case=is_execute_forbiddened_case,
test_domain=test_domain,
global_vars_id=global_vars_id,
alarm_mail_list=alarm_mail_list,
is_ding_ding_notify=is_ding_ding_notify,
ding_ding_access_token=ding_ding_access_token,
ding_ding_notify_strategy=ding_ding_notify_strategy,
is_enterprise_wechat_notify=is_enterprise_wechat_notify,
enterprise_wechat_access_token=
enterprise_wechat_access_token,
enterprise_wechat_notify_strategy=
enterprise_wechat_notify_strategy,
trigger_type=trigger_type, # 更新定时器时，此参数并没有真正起到作用, 仅修改展示字段
test_case_id_list=test_case_id_list,
run_date=run_date,
cron_name=cron_name) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
else:
cron = Cron(
test_case_suite_id_list=test_case_suite_id_list,
is_execute_forbiddened_case=is_execute_forbiddened_case,
test_domain=test_domain,
global_vars_id=global_vars_id,
alarm_mail_list=alarm_mail_list,
is_ding_ding_notify=is_ding_ding_notify,
ding_ding_access_token=ding_ding_access_token,
ding_ding_notify_strategy=ding_ding_notify_strategy,
is_enterprise_wechat_notify=is_enterprise_wechat_notify,
enterprise_wechat_access_token=
enterprise_wechat_access_token,
enterprise_wechat_notify_strategy=
enterprise_wechat_notify_strategy,
trigger_type=trigger_type, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
test_case_id_list=test_case_id_list,
seconds=interval, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
cron_name=cron_name)
# 玄学，更改job的时候必须改args，不能改func
self.scheduler.modify_job(job_id=cron_id,
coalesce=True,
args=[cron])
except BaseException as e:
raise TypeError('更新定时任务失败: %s' % e)
def shutdown(self, force_shutdown=False):
if force_shutdown:
self.scheduler.shutdown(wait=False)
else:
self.scheduler.shutdown(wait=True)
def get_crons(self):
return self.scheduler.get_jobs()
CONF_FILE=os.path.join(CONF_DIR,"config.yaml")
LANG_DIR=os.path.join(SCRIPT_PATH,"language")
LANG_FILE=os.path.join(LANG_DIR,"lm")
DIC_FILE=os.path.join(LANG_DIR,"dic")
'''object for parsedatetime module'''
nt=naturaltime.NaturalTime()
#init the dbus notify2 API
ny=notify.Notify()
broadcast= signal('Interpreter')
#dict containing all listeners the interpreter is hooked to
SCHEDULER = BackgroundScheduler(daemon=True)
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
SCHEDULER.configure(executors=executors)
print SCHEDULER.start()
# SCHEDULER.add_listener(my_listener, SCHEDULER.EVENT_JOB_EXECUTED | SCHEDULER.EVENT_JOB_ERROR)
with open(CONF_FILE,'r') as conf_file:
confs=yaml.load(conf_file)
keyword=confs["KEYWORD"]
keywords=keyword.split(',')
PUSHBULLET_ID=confs["PUSHBULLET_ID"]
PUSHBULLET_KEY=confs["PUSHBULLET_KEY"]
GOOGLE_API_KEY=confs["GOOGLE_API_KEY"]
DEVICE_NAME=confs["DEVICE_NAME"]
coordinates=confs["COORDINATES"]
WOLFRAM_KEY=confs["WOLFRAM_KEY"]
import bottle
import waitress
import controller
import breathe
from pytz import timezone
from apscheduler.schedulers.background import BackgroundScheduler
bottle_app = bottle.app()
scheduler = BackgroundScheduler()
scheduler.configure(timezone=timezone('US/Pacific'))
breather = breathe.Breathe()
my_controller = controller.Controller(bottle_app, breather)
@scheduler.scheduled_job(trigger='cron', hour=20, minute=0)
def on_job():
"""Start at 10:00pm PT"""
print('STARTING BREATHER')
breather.restart()
@scheduler.scheduled_job(trigger='cron', hour=0, minute=1)
def off_job():
"""End at 12:01am PT"""
print("STOPPING BREATHER")
breather.shutdown()
if __name__ == '__main__':
scheduler.start()
waitress.serve(bottle_app, host='0.0.0.0', port=7000)
random.seed()
app = Flask(__name__)
app.config.from_object('config')
db.init_app(app)
CsrfProtect(app)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'
scheduler = BackgroundScheduler()
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
scheduler.configure(executors=executors)
logging.basicConfig(
filename=LOGFILE_LOCATION,
level=logging.DEBUG
)
NOTIFY_MINUTES = 10
SERVER_RUN_HOUR = 3
@login_manager.user_loader
def load_user(id):
return Users.query.get(int(id))
@app.before_request
def before_request():
app = __init__.app
HOST = socket.gethostbyname(socket.gethostname())
IPS = app.config.get('INIT_IPS')
CMDS = app.config.get('INIT_CMDS')
init_key = app.config.get('INIT_KEY')
INIT_USER = app.config.get('INIT_USER')
nodes = app.config.get('NODES_PRODUCE')
sms_url = app.config.get('SMS_URL')
task_servers = app.config.get('TASK_SERVERS')
redis_host = app.config.get('REDIS_HOST')
redis_port = app.config.get('REDIS_PORT')
Redis = redis.StrictRedis(host=redis_host, port=redis_port)
RC = RedisCluster(startup_nodes=nodes, decode_responses=True)
jobstores = {'default': SQLAlchemyJobStore(url=app.config.get('SQLALCHEMY_BINDS')['op'])}
scheduler = BackgroundScheduler({'apscheduler.job_defaults.max_instances': '5','apscheduler.job_defaults.coalesce': 'false'})
scheduler.configure(jobstores=jobstores,timezone=pytz.timezone('Asia/Shanghai'))
def Produce(length=8,chars=string.ascii_letters+string.digits):
return ''.join([choice(chars) for i in range(length)])
def Async_log(user,url):
if 'op.baihe.com' in url:
mysql_op_log = Mysql.mysql_op(user, url)
Proc = Pool()
def Run():
mysql_op_log.op_log()
Proc.apply_async(Run)
Proc.close()
Proc.join()
@check.proce_lock
def auto_init_system():
for ip in Redis.lpop(init_key):
try:
has completed syncing.
"""
details = request.json
slave_id = details['slave_id']
project = details['project']
slave_node = SlaveNode.query.filter_by(id=slave_id).first()
#XXX: Register that the slave has synced xyz project to the db.
print('SlaveNode *%s* synced *%s*'%(slave_node.hostname, project))
return jsonify({'method': 'slave_rsync_complete', 'success': True})
if __name__ == "__main__":
with app.app_context():
db.create_all(app=app)
# Create root user if it does not exist.
root_user = User.query.filter_by(username=app.config['ROOT_USER']).first()
if not root_user:
root_user = User(app.config['ROOT_USER'], app.config['ROOT_PASS'], 'root')
db.session.add(root_user)
db.session.commit()
# Apply configuration to scheduler instance.
scheduler = BackgroundScheduler()
scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc)
scheduler.start()
app.debug = True
app.run(port=app.config['MASTER_PORT'],)
