def get_mongo_job_stores():
from pytz import utc
from apscheduler.jobstores.mongodb import MongoDBJobStore, MongoClient
from apscheduler.executors.pool import ProcessPoolExecutor
from ops.apscheduler.db_context import get_mongo_client
client = get_mongo_client()
jobstores = {
'mongo':
MongoDBJobStore(collection='job',
database='apscheduler',
client=client),
'default':
MongoDBJobStore(collection='job',
database='apscheduler2',
client=client),
}
executors = {
'default': {
'type': 'threadpool',
'max_workers': 20
},
'processpool': ProcessPoolExecutor(max_workers=5)
}
job_defaults = {'coalesce': False, 'max_instances': 3}
scheduler = TornadoScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
return scheduler
def __init__(self):
self.jobstores = {
'mongo':
MongoDBJobStore(collection='job1',
database='saasjob',
client=_mongoclient),
'default':
MemoryJobStore()
}
self.executors = {
'default': ThreadPoolExecutor(1),
'processpool': ProcessPoolExecutor(1)
}
self.job_defaults = {
'coalesce': False,
'misfire_grace_time': 1,
'max_instances': 1
}
self._sched = BackgroundScheduler(jobstores=self.jobstores,
executors=self.executors,
job_defaults=self.job_defaults)
# 添加 任务提交 事件监听
self._sched.add_listener(self.when_job_submitted, EVENT_JOB_SUBMITTED)
# 添加 任务执行完成 事件监听
self._sched.add_listener(self.when_job_executed, EVENT_JOB_EXECUTED)
# 添加 任务异常退出 事件监听
self._sched.add_listener(self.when_job_crashed, EVENT_JOB_ERROR)
self._jobs = {}
self._jobhandlers = {} # format, key: jobid, value: jobhandler
self._jobs_key = ["name", "func", "args", "kwargs"]
self.start()
class Config(object):
# 配置执行job
JOBS = [
{
'id': 'job1',
'func': 'flask_read:job1',
'args': (1, 2),
'trigger': 'interval',
'seconds': 10
}
]
# 存储位置
SCHEDULER_JOBSTORES = {
'mongo': MongoDBJobStore(host='192.168.2.110', database='test', username="******", password="******")
}
# 线程池配置
SCHEDULER_EXECUTORS = {
'default': {'type': 'threadpool', 'max_workers': 20}
}
SCHEDULER_JOB_DEFAULTS = {
'coalesce': False,
'max_instances': 3
}
# 调度器开关
SCHEDULER_API_ENABLED = True
def quantzrepod():
"""
quantz repo 的守护进程，根据数据的更新时间定期更新数据
"""
initialize_db('quantz')
jobstores = {'default': MongoDBJobStore()}
executors = {'default': ThreadPoolExecutor(8)}
job_defaults = {'coalesce': False, 'max_instances': 8}
scheduler = BlockingScheduler( # jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone='Asia/Shanghai')
scheduler.add_job(update_us_wei,
trigger=WeeklyTrigger(3, 19, 0, 'WeiTrigger'))
scheduler.add_job(update_us_initial_jobless,
trigger=WeeklyTrigger(4, 19, 0, 'IcsaTrigger'))
scheduler.add_job(update_us_ccsa,
trigger=WeeklyTrigger(4, 19, 0, 'CcsaTrigger'))
scheduler.add_job(update_stock_basics,
trigger=DailyTrigger(17, 0, 0, 'update_stock_basics'))
scheduler.add_job(update_industry_classification,
trigger=DailyTrigger(17, 30, 0,
'update_industry_classification'))
scheduler.add_job(update_all_daily_trading_info_in_batch,
trigger=DailyTrigger(
18, 00, 0, 'update_all_daily_trading_info_in_batch'))
scheduler.add_job(rank_all_industry,
trigger=DailyTrigger(19, 00, 0, 'rank_all_industry'))
scheduler.start()
class ProductionSecure(BaseConfig):
"""
生产环境安全性配置
"""
SQLALCHEMY_DATABASE_URI = 'mysql+cymysql://root:root@mysql:3306/easy-test'
SQLALCHEMY_ECHO = False
SECRET_KEY = '\x88W\xf09\x91\x07\x98\x85\x87\x96\xb0A\xc68\xf9\xecJJU\x12\xc5V\xbe\x8b\xef\xd7\xd8\xd3\xe6\x95*6'
MONGO_URI = 'mongodb://*****:*****@mongo:27017/easyTest?authSource=admin'
CELERY_BROKER_URL = 'amqp://*****:*****@rabbitmq:5672/my_vhost'
# scheduler
SCHEDULER_JOBSTORES = {
'default': MongoDBJobStore(host='mongo', port=27017, username='******', password='******')
}
# mail
MAIL_SERVER = 'smtp.sina.cn'
MAIL_PORT = 465
MAIL_USE_TLS = False
MAIL_USE_SSL = True
MAIL_DEFAULT_SENDER = '*****@*****.**'
MAIL_USERNAME = '******'
MAIL_PASSWORD = '******'
# mock server
MOCK_SERVER = 'http://api:5000'
API_SERVER = 'http://api:5000'
SITE_DOMAIN = 'http://127.0.0.1'
def __init__(self, settings):
from apscheduler.jobstores.mongodb import MongoDBJobStore
"""
Initializes the repository with the specified settings dict.
Required settings are:
- SCHEDULER_HOST
- SCHEDULER_DATABASE
- SCHEDULER_COLLECTION
"""
if 'SCHEDULER_HOST' in settings:
self.host = settings['SCHEDULER_HOST']
else:
self.host = 'localhost'
if 'SCHEDULER_DATABASE' in settings:
database = settings['SCHEDULER_DATABASE']
else:
database = 'apscheduler'
if 'SCHEDULER_COLLECTION' in settings:
collection = settings['SCHEDULER_COLLECTION']
else:
collection = 'scheduler_jobs'
client = pymongo.MongoClient(self.host)
self.name = 'MongoDB'
self.job_stores = {
'default': MongoDBJobStore(database=database, collection=collection, client=client),
}
# Execute the parent method to obtain the default values
super(Scheduler, self).__init__(settings)
def init_scheduler(self):
store = MongoDBJobStore(database='freenas',
collection='calendar_tasks',
client=self.datastore.client)
self.scheduler = BackgroundScheduler(jobstores={'default': store},
timezone=pytz.utc)
self.scheduler.start()
def create_scheduler(self):
self.jobstores = {
'mongo':
MongoDBJobStore(collection='job1',
database='saasjob',
client=_mongoclient),
'default':
MemoryJobStore()
}
self.executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
self.job_defaults = {
'coalesce': False,
'misfire_grace_time': 1,
'max_instances': 1
}
self._sched = BackgroundScheduler(jobstores=self.jobstores,
executors=self.executors,
job_defaults=self.job_defaults)
# 添加 任务提交 事件监听
self._sched.add_listener(self.when_job_submitted, EVENT_JOB_SUBMITTED)
# # 添加 任务执行完成 事件监听
# self._sched.add_listener(self.when_job_executed, EVENT_JOB_EXECUTED)
# 添加 任务退出 事件监听
self._sched.add_listener(self.when_job_crashed,
EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
def run(self):
"""Start apscheduler tasks"""
jobstores = {'mongo': MongoDBJobStore()}
executors = {
'default': ThreadPoolExecutor(self.poolsize),
'processpool': ProcessPoolExecutor(self.procsize)
}
job_defaults = {'coalesce': False, 'max_instances': 3}
scheduler = GeventScheduler()
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
scheduler.add_job(self.job_worker, 'interval', seconds=0.001)
green_let = scheduler.start()
print('Ctrl+{0} to exit.'.format('Break' if os.name == 'nt' else 'C'))
# Execution will block here util Ctrl+C (Ctrl+Break on Windows).
try:
green_let.join()
except (KeyboardInterrupt, SystemExit):
pass
class Config:
'''
app settings
'''
DEBUG = parser.config['app']['debug']
TESTING = parser.config['app']['testing']
BCRYPT_LEVEL = parser.config['app']['bcrypt_level']
APP_NAME = parser.config['app']['app_name']
SECRET_KEY = parser.config['app']['secret_key']
WTF_CSRF_ENABLED = parser.config['app']['wtf_csrf_enabled']
# APScheduler
JOBS = []
JSON_AS_ASCII = True # 支持json显示中文
SCHEDULER_API_ENABLED = parser.config['scheduler']['api_enabled']
SCHEDULER_JOBSTORES = {
'default': MongoDBJobStore(database=parser.config['scheduler']['mongodb_db'], client=MongoClient(parser.config['scheduler']['mongodb_uri']))
}
SCHEDULER_EXECUTORS = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
SCHEDULER_JOB_DEFAULTS = parser.config['scheduler']['job_defaults']
# mongodb settings
MONGODB_SETTINGS = parser.config['mongodb']['mongodb_settings']
@staticmethod
def init_app(app):
pass
def __init__(self, client_=None):
if client_:
self.client = client_
else:
self.client = client
jobstores = {
'default': MongoDBJobStore(
client=self.client,
database='crawlab_test',
collection='trigger'
)
}
executors = {
'default': ThreadPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 10
}
self.config = [
jobstores, executors, job_defaults
]
self.scheduler = BackgroundScheduler(
jobstores=jobstores,
executors=executors,
job_defaults=job_defaults
)
class Scheduler(object):
mongo = MongoClient(host=MONGO_HOST, port=MONGO_PORT, connect=False)
task_col = 'apscheduler_jobs'
# scheduler jobstore
jobstores = {
'mongo':
MongoDBJobStore(database=MONGO_DB, collection=task_col, client=mongo)
}
# scheduler instance
scheduler = BackgroundScheduler(jobstores=jobstores)
def execute_spider(self, id: str, params: str = None):
print(f'executing spider {id}')
print(f'params: {params}')
self.scheduler.print_jobs(jobstore='mongo')
query = {}
if params is not None:
query['params'] = params
r = requests.get(
'http://%s:%s/api/spiders/%s/on_crawl' %
(FLASK_HOST, FLASK_PORT, id), query)
def update(self):
print('updating...')
# remove all existing periodic jobs
self.scheduler.remove_all_jobs()
self.mongo[MONGO_DB][self.task_col].remove()
periodical_tasks = db_manager.list('schedules', {})
for task in periodical_tasks:
cron = task.get('cron')
cron_arr = cron.split(' ')
second = cron_arr[0]
minute = cron_arr[1]
hour = cron_arr[2]
day = cron_arr[3]
month = cron_arr[4]
day_of_week = cron_arr[5]
self.scheduler.add_job(func=self.execute_spider,
args=(
str(task['spider_id']),
task.get('params'),
),
trigger='cron',
jobstore='mongo',
day_of_week=day_of_week,
month=month,
day=day,
hour=hour,
minute=minute,
second=second)
self.scheduler.print_jobs(jobstore='mongo')
print(f'state: {self.scheduler.state}')
print(f'running: {self.scheduler.running}')
def run(self):
self.update()
self.scheduler.start()
def __init__(self, host, port):
self.host =host
self.port = port
self.client = MongoClient(self.host, self.port)
self.sched = BackgroundScheduler()
self.jobstore = MongoDBJobStore(collection='scanner', database='dns_scanner', client=self.client)
self.sched.add_jobstore(self.jobstore)
self.sched.add_executor(ThreadPoolExecutor(10))
self.mongomanager = MongoManager(self.client)
self.sched.start()
self.joblist = []
self.redisquere = RedisQueue("dns_scanner")
logging.info("Schedule init success")
print "init success"
class DevelopmentSecure(BaseConfig):
"""
开发环境安全性配置
"""
SQLALCHEMY_DATABASE_URI = 'mysql+cymysql://root:[email protected]:8081/easy-test'
SQLALCHEMY_ECHO = False
SECRET_KEY = '\x88W\xf09\x91\x07\x98\x85\x87\x96\xb0A\xc68\xf9\xecJJU\x12\xc5V\xbe\x8b\xef\xd7\xd8\xd3\xe6\x95*6'
MONGO_URI = 'mongodb://*****:*****@127.0.0.1:8090/easyTest?authSource=admin'
CELERY_BROKER_URL = 'amqp://*****:*****@127.0.0.1:8083/my_vhost'
# scheduler
SCHEDULER_JOBSTORES = {
'default': MongoDBJobStore(host='127.0.0.1', port=8090, username='******', password='******')
}
# mail
MAIL_SERVER = 'smtp.sina.cn'
MAIL_PORT = 465
MAIL_USE_TLS = False
MAIL_USE_SSL = True
MAIL_DEFAULT_SENDER = '*****@*****.**'
MAIL_USERNAME = '******'
MAIL_PASSWORD = '******'
# mock server
MOCK_SERVER = 'http://127.0.0.1:5000'
API_SERVER = 'http://127.0.0.1:5000'
SITE_DOMAIN = 'http://127.0.0.1:5000'
def get_scheduler():
"""
Function used to return the `APScheduler <https://apscheduler.readthedocs.io/en/latest/>`_
instance that is used by Eva and plugins.
.. warning::
This function should only be used by Eva. Plugins should access the
scheduler through Eva's singleton object::
from eva import scheduler
# This will fire off the job immediately.
scheduler.add_job(func_name, id="eva_<plugin_id>_job")
.. todo::
Need to add listeners for all event types:
https://apscheduler.readthedocs.io/en/latest/modules/events.html#event-codes
:note: This function most likely needs to be revisited as it may not be
thread-safe. Eva and plugins can modify the config singleon
simultaneously inside and outside of jobs.
:return: The scheduler object used by plugins to schedule long-running jobs.
:rtype: `apscheduler.schedulers.background.BackgroundScheduler
<https://apscheduler.readthedocs.io/en/latest/modules/schedulers/background.html>`_
"""
client = get_mongo_client()
db_name = conf['mongodb']['database']
scheduler = BackgroundScheduler(jobstore=MongoDBJobStore(
database=db_name, collection='scheduler', client=client))
scheduler.add_listener(job_succeeded, EVENT_JOB_EXECUTED)
scheduler.add_listener(job_failed, EVENT_JOB_ERROR)
scheduler.start()
return scheduler
def __init__(self):
if 'scheduler' not in self.__dict__:
# MongoDB的客户端连接配置
client = MongoClient(host=mongodb_host, port=mongodb_port, document_class=dict)
# 做关联
m_job = MongoDBJobStore(database=mongodb_database, client=client, collection=mongodb_job_table)
# Redis存储
# redis_job = RedisJobStore(host=redis_host, port=redis_port, db=redis_db, password=redis_password)
# 初始信息
jobstores = {
'default': m_job,
}
executors = {
'default': ThreadPoolExecutor(20),
# 'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
# 实例调度器对象
scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=timezone('Asia/Shanghai'))
# 启动
scheduler.start()
self.scheduler = scheduler
def dealDataScheduleJob(mon_conn):
#每次重启程序时需要删除队列
mon_conn.kctest.dealDataJob.remove({})
jobstores = {
'mongo':
MongoDBJobStore(collection='dealDataJob',
database='kctest',
client=mon_conn),
'default':
MemoryJobStore()
}
executors = {
'default': ThreadPoolExecutor(4),
'processpool': ProcessPoolExecutor(1)
}
job_defaults = {
'coalesce': False,
'max_instances': 3 #avoid block
}
scheduler = BlockingScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
scheduler.add_job(readWriteBusRunInfo,
'interval',
seconds=15,
jobstore="mongo")
#scheduler.add_job(distinctBusInfo,'interval',seconds=30, jobstore='mongo')
#定时任务
#scheduler.add_job(statistics,"cron",hour=23,minute=59,jobstore='mongo')
#scheduler.add_job(statistics,"interval",seconds=30,jobstore='default')
scheduler.start()
def run_background_scheduler():
jobstores = {'default': MongoDBJobStore(client=client)}
scheduler = BackgroundScheduler(jobstores=jobstores, timezone=utc)
scheduler.start()
jobs.remove_jobs(scheduler)
jobs.add_jobs(scheduler)
return scheduler
def __init__(self, client: MongoClient) -> None:
super().__init__()
self.store = MongoDBJobStore(client=client)
self.scheduler = AsyncIOScheduler()
self.scheduler.add_jobstore(self.store)
self.jobs: list = []
self.jobLengthMax = 0
class SchedulerConfig:
SCHEDULER_API_ENABLED = False # 不开启自带api
SCHEDULER_JOBSTORES = { # 存储位置
'default':
MongoDBJobStore(database='expression',
collection='jobs',
client=client)
}
def __init__(self):
jobstores = {
'default':
MongoDBJobStore(database='apscheduler',
collection='jobs',
host=_MONGO_HOST,
port=27017)
}
self.scheduler = BackgroundScheduler(jobstores=jobstores)
class ProductionConfig(BaseConfig):
if not os.getenv("MONGO_IP"):
load_dotenv(find_dotenv("config.env"))
ACCOUNT = os.getenv("ACCOUNT", "admin")
PASSWORD = os.getenv("PASSWORD", "admin")
MONGO_IP = os.getenv("MONGO_IP", "127.0.0.1")
MONGO_PORT = int(os.getenv("MONGO_PORT", 27017))
MONGO_USER = os.getenv("MONGO_USER", "scan")
MONGO_PWD = os.getenv("MONGO_PWD", "")
MONGO_DB_NAME = os.getenv("MONGO_DB_NAME", "portscan")
REDIS_IP = os.getenv("REDIS_IP", "127.0.0.1")
REDIS_PORT = os.getenv("REDIS_PORT", "6379")
REDIS_PWD = os.getenv("REDIS_PWD", "")
JOBS = []
if MONGO_PWD and MONGO_USER:
SCHEDULER_JOBSTORES = {
'default':
MongoDBJobStore(database='apscheduler',
collection='beholder_jobs',
host='mongodb://%s:%s@%s:%s/' %
(MONGO_USER, MONGO_PWD, MONGO_IP, MONGO_PORT))
}
else:
SCHEDULER_JOBSTORES = {
'default':
MongoDBJobStore(database='apscheduler',
collection='beholder_jobs',
host=MONGO_IP,
port=MONGO_PORT)
}
SCHEDULER_EXECUTORS = {
'default': {
'type': 'threadpool',
'max_workers': 20
}
}
SCHEDULER_JOB_DEFAULTS = {'coalesce': True, 'max_instances': 3}
SCHEDULER_API_ENABLED = True
def __init__(self, job_type, store_executor_alias, process_count):
self.sche = TornadoScheduler()
self.host = MONGO_CONFIG.get('host')
self.mongo_client = MongoClient(self.host)
self.job_type = job_type
self.mongo_job_store = MongoDBJobStore(collection='job',
database=DBNAME,
client=self.mongo_client)
self.store_executor_alise = store_executor_alias
self.process_poll = ProcessPoolExecutor(process_count)
def _init_iob_stores(app):
client = MongoClient(app.config['MONGODB_SETTINGS']['host'],
app.config['MONGODB_SETTINGS']['port'])
# 认证
client[app.config['MONGODB_SETTINGS']['db']].authenticate(
app.config['MONGODB_SETTINGS']['username'],
app.config['MONGODB_SETTINGS']['password'])
return MongoDBJobStore(database=app.config['MONGODB_SETTINGS']['db'],
collection='job',
client=client)
def __init__(self, use_mongo_db=True):
self.scheduler = BackgroundScheduler(timezone=timezone('Asia/Shanghai'))
self.scheduler.configure()
if use_mongo_db:
self.job_store = MongoDBJobStore(database='apscheduler', collection='cronJob', client=db)
self.scheduler.add_jobstore(self.job_store)
self.is_replace_existing = True
else:
self.is_replace_existing = False
class ScheduleManager(FlaskView):
scheduler = BackgroundScheduler()
job_store = MongoDBJobStore(**mongo_params)
scheduler.add_jobstore(job_store)
executor = JobExecutor()
if len(sys.argv) > 1 and sys.argv[1] == '--clear':
scheduler.remove_all_jobs()
scheduler.start()
"""
If you schedule jobs in a persistent job store during your application’s initialization, you MUST define an explicit ID for the job and use replace_existing=True or you will get a new copy of the job every time your application restarts!Tip
"""
@route("scheduleContainer/<string:feedName>", methods=["PUT"])
def scheduleContainer(self, feedName):
job = ScheduledCollection(feedName, **request.get_json())
if job.trigger == 'in':
timing = {
"run_date":
datetime.now() +
timedelta(**{job.increment: int(job.increment_size)})
}
else:
timing = {
job.increment: int(job.increment_size),
}
self.scheduler.add_job(self.executor.startContainer,
job.trigger,
args=[feedName],
id=feedName,
replace_existing=True,
**timing)
return 'ok'
@route("addJob/<string:feedName>", methods=["PUT"])
def addJob(self, feedName):
job = ScheduledCollection(feedName, **request.get_json())
timing = {
job.increment_size:
job.increment,
"run_date":
datetime.now() +
timedelta(**{job.increment: int(job.increment_size)})
}
self.scheduler.add_job(self.executor.publishUrl,
job.trigger,
args=[feedName, job.url],
id=job.url,
**timing)
def getStatus(self):
isRunning = self.scheduler.running
jobs = self.scheduler.get_jobs()
payload = {"isRunning": isRunning, "jobs": jobs}
return json.dumps(payload)
def _set_up_scheduler(self, db: str, timezone: str) -> BackgroundScheduler:
jobstores: Dict = {
'default':
MongoDBJobStore(db=db,
collection='jobs',
client=MongoClient(host=MONGO_URL))
}
scheduler = BackgroundScheduler({'apscheduler.timezone': timezone},
jobstores=jobstores)
scheduler.start()
return scheduler
def create_jobstore(resource_type: str, database, config: dict):
if resource_type == 'pymongo.database':
from apscheduler.jobstores.mongodb import MongoDBJobStore
config['client'] = database.client
config['database'] = database.name
return MongoDBJobStore(**config)
elif resource_type == 'sqlalchemy.engine.interfaces.Connectable':
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
return SQLAlchemyJobStore(engine=database, **config)
raise ValueError(
'Resource type {} is not supported'.format(resource_type))
def __init__(self):
self.scheduler = BackgroundScheduler()
self.job_store = MongoDBJobStore(database=os.getenv(
"CHAIN_DB", 'actionChains'),
collection='client_scheduler_jobs',
**mongo_params)
self.scheduler.add_jobstore(self.job_store)
self.executor = JobExecutor()
if len(sys.argv) > 1 and sys.argv[1] == '--clear':
self.scheduler.remove_all_jobs()
self.scheduler.start()
"""
def setup(db: database.MongoDBConnector):
for collection_name in dict(
filter(lambda i: i[1].get('is_jobstore'),
db.COLLECTIONS_CONFIG.items())):
jobstore = MongoDBJobStore(database=db.database_name,
collection=collection_name,
client=db.client)
scheduler.add_jobstore(jobstore, alias=collection_name)
scheduler.start()
jobs = scheduler.get_jobs()
logger.debug(f"Loaded {len(jobs)} job(s)" +
(f": {', '.join([job.id for job in jobs])}" if jobs else "."))
