def executor(request, mock_scheduler):
if request.param == "threadpool":
from apscheduler.executors.pool import ThreadPoolExecutor
executor_ = ThreadPoolExecutor()
else:
from apscheduler.executors.pool import ProcessPoolExecutor
executor_ = ProcessPoolExecutor()
executor_.start(mock_scheduler, "dummy")
request.addfinalizer(executor_.shutdown)
return executor_
def executor(request, mock_scheduler):
if request.param == 'threadpool':
from apscheduler.executors.pool import ThreadPoolExecutor
executor_ = ThreadPoolExecutor()
else:
from apscheduler.executors.pool import ProcessPoolExecutor
executor_ = ProcessPoolExecutor()
executor_.start(mock_scheduler, 'dummy')
yield executor_
executor_.shutdown()
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.mongodb import MongoDBJobStore
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
from pytz import utc
from btc_panel.config import *
from pymongo import MongoClient
from arctic import Arctic
client = MongoClient(MONGO_HOST)
client.list_database_names()
A = Arctic(client)
jobstores = {'mongo': MongoDBJobStore(client=client)}
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {'coalesce': False, 'max_instances': 5}
scheduler = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
@License : (C) Copyright 2016-2017, iFuture Corporation Limited.
"""
import asyncio
import datetime
import json
import time
import pymongo
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from utils.db_util import mdb, es, INDEX_TEXT, INDEX_MAPPING, create_es_doc
from elasticsearch import helpers
DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
executors = {
"default": ThreadPoolExecutor(10),
"processpool": ProcessPoolExecutor(2)
}
job_defaults = {"coalesce": False, "max_instances": 10}
loop = asyncio.get_event_loop()
scheduler = AsyncIOScheduler({"event_loop": loop},
executors=executors,
job_defaults=job_defaults)
ALL_CATEGORY = ["news"]
# minute
# second
# @scheduler.scheduled_job(trigger='cron', second='*/20', id='sync_to_es')
def sync_to_es():
def _create_default_executor(self):
"""Creates a default executor store, specific to the particular scheduler type."""
return ThreadPoolExecutor()
def send_data_to_kafka(producer,topic):
for stock_symbol in stock_names_set:
msg = getQuotes(stock_symbol)
producer.send(topic=topic,value=json.dumps(msg))
print('the stock symbol is %s and the result is %s' % (str(stock_symbol),str(msg)))
def shutdown_hook(producer,schedulers):
producer.flush(10)
producer.close()
schedulers.shutdown()
if __name__=='__main__':
parser = ConfigParser.ConfigParser()
parser.read(os.getcwd() + '/redis_config.ini')
kafka_cluster = parser.get('kafka_config','cluster')
kafka_topic = parser.get('kafka_config','topic')
producer = KafkaProducer(bootstrap_servers=kafka_cluster)
schedulers = BackgroundScheduler()
schedulers.add_executor(ThreadPoolExecutor(5))
schedulers.add_job(send_data_to_kafka, 'interval', [producer, kafka_topic], seconds=3)
schedulers.start()
atexit.register(shutdown_hook, producer,schedulers)
app.run(host='localhost',port=8081)
# init and run scheduler
import datetime
import logging
from pytz import timezone
from talos.core import config
from talos.core import logging as mylogger
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.executors.pool import ThreadPoolExecutor
from wecube_plugins_itsdangerous.server.wsgi_server import application
CONF = config.CONF
LOG = logging.getLogger(__name__)
jobstores = {'default': MemoryJobStore()}
executors = {'default': ThreadPoolExecutor(5)}
job_defaults = {'coalesce': False, 'max_instances': 1}
def cleanup_cached_dir():
try:
max_delta = 24 * 60 * 60
base_dir = CONF.pakcage_cache_dir
if os.path.exists(base_dir):
for name in list(os.listdir(base_dir)):
fullpath = os.path.join(base_dir, name)
path_stat = os.stat(fullpath)
if time.time() - path_stat.st_atime > max_delta:
LOG.info('remove dir/file: %s, last access: %s', fullpath,
path_stat.st_atime)
if os.path.isdir(fullpath):
help="pauses all instances of cronny",
action="store_true",
required=False,
)
args = parser.parse_args()
timeDate, action = args.d, args.a
stop = args.stop
status, pause = args.status, args.pause
# Config :: MemoryJobStore :: Blocked-Schedule
jobstores = {"memory": {"type": "memory"}}
executors = {
"default": {"type": "threadpool", "max_workers": 1},
"threadpool": ThreadPoolExecutor(max_workers=1),
}
job_defaults = {"coalesce": False, "max_instances": 1}
## Define Scheduler :: configure
scheduler = BackgroundScheduler()
scheduler.configure(
jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc
)
def main(action: str, timeDate: str) -> None:
"""
Parameters:
action (str): A action to feed the scheduler
timeDate (str): The time at which the action on the action should be scheduled
def __init__(self):
self.scheduler = BackgroundScheduler(timezone=self.timezone, executors={'default': ThreadPoolExecutor(30)})
self.scheduler.add_listener(
self._handle_event,
EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES | EVENT_JOB_EXECUTED)
#!/usr/bin/env python
#encoding:utf-8
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.jobstores.redis import RedisJobStore
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
import datetime
jobstores = {
'redis': RedisJobStore(host='172.16.0.121',port=6379,db=0), #用redis作backend
}
# ZRANGE apscheduler.run_times 0 1
# HGETALL apscheduler.jobs
executors = {
'default': ThreadPoolExecutor(10), #默认线程数
'processpool': ProcessPoolExecutor(3) #默认进程
}
sched = BlockingScheduler(jobstores=jobstores, executors=executors)
def aps_test():
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'H')
# @scheduler.scheduled_job('interval', seconds=3,id='job003')
# def test03():
# print('test03......')
#添加任务
sched.add_job(func=aps_test, trigger='cron', second='*/5',id='job001',jobstore='redis',replace_existing=True)
#查看任务状态
# from config.settings import CORE_ID
# from config.settings import MONGO_HOST_SELF
# from config.settings import MONGO_PORT_SELF
from config.settings import coalesce
from config.settings import local_tz
from config.settings import max_instances
from config.settings import path_apscheduler
from config.settings import processpool_executor
from config.settings import threadpool_executor
from core import toLog
from core.log_manager.log_levels import setup_logger
setup_logger('apscheduler', path_apscheduler, logging.DEBUG)
executors = {
'default': ThreadPoolExecutor(threadpool_executor),
'processpool': ProcessPoolExecutor(processpool_executor)
}
job_defaults = {'coalesce': coalesce, 'max_instances': max_instances}
scheduler = TwistedScheduler(timezone=local_tz)
# scheduler.add_jobstore(
# 'mongodb',
# host=MONGO_HOST_SELF,
# port=MONGO_PORT_SELF,
# collection=CORE_ID
# )
scheduler.add_executor(ThreadPoolExecutor(threadpool_executor), 'default')
def run():
# ensure influxdb database exists
ensure_influx_database()
# This call needs to happen on every start of the scheduler to ensure we're not in
# the state where Robotoff is unable to perform tasks because of missing data.
_update_data()
scheduler = BlockingScheduler()
scheduler.add_executor(ThreadPoolExecutor(20))
scheduler.add_jobstore(MemoryJobStore())
# This job takes all of the newly added automatically-processable insights and sets the process_after field on them,
# indicating when these insights should be auto-applied.
scheduler.add_job(mark_insights,
"interval",
minutes=2,
max_instances=1,
jitter=20)
# This job applies all of the automatically-processable insights that have not been applied yet.
scheduler.add_job(process_insights,
"interval",
minutes=2,
max_instances=1,
jitter=20)
# This job exports daily product metrics for monitoring.
scheduler.add_job(save_facet_metrics,
"cron",
day="*",
hour=1,
max_instances=1)
# This job refreshes data needed to generate insights.
scheduler.add_job(_update_data, "cron", day="*", hour="3", max_instances=1)
# This job updates the product insights state with respect to the latest PO dump by:
# - Deleting non-annotated insights for deleted products and insights that
# are no longer applicable.
# - Updating insight attributes.
scheduler.add_job(
functools.partial(refresh_insights, with_deletion=True),
"cron",
day="*",
hour="4",
max_instances=1,
)
# This job generates category insights using ElasticSearch from the last Product Opener data dump.
scheduler.add_job(generate_insights,
"cron",
day="*",
hour="4",
minute=15,
max_instances=1)
scheduler.add_job(
generate_quality_facets,
"cron",
day="*",
hour="5",
minute=25,
max_instances=1,
)
scheduler.add_listener(exception_listener, EVENT_JOB_ERROR)
scheduler.start()
def create_app(config, enable_config_file=False):
"""
创建应用
:param config: 配置信息对象
:param enable_config_file: 是否允许运行环境中的配置文件覆盖已加载的配置信息
:return: 应用
"""
app = create_flask_app(config, enable_config_file)
# 创建Snowflake ID worker
from utils.snowflake.id_worker import IdWorker
app.id_worker = IdWorker(app.config['DATACENTER_ID'],
app.config['WORKER_ID'], app.config['SEQUENCE'])
# 限流器
from utils.limiter import limiter as lmt
lmt.init_app(app)
# 配置日志
from utils.logging import create_logger
create_logger(app)
# 注册url转换器
from utils.converters import register_converters
register_converters(app)
from redis.sentinel import Sentinel
_sentinel = Sentinel(app.config['REDIS_SENTINELS'])
app.redis_master = _sentinel.master_for(
app.config['REDIS_SENTINEL_SERVICE_NAME'])
app.redis_slave = _sentinel.slave_for(
app.config['REDIS_SENTINEL_SERVICE_NAME'])
from rediscluster import StrictRedisCluster
app.redis_cluster = StrictRedisCluster(
startup_nodes=app.config['REDIS_CLUSTER'])
# rpc
app.rpc_reco = grpc.insecure_channel(app.config['RPC'].RECOMMEND)
# Elasticsearch
app.es = Elasticsearch(
app.config['ES'],
# sniff before doing anything
sniff_on_start=True,
# refresh nodes after a node fails to respond
sniff_on_connection_fail=True,
# and also every 60 seconds
sniffer_timeout=60)
# socket.io
app.sio_mgr = socketio.KombuManager(app.config['RABBITMQ'],
write_only=True)
# MySQL数据库连接初始化
from models import db
db.init_app(app)
# APSchduler
from .schedule import statistic
executors = {'default': ThreadPoolExecutor(10)}
app.scheduler = BackgroundScheduler(executors=executors)
# 此处可以添加flask程序之外的定时任务
# app.scheduler.add_job(statistic.fix_statistic, 'cron', hour=3) # 每天的凌晨三点
app.scheduler.add_job(statistic.fix_statistic, 'date',
args=[app]) # 为了测试方便立即执行
app.scheduler.start()
# 添加请求钩子
from utils.middlewares import jwt_authentication
app.before_request(jwt_authentication)
# 注册用户模块蓝图
from .resources.user import user_bp
app.register_blueprint(user_bp)
# 注册新闻模块蓝图
from .resources.news import news_bp
app.register_blueprint(news_bp)
# 注册通知模块
from .resources.notice import notice_bp
app.register_blueprint(notice_bp)
# 搜索
from .resources.search import search_bp
app.register_blueprint(search_bp)
return app
# BackgroundScheduler 是非阻塞的，代码执行完程序退出，主要用于集成到其他（web）程序，因为其他程序本身应该就是无限循环的，不能让程序阻塞
# BackingScheduler 阻塞的定时器（内部无限循环），主要用于独立的程序
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.executors.pool import ThreadPoolExecutor
import logging
logging.basicConfig(filename='scheduler.log')
logging.getLogger('apscheduler').setLevel(logging.WARNING)
# 1, 创建执行器，用于控制任务的并发能力
executor = ThreadPoolExecutor(max_workers=100)
# 2, 创建调度器
scheduler = BackgroundScheduler(exector={'default': executor})
def func1(name, age): #任务
print(1 / 0)
print(name, age)
# 添加任务
""" 三种触发器
date 只执行一次的定时任务
interval 周期执行， 参数是时间间隔
cron 周期执行，参数是时间
"""
scheduler.add_job(func1,
'date',
run_date='2020-05-15 16:26:30',
schemaUpdater.schemaRevisioner.verifySchemaUpToDate()
import logSetup
logSetup.initLogging()
import webserver_process
import datetime
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.executors.pool import ProcessPoolExecutor
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.jobstores.memory import MemoryJobStore
executors = {
'main_jobstore': ThreadPoolExecutor(3),
}
job_defaults = {'coalesce': True, 'max_instances': 1}
jobstores = {
'main_jobstore': MemoryJobStore(),
}
def run_web():
nt.dirNameProxy.startDirObservers()
sched = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
def schedule(config):
arxiv = Arxiv(config)
prl = ApsPRL(config)
prx = ApsPRX(config)
prb = ApsPRB(config)
prresearch = ApsPRResearch(config)
'''
* scheduler
'''
executors = {
'default': ThreadPoolExecutor(10),
# 'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': True,
# 'max_instances': 3,
}
scheduler = BackgroundScheduler(executors=executors,
job_default=job_defaults,
timezone=utc)
utc_hour = (6 - 8) % 24 # beijing_hour: 6
scheduler.add_job(arxiv.main,
id='arxiv',
name='arxiv.main',
trigger='cron',
day='*',
hour=utc_hour,
minute=0)
utc_hour = (7 - 8) % 24 # beijing_hour: 7
scheduler.add_job(prl.main,
id='prl',
name='prl.main',
trigger='cron',
day_of_week='mon,fri',
hour=utc_hour,
minute=0)
scheduler.add_job(prx.main,
id='prx',
name='prx.main',
trigger='cron',
month='*',
day='20',
hour=utc_hour,
minute=15)
scheduler.add_job(prb.main,
id='prb',
name='prb.main',
trigger='cron',
day_of_week='sun',
hour=utc_hour,
minute=30)
scheduler.add_job(prresearch.main,
id='prresearch',
name='prresearch.main',
trigger='cron',
month='*',
day='25',
hour=utc_hour,
minute=45)
# scheduler.add_job(arxiv.main, id='arxiv_test')
scheduler.start()
try:
while True:
time.sleep(3600)
except (KeyboardInterrupt, SystemExit):
scheduler.remove_all_jobs()
scheduler.shutdown()
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2018/6/27 下午7:14 # @Author : haohao.qiang # @Mail : [email protected] # @File : utils.py import logging from expiringdict import ExpiringDict # Python 任务调度框架: APScheduler https://apscheduler.readthedocs.io/en/latest/index.html from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.executors.pool import ThreadPoolExecutor logging.basicConfig( filename='app/logs/app.log', level=logging.INFO, format='%(asctime)s|%(levelname)s|%(filename)s:%(lineno)s|%(message)s', datefmt='%Y-%m-%d %H:%M:%S') executors = {'threadpool': ThreadPoolExecutor(50)} job_defaults = {'coalesce': False, 'max_instances': 50} scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults) scheduler.start() cache = ExpiringDict(max_len=1024, max_age_seconds=1800)
class HttpServer(flask.Flask):
"""Our HTTP/API server."""
EXECUTORS = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
def __init__(self, name, ip, port, *args, **kwargs):
"""Constructor.
Args:
name: (str) name of Flask service
ip: (str) IP address to bind HTTP server
port: (int) TCP port for HTTP server to listen
"""
super(HttpServer, self).__init__(name, *args, **kwargs)
# Fixup the root path for Flask so it can find templates/*
root_path = os.path.abspath(os.path.dirname(__file__))
logging.debug('Setting root_path for Flask: %s', root_path)
self.root_path = root_path
self.targets = config.CollectorConfig()
self.ip = ip
self.port = port
self.start_time = time.time()
self.setup_time = 0
self.scheduler = BackgroundScheduler(daemon=True,
executors=self.EXECUTORS)
self.collection = None
self.add_url_rule('/', 'index', self.index_handler)
self.add_url_rule('/status', 'status', self.status_handler)
self.add_url_rule('/latency', 'latency', self.latency_handler)
self.add_url_rule('/influxdata', 'influxdata', self.influxdata_handler)
self.add_url_rule('/quitquit', 'quitquit', self.shutdown_handler)
logging.info('Starting Llama Collector, version %s', __version__)
def configure(self, filepath):
"""Configure the Collector from file.
Args:
filepath: (str) where the configuration is located
"""
self.targets.load(filepath)
def status_handler(self):
return flask.Response('ok', mimetype='text/plain')
def index_handler(self):
return flask.render_template(
'index.html',
targets=self.targets.targets,
interval=self.interval,
start_time=self.start_time,
setup_time=self.setup_time,
uptime=humanfriendly.format_timespan(time.time() -
self.start_time))
def latency_handler(self):
data = json.dumps(self.collection.stats, indent=4)
return flask.Response(data, mimetype='application/json')
def influxdata_handler(self):
data = json.dumps(self.collection.stats_influx, indent=4)
return flask.Response(data, mimetype='application/json')
def shutdown_handler(self):
"""Shuts down the running web server and other things."""
logging.warn('/quitquit request, attempting to shutdown server...')
self.scheduler.shutdown(wait=False)
fn = flask.request.environ.get('werkzeug.server.shutdown')
if not fn:
raise Error('Werkzeug (Flask) server NOT running.')
fn()
return '<pre>Quitting...</pre>'
def run(self, interval, count, use_udp=False, *args, **kwargs):
"""Start all the polling and run the HttpServer.
Args:
interval: seconds between each poll
count: count of datagram to send each responder per interval
"""
self.interval = interval
self.scheduler.start()
self.collection = Collection(self.targets, use_udp)
self.scheduler.add_job(self.collection.collect,
'interval',
seconds=interval,
args=[count])
super(HttpServer, self).run(host=self.ip,
port=self.port,
threaded=True,
*args,
**kwargs)
self.setup_time = round(time.time() - self.start_time, 0)
from newsreader import NewsReader
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
app = Flask(__name__)
DEBUG = True
MANIFESTO_MODEL_HTTP_PORT = os.environ.get('MANIFESTO_MODEL_HTTP_PORT')
HTTP_PORT = int(os.environ.get('HTTP_PORT'))
MANIFESTO_URL = 'http://*****:*****@app.route('/get_topics')
def get_topics():
"""
use this endpoint to get n news topics
e.g.: http://localhost:5000/get_topics?n=5
:return:
"""
cfg.IntOpt('ProcessPoolExecutor', default=4),
]
CONF = cfg.CONF
CONF.register_opts(aps_job_opts, group='apscheduler')
CONF.register_opts(aps_executor_opts, group='apscheduler')
LOG = log.getLogger(__name__)
JOB_DEFAULTS = {
'misfire_grace_time': CONF.apscheduler.misfire_grace_time,
'coalesce': CONF.apscheduler.coalesce,
'max_instances': CONF.apscheduler.max_instances
}
EXECUTORS = {
'default': ThreadPoolExecutor(CONF.apscheduler.ThreadPoolExecutor),
'processpool': ProcessPoolExecutor(CONF.apscheduler.ProcessPoolExecutor)
}
class JobManager(object):
def __init__(self):
self.scheduler = BackgroundScheduler(executors=EXECUTORS,
job_defaults=JOB_DEFAULTS,
timezone='Asia/Shanghai')
self.jobs = {}
self.scheduler.start()
def add_job_store(self):
pass
from .base import *
from datetime import datetime, timedelta
from apscheduler.executors.pool import ThreadPoolExecutor
SCRAPY_CRAWLERS = scrapy_crawlers_list
SELENIUM_CRAWLERS = selenium_crawlers_list
SCRAPY_CRAWLERS_CONF = scrapy_crawlers_conf
SELENIUM_CRAWLERS_CONF = selenium_crawlers_conf
MINIO = minio_conf
MAIL = mail_conf
DATABASE = database_conf
ERP = erp_conf
EXECUTORS = {'default': ThreadPoolExecutor(max_workers=10)}
TASKS = {
'web_crawling': {
'trigger': 'cron',
'hour': 21,
'minutes': 0,
},
'check_new_events': {
'trigger': 'cron',
'hour': 22,
'minute': 0,
},
'summary': {
'trigger': 'cron',
'hour': 23,
def task2(arg1, arg2):
print("----------------------------------------------")
print("开始执行task2")
time.sleep(random.randint(1, 5))
print("task2执行完成")
print("----------------------------------------------" + "\n")
return arg1 * arg2
@single_task
def task3(arg1, arg2):
print("----------------------------------------------")
print("开始执行task2")
time.sleep(random.randint(1, 5))
print("task2执行完成")
print("----------------------------------------------" + "\n")
return arg1 / arg2
if __name__ == '__main__':
executors = {
"default": ThreadPoolExecutor(20), # 设置一个名为 default的线程池执行器， 最大线程设置为20个
"processpool": ProcessPoolExecutor(5), # 设置一个名为 processpool的进程池执行器，最大进程数设为5个
}
print("开始执行定时任务")
scheduler = BlockingScheduler(executors=executors, timezone="Asia/Shanghai")
scheduler.add_job(task1, args=(5, 5), trigger="interval", seconds=5)
scheduler.add_job(task2, args=(5, 5), trigger="interval", seconds=5)
scheduler.add_job(task3, args=(5, 5), trigger="interval", seconds=5)
scheduler.start()
def fuckup(p_command=None):
Main.rootdir = os.path.abspath('.')
#Initialize application configure
filename = "application-config.yml"
Configure.load(p_dir=Main.rootdir + "/" + filename,
p_command=p_command)
nodename = Configure.configure().value("worknode.name")
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 80))
Main.ipAddr = s.getsockname()[0]
finally:
s.close()
#Initialize log
Logger()
#Initialize elasticsearch client
Main.es_client = ESHandler()
#Initialize worker monitor
monitor = MultiProcessJobWatcher()
executors = {
'default': ThreadPoolExecutor(1),
'processpool': ProcessPoolExecutor(1)
}
job_defaults = {'coalesce': True, 'max_instances': 1}
mosche = BackgroundScheduler(executors=executors,
job_defaults=job_defaults,
timezone=utc)
mosche.add_job(monitor,
'interval',
seconds=Configure.configure().value(
"worknode.workerMonitorInterval"))
#Initialize worker leader
leader = Leader(p_addr=Main.ipAddr,
p_node_name=nodename,
p_monitor=monitor)
#Initialize node register and health info report schedule
scheduleserveraddr = Configure.configure().value(
"server.healthServer.host")
scheduleserverport = Configure.configure().value(
"server.healthServer.port")
scheduleserver = {
"host": scheduleserveraddr,
"port": scheduleserverport
}
Main.communicator = Communicator(p_schedule_server=scheduleserver,
p_leader=leader)
#Initialize node job accept service
ServerWrapper.listen(p_name=nodename,
p_prefix="server.nodeServer",
p_handler=leader)
tornado.ioloop.IOLoop.current().start()
try:
# This is here to simulate application activity (which keeps the main thread alive).
while True:
time.sleep(2)
except (KeyboardInterrupt, SystemExit):
# Not strictly necessary if daemonic mode is enabled but should be done if possible
parellelSchedule.shutdown()
engine_options={"pool_recycle": 5})
}
jobstores_gevent = {
'default':
SQLAlchemyJobStore(
url='mysql+pymysql://%s:%s@%s:%s/%s?charset=utf8' %
(JOBSTORES_SQL_USER, JOBSTORES_SQL_PASSWORD, JOBSTORES_SQL_HOST,
JOBSTORES_SQL_PORT, JOBSTORES_SQL_DBNAME),
tablename='gevent_apscheduler_jobs',
pickle_protocol=0,
engine_options={"pool_recycle": 5})
}
executors = {
'default': ThreadPoolExecutor(THREAD_POOL_EXECUTER_NUM),
}
gevent_executors = {'default': ThreadPoolExecutor(THREAD_POOL_EXECUTER_NUM)}
job_defaults = {
'coalesce': JOB_COALESCE,
'max_instances': JOB_MAX_INSTANCE,
"misfire_grace_time": 500
}
def take_tasklog_notes(taskid, excute_description, message):
current_time = time.time()
manager_log = SfoManagerTaskLogMethod.create_manager_task_log(
taskid, message, excute_description, timestamp_format(current_time))
def init_app(with_scheduler: bool = True) -> None:
"""Initialize the application (database, scheduler, etc) and recover jobs``.
:func:`init_app` should only be called after **all** application configuration has been resolved.
Most of that happens implicitly in :mod:`supa`,
but some of needs to be done after processing command line options.
For instance, :func:`init_app` assumes :data:`settings`
(the :attr:`~Settings.database_file` attribute of it)
has been updated **after** command line processing,
that is, if the setting was changed on the command line.
That way the command line options have had the ability
to override the default values, those of the env file and environment variables.
.. note:: Only import :data:`supa.db.Session` after the call to :func:`init_app`.
If imported earlier, :data:`supa.db.Session` will refer to :class:`supa.db.UnconfiguredSession`
and you will get a nice exception upon usage.
Likewise only import :data:`scheduler` after the call to :func:`init_app`.
If imported earlier, :data:`scheduler` will refer to :class:`UnconfiguredScheduler`
and you will get an equally nice exception.
If the scheduler is to be initialized as well (see: :attr:`with_scheduler`)
it will also be started.
Args:
with_scheduler: if True, initialize and start scheduler. If False, don't.
"""
random.seed()
# Initialize the database
database_file = resolve_database_file(settings.database_file)
if not database_file.exists():
logger.warn(
"`database_file` did not exist. Created new SQLite DB file. Is this really what you wanted?",
database_file=database_file,
)
engine = create_engine(f"sqlite:///{database_file}")
import supa.db.session
Base.metadata.create_all(engine)
session_factory = sessionmaker(bind=engine)
supa.db.session.Session = scoped_session(session_factory)
if with_scheduler:
# Initialize and start the scheduler
jobstores = {"default": MemoryJobStore()}
logger.info("Configuring scheduler executor.",
scheduler_max_workers=settings.scheduler_max_workers)
executors = {
"default": ThreadPoolExecutor(settings.scheduler_max_workers)
}
# misfire_grace_time (int) – the time (in seconds) how much this job’s execution is allowed to be late
# (None means “allow the job to run no matter how late it is”)
job_defaults = {
"coalesce": False,
"max_instances": 1,
"misfire_grace_time": None
}
global scheduler
scheduler = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=pytz.utc)
scheduler.start()
from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from pytz import utc
from server.settings import SCHEDULER_THREAD_COUNT, SQLALCHEMY_DATABASE_URI
app = Flask(__name__)
db = SQLAlchemy(app)
app.config.from_object('server.settings')
migrate = Migrate(app, db)
jobstores = {'default': SQLAlchemyJobStore(url=SQLALCHEMY_DATABASE_URI)}
executors = {'default': ThreadPoolExecutor(SCHEDULER_THREAD_COUNT)}
scheduler = BackgroundScheduler(jobstores=jobstores,
executors=executors,
timezone=utc,
daemon=True)
scheduler.start()
def __init__(self):
time_zone = pytz.timezone('Asia/Shanghai')
executors = {
'default': ThreadPoolExecutor(20)
}
self.sche = BackgroundScheduler(timezone=time_zone, executors=executors)
"""
from apscheduler.schedulers.background import BackgroundScheduler, BlockingScheduler
from apscheduler.jobstores.redis import RedisJobStore
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
def print_args(*args):
"""要定时执行的函数"""
for arg in args:
print(arg)
# 执行器，常用的就线程池和进程池两种
thread_pool = ThreadPoolExecutor(30)
process_pool = ProcessPoolExecutor(5)
executors = {'thread': thread_pool, 'process': process_pool}
#存储器 默认使用内存，对定时任务丢失什么的不敏感，对定时任务执行要求低
redis_store = RedisJobStore(host="172.16.120.120", port='6379')
sqlite_store = SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')
jobstores = {'redis': redis_store, 'default': sqlite_store}
#删除被持久化的定时任务， redis_store.remove_all_jobs()
#调度器
#调度器设置
job_defaults = {
'coalesce': False,
'max_instances': 5,
'misfire_grace_time': 60
def setUp(self):
self.scheduler = BackgroundScheduler()
self.scheduler.add_jobstore(MemoryJobStore(), alias='in_memory')
self.scheduler.add_executor(ThreadPoolExecutor(1), alias='secondary_executor')
self.scheduler.start()
from database import SessionLocal, engine
from datetime import datetime
from loguru import logger
from apscheduler.executors.pool import ThreadPoolExecutor#, ProcessPoolExecutor
from app_log import make_logger
from decouple import config, Csv
import asyncio
import sqlalchemy
import models
import schemas
import requests
import random
# Executors for the app scheduler. Use threads, then fall back on processes
executors = {
'default' : ThreadPoolExecutor(max_workers=30),
# 'processpool' : ProcessPoolExecutor(max_workers=2)
}
# Initialise the logger
log: Logger = make_logger()
# # set the app scheduler log level
logging.basicConfig()
logging.getLogger('apscheduler').setLevel(logging.ERROR)
# Create a background scheduler
scheduler: BackgroundScheduler = BackgroundScheduler(executers=executors)
# start the scheduler
scheduler.start()
def _initialize_funcs(self):
"""Setup the filament modules functions.
Functions
---------
set_filter: func
accepts the comma separated list of kernel events
for whose the filter should be applied
set_interval: func
establishes the fixed repeating interval in seconds
columns: func
configure the column set for the table
add_row: func
adds a new row to the table
sort_by: func
sorts the table by specific column
"""
def set_filter(*args):
self._filters = args
self._filament_module.set_filter = set_filter
def set_interval(interval):
if not type(interval) is int:
raise FilamentError('Interval must be an integer value')
self._interval = interval
self._filament_module.set_interval = set_interval
def columns(cols):
if not isinstance(cols, list):
raise FilamentError('Columns must be a list, '
'%s found' % type(cols))
self._cols = cols
self._tabular = Tabular(self._cols)
self._tabular.padding_width = 10
self._tabular.junction_char = '|'
def add_row(row):
if not isinstance(row, list):
raise FilamentError(
'Expected list type for the row, found %s' % type(row))
self._tabular.add_row(row)
def sort_by(col, sort_desc=True):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if col not in self._cols:
raise FilamentError('%s column does not exist' % col)
self._sort_by = col
self._sort_desc = sort_desc
def limit(l):
if len(self._cols) == 0:
raise FilamentError('Expected at least 1 column but 0 found')
if not type(l) is int:
raise FilamentError('Limit must be an integer value')
self._limit = l
def title(text):
self._tabular.title = text
self._filament_module.columns = columns
self._filament_module.title = title
self._filament_module.sort_by = sort_by
self._filament_module.limit = limit
self._filament_module.add_row = add_row
self._filament_module.render_tabular = self.render_tabular
on_init = self._find_filament_func('on_init')
if on_init and self._zero_args(on_init):
self._filament_module.on_init()
if self._find_filament_func('on_interval'):
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=4))
self.scheduler.start()
def on_interval():
try:
self._filament_module.on_interval()
except Exception:
self._logger.error(
'Unexpected error on interval elapsed %s' %
traceback.format_exc())
self.scheduler.add_job(on_interval,
'interval',
seconds=self._interval,
max_instances=4,
misfire_grace_time=60)
if len(self._cols) > 0:
try:
self._ansi_term.setup_console()
except TermInitializationError:
panic('fibratus run: ERROR - console initialization failed')
class Config:
DEBUG = str_to_bool(os.getenv('DEBUG', True))
TESTING = False
SECRET_KEY = os.getenv(
'SECRET_KEY',
get_secret_key_from_file(os.getenv('SECRET_PATH'), 'secrets.yml'))
ERROR_INCLUDE_MESSAGE = str_to_bool(
os.getenv('ERROR_INCLUDE_MESSAGE', True))
RESTPLUS_MASK_SWAGGER = False
SWAGGER_UI_REQUEST_DURATION = True
JOBS_SOURCE_REFRESH_INTERVAL = None
PUSHGATEWAYS = {}
SQLALCHEMY_ENGINE_OPTIONS = {
'poolclass': NullPool,
'pool_reset_on_return': None
}
SQLALCHEMY_TRACK_MODIFICATIONS = False
SCHEDULER_API_ENABLED = False
SCHEDULER_EXECUTORS = {
'default':
ThreadPoolExecutor(
max_workers=int(os.getenv('THREADPOOL_MAX_WORKERS', '20'))),
'processpool':
ProcessPoolExecutor(
max_workers=int(os.getenv('PROCESSPOOL_MAX_WORKERS', '10')))
}
SCHEDULER_JOB_DEFAULTS = {
'misfire_grace_time': int(os.getenv('MISFIRE_GRACE_TIME', '5'))
}
SCHEDULER_JOB_DELAY_SECONDS = int(
os.getenv('JOB_INITIAL_DELAY_SECONDS', '10'))
SUSPEND_JOB_ON_FAILURE = str_to_bool(
os.getenv('SUSPEND_JOB_ON_FAILURE', False))
def __init__(self):
if os.getenv('VCAP_SERVICES') is None:
os.environ['VCAP_SERVICES'] = get_services_from_file(
os.getenv('SERVICES_PATH'), 'env/services.yml')
env = AppEnv()
self.APIALCHEMY_BINDS = {}
self.apialchemy_binds = api_helper.parse_services_for_binds(
service_prefix['APIALCHEMY'], env.services)
self.SQLALCHEMY_BINDS = {}
self.sqlalchemy_binds = sqlalchemy_helper.parse_services_for_binds(
service_prefix['SQLALCHEMY'], env.services)
self.SCHEDULER_JOBS = []
if os.getenv('JOBS_SOURCE_SERVICE') is not None:
self.JOBS_SOURCE_REFRESH_INTERVAL = os.getenv(
'JOBS_SOURCE_REFRESH_INTERVAL', '60')
self.service_jobs = apscheduler_helper.get_jobs_from_services(
job_prefix, template_prefix, env.services)
self.source_jobs = {}
self.jobs = self.service_jobs
def add_jobs_from_source(self, aa):
if self.JOBS_SOURCE_REFRESH_INTERVAL is not None:
self.source_jobs = apscheduler_helper.get_jobs_from_source(
aa, (service_prefix['APIALCHEMY'], self.APIALCHEMY_BINDS))
self.jobs = {**self.service_jobs, **self.source_jobs}
@property
def apialchemy_binds(self):
return self._apialchemy_binds
@apialchemy_binds.setter
def apialchemy_binds(self, value):
self._apialchemy_binds = value
self.APIALCHEMY_BINDS = api_helper.build_bind_dict(
self._apialchemy_binds, self.SECRET_KEY)
def set_pushgateways(self, aa):
self.PUSHGATEWAYS = prometheus_helper.get_pushgateways(
aa, (service_prefix['APIALCHEMY'], self.APIALCHEMY_BINDS))
@property
def sqlalchemy_binds(self):
return self._sqlalchemy_binds
@sqlalchemy_binds.setter
def sqlalchemy_binds(self, value):
self._sqlalchemy_binds = value
self.SQLALCHEMY_BINDS = sqlalchemy_helper.build_bind_dict(
self._sqlalchemy_binds, self.SECRET_KEY)
@property
def jobs(self):
return self._jobs
@jobs.setter
def jobs(self, value):
self._jobs = value
self.SCHEDULER_JOBS = apscheduler_helper.build_job_list(
self._jobs, (service_prefix['APIALCHEMY'], self._apialchemy_binds),
(service_prefix['SQLALCHEMY'], self._sqlalchemy_binds))
