class EmailScheduler(): jobstores = { 'default': SQLAlchemyJobStore( f'postgresql://192.168.5.172/billtrak?user=dj&password={os.getenv("dbpw")}' ) } executors = { 'default': { 'type': 'threadpool', 'max_workers': 20 }, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = {'coalesce': False, 'max_instances': 3} scheduler = BackgroundScheduler() def getcron(self): return cron.CronTrigger() scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) def getscheduler(self): return self.scheduler
def start_scheduler(settings): assert settings['scheduler.store'] in ('redis', 'sqlalchemy'),\ 'Uknown job store, must by one of redis or sqlalchemy' if settings['scheduler.store'] == 'redis': jobstores = { 'default': RedisJobStore(db=settings['scheduler.db']) } else: jobstores = { 'default': SQLAlchemyJobStore(url=settings['scheduler.url']) } executors = { 'default': { 'type': settings['scheduler.executors.type'], 'max_workers': settings['scheduler.executors.max_workers'] }, 'processpool': ProcessPoolExecutor( max_workers=settings['scheduler.executors.processpool.max_workers'] ) } job_defaults = { 'coalesce': False, 'max_instances': settings['scheduler.job_defaults.max_instances'] } scheduler.configure( jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=timezone('UTC') ) if settings['scheduler.autostart'] == 'true': scheduler.start()
def run(self): sql_session = self.sql_session self.period = self.get_monitoring_period() executors = { 'default': {'type': 'threadpool', 'max_workers': 20}, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = { 'coalesce': False, 'max_instances': 3 } self.scheduler = BackgroundScheduler() self.scheduler.configure(executors=executors, job_defaults=job_defaults) clusters = sql_session.query(GnCluster).filter(GnCluster.status=='Running').all() for cluster in clusters: if cluster.type == 'docker': docker_url = 'http://%s/monitor' % (str(cluster.ip)) self.scheduler.add_job(lambda : self.docker_monitor(docker_url), trigger='interval', seconds=int(self.period)) elif cluster.type == 'kvm': kvm_url = 'http://%s/monitor' % (str(cluster.ip)) self.scheduler.add_job(lambda : self.kvm_monitor(kvm_url), trigger='interval', seconds=int(self.period)) elif cluster.type == 'hyperv': hyperv_url = 'http://%s/monitor' % (str(cluster.ip)) self.scheduler.add_job(lambda : self.hyperv_monitor(hyperv_url), trigger='interval', seconds=int(self.period)) else: print 'type = %s is not scheduling' % cluster.type self.scheduler.start() sql_session.commit()
class Config: ''' app settings ''' DEBUG = parser.config['app']['debug'] TESTING = parser.config['app']['testing'] BCRYPT_LEVEL = parser.config['app']['bcrypt_level'] APP_NAME = parser.config['app']['app_name'] SECRET_KEY = parser.config['app']['secret_key'] WTF_CSRF_ENABLED = parser.config['app']['wtf_csrf_enabled'] # APScheduler JOBS = [] JSON_AS_ASCII = True # 支持json显示中文 SCHEDULER_API_ENABLED = parser.config['scheduler']['api_enabled'] SCHEDULER_JOBSTORES = { 'default': MongoDBJobStore(database=parser.config['scheduler']['mongodb_db'], client=MongoClient(parser.config['scheduler']['mongodb_uri'])) } SCHEDULER_EXECUTORS = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } SCHEDULER_JOB_DEFAULTS = parser.config['scheduler']['job_defaults'] # mongodb settings MONGODB_SETTINGS = parser.config['mongodb']['mongodb_settings'] @staticmethod def init_app(app): pass
def dealDataScheduleJob(mon_conn): #每次重启程序时需要删除队列 mon_conn.kctest.dealDataJob.remove({}) jobstores = { 'mongo': MongoDBJobStore(collection='dealDataJob', database='kctest', client=mon_conn), 'default': MemoryJobStore() } executors = { 'default': ThreadPoolExecutor(4), 'processpool': ProcessPoolExecutor(1) } job_defaults = { 'coalesce': False, 'max_instances': 3 #avoid block } scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) scheduler.add_job(readWriteBusRunInfo, 'interval', seconds=15, jobstore="mongo") #scheduler.add_job(distinctBusInfo,'interval',seconds=30, jobstore='mongo') #定时任务 #scheduler.add_job(statistics,"cron",hour=23,minute=59,jobstore='mongo') #scheduler.add_job(statistics,"interval",seconds=30,jobstore='default') scheduler.start()
def __init__(self): self.jobstores = { 'mongo': MongoDBJobStore(collection='job1', database='saasjob', client=_mongoclient), 'default': MemoryJobStore() } self.executors = { 'default': ThreadPoolExecutor(1), 'processpool': ProcessPoolExecutor(1) } self.job_defaults = { 'coalesce': False, 'misfire_grace_time': 1, 'max_instances': 1 } self._sched = BackgroundScheduler(jobstores=self.jobstores, executors=self.executors, job_defaults=self.job_defaults) # 添加 任务提交 事件监听 self._sched.add_listener(self.when_job_submitted, EVENT_JOB_SUBMITTED) # 添加 任务执行完成 事件监听 self._sched.add_listener(self.when_job_executed, EVENT_JOB_EXECUTED) # 添加 任务异常退出 事件监听 self._sched.add_listener(self.when_job_crashed, EVENT_JOB_ERROR) self._jobs = {} self._jobhandlers = {} # format, key: jobid, value: jobhandler self._jobs_key = ["name", "func", "args", "kwargs"] self.start()
def get_mongo_job_stores(): from pytz import utc from apscheduler.jobstores.mongodb import MongoDBJobStore, MongoClient from apscheduler.executors.pool import ProcessPoolExecutor from libs.db_context import get_mongo_client client = get_mongo_client() jobstores = { 'mongo': MongoDBJobStore(collection='job', database='apscheduler', client=client), 'default': MongoDBJobStore(collection='job', database='apscheduler2', client=client), } executors = { 'default': { 'type': 'threadpool', 'max_workers': 20 }, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = {'coalesce': False, 'max_instances': 3} scheduler = TornadoScheduler() scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) return scheduler
class Config: SECRET_KEY = 'QWERTYUIOPASDFGHJ' MAIL_SERVER = 'smtp.126.com' MAIL_PORT = 587 MAIL_USE_TLS = True MAIL_USERNAME = '******' MAIL_PASSWORD = '******' FLASKY_MAIL_SUBJECT_PREFIX = 'SUBJECT_PREFIX' FLASKY_MAIL_SENDER = 'MAIL_SENDER' FLASKY_ADMIN = 'ADMIN' SSL_REDIRECT = False # logging level LOGGING_LEVEL = logging.INFO AUTO_HOME = os.getcwd().replace('\\', '/') + '/.beats' AUTO_ROBOT = [] executors = { 'default': { 'type': 'threadpool', 'max_workers': 20 }, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = {'coalesce': False, 'max_instances': 10} @staticmethod def init_app(app): pass
def main(): # Clean logger.info("Clean memcached before init") memcached_host, memcached_port = get_memcached_config() mem_nfv = MemcachedNFV(memcached_host, memcached_port) mem_nfv.clean_memcached() mem_nfv.disconnect() # Init server list init_server_cached_list_api() # Scheduler for get statistic data_config = read_config_json_file(NFV_CONFIG_PATH) interval = int(data_config['interval']) executors = { 'default': { 'type': 'threadpool', 'max_workers': 20 }, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = {'coalesce': False, 'max_instances': 3} scheduler = BlockingScheduler() scheduler.configure(executors=executors, job_defaults=job_defaults) scheduler.add_job(scheduler_get_statistic_job, 'interval', seconds=interval) scheduler.start()
def __init__(self, background=False, deamon=True, **kwargs): logging.basicConfig(format="[%(asctime)s] %(message)s", atefmt="%Y-%m-%d %H:%M:%S") logging.getLogger('apscheduler').setLevel(logging.DEBUG) if background: self.sched = BackgroundScheduler(deamon=deamon) # background else: self.sched = BlockingScheduler(deamon=deamon) # foreground # TODO: Read from configuration file. self.sched.configure( jobstores={ # "sqlite": SQLAlchemyJobStore(url='sqlite:///app/database/example.db'), # "default": MemoryJobStore() "default": SQLAlchemyJobStore(url='sqlite:///app/database/example.db') }, executors={ 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) }, job_defaults={ 'coalesce': False, 'max_instances': 3 }, timezone=get_localzone() # Asia/Seoul ) self.retried = 0 self.logger = logging.getLogger('apscheduler') super(JobLauncher, self).__init__()
def runScheduler(): runProxyFetch() timezone = ConfigHandler().timezone scheduler_log = LogHandler("scheduler") scheduler = BlockingScheduler(logger=scheduler_log, timezone=timezone) scheduler.add_job(runProxyFetch, 'interval', minutes=4, id="proxy_fetch", name="proxy采集") scheduler.add_job(runProxyCheck, 'interval', minutes=2, id="proxy_check", name="proxy检查") executors = { 'default': { 'type': 'threadpool', 'max_workers': 20 }, 'processpool': ProcessPoolExecutor(max_workers=8) # default 5 } job_defaults = {'coalesce': False, 'max_instances': 10} scheduler.configure(executors=executors, job_defaults=job_defaults, timezone=timezone) scheduler.start()
def __init__(self, elastic, crawler_dir="crawlers", crawler_args={}, **cron_defaults): """Initializes the scheduler by binding it to it's elasticsearch db. Args: elastic (elasticsearch.Elasticsearh): The es-client to save the crawling jobs in. crawler_dir (str): the directory, where the crawlers will be found. Defaults to "crawlers". job_defaults (dict): a dictionary of keyword arguments for the schedulers job_defaults. **cron_defaults (dict): a dictionary of keyword arguments for the schedulers job_defaults. Returns: Scheduler: a fresh Scheduler instance. """ jobstores = { "default": { "type": "memory" }, "elastic": InjectorJobStore(kwargs=crawler_args, client=elastic) } executors = { "default": ThreadPoolExecutor(10), "processpool": ProcessPoolExecutor(10) } job_defaults = { "misfire_grace_time": 5 * 60, # 5min "coalesce": True, } self.cron_defaults = utility.DefaultDict( { # standard is every day at 00:00:00 "hour": 0, "minute": 0, "second": 0 }, **cron_defaults) self.scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) self.crawlers = _detect_crawlers() # set up the validator schema. self.job_validator = cerberus.Validator(SCHEMATA["job"]({ "trigger_ids": list(self.TRIGGERS) }), allow_unknown=True) self.scheduler.start()
def __init__(self): self.run_date = datetime.datetime.now() + datetime.timedelta(seconds=3) self.run_date = self.run_date.strftime('%Y-%m-%d %H:%M:%S') self.tm = time.strftime('%Y%m%d%H%M%S',time.localtime()) self.scheduler = BackgroundScheduler() self.executors = {'default': ThreadPoolExecutor(10), 'processpool': ProcessPoolExecutor(5)} self.job_defaults = {'coalesce': False, 'max_instances': 1} self.scheduler.configure(timezone=pytz.timezone('Asia/Shanghai'),job_defaults=self.job_defaults,executors=self.executors)
class ConfigTask(object): """ 定时任务配置 """ jobstores = {'default': SQLAlchemyJobStore(url="sqlite:///" + os.path.join(basedir, "data.sqlite"))} executors = {'default': ThreadPoolExecutor(10), 'processpool': ProcessPoolExecutor(3)} def __init__(self): self.scheduler = BackgroundScheduler(jobstores=self.jobstores, executors=self.executors)
def __init__(self, time_offset, sampling_freq=5): self.sampling_interval = 1.0 / sampling_freq # 5 Hz by default self.total_mem = psutil.virtual_memory().total self.manager = Manager() self.samples = self.manager.list() self.time_offset = time_offset self.scheduler = BackgroundScheduler( executors={'default': ProcessPoolExecutor(1)}) self.job = None
def run(self): executors = { 'default': {'type': 'threadpool', 'max_workers': 20}, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = { 'coalesce': False, 'max_instances': 3 } self.scheduler = BackgroundScheduler() self.scheduler.configure(executors=executors, job_defaults=job_defaults) self.scheduler.add_job(lambda : self.backup(), trigger='cron', hour=0, minute=10) self.scheduler.start()
def getBlockingScheduler(): executors = { 'default': ThreadPoolExecutor(10), 'processpool': ProcessPoolExecutor(5) } job_defaults = {'coalesce': False, 'max_instances': 5} scheduler = BlockingScheduler(executors=executors, job_defaults=job_defaults) return scheduler
def getSchedule(self): executors = { 'default': ThreadPoolExecutor(3), 'processpool': ProcessPoolExecutor(1) } job_defaults = {'coalesce': True, 'max_instances': 1} scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults, timezone=utc) return scheduler
def __init__(self, job_type, store_executor_alias, process_count): self.sche = TornadoScheduler() self.host = MONGO_CONFIG.get('host') self.mongo_client = MongoClient(self.host) self.job_type = job_type self.mongo_job_store = MongoDBJobStore(collection='job', database=DBNAME, client=self.mongo_client) self.store_executor_alise = store_executor_alias self.process_poll = ProcessPoolExecutor(process_count)
def executor(request, mock_scheduler): if request.param == 'threadpool': from apscheduler.executors.pool import ThreadPoolExecutor executor_ = ThreadPoolExecutor() else: from apscheduler.executors.pool import ProcessPoolExecutor executor_ = ProcessPoolExecutor() executor_.start(mock_scheduler, 'dummy') yield executor_ executor_.shutdown()
def run(self): executors = { 'default': {'type': 'threadpool', 'max_workers': 20}, 'processpool': ProcessPoolExecutor(max_workers=5) } job_defaults = { 'coalesce': False, 'max_instances': 3 } self.scheduler = BackgroundScheduler() self.scheduler.configure(executors=executors, job_defaults=job_defaults) # this job will run every first day of month at am 01:01:00 self.scheduler.add_job(lambda : self.invoice_calc(), trigger='cron', day=1, hour=1, minute=0) self.scheduler.start()
def main(): args = parse_args() print(args) logfile = path.expanduser(args.logfile) if not path.exists(path.dirname(logfile)): os.makedirs(path.dirname(logfile)) root_logger = logging.getLogger() formatter = logging.Formatter( "%(asctime)s:%(levelname)s:%(name)s:%(message)s") handler = logging.handlers.RotatingFileHandler(args.logfile, maxBytes=args.logsize, backupCount=args.logcount) handler.setFormatter(formatter) root_logger.addHandler(handler) root_logger.setLevel(args.loglevel) jobstores = {'memory': MemoryJobStore()} executors = { 'default': ProcessPoolExecutor(args.processes), 'threadpool': ThreadPoolExecutor(args.threads) } job_defaults = {'max_instances': 10000} scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) ''' Add jobs here ''' x = 1 for x in range(1, 10000): interval = random.randint(30, 120) scheduler.add_job(handle_job, 'interval', seconds=interval, kwargs={ 'id': str(x), 'interval': str(interval) }) x += 1 print("\nStarting Scheduler...") scheduler.start() while True: time.sleep(1) print("Scheduleder started") print("Shutting down... please wait!") scheduler.shutdown() logging.shutdown()
def executor(request, mock_scheduler): if request.param == 'threadpool': from apscheduler.executors.pool import ThreadPoolExecutor executor_ = ThreadPoolExecutor() else: from apscheduler.executors.pool import ProcessPoolExecutor executor_ = ProcessPoolExecutor() executor_.start(mock_scheduler, 'dummy') request.addfinalizer(executor_.shutdown) return executor_
def __init__(self): self.scheduler = BlockingScheduler(executors={ 'default': ThreadPoolExecutor(15), 'processpool': ProcessPoolExecutor(1) }, job_defaults={ 'coalesce': False, 'max_instances': 1 })
def __init__(self, parent=None): super(TimingTasksManager, self).__init__(parent) executors = { 'default': ThreadPoolExecutor(10), 'processpool': ProcessPoolExecutor(3) } self.scheduler = BackgroundScheduler(executors=executors) self._load_jobs() self.scheduler.start() log.info(u'成功启动定时任务')
def config_Scheduler(self, timezone): jobstores = { 'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite') } executors = {'processpool': ProcessPoolExecutor(max_workers=5)} job_defaults = {'coalesce': False, 'max_instances': 3} scheduler = BackgroundScheduler() scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=timezone) sched.add_job(job_function, 'interval', seconds=1)
def __init__(self, ctx): self.ctx = ctx log1.setLevel(ctx.logLevel) log2.setLevel(ctx.logLevel) self.executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } self.job_defaults = {'coalesce': False, 'max_instances': 1} self.scheduler = BackgroundScheduler(executors=self.executors, job_defaults=self.job_defaults) self.start()
def init_scheduler(): """初始化调度器""" def my_listener(event): """事件监听""" if event.exception: logger.exception('========== The job crashed :( ==========') logger.exception(str(event.exception)) else: logger.info('============ The job worked :) ===========') job_sqlite_path = os.path.join(CommonConf.SQLITE_DIR, 'jobs.sqlite') # 每次启动任务时删除数据库 os.remove(job_sqlite_path) if os.path.exists(job_sqlite_path) else None jobstores = { 'default': SQLAlchemyJobStore( url='sqlite:///' + job_sqlite_path) # SQLAlchemyJobStore指定存储链接 } executors = { 'default': { 'type': 'threadpool', 'max_workers': 10 }, # 最大工作线程数20 'processpool': ProcessPoolExecutor(max_workers=1) # 最大工作进程数为5 } job_defaults = {'coalesce': True, 'max_instances': 3} scheduler._logger = logger scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults) # 查找数据库中没能成功上传的数据,重新上传 scheduler.add_job(ThirdEtcApi.reupload_etc_deduct_from_db, trigger='cron', hour='*/1', id='reupload_etc_deduct_from_db') # 检测天线心跳状态, 心跳停止过长,重启天线 scheduler.add_job(RsuStatus.check_rsu_heartbeat, trigger='cron', minute='*/3', id='check_rsu_heartbeat', kwargs={'callback': ThirdEtcApi.tianxian_heartbeat}, max_instances=2) # 平台参数下载-发行方黑名单接口 ThirdEtcApi.download_fxf_blacklist() # 先立即执行一次 scheduler.add_job(ThirdEtcApi.download_fxf_blacklist, trigger='cron', hour='*/12', id='download_fxf_blacklist') scheduler.add_listener(my_listener, events.EVENT_JOB_EXECUTED | events.EVENT_JOB_ERROR) logger.info("启动调度器...") scheduler.start()
def __init__(self): jobstores = {'default': MemoryJobStore()} executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } job_defaults = {'coalesce': False, 'max_instances': 3} self.__sched = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) self.__sched.start()
class DevelopmentConfig(BaseConfig): # 本项目使用的域名与端口号 PROJECT_PORT = 5000 PROJECT_DOMAIN = f"http://*****:*****@127.0.0.1:13306/common_web_service?charset=utf8mb4' SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_ENCODING = "utf8mb4" # 添加celery配置 broker_url = 'redis://localhost:6379' result_backend = 'redis://localhost:6379' imports = ('proStruct.services.flask_linux_crontab') # 配置redis REDIS_HOST = '127.0.0.1' REDIS_PORT = 6379 REDIS_PASSWORD = None REDIS_DB = None # 测试账号 TEST_APP_ID = 'dc601e113be8a2e622f9f9a3f363eb93' TEST_ACCOUNT = '15845623256' TEST_PASSWORD = '******' # aaasss123 # 日志配置: 线上需要重新设置 LOG_FILE_PATH = os.path.join(project_root_path, modifyPath('logs/web/web_common.log')) LOG_LEVEL = logging.INFO LOG_FILE_SIZE = 10 * 1204 * 1024 LOG_FILE_COUNT = 10 # apscheduler 定时任务调度配置 JOB_STORES = { "redis": RedisJobStore(host=REDIS_HOST, port=REDIS_PORT), # 设置一个名为redis的job存储,后端使用 redis # 一个名为 default 的 job 存储,后端使用数据库(使用 Sqlite) # "default": SQLAlchemyJobStore(url="sqlite:///flask_linux_crontab.sqlite") "backend_db": SQLAlchemyJobStore(url=SQLALCHEMY_DATABASE_URI) } JOB_EXECUTORS = { "default": ThreadPoolExecutor(1), # 设置一个名为 default的线程池执行器, 最大线程设置为20个 # TODO 线程过多,会出现同一个任务被多次执行的情况 "processpool": ProcessPoolExecutor(1), # 设置一个名为 processpool的进程池执行器,最大进程数设为5个 } # 开启job合并,设置job最大实例上限为3 JOB_DEFAULT = {'coalesce': False, 'max_instances': 3}