def handle(self, *args, **options): scheduler = BackgroundScheduler(timezone=settings.TIME_ZONE) scheduler.add_jobstore(DjangoJobStore(), "default") scheduler.add_job(fetch_tweets, trigger=CronTrigger(hour='*/3'), id="fetch_tweets", max_instances=1, replace_existing=True) logger.info("Added job 'fetch_tweets'.") scheduler.add_job(delete_old_job_executions, trigger=CronTrigger(day_of_week="mon", hour="00", minute="00"), id="delete_old_job_executions", max_instances=1, replace_existing=True) logger.info("Added weekly job: 'delete_old_job_executions'.") try: logger.info("Starting scheduler...") scheduler.start() except KeyboardInterrupt: logger.info("Stopping scheduler...") scheduler.shutdown() logger.info("Scheduler shut down successfully!")
def test_run_selected_jobs_enforces_timeout(self, rf, monkeypatch): monkeypatch.setattr(settings, "APSCHEDULER_RUN_NOW_TIMEOUT", 1) # Shorten timeout to reduce test runtime scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore()) scheduler.start() job = scheduler.add_job(print, trigger="interval", seconds=60) admin = DjangoJobAdmin(DjangoJob, None) r = rf.get("/django_apscheduler/djangojob/") # Add support for Django messaging framework r._messages = mock.MagicMock(BaseStorage) r._messages.add = mock.MagicMock() with mock.patch( "django_apscheduler.admin.BackgroundScheduler.add_listener"): admin.run_selected_jobs(r, DjangoJob.objects.filter(id=job.id)) assert DjangoJobExecution.objects.count() == 0 r._messages.add.assert_called_with( 40, format_html( "Maximum runtime of {} seconds exceeded! Not all jobs could be completed successfully. " "Pending jobs: {}", admin._job_execution_timeout, ",".join({job.id}), ), "", ) scheduler.shutdown()
def run_selected_jobs(self, request, queryset): scheduler = BackgroundScheduler() scheduler.add_jobstore(self._memory_jobstore) scheduler.add_listener(self._handle_execution_event, events.EVENT_JOB_EXECUTED) scheduler.start() num_jobs_scheduled = 0 self._jobs_executed = [] start_time = timezone.now() for item in queryset: django_job = self._django_jobstore.lookup_job(item.id) if not django_job: msg_dict = {"job_id": item.id} msg = _( "Could not find job {job_id} in the database! Skipping execution..." ) self.message_user( request, format_html(msg, **msg_dict), messages.WARNING ) continue scheduler.add_job( django_job.func_ref, trigger=None, # Run immediately args=django_job.args, kwargs=django_job.kwargs, id=django_job.id, name=django_job.name, misfire_grace_time=django_job.misfire_grace_time, coalesce=django_job.coalesce, max_instances=django_job.max_instances, ) num_jobs_scheduled += 1 while len(self._jobs_executed) < num_jobs_scheduled: # Wait for selected jobs to be executed. if timezone.now() > start_time + timedelta( seconds=self._job_execution_timeout ): msg = _( "Maximum runtime exceeded! Not all jobs could be completed successfully." ) self.message_user(request, msg, messages.ERROR) scheduler.shutdown(wait=False) return None time.sleep(0.1) for job_id in self._jobs_executed: msg_dict = {"job_id": job_id} msg = _("Executed job '{job_id}'!") self.message_user(request, format_html(msg, **msg_dict)) scheduler.shutdown() return None
def handle(self, *args, **options): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") # run this job every 24 hours scheduler.add_job(main(), trigger=CronTrigger(hour="24"), # every 24 hours id="my_job", jobstore='default', replace_existing=True, ) print("Added job 'my_job'.") scheduler.add_job( delete_old_job_executions, trigger=CronTrigger( day_of_week="mon", hour="00", minute="00" ), # Midnight on Monday, before start of the next work week. id="delete_old_job_executions", max_instances=1, replace_existing=True, ) print( "Added weekly job: 'delete_old_job_executions'." ) try: print("Starting scheduler...") scheduler.start() except KeyboardInterrupt: print("Stopping scheduler...") scheduler.shutdown() print("Scheduler shut down successfully!")
def start(): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") #test scheduler.add_job(update_rankings, 'interval', hours=48) register_events(scheduler) scheduler.start()
def test_run_selected_jobs_creates_job_execution_entry( self, rf, monkeypatch): monkeypatch.setattr(settings, "APSCHEDULER_RUN_NOW_TIMEOUT", 1) # Shorten timeout to reduce test runtime scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore()) scheduler.start() job = scheduler.add_job(print, trigger="interval", seconds=60) admin = DjangoJobAdmin(DjangoJob, None) r = rf.get("/django_apscheduler/djangojob/") # Add support for Django messaging framework r._messages = mock.MagicMock(BaseStorage) r._messages.add = mock.MagicMock() assert not DjangoJobExecution.objects.filter(job_id=job.id).exists() admin.run_selected_jobs(r, DjangoJob.objects.filter(id=job.id)) assert DjangoJobExecution.objects.filter(job_id=job.id).exists() r._messages.add.assert_called_with(20, f"Executed job '{job.id}'!", "") scheduler.shutdown()
def get_singel_api_task_time(request): username = request.session.get('user', '') tasks = process_apis_task.objects.all() if request.method == "POST": singel_task_date = request.POST.get('date') singel_task_frequency = request.POST.get('frequency') if singel_task_date & singel_task_frequency != '': try: scheduler = BackgroundScheduler() # 指定时间执行 scheduler.add_jobstore(start_singel_apis_task, 'date', run_date=singel_task_date) #调度开始 scheduler.start() print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) except (KeyboardInterrupt, SystemExit): #键盘打断则退出 scheduler.shutdown() else: try: scheduler = BackgroundScheduler() # 指定间隔时间执行一次 scheduler.add_jobstore(start_singel_apis_task, 'interval', minutes=singel_task_frequency) #调度开始 scheduler.start() except (KeyboardInterrupt, SystemExit): #键盘打断则退出 scheduler.shutdown() return render(request, "singel_periodic_task.html", { "user": username, "tasks": tasks })
def create_scheduler(): # manage = SchedulerManage() scheduler = BackgroundScheduler(daemonic=True) scheduler.add_jobstore(DjangoJobStore(), "default") date = dt.datetime.now() # 报警 scheduler.add_job(cal_kde_value, "date", run_date=date, id='alarm_proj', args=[], replace_existing=True) scheduler.add_job(his_model_update, "date", run_date=date, id='his_model_up', args=[], replace_existing=True) scheduler.add_job(seperate_operate_record.main, "date", run_date=date, id='operate_parsing', args=[], replace_existing=True) scheduler.add_job(clear_database, 'cron', hour='16', minute='04', id='clear_database', replace_existing=True) # scheduler.add_job(operate_resolve, "date", run_date=date, id='alarm_proj', args=[], replace_existing=True) # scheduler.add_job(seperate_operate_record.main, "interval", minutes=1, id='operate_proj', args=[]) # scheduler.add_job(time_task, "interval", seconds=5, id='mytask2', args=['mytask2',], replace_existing=True) scheduler.add_job(so_run, "interval", minutes=1, id='operate_match', args=[], replace_existing=True) # try: # group, int_list, scats_input = get_scats_int() # except Exception as e: # logger.error(e) # print(e) # else: # logger.info("get scats basic inf successfully!") # scheduler.add_job(thread_creat, "interval", minutes=5, id='scats_salklist', args=[group, int_list, scats_input], # replace_existing=True) # scheduler.add_job(RequestDynaDataFromInt, "interval", minutes=5, id='scats_volumns', args=[int_list], # replace_existing=True) # scheduler.add_job(get_operate, "interval", minutes=3, id='scats_operate', args=[], # replace_existing=True) scheduler.start() logger.info('start scheduler task') print("=======================定时任务启动==========================") print(scheduler.get_jobs()) print(scheduler.state) logger.info('start task register,check on admin platform!') register_events(scheduler)
def start(): print("scheduler initializing...") scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") scheduler.add_job(check_and_send_mail, 'interval', hours=1, name='checking_and_sending_mails', jobstore='default') register_events(scheduler) scheduler.start() print("Scheduler started...", file=sys.stdout)
def __init__(self,interval=2): global se_scheduler se_scheduler = BackgroundScheduler() se_scheduler.add_jobstore("mongodb", database="felix_se", collection="ScheduledJobs") se_scheduler.start() super(SESchedulerService, self).__init__("SESchedulerService",interval) self.first_time = True
def start(): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") # run this job every 24 hours scheduler.add_job(deactivate_expired_accounts, 'interval', seconds=10, name='clean_accounts', jobstore='default') register_events(scheduler) scheduler.start() print("Scheduler started...", file=sys.stdout)
def start(): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") # run this job every 24 hours scheduler.add_job(check_obsolete_orders, 'interval', hours=24, name='clean_orders', jobstore='default', id="check_obsolete_orders", replace_existing=True) register_events(scheduler) scheduler.start() print("Scheduler started...", file=sys.stdout)
def start(): backscheduler = BackgroundScheduler() backscheduler.add_jobstore(DjangoJobStore(), "default") backscheduler.add_job(scheduleScrape(), trigger="interval", name='scrape_all', seconds=300, jobstore="default") backscheduler.start()
def add_sqlcronjob(job_id, run_date, workflowId, url): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") scheduler.add_job(execute_job, 'date', run_date=run_date, args=[workflowId, url], id=job_id, replace_existing=True) register_events(scheduler) try: scheduler.start() except SchedulerAlreadyRunningError: logger.debug("Scheduler is already running!") logger.debug('add_sqlcronjob:' + job_id + " run_date:" + run_date.strftime('%Y-%m-%d %H:%M:%S'))
def start(): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), 'djangojobstore') register_events(scheduler) @scheduler.scheduled_job('cron', minute='*/5', name='auto_mail') def auto_mail(): send_meilmail() scheduler.start()
def __init__(self, interval=2): global se_scheduler se_scheduler = BackgroundScheduler() se_scheduler.add_jobstore("mongodb", database="felix_se", collection="ScheduledJobs") se_scheduler.start() super(SESchedulerService, self).__init__("SESchedulerService", interval) self.first_time = True
class ApsScheduler(Scheduler): def __init__(self): self._scheduler = BackgroundScheduler() def start(self): self._scheduler.add_jobstore(DjangoJobStore(), 'default') self._scheduler.start() def add_job(self, func: Callable, func_kwargs: dict, cron_schedule: CronSchedule) -> str: job = self._scheduler.add_job(func=func, kwargs=func_kwargs, trigger='cron', start_date=cron_schedule.start_date, end_date=cron_schedule.end_date, year=cron_schedule.year, month=cron_schedule.month, day=cron_schedule.day, week=cron_schedule.week, day_of_week=cron_schedule.day_of_week, second=cron_schedule.second, minute=cron_schedule.minute, hour=cron_schedule.hour) return job.id def modify(self, cron_schedule: CronSchedule): for job_id in self._get_all_affected_job_ids(cron_schedule.pk): self._scheduler.reschedule_job( job_id, trigger='cron', start_date=cron_schedule.start_date, end_date=cron_schedule.end_date, year=cron_schedule.year, month=cron_schedule.month, day=cron_schedule.day, week=cron_schedule.week, day_of_week=cron_schedule.day_of_week, second=cron_schedule.second, minute=cron_schedule.minute, hour=cron_schedule.hour) def delete(self, cron_schedule_id: int): for job_id in self._get_all_affected_job_ids(cron_schedule_id): self._scheduler.remove_job(job_id) def delete_job(self, job_id): self._scheduler.remove_job(job_id) @staticmethod def _get_all_affected_job_ids(cron_schedule_id): objs = Timelapse.objects.filter(schedule_id=cron_schedule_id) for obj in objs: assert isinstance(obj, Timelapse) yield obj.schedule_job_id
def start(): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") scheduler.add_job(report, 'interval', minutes=1, name='report_accounts', jobstore='default') register_events(scheduler) scheduler.start() print("Scheduler has started")
def __init__(): global g_main_scheduler g_main_scheduler = BackgroundScheduler(timezone=utc) g_main_scheduler.add_jobstore('sqlalchemy', url='sqlite:///jobs.sqlite') g_main_scheduler.start() log_jobs() import threading myLogger("g_main_scheduler in this thread:{}".format( threading.current_thread()))
def wechat_sendfile_server(): data = {} itchat.auto_login() scheduler = BackgroundScheduler() # 调度器使用DjangoJobStore() scheduler.add_jobstore(DjangoJobStore(), "default") # trigger = DateTrigger(run_date='2019-05-13 15:25:30') job = scheduler.add_job(send_file_by_time, trigger='date', next_run_time='2019-05-13 14:06:30') # job = scheduler.add_job(send_file_by_time, trigger) scheduler.start() job.remove()
class SchedulerProcess(ZMQProcess): """ØMQ tasks scheduler process""" def __init__(self, zmq_address, handler, auth, clients, registry): # pylint: disable=too-many-arguments super().__init__(zmq_address, handler, auth, clients, registry) self.scheduler = BackgroundScheduler() self.jobstore = MemoryJobStore() def run(self): if self.scheduler is not None: self.scheduler.add_jobstore(self.jobstore, 'default') self.scheduler.start() ZMQProcess.run(self)
def create_scheduler(): from apscheduler.schedulers.background import BackgroundScheduler scheduler = BackgroundScheduler() mongo_client_kwargs = dict( host=CONF.MONGO_HOST, port=CONF.MONGO_PORT, ) scheduler.add_jobstore( 'mongodb', collection=CONF.SCHEDULER_MONGO_COLLECTION, **mongo_client_kwargs ) return scheduler
def main(): context = daemon.DaemonContext( working_directory='/opt/sendlater', pidfile=daemon.pidfile.PIDLockFile('/var/run/sendlater.pid'), ) with context: executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5), } scheduler = BackgroundScheduler(executors=executors) scheduler.add_jobstore( 'redis', jobs_key='sendlater:jobs', run_times_key='sendlater:run_times', ) scheduler.start() datetime_parser = parsedatetime.Calendar() def process(text): data = json.loads(text) recipient = data.get('recipient') sender = data.get('sender') when = data.get('when') message = data.get('message') parsed_tuples = datetime_parser.nlp(when) parsed_tuple = parsed_tuples[0] dt, flags, start_pos, end_pos, matched_text = parsed_tuple trigger = DateTrigger(dt) scheduler.add_job( send_message, trigger=trigger, args=[recipient, sender, when, message], ) with open('/opt/sendlater/logs', 'a') as f: f.write('SCHEDULED {} {} {} {}\n'.format(recipient, sender, when, message)) r = redis.Redis() while True: queue_name, text = r.blpop('sendlater:messages') try: process(text) except: pass
def create_scheduler(message_queue): manage = SchedulerManage() scheduler = BackgroundScheduler(daemonic=True) scheduler.add_jobstore(DjangoJobStore(), "default") date = dt.datetime.now() scheduler.add_job(main, "date", run_date=date, id='alarm_proj', args=[message_queue], replace_existing=True) # scheduler.add_job(manage.message_accept, "date", run_date=date, id='alarm_proj', args=[message_queue], replace_existing=True) # scheduler.add_job(seperate_operate_record.main, "interval", minutes=1, id='operate_proj', args=[]) # scheduler.add_job(time_task, "interval", seconds=5, id='mytask2', args=['mytask2',], replace_existing=True) scheduler.add_job(so_run, "interval", minutes=1, id='operate_match', args=[message_queue], replace_existing=True) try: group, int_list, scats_input = get_scats_int() except Exception as e: logger.error(e) print(e) else: logger.info("scats基础信息获取成功") scheduler.add_job(thread_creat, "interval", minutes=5, id='scats_salklist', args=[group, int_list, scats_input], replace_existing=True) scheduler.add_job(RequestDynaDataFromInt, "interval", minutes=5, id='scats_volumns', args=[int_list], replace_existing=True) scheduler.add_job(get_operate, "interval", minutes=3, id='scats_operate', args=[], replace_existing=True) scheduler.start() logger.error('定时任务开始') print("=======================定时任务启动==========================") print(scheduler.get_jobs())
def add_sqlcronjob(request): workflowId = request.POST.get('workflowid') run_date = request.POST.get('run_date') if run_date is None or workflowId is None: context = {'errMsg': '时间不能为空'} return render(request, 'error.html', context) elif run_date < datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'): context = {'errMsg': '时间不能小于当前时间'} return render(request, 'error.html', context) workflowDetail = workflow.objects.get(id=workflowId) if workflowDetail.status not in ['审核通过', '定时执行']: context = {'errMsg': '必须为审核通过或者定时执行状态'} return render(request, 'error.html', context) run_date = str(datetime.datetime.strptime(run_date, "%Y-%m-%d %H:%M:%S")) url = getDetailUrl(request) + str(workflowId) + '/' job_id = Const.workflowJobprefix['sqlreview'] + '-' + str(workflowId) try: scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") # mysql_add = "mysql://*****:*****@localhost:3308/archer3" # scheduler.add_jobstore(SQLAlchemyJobStore(url=mysql_add)) scheduler.add_job(execute_job, 'date', run_date=run_date, args=[workflowId, url], id=job_id, replace_existing=True) register_events(scheduler) try: scheduler.start() logger.debug("Scheduler started!") except SchedulerAlreadyRunningError: logger.debug("Scheduler is already running!") workflowDetail.status = Const.workflowStatus['tasktiming'] workflowDetail.save() except Exception as e: context = {'errMsg': '任务添加失败,错误信息:' + str(e)} return render(request, 'error.html', context) else: # logger.debug('add_sqlcronjob:' + job_id + "run_date:" + run_date.strftime('%Y-%m-%d %H:%M:%S')) logger.debug('add_sqlcronjob:' + job_id + "run_date:" + run_date) return HttpResponseRedirect( reverse('sql:detail', kwargs={ 'workflowId': workflowId, 'workflowType': 0 }))
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) Config = get_config() mysql_url = (f'mysql://{Config.MYSQL_USER}:{Config.MYSQL_PASSWORD}' f'@{Config.MYSQL_HOST}:{Config.MYSQL_PORT}' f'/{Config.MYSQL_DATABASE}') jobstore = SQLAlchemyJobStore(url=mysql_url) scheduler = BackgroundScheduler() scheduler.add_jobstore(jobstore) scheduler.start() self.scheduler = scheduler
def start_worker(config): connection_string = "mysql://%s:%s@%s/%s" % ( config.get('mysql', 'user'), config.get('mysql', 'passwd'), config.get('mysql', 'host'), config.get('mysql', 'database') ) scheduler = BackgroundScheduler() scheduler.add_jobstore(SQLAlchemyJobStore(url=connection_string), alias='db') scheduler.start() scheduler.add_job( update_all_job, args=[config], replace_existing=True, id='worker', trigger='cron', minute='*/5', jobstore='db', timezone='UTC')
class Scheduler(metaclass=Singleton): def __init__(self): _settings, _ = import_settings() self._scheduler = BackgroundScheduler() if 'REDIS_URL' in _settings: redis_config = gen_config_dict(_settings) self._scheduler.add_jobstore('redis', **redis_config) def __getattr__(self, item): return getattr(self._scheduler, item) @staticmethod def get_instance(): return Scheduler()
def get_paused_scheduler(cls): """Return paused scheduler for feeding jobs :return: scheduler instance """ # As we would like to feed default jobs, not to run any job, make sure we start scheduler in paused mode. if cls._scheduler is not None: return cls._scheduler else: scheduler = BackgroundScheduler() scheduler.add_jobstore('sqlalchemy', url=get_postgres_connection_string()) scheduler.start(paused=True) return scheduler
def start(): scheduler = BackgroundScheduler() scheduler.add_jobstore(DjangoJobStore(), "default") # run this job every 1 hours scheduler.add_job(update_tweets_hourly, 'interval', minutes=60, name='update_tweets_hourly', id="update_tweets_hourly", max_instances=1, replace_existing=True, jobstore='default') register_events(scheduler) scheduler.start() print("Scheduler started...", file=sys.stdout)
def add_periodic_job(sched_sql_loc, function_to_run, time_df, id_modifier, args): """ Adds a job to the scheduler database. This function must have the same arguements for all runs. :param: sched_sql_loc: location of the sql job database generated by this program and used by the scheduler :type: sched_sql_loc: string :param: function_to_run: The function that is being scheduled :type: csv_path_in: function :param: time_df: pandas dataframe that contains when the function should run :type: time_df: pandas dataframe :param: id_modifier: string that will be added to the id for add job :type: string :param: args: list of arguements that are used by function_to_run :type: args: list """ # opne the scheduler object and associate the job database with it scheduler = BackgroundScheduler() scheduler.add_jobstore('sqlalchemy', url='sqlite:///%s' % sched_sql_loc) sched_time_hours = time_df['hours'].values sched_time_minutes = time_df['minutes'].values sched_time_seconds = time_df['seconds'].values sched_day_code = time_df['day_code'].values sched_index = time_df.index.values for ind in range(len(sched_time_hours)): # misfire_grace_time - seconds after the designated runtime that # the job is still allowed to be run tempargs = args.copy() tempargs.append(sched_index[ind]) scheduler.add_job(function_to_run, 'cron', day_of_week=sched_day_code[ind], hour=int(sched_time_hours[ind]), minute=int(sched_time_minutes[ind]), second=int(sched_time_seconds[ind]), misfire_grace_time=120, id=(id_modifier + str(sched_index[ind])), args=tempargs) scheduler.print_jobs() scheduler.start() scheduler.shutdown()
class SchedulerManager(SingletonMixin): def __init__(self): self._scheduler = BackgroundScheduler() def start(self, db_url): self._scheduler.add_jobstore('sqlalchemy', db_url) self._scheduler.start() def stutdown(self): self._scheduler.shutdown() def add_interval_job(self, func, args, seconds=60): self._scheduler.add_job(func, 'interval', args, seconds) def add_cron_job(self, func, hour, minute): self._scheduler.add_job(func, 'cron', hour, minute)
def schedule_job(message): log.info("Received request to schedule the exporter job %s with job_id %s" % (message['exporter_name'], message['exporter_id'])) metrics_list = message['metrics_list'] for i in metrics_list: scheduler = BackgroundScheduler() scheduler.add_jobstore('sqlalchemy', url=CONF.gexporter.scheduler_db_url) job_id = message['exporter_id'] scheduler.add_job(create_job, args=[i, message["time_interval"], job_id], trigger='interval', minutes=int(message["time_interval"]), id=job_id) try: print("Starting scheduler") scheduler.start() except Exception as ex: log.error("last cycle of Scheduler has hit an exception") """
def __init__(self): """ Constructor of the service. """ self.config = ConfParser("ro.conf") self.scheduler = self.config.get("scheduler") interval = int(self.scheduler.get("frequency")) master_ro = self.config.get("master_ro") mro_enabled = ast.literal_eval(master_ro.get("mro_enabled")) db_name = "felix_ro" if mro_enabled: db_name = "felix_mro" global ro_scheduler ro_scheduler = BackgroundScheduler() ro_scheduler.add_jobstore( MongoDBJobStore(database=db_name, collection="scheduler.jobs")) ro_scheduler.start() super(ROSchedulerService, self).__init__( "ROSchedulerService", interval) self.first_time = True
class CommandScheduler(object): def __init__(self, start=True): executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } self.scheduler = BackgroundScheduler(executors=executors) self.scheduler.add_jobstore('redis', jobs_key='shh:jobs', run_times_key='shh:run_times' ) if start: self.scheduler.start() self.datetime_parser = parsedatetime.Calendar() def parse(self, at): return self.datetime_parser.parseDT(at)[0] def already_scheduled(self, what): # TODO(Bieber): Improve efficiency with a dict for scheduled_at, scheduled_what in self.get_jobs(): if what == scheduled_what: return True return False def schedule(self, at, what): dt = self.parse(at) trigger = DateTrigger(dt) self.scheduler.add_job(execute_command, trigger=trigger, args=[what.strip()], ) def get_jobs(self): jobs = self.scheduler.get_jobs() return [(job.next_run_time, job.args[0]) for job in jobs]
def add_schedule_backup_job(): # if __name__ == '__main__': os_user = config.OS_USER os_password = config.OS_APPS_PASSWD scheduler = BackgroundScheduler() # 默认内存的jobstore url = "sqlite:////home/apps/dbajob.sqlite" scheduler.add_jobstore("sqlalchemy", url=url, alias="sqlite_js") scheduler.print_jobs() print "a" scheduler.remove_all_jobs(jobstore="sqlite_js") scheduler.print_jobs() print "remove" # v_current_jobs = scheduler.get_jobs() # print v_current_jobs # if v_current_jobs: # 如果job存在的话,先请客 # scheduler.remove_job('backup') # 连接配置中心库,获取数据库备份周期等信息 db = Connection("/tmp/mysql3306.sock", config.DB_NAME, config.DB_USER, config.DB_PASSWD, time_zone="+8:00") v_sql = r"""SELECT a.instance_id,b.ip,b.port,a.backup_interval_type,a.backup_start_time from mysql_ins_bak_setup a,tag b where a.instance_id=b.id """ print v_sql bak_server_list = db.query(v_sql) if bak_server_list: # 有server需要配置 i = 0 # 把还没有开始的调度任务,置为手工结束 backup_result_type=4 v_manual_end_sql = "update mysql_ins_bak_log set backup_result_type=4 where backup_result_type=0" db.execute(v_manual_end_sql) for bak_server in bak_server_list: instance_id = bak_server["instance_id"] from_host = bak_server["ip"] # print from_host mysql_port = bak_server["port"] backup_interval_type = bak_server["backup_interval_type"] backup_start_time = bak_server["backup_start_time"] str_start_date = time.strftime("%Y-%m-%d") + " " + backup_start_time print str_start_date v_job_id = "backup_%s_%s" % (from_host, str(mysql_port)) if backup_interval_type == 1: # every day # scheduler.add_interval_job(backup, days=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], jobstore='file') scheduler.add_job( backup, "interval", id=v_job_id, days=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], replace_existing=True, jobstore="sqlite_js", ) elif backup_interval_type == 2: # every week weeks=1 scheduler.add_job( backup, "interval", id=v_job_id, weeks=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], replace_existing=True, jobstore="sqlite_js", ) elif backup_interval_type == 3: # every hour hours=1 scheduler.add_job( backup, "interval", id=v_job_id, hours=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], replace_existing=True, jobstore="sqlite_js", ) else: pass # 开始在数据库记录备份的调度任务状态 0:调度任务已启动,实际备份还没有开始 v_sche_start_sql = """insert into mysql_ins_bak_log(instance_id,backup_result_type) values(%d,0)""" % ( instance_id ) db.execute(v_sche_start_sql) i = i + 1 scheduler.print_jobs() print "b" scheduler.start() scheduler.print_jobs() print "c" db.close()
import requests logging.basicConfig() #import click app = Flask(__name__) # Load default config and override config from an environment variable app.config.update(dict( DATABASE=os.path.join(app.root_path, 'db', 'notifications.db'), SECRET_KEY='secret', USERNAME='******', PASSWORD='******' )) # scheduler sched = BackgroundScheduler(timezone='America/Los_Angeles') sched.add_jobstore('sqlalchemy', url='sqlite:///db/notifications.db') sched.add_executor('threadpool') sched.start() def connect_db(): """Connects to the specific database.""" rv = sqlite3.connect(app.config['DATABASE']) rv.row_factory = dict_factory return rv @app.cli.command('initdb') @click.option('--file',help='the schema file') def initdb_command(file): initdb(file) def initdb(file):
return True if __name__ == '__main__': app_dir = os.path.dirname(os.path.realpath(__file__)) filename= app_dir + '/../log/scheduler.log' f = open(filename, 'w+') f.close() logging.basicConfig( filename=filename, format='%(asctime)s - %(name)s - %(levelname)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG) logging.info('TandemApi Scheduler Started') scheduler = BackgroundScheduler() scheduler.add_jobstore('mongodb', database="tandem", collection='jobs') if len(sys.argv) > 1 and sys.argv[1] == '--clear': scheduler.remove_all_jobs() scheduler.start() print('To clear the alarms, run this example with the --clear argument.') print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) client = MongoClient('localhost', 27017) db = client['tandem'] new_jobs_queue = db['new_jobs_queue'] try: while True: time.sleep(2) addJobs(new_jobs_queue)
class ScheduleCommand(Command): schedule_sender = None def __init__(self, logger, message_sender): super().__init__(logger, message_sender) # Message scheduler configuration self.scheduler = BackgroundScheduler() configuration = self.message_sender.get_configuration() sdb_url = configuration.get('Message Scheduler', 'db_path') self.scheduler.add_jobstore('sqlalchemy', url=sdb_url) self.scheduler.start() ScheduleCommand.schedule_sender = message_sender def process(self, chat_id, user_id, username, arguments): if len(arguments) < 1: return self.help() operation = arguments[0] command_arguments = arguments[1:] if operation == "add": return self.add_message(username, chat_id, command_arguments) elif operation == "time": return self.get_local_time() elif operation == "remove": return self.remove_message(username, command_arguments) else: return self.help() def get_local_time(self): date_str = str(datetime.today().strftime('%Y-%m-%d %H:%M:%S')) return "My time is {}".format(date_str) def add_message(self, username, chat_id, arguments): if len(arguments) <2: return self.help() date = None message = None if arguments[0] == 'relative': if len(arguments)<3: return self.help() date = self.on_delta_parse(arguments[1]) message = arguments[2] else: date = self.on_date_parse(arguments[0]) message = arguments[1] if message.startswith('"') and message.endswith('"'): message = message[1:-1] reference = self.user_reference(username) if date is None: return "{}Date format not recognized".format(reference) current_date = datetime.today() if date < current_date: current_str = self.get_human_string_date(current_date) return "{}Sorry, I can't travel to the past, my current date is: {}".format(reference, current_str) message_id = str(random.randrange(0, 999999999)) self.scheduler.add_job(ScheduleCommand.send_programmed_message, \ 'date', run_date=date, args=[username, \ chat_id, message], id = message_id) date_str = self.get_human_string_date(date) return "{}The message [{}] has been successfully scheduled on {}"\ .format(reference, message_id, date_str) def send_programmed_message(username, chat_id, message): reference = ScheduleCommand.user_reference(username) text = "{}{}".format(reference, message) ScheduleCommand.schedule_sender.send_message(chat_id, text) send_programmed_message = staticmethod(send_programmed_message) def remove_message(self, username, arguments): if len(arguments) < 1: return self.help() founds = 0 ids = self.get_comma_arguments(arguments[0]) for message_id in ids: job = self.scheduler.get_job(message_id) if job == None: continue if job.args[0] != username: continue self.scheduler.remove_job(message_id) founds += 1 reference = self.user_reference(username) if founds == len(ids): if len(ids) ==1: return "{}The scheduled message was canceled.".format(reference) else: return "{}The scheduled messages were canceled.".format(reference) elif founds > 0: return "{}Some scheduled messages were canceled, but some others were not found or you aren't the owner.".format(reference) else: if len(ids) == 1: return "{}The scheduled message was not found or you aren't the owner.".format(reference) else: return "{}The scheduled message were not found or you aren't the owner.".format(reference) def get_human_string_date(self,datetime): return str(datetime.strftime('%Y-%m-%d %H:%M:%S')) def on_delta_parse(self, text_date): regex = re.compile(r'^((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?$') parts = regex.match(text_date) if not parts: return None parts = parts.groupdict() time_params = {} valid_params = 0 for (name, param) in parts.items(): if param: time_params[name] = int(param) valid_params = valid_params + 1 if valid_params == 0: return None return datetime.now() + timedelta(**time_params) def on_date_parse(self, text_date): try: return parser.parse(text_date, dayfirst = True, yearfirst = True) except ValueError: return None def user_reference(username): if username is None: return "" else: return "@{}: ".format(username) user_reference = staticmethod(user_reference) def help(self): self.logger.info("Printing help for schedule message command.") return self.get_file_help(__file__, "schedule_message.man") def name(self): return "schedule_message" def description(self): return "Schedule a message to be sent in a specified date."
def initializeScheduler(self): scheduler = BackgroundScheduler() connect_url = self.config.get(self.ORCH_SECTION,"sched_connect_url") scheduler.add_jobstore('sqlalchemy',url=connect_url) return scheduler
def connect(dbapi_connection, connection_record): connection_record.info['pid'] = os.getpid() @event.listens_for(Engine, "checkout") def checkout(dbapi_connection, connection_record, connection_proxy): pid = os.getpid() if 'pid' not in connection_record.info or connection_record.info['pid'] != pid: connection_record.connection = connection_proxy.connection = None raise exc.DisconnectionError( "Connection record belongs to pid %s, " "attempting to check out in pid %s" % (connection_record.info['pid'], pid) ) scheduler = BackgroundScheduler() scheduler.add_jobstore('sqlalchemy', url='mysql+pymysql://{}:{}@{}/{}'.format(config.db_user, config.db_pass, config.db_host, config.db_name)) def post_diamondtime(): now = datetime.now(timezone('US/Eastern')) episode_date = now + relativedelta(weekday=TU, hour=22, minute=0, second=0, microsecond=0) title = u'Weekly Diamond Time Submissions - For Episode on {0:%b %d, %Y}'.format(episode_date) f = open('diamondtime-text.md', 'r') text = f.read() sub = r.submit('diamondclub',title,text,save=False) sub.set_contest_mode(True) sub.set_flair(flair_text=u'Diamond Time',flair_css_class=u'diamondtime') sub.sticky(bottom=False) sub.distinguish(as_made_by=u'mod')
def init_scheduler(): global _scheduler if _scheduler is None: _scheduler = BackgroundScheduler() _scheduler.add_jobstore('sqlalchemy', url=SQLALCHEMY_DATABASE_URI) return _scheduler
def add_single_backup_job(v_setup_id): # mysql_ins_bak_setup 表的当前更新或插入的备份实例id os_user = config.OS_USER os_password = config.OS_APPS_PASSWD scheduler = BackgroundScheduler() # 默认内存的jobstore url = "sqlite:////home/apps/dbajob.sqlite" scheduler.add_jobstore("sqlalchemy", url=url, alias="sqlite_js") scheduler.print_jobs() print "a" # 连接配置中心库,获取数据库备份周期等信息 db = Connection("/tmp/mysql3306.sock", config.DB_NAME, config.DB_USER, config.DB_PASSWD, time_zone="+8:00") v_sql = r"""SELECT a.instance_id,b.ip,b.port,a.backup_interval_type,a.backup_start_time from mysql_ins_bak_setup a,tag b where a.instance_id=b.id and a.id=%d""" % ( v_setup_id ) print v_sql bak_server = db.get(v_sql) instance_id = bak_server["instance_id"] from_host = bak_server["ip"] # print from_host mysql_port = bak_server["port"] backup_interval_type = bak_server["backup_interval_type"] backup_start_time = bak_server["backup_start_time"] str_start_date = time.strftime("%Y-%m-%d") + " " + backup_start_time print str_start_date v_job_id = "backup_%s_%s" % (from_host, str(mysql_port)) if backup_interval_type == 1: # every day # scheduler.add_interval_job(backup, days=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], jobstore='file') scheduler.add_job( backup, "interval", id=v_job_id, days=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], replace_existing=True, jobstore="sqlite_js", ) elif backup_interval_type == 2: # every week weeks=1 scheduler.add_job( backup, "interval", id=v_job_id, weeks=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], replace_existing=True, jobstore="sqlite_js", ) elif backup_interval_type == 3: # every hour hours=1 scheduler.add_job( backup, "interval", id=v_job_id, hours=1, start_date=str_start_date, args=[from_host, mysql_port, os_user, os_password], replace_existing=True, jobstore="sqlite_js", ) else: pass scheduler.print_jobs() print "b" scheduler.start() scheduler.print_jobs() print "c" # 开始在数据库记录备份的调度任务状态 0:调度任务已启动,实际备份还没有开始 v_sche_start_sql = """insert into mysql_ins_bak_log(instance_id,backup_result_type) values(%d,0)""" % ( instance_id ) db.execute(v_sche_start_sql) db.close()
class HackathonScheduler(object): """An helper class for apscheduler""" jobstore = "ohp" def get_scheduler(self): """Return the apscheduler instance in case you have to call it directly :return the instance of APScheduler .. notes:: the return value might be None in flask debug mode """ return self.__apscheduler def add_once(self, feature, method, context=None, id=None, replace_existing=True, run_date=None, **delta): """Add a job to APScheduler and executed only once Job will be executed at 'run_date' or after certain timedelta. :Example: scheduler = RequiredFeature("scheduler") # execute task once in 5 minutes: context = Context(user_id=1) scheduler.add_once("user_manager","get_user_by_id",context=context, minutes=5) # 5 minutes later, user_manager.get_user_by_id(context) will be executed :type feature: str|unicode :param: the feature that are used to look for instance through hackathon_factory. All features are defined in __init__.py :type method: str|unicode :param method: the method name defined in the instance :type context: Context, see hackathon/__init__.py :param context: the execution context. Actually the parameters of 'method' :type id: str :param id: id for APScheduler job. Random id will be generated if not specified by caller :type replace_existing: bool :param replace_existing: if true, existing job with the same id will be replaced. If false, exception will be raised :type run_date: datetime | None :param run_date: job run date. If None, job run date will be datetime.now()+timedelta(delta) :type delta: kwargs for timedelta :param delta: kwargs for timedelta. For example: minutes=5. Will be ignored if run_date is not None """ if not run_date: run_date = get_now() + timedelta(**delta) if self.__apscheduler: self.__apscheduler.add_job(scheduler_executor, trigger='date', run_date=run_date, id=id, max_instances=1, replace_existing=replace_existing, jobstore=self.jobstore, args=[feature, method, context]) def add_interval(self, feature, method, context=None, id=None, replace_existing=True, next_run_time=undefined, **interval): """Add an interval job to APScheduler and executed. Job will be executed firstly at 'next_run_time'. And then executed in interval. :Example: scheduler = RequiredFeature("scheduler") context = Context(user_id=1) scheduler.add_interval("user_manager","get_user_by_id", context=context, minutes=10) # user_manager.get_user_by_id(context) will be called every 10 minutes :type feature: str|unicode :param: the feature that are used to look for instance through hackathon_factory. All features are defined in __init__.py :type method: str|unicode :param method: the method name defined in the instance :type context: Context, see hackathon/__init__.py :param context: the execution context. Actually the parameters of 'method' :type id: str :param id: id for APScheduler job. Random id will be generated if not specified by caller :type replace_existing: bool :param replace_existing: if true, existing job with the same id will be replaced. If false, exception will be raised :type next_run_time: datetime | undefined :param next_run_time: the first time the job will be executed. leave undefined to don't execute until interval time reached :type interval: kwargs for "interval" trigger :param interval: kwargs for "interval" trigger. For example: minutes=5. """ if self.__apscheduler: self.__apscheduler.add_job(scheduler_executor, trigger='interval', id=id, max_instances=1, replace_existing=replace_existing, next_run_time=next_run_time, jobstore=self.jobstore, args=[feature, method, context], **interval) def remove_job(self, job_id): """Remove job from APScheduler job store :type job_id: str | unicode :param job_id: the id of job """ if self.__apscheduler: try: self.__apscheduler.remove_job(job_id, self.jobstore) except JobLookupError: log.debug("remove job failed because job %s not found" % job_id) except Exception as e: log.error(e) def has_job(self, job_id): """Check the existence of specific job """ if self.__apscheduler: job = self.__apscheduler.get_job(job_id, jobstore=self.jobstore) return job is not None return False def __init__(self, app): """Initialize APScheduler :type app: Flask :param app: the Flask app """ self.app = app self.__apscheduler = None # NOT instantiate while in flask DEBUG mode or in the main thread # It's to avoid APScheduler being instantiated twice if not app.debug or os.environ.get("WERKZEUG_RUN_MAIN") == "true": self.__apscheduler = BackgroundScheduler(timezone=utc) # add MySQL job store job_store_type = safe_get_config("scheduler.job_store", "memory") if job_store_type == "mysql": log.debug("add aps_cheduler job store based on mysql") self.__apscheduler.add_jobstore('sqlalchemy', alias=self.jobstore, url=get_config("scheduler.job_store_url")) elif job_store_type == "mongodb": log.debug("add aps_cheduler job store based on mongodb") self.__apscheduler.add_jobstore('mongodb', alias=self.jobstore, database=safe_get_config("scheduler.database", "apscheduler"), collection=safe_get_config("scheduler.collection", "jobs"), host=safe_get_config("scheduler.host", "localhost"), port=safe_get_config("scheduler.port", 27017)) # add event listener self.__apscheduler.add_listener(scheduler_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR | EVENT_JOB_ADDED) log.info("APScheduler loaded") self.__apscheduler.start()
class BileanScheduler(object): """Billing scheduler based on apscheduler""" job_types = ( NOTIFY, DAILY, FREEZE, ) = ( 'notify', 'daily', 'freeze', ) trigger_types = (DATE, CRON) = ('date', 'cron') def __init__(self, **kwargs): super(BileanScheduler, self).__init__() self._scheduler = BackgroundScheduler() self.notifier = notifier.Notifier() self.engine_id = kwargs.get('engine_id', None) self.context = kwargs.get('context', None) if not self.context: self.context = bilean_context.get_admin_context() if cfg.CONF.bilean_task.store_ap_job: self._scheduler.add_jobstore(cfg.CONF.bilean_task.backend, url=cfg.CONF.bilean_task.connection) def init_scheduler(self): """Init all jobs related to the engine from db.""" jobs = db_api.job_get_all(self.context, engine_id=self.engine_id) if not jobs: LOG.info(_LI("No job found from db")) return True for job in jobs: if self.bilean_scheduler.is_exist(job.id): continue task_name = "_%s_task" % (job.job_type) task = getattr(self, task_name) self.bilean_task.add_job(task, job.id, job_type=job.job_type, params=job.parameters) def add_job(self, task, job_id, trigger_type='date', **kwargs): """Add a job to scheduler by given data. :param str|unicode user_id: used as job_id :param datetime alarm_time: when to first run the job """ mg_time = cfg.CONF.bilean_task.misfire_grace_time job_time_zone = cfg.CONF.bilean_task.time_zone user_id = job_id.split('-')[1] if trigger_type == 'date': run_date = kwargs.get('run_date') if run_date is None: msg = "Param run_date cannot be None for trigger type 'date'." raise exception.InvalidInput(reason=msg) self._scheduler.add_job(task, 'date', timezone=job_time_zone, run_date=run_date, args=[user_id], id=job_id, misfire_grace_time=mg_time) return True # Add a cron type job hour = kwargs.get('hour', None) minute = kwargs.get('minute', None) if not hour or not minute: hour, minute = self._generate_timer() self._scheduler.add_job(task, 'cron', timezone=job_time_zone, hour=hour, minute=minute, args=[user_id], id=job_id, misfire_grace_time=mg_time) return True def modify_job(self, job_id, **changes): """Modifies the properties of a single job. Modifications are passed to this method as extra keyword arguments. :param str|unicode job_id: the identifier of the job """ self._scheduler.modify_job(job_id, **changes) def remove_job(self, job_id): """Removes a job, preventing it from being run any more. :param str|unicode job_id: the identifier of the job """ self._scheduler.remove_job(job_id) def start(self): LOG.info(_('Starting Billing scheduler')) self._scheduler.start() def stop(self): LOG.info(_('Stopping Billing scheduler')) self._scheduler.shutdown() def is_exist(self, job_id): """Returns if the Job exists that matches the given ``job_id``. :param str|unicode job_id: the identifier of the job :return: True|False """ job = self._scheduler.get_job(job_id) return job is not None def _notify_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) msg = {'user': user.id, 'notification': 'The balance is almost use up'} self.notifier.info('billing.notify', msg) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'notify')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) self._add_freeze_job(user) def _daily_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'daily')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def _freeze_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'freeze')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def _add_notify_job(self, user): if not user.rate: return False total_seconds = user['balance'] / user['rate'] prior_notify_time = cfg.CONF.bilean_task.prior_notify_time * 3600 notify_seconds = total_seconds - prior_notify_time notify_seconds = notify_seconds if notify_seconds > 0 else 0 run_date = timeutils.utcnow() + timedelta(seconds=notify_seconds) job_params = {'run_date': run_date} job_id = self._generate_job_id(user['id'], self.NOTIFY) self.add_job(self._notify_task, job_id, params=job_params) # Save job to database job = {'id': job_id, 'job_type': self.NOTIFY, 'engine_id': self.engine_id, 'parameters': {'run_date': run_date}} db_api.job_create(self.context, job) def _add_freeze_job(self, user): if not user.rate: return False total_seconds = user.balance / user.rate run_date = timeutils.utcnow() + timedelta(seconds=total_seconds) job_params = {'run_date': run_date} job_id = self._generate_job_id(user.id, self.FREEZE) self.add_job(self._freeze_task, job_id, params=job_params) # Save job to database job = {'id': job_id, 'job_type': self.FREEZE, 'engine_id': self.engine_id, 'parameters': {'run_date': run_date}} db_api.job_create(self.context, job) return True def _add_daily_job(self, user): job_id = self._generate_job_id(user.id, self.DAILY) params = {'hour': random.randint(0, 23), 'minute': random.randint(0, 59)} self.add_job(self._daily_task, job_id, trigger_type='cron', params=params) # Save job to database job = {'id': job_id, 'job_type': self.DAILY, 'engine_id': self.engine_id, 'parameters': params} db_api.job_create(self.context, job) return True def _delete_all_job(self, user): for job_type in self.job_types: job_id = self._generate_job_id(user.id, job_type) if self.is_exist(job_id): self.remove_job(job_id) try: db_api.job_delete(self.context, job_id) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def update_user_job(self, user): """Update user's billing job""" if user.status not in [user.ACTIVE, user.WARNING]: self._delete_all_job(user.id) return for job_type in self.NOTIFY, self.FREEZE: job_id = self._generate_job_id(user.id, job_type) if self.is_exist(job_id): self.remove_job(job_id) try: db_api.job_delete(self.context, job_id) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) daily_job_id = self._generate_job_id(user.id, self.DAILY) if not self.is_exist(daily_job_id): self._add_daily_job(user) if user.status == user.ACTIVE: self._add_notify_job(user) else: self._add_freeze_job(user) def _generate_timer(self): """Generate a random timer include hour and minute.""" hour = random.randint(0, 23) minute = random.randint(0, 59) return hour, minute def _generate_job_id(self, user_id, job_type): """Generate job id by given user_id and job type""" return "%s-%s" % (job_type, user_id)
class MainRunner(object): class FilterAllLog(logging.Filter): # default we will filter logger from apscheduler.executors.default, apscheduler.scheduler, # you can config filter logger in config.json def filter(self, record): return "" def __init__(self, input_cmd_config_fp, input_job_config_fp, input_config_fp): # init value cmd_config_fp = os.path.abspath(input_cmd_config_fp) job_config_fp = os.path.abspath(input_job_config_fp) config_fp = os.path.abspath(input_config_fp) # load configuration json files self.cmd_config = CommonUtil.load_json_file(cmd_config_fp) self.job_config = CommonUtil.load_json_file(job_config_fp) self.config = CommonUtil.load_json_file(config_fp) # init schedulers self.scheduler = BackgroundScheduler() self.scheduler.add_jobstore('sqlalchemy', url=self.config['job_store_url']) self.scheduler.start() # init variables mananger = Manager() self.sync_queue = mananger.Queue() self.async_queue = mananger.Queue() self.current_job_list = [] # Slack Sending Queue # TODO: prevent the Slack bot is disable, the sending queue will use too much memory. self.slack_sending_queue = mananger.Queue(50) # init logger self.set_logging(self.config['log_level'], self.config['log_filter']) def set_logging(self, log_level, log_filter_list): default_log_format = '%(asctime)s %(levelname)s [%(name)s.%(funcName)s] %(message)s' default_datefmt = '%Y-%m-%d %H:%M' if log_level.lower() == "debug": logging.basicConfig(level=logging.DEBUG, format=default_log_format, datefmt=default_datefmt) else: logging.basicConfig(level=logging.INFO, format=default_log_format, datefmt=default_datefmt) my_filter = self.FilterAllLog() for target_logger in log_filter_list: logging.getLogger(target_logger).addFilter(my_filter) def scheduler_del_job(self, **kwargs): input_cmd_str = kwargs.get("input_cmd_str", "") cmd_str_list = input_cmd_str.split(" ") if len(cmd_str_list) == 2: job_id = cmd_str_list[1] current_job_list = self.scheduler.get_jobs() current_job_id_list = [j.id for j in current_job_list] if job_id in current_job_id_list: self.scheduler.remove_job(job_id) else: logging.error("Cannot find the specify job id [%s]" % job_id) else: logging.error("Incorrect cmd format! [%s]" % input_cmd_str) def scheduler_list_job(self, **kwargs): self.scheduler.print_jobs() def scheduler_shutdown(self, **kwargs): self.scheduler.shutdown() sys.exit(0) def list_all_commands(self, **kwargs): print "Current supported commands as below:" print "-" * 80 for cmd_str in self.cmd_config['cmd-settings']: print '{:30s} {:50s} '.format(cmd_str, self.cmd_config['cmd-settings'][cmd_str]['desc']) print "-" * 80 def scheduler_job_handler(self, input_cmd_obj, input_cmd_str): cmd_match_pattern = input_cmd_obj.keys()[0] func_point = getattr(self, input_cmd_obj[cmd_match_pattern]['func-name']) func_point(cmd_configs=input_cmd_obj[cmd_match_pattern]['configs'], input_cmd_str=input_cmd_str) def cmd_queue_composer(self, input_cmd_str): for cmd_pattern in self.cmd_config['cmd-settings']: re_compile_obj = re.compile(cmd_pattern) re_match_obj = re_compile_obj.search(input_cmd_str) if re_match_obj: current_command_obj = self.cmd_config['cmd-settings'][cmd_pattern] logging.debug("job matched [%s]" % cmd_pattern) target_queue_type = current_command_obj.get('queue-type', None) if target_queue_type == "async": self.async_queue.put({"cmd_obj": current_command_obj, "cmd_pattern": cmd_pattern, "input_cmd_str": input_cmd_str}) elif target_queue_type == "sync": self.sync_queue.put({"cmd_obj": current_command_obj, "cmd_pattern": cmd_pattern, "input_cmd_str": input_cmd_str}) else: self.scheduler_job_handler({cmd_pattern: current_command_obj}, input_cmd_str) break def load_default_jobs(self, input_scheduler, input_job_config): current_jobs = input_scheduler.get_jobs() current_jobs_name = [job.name for job in current_jobs] for job_name in input_job_config: if input_job_config[job_name]['default-loaded']: if job_name not in current_jobs_name: func_point = getattr(importlib.import_module(input_job_config[job_name]['module-path']), job_name) self.scheduler.add_job(func_point, input_job_config[job_name]['trigger-type'], id=job_name, seconds=input_job_config[job_name]['interval'], max_instances=input_job_config[job_name]['max-instances'], kwargs={ 'async_queue': self.async_queue, 'sync_queue': self.sync_queue, 'slack_sending_queue': self.slack_sending_queue, 'configs': input_job_config[job_name]['configs'], 'cmd_config': self.cmd_config} ) def job_exception_listener(self, event): if event.exception: logging.error("Job [%s] crashed [%s]" % (event.job_id, event.exception)) logging.error(event.traceback) def add_event_listener(self): self.scheduler.add_listener(self.job_exception_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) def run(self): # load default job into scheduler if the job is not exist self.load_default_jobs(self.scheduler, self.job_config) # add event listener into scheduler self.add_event_listener() # enter the loop to receive the interactive command while True: user_input = raw_input() self.cmd_queue_composer(user_input) time.sleep(3)
class isardScheduler(): def __init__(self): ''' JOB SCHEDULER ''' #<<<<<<< HEAD #~ host=app.config['RETHINKDB_HOST'], #~ port=app.config['RETHINKDB_PORT'], #~ auth_key=app.config['RETHINKDB_AUTH'] #~ rConn=r.connect(host=app.config['RETHINKDB_HOST'], #~ port=app.config['RETHINKDB_PORT'], #~ auth_key=app.config['RETHINKDB_AUTH'], #~ db=app.config['RETHINKDB_DB']) self.rStore=RethinkDBJobStore() #======= # ~ self.rStore=RethinkDBJobStore(host=app.config['RETHINKDB_HOST'], # ~ port=app.config['RETHINKDB_PORT'], # ~ auth_key=app.config['RETHINKDB_AUTH']) #>>>>>>> fe171dc30ddd8a2dabafa7b2085cbb60e6432c35 self.scheduler = BackgroundScheduler(timezone=pytz.timezone('UTC')) self.scheduler.add_jobstore('rethinkdb',self.rStore, database='isard', table='scheduler_jobs',host=app.config['RETHINKDB_HOST'], port=app.config['RETHINKDB_PORT'], auth_key=app.config['RETHINKDB_AUTH']) self.scheduler.remove_all_jobs() #~ scheduler.add_job(alarm, 'date', run_date=alarm_time, args=[datetime.now()]) #~ app.sched.shutdown(wait=False) self.turnOn() def add_scheduler(self,kind,action,hour,minute): id=kind+'_'+action+'_'+str(hour)+str(minute) function=getattr(isardScheduler,action) if kind == 'cron': self.scheduler.add_job(function, kind, hour=int(hour), minute=int(minute), jobstore=self.rStore, replace_existing=True, id=id) if kind == 'interval': self.scheduler.add_job(function, kind, hours=int(hour), minutes=int(minute), jobstore=self.rStore, replace_existing=True, id=id) if kind == 'date': alarm_time = datetime.now() + timedelta(hours=int(hour),minutes=int(minute)) self.scheduler.add_job(function, kind, run_date=alarm_time, jobstore=self.rStore, replace_existing=True, id=id) with app.app_context(): r.table('scheduler_jobs').get(id).update({'kind':kind,'action':action,'name':action.replace('_',' '),'hour':hour,'minute':minute}).run(db.conn) return True ''' Scheduler actions ''' def stop_domains(): with app.app_context(): r.table('domains').get_all('Started',index='status').update({'status':'Stopping'}).run(db.conn) def stop_domains_without_viewer(): with app.app_context(): r.table('domains').get_all('Started',index='status').filter({'viewer':{'client_since':False}}).update({'status':'Stopping'}).run(db.conn) def delete_old_stats(reduce_interval=300,delete_interval=86400): # 24h with app.app_context(): # domains_status r.table('domains_status_history').filter(r.row['when'] < int(time.time()) - delete_interval).delete().run(db.conn) reduced=[] cursor = r.table('domains_status').filter(r.row['when'] < int(time.time()) - reduce_interval).order_by('when').run(db.conn) r.table('domains_status').filter(r.row['when'] < int(time.time()) - reduce_interval).delete().run(db.conn) i=0 for c in cursor: if i % 50 == 0: reduced.append(c) i+=1 r.table('domains_status_history').insert(reduced).run(db.conn) # Hypervisors_status r.table('hypervisors_status_history').filter(r.row['when'] < int(time.time()) - delete_interval).delete().run(db.conn) reduced=[] cursor = r.table('hypervisors_status').filter(r.row['when'] < int(time.time()) - reduce_interval).order_by('when').run(db.conn) r.table('hypervisors_status').filter(r.row['when'] < int(time.time()) - reduce_interval).delete().run(db.conn) i=0 for c in cursor: if i % 50 == 0: reduced.append(c) i+=1 r.table('hypervisors_status_history').insert(reduced).run(db.conn) # Hypervisors_events (does not grow at the same speed) r.table('hypervisors_events').filter(r.row['when'] < int(time.time()) - delete_interval).delete().run(db.conn) def turnOff(self): self.scheduler.shutdown() def turnOn(self): self.scheduler.start() def removeJobs(self): self.scheduler.remove_all_jobs() ''' BULK ACTIONS ''' def bulk_action(self,table,tbl_filter,tbl_update): with app.app_context(): log.info('BULK ACTION: Table {}, Filter {}, Update {}'.format(table,filter, update)) r.table(table).filter(filter).update(update).run(db.conn)
login_manager = LoginManager() login_manager.init_app(app) bcrypt = Bcrypt(app) toolbar = DebugToolbarExtension(app) bootstrap = Bootstrap(app) db = SQLAlchemy(app) #################### #### Scheduler #### #################### from apscheduler.schedulers.background import BackgroundScheduler scheduler = BackgroundScheduler() scheduler.add_jobstore("sqlalchemy", url=app.config["SQLALCHEMY_DATABASE_URI"]) ################### ### blueprints #### ################### # from project.user.views import user_blueprint # app.register_blueprint(user_blueprint) from project.main.views import main_blueprint from project.song.views import song_blueprint from project.event.views import event_blueprint from project.job.views import job_blueprint app.register_blueprint(main_blueprint)
import logging logging.basicConfig(filename='example.log',level=logging.DEBUG) # configuration USERNAME = '******' PASSWORD = '******' db_url = 'sqlite:////tmp/test.db' app = Flask(__name__) app.debug = True app.secret_key = '\xf0\xc0\xd7\xbf\x82\xb4\xbbz2N\xa31\xc7k\xb0\x1e\xc9\x1e\xe9\x9egtOn' app.config['SQLALCHEMY_DATABASE_URI'] = db_url db = SQLAlchemy(app) scheduler = BackgroundScheduler(timezone=pytz.utc) scheduler.add_jobstore('sqlalchemy', url = db_url) scheduler.start() # database models class User(db.Model): id = db.Column(db.Integer, primary_key=True) phone_number = db.Column(db.String(11), unique=True) timezone = db.Column(db.String(32)) verification_code = db.Column(db.Integer) is_deleted = db.Column(db.Boolean) def __init__(self, phone_number, timezone): self.phone_number = phone_number self.timezone = timezone self.verification_code = random.randint(100000, 999999)
from django_apscheduler.jobstores import DjangoJobStore, register_events from apscheduler.executors.pool import ThreadPoolExecutor from scrapyd_api import ScrapydAPI from gerapy.server.core.utils import scrapyd_url from gerapy.server.core.models import Task, Client logger = logging.getLogger(__name__) db_time_format = "%Y-%m-%d %H:%M:%S" executors = { 'default': ThreadPoolExecutor(20) } scheduler = BackgroundScheduler(executors=executors) scheduler.add_jobstore(DjangoJobStore(), "default") @scheduler.scheduled_job("interval", seconds=60, id="scheduler_job") def scheduler_job(): """ 每分钟检查一次定时任务 :return: """ models = Task.objects.all() for model in models: scheduler_at = model.scheduler_at updated_at = model.updated_at scheduler_at_time_stamp = scheduler_at * 60 updated_at_time_stamp = time.mktime(updated_at.timetuple()) if time.time() - updated_at_time_stamp > scheduler_at_time_stamp:
from functions import safe_get_config, get_config from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.events import EVENT_JOB_EXECUTED, EVENT_JOB_ERROR from . import app import os from log import log def scheduler_listener(event): if event.code == EVENT_JOB_ERROR: print('The job crashed :(') log.warn("The schedule job crashed because of %s" % repr(event.exception)) else: print('The job executed :)') log.debug("The schedule job %s executed and return value is '%s'" % (event.job_id, event.retval)) if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': scheduler = BackgroundScheduler() # job store if safe_get_config("scheduler.job_store", "memory") == "mysql": scheduler.add_jobstore('sqlalchemy', url=get_config("scheduler.job_store_url")) # listener # do we need listen EVENT_JOB_MISSED? scheduler.add_listener(scheduler_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) scheduler.start()