class JobScheduler(object): def __init__(self, config): self.scheduler = BackgroundScheduler() self.scheduler.add_job( self.job_process, 'interval', seconds=config['JOB_POLL_INTERVAL_SECONDS'], max_instances=1) self.scheduler.add_job( self.notification_job_process, 'interval', seconds=config['DELIVERY_POLL_INTERVAL_SECONDS'], max_instances=1) self.config = config def start(self): self.scheduler.start() def shutdown(self): self.scheduler.shutdown(wait=True) def job_process(self): process_jobs(self.config) def notification_job_process(self): process_notification_job(self.config)
def go(managedNamespace): statusMgr = statusDbManager.StatusResource() managedNamespace.run = True managedNamespace.serverRun = True checkInitDbs() server_process = multiprocessing.Process(target=serverProcess, args=(managedNamespace,)) sched = BackgroundScheduler() scheduleJobs(sched, managedNamespace) server_process.start() sched.start() loopCtr = 0 while managedNamespace.run: time.sleep(0.1) if loopCtr % 100 == 0: for job in sched.get_jobs(): statusMgr.updateNextRunTime(job.name, job.next_run_time.timestamp()) loopCtr += 1 sched.shutdown() server_process.join()
class MyScheduler(object): """scheduler class, difine scheduler and jobs""" def __init__(self): super(MyScheduler, self).__init__() self.scheduler = BackgroundScheduler() self.wish_time = setting.WISH_DELAY self.book_time = setting.BOOK_LAST_TIME def start(self): self.scheduler.start() def shutdown(self): self.scheduler.shutdown() def add_check_wish_status_job(self, wish_id): # print datetime.now() run_date = datetime.now() + timedelta(seconds=self.wish_time) self.scheduler.add_job(_reset_wish_status, 'date', args=[wish_id], run_date=run_date) def add_set_book_removed_job(self, book_id): run_date = datetime.now() + timedelta(days=self.book_time) self.scheduler.add_job(_set_book_removed, 'date', args=[book_id], run_date=run_date) def add_xapian_reindex_job(self, xapian_tool): self.scheduler.add_job(xapian_tool.index, 'interval', minutes=15)
class TimerTrigger(BaseTrigger): name = "timer" log = logging.getLogger("zuul.Timer") def __init__(self, trigger_config={}, sched=None, connection=None): super(TimerTrigger, self).__init__(trigger_config, sched, connection) self.apsched = BackgroundScheduler() self.apsched.start() def _onTrigger(self, pipeline_name, timespec): for project in self.sched.layout.projects.values(): event = TriggerEvent() event.type = "timer" event.timespec = timespec event.forced_pipeline = pipeline_name event.project_name = project.name self.log.debug("Adding event %s" % event) self.sched.addEvent(event) def stop(self): self.apsched.shutdown() def getEventFilters(self, trigger_conf): def toList(item): if not item: return [] if isinstance(item, list): return item return [item] efilters = [] for trigger in toList(trigger_conf): f = EventFilter(trigger=self, types=["timer"], timespecs=toList(trigger["time"])) efilters.append(f) return efilters def postConfig(self): for job in self.apsched.get_jobs(): job.remove() for pipeline in self.sched.layout.pipelines.values(): for ef in pipeline.manager.event_filters: if ef.trigger != self: continue for timespec in ef.timespecs: parts = timespec.split() if len(parts) < 5 or len(parts) > 6: self.log.error( "Unable to parse time value '%s' " "defined in pipeline %s" % (timespec, pipeline.name) ) continue minute, hour, dom, month, dow = parts[:5] if len(parts) > 5: second = parts[5] else: second = None trigger = CronTrigger(day=dom, day_of_week=dow, hour=hour, minute=minute, second=second) self.apsched.add_job(self._onTrigger, trigger=trigger, args=(pipeline.name, timespec))
def post(self, action, position = ''): global scheduler self.checkStartup() if action == 'play': runCommand('mpc play ' + position) #Settings.set('radio', 'state', 'play') if scheduler is None: scheduler = BackgroundScheduler() scheduler.add_job(self.checkStatus, 'interval', seconds=30, id='checkStatus', replace_existing=True) scheduler.start() elif action == 'stop': runCommand('mpc stop') #Settings.set('radio', 'state', 'stop') if scheduler is not None: scheduler.remove_job('checkStatus') scheduler.shutdown() scheduler = None return {'playMode': 'stopped'} elif action =='pause': runCommand('mpc pause') elif action =='next': runCommand('mpc next') elif action =='previous': runCommand('mpc prev') else: return {'playMode': 'invalid'} (out, err) = runCommand('mpc status') if err: return {'error', err}, 500 return {'playMode': Parser.parsePlayMode(out)}
def main(args): scheduler = BackgroundScheduler(coalesce=True, misfire_grace_time=4) taskparser = TaskParser(args['f']) taskparser.parse() yml_handler = YmlFileEventHandler(patterns=["*.yml"]) yml_handler.set_scheduler(scheduler) yml_handler.set_parser(taskparser) file_observer = Observer() file_observer.schedule(yml_handler, path=args['f'], recursive=False) file_observer.start() # Initial parsing of the task folder for t in taskparser.task_list: addJob(t, scheduler) # Spread tasks from each other to prevent overload/task miss time.sleep(1) scheduler.start() # Update jobs while running while True: try: time.sleep(15) except KeyboardInterrupt: break scheduler.shutdown()
class RealTimeQuote(object): def __init__(self, cf, codelist, eventEngine_): self._codelist = codelist logger.info("codelist:%s", self._codelist) self._eventEngine = eventEngine_ #self._eventEngine.register(EVENT_TIMER, self.TimerCall) self._sched = BackgroundScheduler() def start(self): self._sched.add_job(self.TimerCall, 'interval', seconds=3) self._sched.start() logger.info('RealTimeQuote start') def stop(self): logger.info('RealTimeQuote stop') self._sched.shutdown() def TimerCall(self): ''' 定时根据代码列表获取最新行情 :return: ''' if len(self._codelist) < 1: return rtQuote = GetRealTimeQuote(self._codelist) for i in range(rtQuote.shape[0]): itQuote = rtQuote.ix[i] if float(itQuote['amount']) <= 0.01: continue event = Event(type_=EVENT_MARKETDATA_CONTRACT + itQuote['code']) event.dict_['tick'] = itQuote self._eventEngine.put(event)
def go(): preflight() sched = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults) # startTime = datetime.datetime.now()+datetime.timedelta(seconds=60*60) # startTime = datetime.datetime.now()+datetime.timedelta(seconds=60*15) # startTime = datetime.datetime.now()+datetime.timedelta(seconds=60*5) # startTime = datetime.datetime.now()+datetime.timedelta(seconds=20) startTime = datetime.datetime.now()+datetime.timedelta(seconds=10) scheduleJobs(sched, startTime) sched.start() # spinwait for ctrl+c, and exit when it's received. loops = 0 while runStatus.run: time.sleep(0.1) # loops += 1 # if loops > 100: # logging_tree.printout() # loops = 0 print("Scraper stopping scheduler") sched.shutdown() nt.dirNameProxy.stop()
def mail2diaspora(config_pathname): # configure logging logger = logging.getLogger(__name__) configure_logging(logging.INFO) logger.info("Start mail2diaspora application") config.initialize(config_pathname) os.chdir(config.get(config.TEMP)) # cron email fetcher scheduler = BackgroundScheduler() scheduler.add_job( diaspora.mail_poll, "interval", seconds=config.getInt(config.MAIL_POLLING) ) scheduler.start() print("Press Ctrl+{0} to exit".format("Break" if os.name == "nt" else "C")) try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(2) except (KeyboardInterrupt, SystemExit): # Not strictly necessary if daemonic mode is enabled but should be done if possible scheduler.shutdown() logger.info("Stop mail2diaspora application")
def build(cube_name_list, buildType, start_time=None, end_time=None): run_cube_job_id = '1' check_cube_job_id = '2' scheduler = BackgroundScheduler() CubeWorker.job_instance_dict = {} for cube_name in cube_name_list: CubeWorker.job_instance_dict[cube_name] = None CubeWorker.scheduler = scheduler CubeWorker.run_cube_job_id = run_cube_job_id CubeWorker.check_cube_job_id = check_cube_job_id # start the run cube job immediately CubeWorker.run_cube_job(buildType, start_time, end_time) scheduler.add_job(CubeWorker.run_cube_job, 'interval', seconds=30, id=run_cube_job_id, args=[buildType, start_time, end_time]) scheduler.add_job(CubeWorker.check_cube_job, 'interval', seconds=30, id=check_cube_job_id) scheduler.start() while True: if CubeWorker.all_finished(): print "all cube jobs are finished" scheduler.remove_job(check_cube_job_id) scheduler.remove_job(run_cube_job_id) scheduler.shutdown() status = CubeWorker.get_status() print 'Build exited with status %s' % status return status == CubeWorkerStatus.SUCCESS time.sleep(settings.KYLIN_CHECK_STATUS_INTERVAL)
def go(managedNamespace): log.info("Go()") resetter = xascraper.status_monitor.StatusResetter() resetter.resetRunState() # statusMgr = manage.statusDbManager.StatusResource() managedNamespace.run = True managedNamespace.serverRun = True server_process = multiprocessing.Process(target=serverProcess, args=(managedNamespace,)) if "debug" in sys.argv: log.info("Not starting scheduler due to debug mode!") sched = None else: sched = BackgroundScheduler({ 'apscheduler.jobstores.default': { 'type': 'memory' }, 'apscheduler.executors.default': { 'class': 'apscheduler.executors.pool:ThreadPoolExecutor', 'max_workers' : 5 }, 'apscheduler.job_defaults.coalesce' : True, 'apscheduler.job_defaults.max_instances' : 1, 'apscheduler.job_defaults.misfire_grace_time ' : 60 * 60 * 2, }) logging.getLogger('apscheduler').setLevel(logging.DEBUG) sched.add_listener(job_evt_listener, apscheduler.events.EVENT_JOB_EXECUTED | apscheduler.events.EVENT_JOB_ERROR | apscheduler.events.EVENT_JOB_MISSED | apscheduler.events.EVENT_JOB_MAX_INSTANCES ) scheduleJobs(sched, managedNamespace) sched.start() log.info("Scheduler is running!") log.info("Launching server process") server_process.start() loopCtr = 0 log.info("Entering idle loop.") while managedNamespace.run: time.sleep(0.1) # if loopCtr % 100 == 0: # for job in sched.get_jobs(): # print("Job: ", job.name, job.next_run_time.timestamp()) # # statusMgr.updateNextRunTime(job.name, job.next_run_time.timestamp()) loopCtr += 1 if sched: sched.shutdown() log.info("Joining on web thread.") server_process.join()
def schedule(): scheduler = BackgroundScheduler() scheduler.add_job(process, 'interval', seconds=120) scheduler.start() print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(2) except (KeyboardInterrupt, SystemExit): scheduler.shutdown() # Not strictly necessary if daemonic mode is enabled but should be done if possible
def addJobToScheduler(): timeArgv = int(sys.argv[3]) sc = BackgroundScheduler() sc.add_job(backUpFile, 'interval', seconds=timeArgv) sc.start() try: while True: time.sleep(2) except(KeyboardInterrupt, SystemExit): sc.shutdown()
def main(): logging.basicConfig() sys.setrecursionlimit(1500) scheduler = BackgroundScheduler() scheduler.add_job(url_scraper.scrapeurl, 'interval', hours=8) scheduler.start() while True: time.sleep(10) scheduler.shutdown()
def update_monitoring_cache(): """Update the monitoring cache.""" from apscheduler.schedulers.background import BackgroundScheduler scheduler = BackgroundScheduler() self.register_scheduler_jobs(current_app, run_once=True) for args, kwargs in current_app.jobs: scheduler.add_job(*args, **kwargs) scheduler.start() while scheduler.get_jobs() > 0: continue scheduler.shutdown(wait=True)
def scheduler_scan_all_article(): print "scheduler_scan_all_article is start!" #定时任务启动 scheduler = BackgroundScheduler() scheduler.add_job(scan_all_article, 'cron', hour='0-23', minute=50) scheduler.start() try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(2) except (KeyboardInterrupt, SystemExit): scheduler.shutdown()
class Scheduler: def __init__(self): self.jobs = {} self.scheduler = BackgroundScheduler({ 'apscheduler.job_defaults.coalesce': 'true', }) def addJob(self, policy, job_function, job_name): # Check that a job with the same name has not already be registered if self.jobs.has_key(job_name): raise Exception ('A job named "' + job_name + '" has already been scheduled') # Get the period trigger to use trigger = self._getTrigger(policy) # Add the job to the scheduler j = self.scheduler.add_job(job_function, name = job_name, max_instances = 1, trigger=trigger) getLogger(__name__).debug('Job "' + job_name +'" has been added to scheduler, it has the id ' + j.id + '. It is scheduled every ' + policy) # Store job if so that we can update it if needed self.jobs[job_name] = j.id def rescheduleJob(self, policy, job_name): # Check that the job with job_name well exist if not(self.jobs.has_key(job_name)): raise Exception ('Job named "' + job_name + '" can not be rescheduled because it is not registered in scheduler') # Get the period trigger to use trigger = self._getTrigger(policy) # Reschedule the job with the new trigger getLogger(__name__).debug('Reschedule job "' + job_name +'" having id ' + self.jobs.get(job_name)) self.scheduler.reschedule_job(self.jobs.get(job_name), trigger=trigger) def start(self): self.scheduler.start() getLogger(__name__).info('Start scheduler') def stop(self): self.scheduler.shutdown() getLogger(__name__).info('Stop scheduler') def _getTrigger(self, policy): comp = re.compile("^([0-9]*)([smhd])?$") match = comp.match(policy) policy = match.groups() if (policy[0] == '' or policy[0] == None): raise Exception ("The periodicity of your task is not well defined.") else: period = int(policy[0]) return IntervalTrigger(seconds=period)
def restart_schedule(): # flask 实例重启后,原来的schedule 进程已经不存在了,不需要shutdown了,若应用已经起来了, # Schedule 已经初始化后,需要shutdown 它 scheduler = BackgroundScheduler() try: scheduler.shutdown(wait=False) except Exception, e: print "Scheduler is not running"
class Schedule(object): def __init__(self): self.scheduler = BackgroundScheduler() def tick(self): pass def start(self): self.scheduler.add_job(self.tick, 'interval', seconds=3*60*60) self.scheduler.start() atexit.register(lambda: self.shutdown()) def shutdown(self): self.scheduler.shutdown(wait=False)
def run(self): """Run watcher""" self.logger.info("Running watcher ...") scheduler = BackgroundScheduler() scheduler.add_job(self.watching, "interval", seconds=self.config["interval"]) scheduler.start() try: # This is here to simulate application activity (which keeps the main thread alive). while self._running: time.sleep(2) scheduler.shutdown() except (KeyboardInterrupt, SystemExit): # Not strictly necessary if daemonic mode is enabled but should be done if possible scheduler.shutdown()
class JobMonitor(telepot.helper.Monitor): def __init__(self, seed_tuple, server, db): super(JobMonitor, self).__init__(seed_tuple, capture=[{'_': lambda msg: True}]) self.server = server self.db = db self.logger = logging.getLogger('torrentbot') self.sched = BackgroundScheduler() self.sched.start() self.sched.add_job(self.torrentMonitor, 'interval', minutes=3) self.logger.debug('jobmonitor logger init ...') def on_chat_message(self, msg): pass def on_callback_query(self, msg): pass def on_close(self, e): self.logger.debug('jobmonitor will shutdown') self.shutdown() def shutdown(self): self.sched.shutdown() def torrentMonitor(self): self.logger.debug('========== DB ==========') fromDB = self.db.uncompleted() self.logger.debug(fromDB) self.logger.debug('========== Server ==========') fromServer = self.server.completed() self.logger.debug(fromServer) # extract complete torrents self.logger.debug('========== updateList ==========') updateList = [] for dbt in fromDB: for st in fromServer: if dbt['id'] == st['id']: updateList.append({'id':dbt['id'], 'chat_id':dbt['chat_id'], 'title': st['title']}) self.logger.debug(updateList) for t in updateList: self.bot.sendMessage(t['chat_id'], 'downloaded\n' + t['title']) # self.db.completeTorrent(t['chat_id'], t['id']) self.server.delete(t['id']) self.db.deleteTorrent(t['chat_id'], t['id'])
def start_schedule(self): sched = BackgroundScheduler() job = sched.add_job(self.do_post, 'date', run_date=chun.next_sunrise) sched.add_listener(self.job_executed_listener, EVENT_JOB_EXECUTED) sched.start() try: while True: self.running = True while self.running: time.sleep(10) job.remove() job = sched.add_job(self.do_post, 'date', run_date=self.next_sunrise, misfire_grace_time=120) except (KeyboardInterrupt, SystemExit): sched.shutdown()
def begin(): init() run_robot() scheduler = BackgroundScheduler() scheduler.add_job(run_360, 'interval', seconds=1800) scheduler.add_job(run_wooyun, 'interval', seconds=300) scheduler.add_job(run_freebuf, 'interval', seconds=3600) scheduler.add_job(run_time_report, 'interval', seconds=43200) scheduler.start() try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(2) except (KeyboardInterrupt, SystemExit): scheduler.shutdown() # Not strictly necessary if daemonic mode is enabled but should be done if possible
def main(): scheduler = BackgroundScheduler(); scheduler.add_job(onOpen,'cron',hour=9, minute=29, misfire_grace_time=10, coalesce=True); scheduler.add_job(onClose,'cron',hour=14, minute=58, misfire_grace_time=10, coalesce=True); scheduler.start(); while True: time.sleep(1); pass; scheduler.shutdown(); pass;
def handle(self, *args, **options): source_id = options['source_id'] # start BackgroundScheduler and add an interval job that gets ran every 30 minutes scheduler = BackgroundScheduler() scheduler.start() scheduler.add_job(self.job, 'interval', id='scraper', minutes=30, kwargs={'source_id': source_id}) try: # this keeps the thread alive while True: sleep(1) except (KeyboardInterrupt, SystemExit): scheduler.shutdown()
def __init__(self, profile): self._logger = logging.getLogger(__name__) self.q = Queue.Queue() self.profile = profile self.notifiers = [] self._logger.debug('Initializing Notifier...') if 'gmail_address' in profile and 'gmail_password' in profile: self.notifiers.append(self.NotificationClient( self.handleEmailNotifications, None)) else: self._logger.warning('gmail_address or gmail_password not set ' + 'in profile, Gmail notifier will not be used') if 'ssh_auth_log' in profile: self.notifiers.append(self.NotificationClient( self.handleSSHAuthNotifications, None)) else: self._logger.warning('ssh_auth_log not set,' + 'SSH login notifier will not be used') job_defaults = { 'coalesce': True, 'max_instances': 1 } sched = BackgroundScheduler(timezone="UTC", job_defaults=job_defaults) sched.start() sched.add_job(self.gather, 'interval', seconds=30) atexit.register(lambda: sched.shutdown(wait=False)) # put the scheduler in Notifier object for reference self._sched = sched
def configure_scheduler_from_config(settings): scheduler = BackgroundScheduler() scheduler.start() # run `purge_account` job at 0:00 scheduler.add_job( purge_account, id='purge_account', name='Purge accounts which where not activated', trigger='cron', hour=0, minute=0 ) # run `purge_token` job at 0:30 scheduler.add_job( purge_token, id='purge_token', name='Purge expired tokens', trigger='cron', hour=0, minute=30 ) scheduler.add_listener(exception_listener, EVENT_JOB_ERROR) atexit.register(lambda: scheduler.shutdown())
def test4(): """定时执行任务,关闭调度器""" start_time = time.time() scheduler = BackgroundScheduler() scheduler.add_job(my_job, 'interval', args=('123',),seconds=1, id='my_job_id') # 每隔1秒执行一次my_job函数,args为函数my_job的输入参数;id:可省略; scheduler.start() # 程序运行到这里,任务没有运行完也会往后执行,既执行后面的任务,又执行这个任务。 print('运行到了这里1') #默认情况下调度器会等待所有正在运行的作业完成后,关闭所有的调度器和作业存储。如果你不想等待,可以将wait选项设置为False。 time.sleep(1) print('关闭所有的调度器和作业存储') #scheduler.shutdown() scheduler.shutdown(wait=False) print("输出所有格式化的作业列表:{}".format(scheduler.print_jobs())) # 输出所有格式化的作业列表。
def handle(self, **options): print "This is a command" print('Tick! The time is: %s' % datetime.now()) #if __name__ == '__main__': scheduler = BackgroundScheduler() scheduler.add_job(handle, 'interval', seconds=3) scheduler.start() print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(10) except (KeyboardInterrupt, SystemExit): scheduler.shutdown() # Not strictly necessary if daemonic mode is enabled but should be done if possible
class Greeter(pykka.ThreadingActor): '''Greeter''' def __init__(self, greeting='Hi there!'): super(Greeter, self).__init__() self.printer = Printer() self.greeting = greeting self.sched = BackgroundScheduler() self.job = self.sched.add_job(self.printer.display, 'interval', minutes=0.01) self.sched.start() def on_receive(self, message): if (message['msg']=='Stop!'): self.sched.shutdown() self.stop() else: self.printer.set_mesg(message['msg'])
s = Scheduler() def output(): for i in 'hello': sleep(0.4) print i s.add_job(output, 'interval', seconds=3) s.start() print 'started' sleep(10) print 'trying to shutdown' #s.shutdown(wait=False) #s.shutdown(wait=True) s.shutdown() print 'shutdown complete' #sleep(5) # 结论: # shutdown时,无论是否wait,都不会强中断job线程。 # 假设shutdown时,有正在执行的任务,那么: # wait为True,则shutdown等job全执行完再执行后面的。好比饭店要关门了,店长说:“要关门啦!大家吃完后就得走啦,我等你们吃完再关门,然后去接孩子。” # wait为False,则shutdown告诉job线程“一会自己关上,我先去干别的”,好比“大家吃完就走哈,我不等你们了,我得先去接孩子!” # wait默认为True
conn = conn_mysqldb() t = 0 def insertFunc(): global t cursor = conn.cursor() t+=1 sql = "Insert into map_info values(%d,'TEST_%d','%d','%d')"%(0,t,t*10,t*100) cursor.execute(sql) conn.commit() cursor.close() print("Commit Complete, TEST_%d %d %d"%(t,t*10,t*100)) sched.add_job(insertFunc, "cron",minutes = 15) try: while True: print("Running Main Process..") time.sleep(1) except: sched.shutdown() """ 이 스케쥴러는 실시간 전력 데이터를 받아와서 DB에 저장하는 스케쥴러 * 1시간마다 실시간 전력 데이터를 어딘가로부터 받아와 우리 DB 서버에 저장 다른 스케쥴러는 Tensorflow Serving 용으로 만들어야함 * Tensorflow Serving 에서 사용량을 보냄 --> 예측 --> 받아서 저장 (24시간) """
class Controller(object): """The Controller in a MVC model""" def __init__(self, db, bottle_ip='127.0.0.1', pigpio_host="", pigpio_port=8888): """- Create the controller, its Viewer and connect to database (Model) - Select all the hardware (sensors/switches) for the systems under its control - Launch the scheduler - host:port is used to connect to the pigpio server running on the Raspberry Pi. Need to execute 'sudo pigpiod' to get that daemon running if it is not automatically started at boot time :param db: instance of a Ponicwatch_Db :param bottle_ip: IP to reach the webpages. Important to set properly for remote access. """ global _simulation # if no PGIO port as we are not running on a Raspberry Pi self.debug = DEBUG self.bottle_ip = bottle_ip # keep a link to the database i.e. M in MVC self.db = db self.db.allow_close = False # finding the Controller User entry --> currently 'hard coded' as 'ctrl'/'passwd' --> to improve later self.user = User(self, id=1) self.name = self.user[ "name"] # this name is used to identify the log messages posted by this controller # opening the LOGger with the debug level for this application run self.log = Ponicwatch_Log(controller=self, debug=DEBUG) # Create the background scheduler that will execute the actions (using the APScheduler library) self.scheduler = BackgroundScheduler() # select all the systems, sensors, switchs to monitor and the necessary hardware drivers self.pig = pigpio.pi( pigpio_host, pigpio_port) if not _simulation else pigpio_simu.pi() if not self.pig.connected: if self.debug >= 2: print("WARNING: not connected to a RasPi") self.pig = pigpio_simu.pi() _simulation = True # some plural are "fake" to respect the logic of: <cls name> + 's' --> dictionary of <cls> PonicWatch Object self.systems, self.sensors, self.switchs, self.hardwares, self.interrupts = {}, {}, {}, {}, {} # system_id <= 0: inactive link --> ignore this row self.db.curs.execute( "SELECT * from tb_link where system_id > 0 order by system_id desc, order_for_creation" ) self.links = self.db.curs.fetchall() for system_id, sensor_id, switch_id, hardware_id, order_for_creation, interrupt_id in self.links: # (1) create all necessary objects # (2) and register the system and hardware to a sensor/switch if system_id not in self.systems: self.systems[system_id] = System(self, id=system_id) if hardware_id and hardware_id not in self.hardwares: self.hardwares[hardware_id] = Hardware( controller=self, id=hardware_id, system_name=self.systems[system_id]["name"]) if sensor_id and sensor_id not in self.sensors: self.sensors[sensor_id] = Sensor( controller=self, id=sensor_id, system_name=self.systems[system_id]["name"], hardware=self.hardwares[hardware_id]) if switch_id and switch_id not in self.switchs: self.switchs[switch_id] = Switch( controller=self, id=switch_id, system_name=self.systems[system_id]["name"], hardware=self.hardwares[hardware_id]) if interrupt_id and interrupt_id not in self.interrupts: self.interrupts[interrupt_id] = Interrupt( controller=self, id=interrupt_id, system_name=self.systems[system_id]["name"], hardware=self.hardwares[hardware_id]) self.db.allow_close = True self.db.close() def add_cron_job(self, callback, timer): """ Add a new scheduled task :param callback: :param timer: a string '* * * * * *' OR a JSON object as { 't': ['* * * * * *', t2, t3...]} :return: """ if timer[0] == '{': # convert the JSON timer string to a python dictionary try: cron_times = json.loads(timer) cron_times = cron_times['t'] except: # json.JSONDecodeError: print("Alarm: timer is not a JSON string!", timer) else: cron_times = [timer] i = 0.0 for cron_time in cron_times: # When do we need to read the sensor or activate a switch? # ┌───────────── sec (0 - 59) # | ┌───────────── min (0 - 59) # | │ ┌────────────── hour (0 - 23) # | │ │ ┌─────────────── day of month (1 - 31) # | │ │ │ ┌──────────────── month (1 - 12) # | │ │ │ │ ┌───────────────── day of week (0 - 6) (0 to 6 are Sunday to # | │ │ │ │ │ Saturday, or use names; 7 is also Sunday) # | │ │ │ │ │ # | │ │ │ │ │ # * * * * * * _sec, _min, _hrs, _dom, _mon, _dow = cron_time.split( ) # like "*/5 * * * * *" --> every 5 seconds self.scheduler.add_job(callback, 'cron', second=_sec, minute=_min, hour=_hrs, day=_dom, month=_mon, day_of_week=_dow) self.log.add_log(log_type='SCHEDULER', system_name='@startup', param={ 'error_code': 0, 'text_value': cron_time, 'float_value': i }) i += 1.0 def run(self): """Starts the APScheduler task and the Bottle HTTP server""" self.running = True with open("ponicwatch.pid", "wt") as fpid: print(os.getpid(), file=fpid) def stop_handler(signum, frame): """ allow: kill -10 `cat ponicwatch.pid` """ self.stop() sys.exit() signal.signal(signal.SIGUSR1, stop_handler) self.scheduler.start() self.log.add_info("Controller {} is now running.".format(__version__), fval=1.0) # http_view.controller = self try: http_view.run(host=self.bottle_ip) except (KeyboardInterrupt, SystemExit): pass finally: self.stop() def stop(self, from_bottle=False): try: self.scheduler.shutdown( ) # Not strictly necessary if daemonic mode is enabled but should be done if possible except SchedulerNotRunningError: pass for hw in self.hardwares.values(): hw.cleanup() self.log.add_info( "Controller {} has been stopped.".format(__version__), fval=0.0) if not from_bottle: bottle_stop() # if_expression manipulation to provide conditional execution to PWO based on their 'if' string def make_expression(self, submitted_by, if_expression): """Replace the Sensor/Switch/Hardware reference to its value Error will be caught by the calling function: SyntaxError, ValueError, NameError :param submitted_by: the pwo requesting the 'if_expression' evalution (needed for logging in case of error) :param if_expression: string or tuple from the 'init' dictionary under the key 'if' :return result: python evaluation of the input string with all pwo references have been replaced by their value """ if isinstance(if_expression, str): # expects a string starting by a pwo reference and then a boolean test # ex: "Sensor[2]>=40.0" or "Switch[1]==0" pwo_cls, _ = if_expression.split('[', 1) id, test = _.split(']', 1) pwo = self.get_pwo(pwo_cls, id) try: _expression = str( pwo.value ) + test # if the direct read is possible then use it except AttributeError: _expression = str( pwo["value"]) + test # else take the latest read value elif isinstance(if_expression, list): # expects list: format string followed by the pwo references or keyword 'now' # example: [ "{}>10. and {}==1", "Sensor[1]", "Switch[2]" ] _format, pwo_values = if_expression[0], [] for pwo_ref in if_expression[1:]: if pwo_ref.lower() == "now": pwo_values.append( datetime.now() ) # expects {:%H} for hour --> "8<={:%H}<=20".format(datetime.now()) else: pwo_cls, id = pwo_ref.split('[', 1) pwo = self.get_pwo(pwo_cls, id[:-1]) # drop the last ']' try: pwo_values.append(pwo.value) except AttributeError: pwo_values.append(pwo["value"]) _expression = _format.format(*pwo_values) else: msg = "Error! Unknown if_expression type: {} for {}".format( type(if_expression), submitted_by) self.log.add_error(msg=msg, err_code=submitted_by["id"], fval=-2.6) _expression = None return _expression def eval_expression(self, submitted_by, if_expression, make_expression=None): """evaluate the expression found in the init_dit['if'] of a PWO""" try: _expression = make_expression or self.make_expression( submitted_by, if_expression) result = eval(_expression) except (SyntaxError, NameError, ValueError) as err: self.log.add_error( msg="if_expression {} cannot be evaluated: {} for {}".format( if_expression, err, submitted_by), err_code=submitted_by["id"], fval=-2.7) result = None if self.debug >= 3: print("eval({}) == {}".format(_expression, result)) return result ### helper functions def get_pwo(self, cls, id): """return a PonicWatch Object from the controller's PWO dictionary :param cls: either a string as class name or an PW object :param id: the pwo's id, must be integer as pwo dictionary key :return pwo: the matching pwo or None """ pwo_dict_name = (cls if isinstance(cls, str) else cls.__class__.__name__).lower() + 's' pwo_dict = getattr(self, pwo_dict_name) pwo = None try: pwo = pwo_dict[int(id)] except KeyError: msg = "No pwo id {} found in {}".format(id, pwo_dict_name) if self.debug >= 3: print(msg) self.log.add_error(msg=msg, err_code=id, fval=-1.1) return pwo def print_list(self): """CLI: Print the list of all created objects in the __init__ phase""" print("--- System ---") for k, v in self.systems.items(): print(k, v) print("--- Hardware ---") for k, v in self.hardwares.items(): print(k, v, v["init"]) print("--- Switches ---") for k, v in self.switchs.items(): print(k, v, v["init"]) print("--- Sensors ---") for k, v in self.sensors.items(): print(k, v, v["init"]) print("--- Interruptions ---") for k, v in self.interrupts.items(): print(k, v) print("--- Links ---") for system_id, sensor_id, switch_id, hardware_id, order_for_creation, interrupt_id in self.links: print("system_id", system_id or '-', "hardware_id", hardware_id or '-', "switch_id", switch_id or '-', "sensor_id", sensor_id or '-', "interrupt_id", interrupt_id or '-', sep='\t') def ponicwatch_notification(self): """ Regular email to inform the system manager of its status :return: """ images = [] # list of .png to attach to the email html = http_default() # email's body objects = [] # working list of all Ponicwatch objects (pwo) for s in self.systems.values(): objects.append(s) for s in self.switchs.values(): objects.append(s) for s in self.sensors.values(): objects.append(s) for pwo in objects: html += one_pw_object_html(pwo, only_html=True) if os.path.isfile(get_image_file(pwo)): images.append(get_image_file(pwo)) # print(html) send_email("Ponicwatch Notification - %s" % ",".join([s["name"] for s in self.systems.values()]), from_=self.user["email"], to_=[ "*****@*****.**", ], message_HTML=html, images=images, login=self.user["email"], passwd=self.user["password"])
class AutoExportingCounters(object): """ A wrapper around collections.Counter that adds periodic backup. NOTE: Not to be confused with remote_datatypes. Counter which wraps a live redis counter. This counter is save only, and only offers weak durability. This is appropriate for monitoring and performance measurements, not for operational counters that require strict semantics. At the specified interval and program exit, the value in the counters will be sent to the provided channel. """ def __init__(self, name, host=None, export_interval_secs=None, counter_type=None, config=None, redis=None, counter_names=None, timer_names=None): config = config or forge.get_config() self.channel = forge.get_metrics_sink(redis) self.export_interval = export_interval_secs or config.core.metrics.export_interval self.name = name self.host = host or get_random_id() self.type = counter_type or name self.counter_schema = set(counter_names) self.timer_schema = set(timer_names) self.counts = None self.lock = threading.Lock() self.scheduler = None self.reset() assert self.channel assert(self.export_interval > 0) # noinspection PyUnresolvedReferences def start(self): from apscheduler.schedulers.background import BackgroundScheduler import atexit self.scheduler = BackgroundScheduler(daemon=True) self.scheduler.add_job(self.export, 'interval', seconds=self.export_interval) self.scheduler.start() atexit.register(lambda: self.stop()) def reset(self): with self.lock: old, self.counts = self.counts, Counters({key: 0 for key in self.counter_schema}) self.counts.update({key + '.t': 0 for key in self.timer_schema}) self.counts.update({key + '.c': 0 for key in self.timer_schema}) self.counts['type'] = self.type self.counts['name'] = self.name self.counts['host'] = self.host return old def stop(self): if self.scheduler: self.scheduler.shutdown(wait=False) self.scheduler = None self.export() def export(self): try: # To avoid blocking increments on the redis operation # we only hold the long to do a copy. thread_copy = dict(self.reset().items()) self.channel.publish(thread_copy) log.debug(f"{pprint.pformat(thread_copy)}") return thread_copy except Exception: log.exception("Exporting counters") def increment(self, name, increment_by=1): try: if name not in self.counter_schema: raise ValueError(f"{name} is not an accepted counter for this module: f{self.counter_schema}") with self.lock: self.counts[name] += increment_by return increment_by except Exception: # Don't let increment fail anything. log.exception("Incrementing counter") return 0 def increment_execution_time(self, name, execution_time): try: if name not in self.timer_schema: raise ValueError(f"{name} is not an accepted counter for this module: f{self.timer_schema}") with self.lock: self.counts[name + ".c"] += 1 self.counts[name + ".t"] += execution_time return execution_time except Exception: # Don't let increment fail anything. log.exception("Incrementing counter") return 0
class HttpServer(flask.Flask): """Our HTTP/API server.""" EXECUTORS = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } def __init__(self, name, ip, port, *args, **kwargs): """Constructor. Args: name: (str) name of Flask service ip: (str) IP address to bind HTTP server port: (int) TCP port for HTTP server to listen """ super(HttpServer, self).__init__(name, *args, **kwargs) # Fixup the root path for Flask so it can find templates/* root_path = os.path.abspath(os.path.dirname(__file__)) logging.debug('Setting root_path for Flask: %s', root_path) self.root_path = root_path self.targets = config.CollectorConfig() self.ip = ip self.port = port self.start_time = time.time() self.setup_time = 0 self.scheduler = BackgroundScheduler(daemon=True, executors=self.EXECUTORS) self.collection = None self.add_url_rule('/', 'index', self.index_handler) self.add_url_rule('/status', 'status', self.status_handler) self.add_url_rule('/latency', 'latency', self.latency_handler) self.add_url_rule('/influxdata', 'influxdata', self.influxdata_handler) self.add_url_rule('/quitquit', 'quitquit', self.shutdown_handler) logging.info('Starting Llama Collector, version %s', __version__) def configure(self, filepath): """Configure the Collector from file. Args: filepath: (str) where the configuration is located """ self.targets.load(filepath) def status_handler(self): return flask.Response('ok', mimetype='text/plain') def index_handler(self): return flask.render_template( 'index.html', targets=self.targets.targets, interval=self.interval, start_time=self.start_time, setup_time=self.setup_time, uptime=humanfriendly.format_timespan(time.time() - self.start_time)) def latency_handler(self): data = json.dumps(self.collection.stats, indent=4) return flask.Response(data, mimetype='application/json') def influxdata_handler(self): data = json.dumps(self.collection.stats_influx, indent=4) return flask.Response(data, mimetype='application/json') def shutdown_handler(self): """Shuts down the running web server and other things.""" logging.warn('/quitquit request, attempting to shutdown server...') self.scheduler.shutdown(wait=False) fn = flask.request.environ.get('werkzeug.server.shutdown') if not fn: raise Error('Werkzeug (Flask) server NOT running.') fn() return '<pre>Quitting...</pre>' def run(self, interval, count, use_udp=False, dst_port=util.DEFAULT_DST_PORT, timeout=util.DEFAULT_TIMEOUT, *args, **kwargs): """Start all the polling and run the HttpServer. Args: interval: seconds between each poll count: count of datagram to send each responder per interval use_udp: utilize UDP probes for testing dst_port: port to use for testing (only UDP) timeout: how long to wait for probes to return """ self.interval = interval self.scheduler.start() self.collection = Collection(self.targets, use_udp) self.scheduler.add_job(self.collection.collect, 'interval', seconds=interval, args=[count, dst_port, timeout]) super(HttpServer, self).run(host=self.ip, port=self.port, threaded=True, *args, **kwargs) self.setup_time = round(time.time() - self.start_time, 0)
class JobManager: """ Central overseer that manages the measurement jobs. """ _instance: Optional['JobManager'] = None @classmethod def initialize( cls, datadir: Path ) -> 'JobManager': if cls._instance is None: cls._instance = JobManager( datadir ) return cls._instance @classmethod def get_instance( cls ) -> 'JobManager': if cls._instance is None: raise RuntimeError( "Attempted to obtain manager before initializing it." ) return cls._instance @classmethod def run_job( cls, job: Job ) -> None: """ Executes the specified job once. :param job: The job to execute. """ _LOGGER.debug( "Running job '%s'.", job.id ) timestamp = datetime.now( pytz.utc ) try: output = speedtest.run_test( server_id = job.server_id, server_name = job.server_name ) result = { 'success': True, 'time': timestamp.isoformat(), 'result': output } except speedtest.TestError as e: _LOGGER.exception( "Test could not be completed." ) result = { 'success': False, 'timestamp': timestamp.isoformat(), 'error': str( e ), 'stdout': e.stdout, 'stderr': e.stderr, } with open( cls.get_instance().output_file( job ), 'a' ) as f: f.write( json.dumps( result ) ) f.write( ',\n' ) # Line break to make it slightly more readable _LOGGER.debug( "Finished running job '%s'.", job.id ) def __init__( self, datadir: Path ): """ Initializes a new manager that uses the specified directory to store data. :param datadir: The path of the directory where data should be stored. """ _LOGGER.debug( "Initializing manager." ) try: _LOGGER.info( "Using %s", speedtest.get_version() ) # Also implicitly check installed except speedtest.TestError: _LOGGER.exception( "Obtaining Speedtest CLI version caused an error." ) _LOGGER.critical( "The Speedtest CLI could not accessed. Is it installed in this system?" ) sys.exit( 1 ) database_path = datadir / 'jobs.db' self.storage = datadir / 'results' self.storage.mkdir( mode = 0o770, exist_ok = True ) self.engine = create_engine( f'sqlite:///{database_path}' ) Base.metadata.create_all( self.engine ) self.Session = orm.sessionmaker( bind = self.engine ) jobstores = { 'default': SQLAlchemyJobStore( engine = self.engine ) } executors = { 'default': ThreadPoolExecutor( 1 ) } job_defaults = { 'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 5 * 60, # Can be up to 5 minutes late } self.scheduler = BackgroundScheduler( jobstores = jobstores, executors = executors, job_defaults = job_defaults, timezone = pytz.utc ) self.scheduler.add_listener( self.job_stopped, mask = events.EVENT_JOB_REMOVED ) _LOGGER.debug( "Manager initialized." ) def start( self ): """ Starts processing jobs. """ _LOGGER.info( "Manager starting." ) self.scheduler.start() _LOGGER.debug( "Manager started." ) def shutdown( self, wait: bool = True ): """ Shuts down the manager, stopping job processing. :param wait: If True, waits for all currently executing jobs to finish before returning. """ _LOGGER.info( "Manager stopping." ) self.scheduler.shutdown( wait = wait ) _LOGGER.debug( "Manager stopped." ) def job_stopped( self, event: events.JobEvent ) -> None: id: str = event.job_id with self.transaction() as session: job: JobMetadata = session.query( JobMetadata ).filter_by( id = id ).first() job.running = False def output_file( self, job: Job ) -> Path: """ Determines the path to the output file of the job identified by the given ID. :param job: The job to get the path for. :return: The path of the output file for the given job. """ return self.storage / f'{job.id}.result' # Not really proper JSON def load_results( self, job: Job ) -> Sequence[JSONData]: """ Loads the results obtained so far for the given job. :param job: The job to load results for. :return: The results of the given job, as a list of JSON objects. """ output_file = self.output_file( job ) if not output_file.exists(): return [] with open( output_file, 'r' ) as f: results = f.read() results = '[' + results[:-2] + ']' # Remove trailing comma and line break and add brackets return json.loads( results ) @contextmanager def transaction( self ) -> orm.Session: """ Provide a transactional scope around a series of operations. """ session: orm.Session = self.Session() try: yield session session.commit() except: session.rollback() raise finally: session.close() def new_job( self, job: Job ) -> None: """ Registers the given job. :param job: The job to register. :raises IDExistsError: if the ID of the given job is already in use. """ _LOGGER.info( "Registering job '%s'.", job.id ) _LOGGER.debug( "Job '%s' (%s) has target %d|'%s', starts at %s and ends at %s with interval %s.", job.id, job.title, job.server_id, job.server_name, job.start, job.end, job.interval ) with self.transaction() as session: try: if session.query( JobMetadata ).filter_by( id = job.id ).count() > 0: raise IDExistsError( "There is already metadata for the given ID." ) new_job = JobMetadata( job ) session.add( new_job ) if job.interval: _LOGGER.debug( "Creating an interval-triggered job." ) if job.end is not None and job.end < ( now := datetime.now( pytz.utc ) ): raise PastEndError( now, job ) trigger = IntervalTrigger( seconds = int( job.interval.total_seconds() ), start_date = job.start if job.start is not None else datetime.now( pytz.utc ), end_date = job.end ) else: _LOGGER.debug( "Creating a date-triggered job." ) trigger = DateTrigger( run_date = job.start )
def run(self, output_fn, **kwargs): '''处理数据库中的任务队列''' # 引入MySQL配置 from Functions import AppServer db_name = AppServer().getConfValue('Databases', 'MysqlDB') db_user = AppServer().getConfValue('Databases', 'MysqlUser') db_pass = AppServer().getConfValue('Databases', 'MysqlPass') db_ip = AppServer().getConfValue('Databases', 'MysqlHost') db_port = AppServer().getConfValue('Databases', 'MysqlPort') dbconn = 'mysql://%s:%s@%s:%s/%s' % (db_user, db_pass, db_ip, int(db_port), db_name) from MySQL import writeDb # 尝试清空DB数据库中记录的JOB信息 try: sql = """delete from apscheduler_jobs ;""" writeDb(sql, ) except: True # 动态引入任务函数 moduleSrc = 'TaskFunctions' dylib = importlib.import_module(moduleSrc) # 重新加载job队列[两种类型调度器按情况选择] from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor job_defaults = {'max_instances': 1} executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } scheduler = BackgroundScheduler(timezone='Asia/Shanghai', executors=executors, job_defaults=job_defaults) # sqlite or mysql scheduler.add_jobstore('sqlalchemy', url='%s' % dbconn) from MySQL import readDb sql = """ Select id,timedesc from taskconf where status='1' """ result = readDb(sql, ) for taskobject in result: Taskid = 'TaskID_%s' % taskobject.get('id') FunName = 'TaskFunc_%s' % taskobject.get('id') function = getattr(dylib, FunName) cronlist = taskobject.get('timedesc').strip().split(' ') print cronlist if len(cronlist) == 5: scheduler.add_job(func=function, trigger='cron', month=cronlist[4], day=cronlist[3], hour=cronlist[2], minute=cronlist[1], second=cronlist[0], id=Taskid) elif len(cronlist) == 6: scheduler.add_job(func=function, trigger='cron', day_of_week=cronlist[5], month=cronlist[4], day=cronlist[3], hour=cronlist[2], minute=cronlist[1], second=cronlist[0], id=Taskid) else: continue scheduler.start() fd = open(output_fn, 'a') try: dtnow = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') line = '\n[%s]: System Starting all Tasks...\n' % dtnow fd.write(line) fd.flush() while 1: pass except KeyboardInterrupt: #捕获键盘ctrl+c,在此脚本中不生效,console下可用 dtnow = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') line = '\n[%s]: System Stoping all Tasks, please wait...\n' % dtnow fd.write(line) fd.close() scheduler.shutdown() time.sleep(1) os._exit(0)
class TimerPlus(object): def __init__(self): self._sche = BackgroundScheduler() def add_jobs(self, job_container_, **kw): """ define job store 1. csv: filepath 2. pgsql: db_cfg, table """ self._job_container = job_container_.lower() container_ = [k for (k, v) in JobContainer.items()] assert self._job_container in container_, f'job_container_ should be in {container_}' for arg in JobContainer[self._job_container]: assert arg in kw, f'{arg} should be included.' if self._job_container == 'csv': assert os.path.exists( kw['filepath']), f"{kw['filepath']} is not existed." config_dict_ = read_yaml( config_name='cfg/csv.yaml', package='jt.timerplus') jobs = Utils.read_csv(kw['filepath'], config_dict_.get( 'import_data'), **config_dict_.get('config')) elif self._job_container == 'pgsql': _con = PgSQLLoader(kw['db_cfg']) _dbtabale = kw['table'] jobs = _con.read(f'''select job_id,job_name,job_type,job_args, trigger,trigger_args,is_trade_date,exchange from {_dbtabale}''') # loop adding job for row in jobs.itertuples(): assert getattr( row, 'job_type') in JOBTYPE, f'job type should be in {JOBTYPE}!' if isinstance(getattr(row, 'job_args'), str): t_job_args = json.loads(getattr(row, 'job_args')) else: t_job_args = None self._sche.add_job(func=self._distributor, args=[getattr(row, 'job_type'), getattr(row, 'job_name'), t_job_args, getattr(row, 'is_trade_date'), getattr(row, 'exchange')], trigger=getattr(row, 'trigger'), id=getattr(row, 'job_id'), **json.loads(getattr(row, 'trigger_args'))) self._sche.add_listener( self.__my_listener, EVENT_JOB_ERROR | EVENT_JOB_MISSED) def print_jobs(self): self._sche.print_jobs() def start(self): self._sche.start() def shutdown(self): self._sche.shutdown() def _distributor(self, job_type_=None, job_name_=None, job_args_=None, is_trade_date_='n', exchange_='sz'): today_ = datetime2string(datetime.datetime.now(), r'%Y%m%d') if is_trade_date_.lower() == 'y': if not calendar.is_trading_date(today_, exchange_): return if job_type_ == 'matlab': self.__matlab_executor(job_name_, job_args_) elif job_type_ == 'python': self.__python_executor(job_name_, job_args_) elif job_type_ == 'sqlprocedure': self.__sqlprocedure_executor(job_name_, job_args_) elif job_type_ == 'cmd': pass def __matlab_executor(self, job_name_, job_args_): """ execute the user defined matlab function/scripts """ eng = matlab.engine.start_matlab() func = getattr(eng, job_name_) assert func is not None, f'job_name : {job_name_} is not found!' if job_args_ is None: func(nargout=0) else: func(*job_args_, nargout=0) eng.quit() def __python_executor(self, job_name_, job_args_): assert isinstance(job_name_, str), 'job_name_ should be a String!' module_name = job_name_[0:job_name_.rfind('.')] spec = importlib.util.find_spec(module_name) assert spec is not None, f'job_name : {job_name_} is not found!' module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) func_name = job_name_[job_name_.rfind('.')+1:len(job_name_)] func = getattr(module, func_name) assert func is not None, f'job_name : {job_name_} is not found!' assert hasattr(func, '__call__'), f'{job_name_} is not callable!' if job_args_ is None: func() else: func(**job_args_) def __cmd_executor(self, *args, **kw): pass def __sqlprocedure_executor(self, job_name_, job_args_): assert 'db_env' in job_args_, 'db_env should be in job_args' pgl = PgSQLLoader(job_args_.get('db_env')) pgl.call_procedure(job_name_, job_args_.get( 'args', []), job_args_.get('proc_has_return', 'n')) def __my_listener(self, event): """ add listener for job error and missed """ if event.exception: logger.exception('%s error.', str(event.job)) else: logger.info('%s miss.', str(event.job))
def create_app(): """Create and configure an instance of the Flask application.""" app = Flask(__name__, instance_relative_config=True) app.config.from_object(config) app.config.from_pyfile('config.py') # set logging level logging.basicConfig(level=app.config['LOGGING_LEVEL']) # ensure the instance folder exists try: os.makedirs(app.instance_path) except OSError: pass # register the database commands from providentia.db import this as db db.init_app(app) # apply the blueprints to Providentia logging.debug('Applying blueprints to routes.') from providentia.views import new_job, dataset, database, benchmark, analysis, classifier, logs, queries, \ kate, review_trends, city_sentiment, sim1, sim2, sim3 app.register_blueprint(benchmark.bp, url_prefix='/api/benchmark') app.register_blueprint(dataset.bp, url_prefix='/api/dataset') app.register_blueprint(database.bp, url_prefix='/api/database') app.register_blueprint(analysis.bp, url_prefix='/api/analysis') app.register_blueprint(new_job.bp, url_prefix='/api/new-job') app.register_blueprint(classifier.bp, url_prefix='/api/classifier') app.register_blueprint(logs.bp, url_prefix='/api/logs') app.register_blueprint(queries.bp, url_prefix="/api/queries") app.register_blueprint(kate.bp, url_prefix="/api/result/kate") app.register_blueprint(review_trends.bp, url_prefix="/api/result/review-trends") app.register_blueprint(city_sentiment.bp, url_prefix="/api/result/city-sentiment") app.register_blueprint(sim1.bp, url_prefix="/api/result/sim1") app.register_blueprint(sim2.bp, url_prefix="/api/result/sim2") app.register_blueprint(sim3.bp, url_prefix="/api/result/sim3") # establish analysis database for this app logging.debug('Establishing database connections.') from providentia.db import this app.teardown_appcontext(this.close_db) # restart any incomplete jobs from providentia.repository.tbl_benchmark import reset_processing_jobs with app.app_context(): reset_processing_jobs() # test connections to benchmark databases test_database_connections(app) # register CORS logging.debug('Registering CORS filter.') CORS(app, resources={r"/api/*": {"origins": app.config['CORS_ORIGINS']}}) # enable scheduler logging.debug('Starting background jobs.') from apscheduler.schedulers.background import BackgroundScheduler from providentia.analysis.periodic_jobs import log_server_state, execute_waiting from providentia.classifier import sentiment, fake from datetime import datetime, timedelta classifier_start_train = datetime.now() + timedelta(0, 10) scheduler = BackgroundScheduler() scheduler.add_job(func=execute_waiting, id='execute_waiting', trigger='interval', seconds=10) scheduler.add_job(func=log_server_state, id='log_server_state', trigger='interval', seconds=1) # train the classifier model if it is enabled if app.config['ENABLE_SENTIMENT'] is True: logging.debug( '[SENTIMENT] Checking necessary NLTK resources are installed') check_nltk_deps() # Check if model exists else train one if os.path.exists( "./models/naivebayes.pickle") is True and os.path.exists( "./models/features.pickle") is True: try: sentiment.deserialize_model() logging.info("Sentiment classifier ready!") except OSError as e: logging.error( "Unable to deserialize Naive Bayes model! Creating a new one.", e) scheduler.add_job(func=sentiment.train_model, id='train_sentiment', trigger='date', next_run_time=classifier_start_train, args=[app.config['SENTIMENT_DATA'], app]) else: scheduler.add_job(func=sentiment.train_model, id='train_sentiment', trigger='date', next_run_time=classifier_start_train, args=[app.config['SENTIMENT_DATA'], app]) filter_apscheduler_logs() scheduler.start() # shut down the scheduler when exiting the app atexit.register(lambda: scheduler.shutdown()) return app
patch_cursor = cve_collection.find_one( {'id': patch_id}) if ap.handle_Patch_Update( patch_cursor, package_cursor['package_name']): print("Successfully applied patch") else: print("Admin patch requested") except: pass gv.collect_Checkable_Packages() if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': scheduler = BackgroundScheduler(daemon=True) scheduler.start() scheduler.add_job(func=call_database_updater, trigger=IntervalTrigger(hours=24), id='refreshing_database', name='Database_Refresh', replace_existing=True) atexit.register(lambda: scheduler.shutdown()) if __name__ == "__main__": log = logging.getLogger('werkzeug') log.setLevel(logging.ERROR) print("\t\t\t**** Use CTRL+C then Enter key to exit ****") print("\t\t\t\tFlask logging mode off") print("\t\t\t**** Access to UI at 127.0.0.1:5000 ****\n") app.run(debug=True)
class App: # Static variables folder_location = "" # Future static var for location of database for multiple apps logging.basicConfig(filename='app.log', level=logging.INFO) def __init__(self, master): # Initialize Auxillaries pygame.mixer.init() # Reset/Initialize Counts for Button Text and instant reports self.ad_num = 0 self.civ_num = 0 self.ret_num = 0 self.active_duty_numbers = 0 self.civilian_numbers = 0 self.retiree_numbers = 0 # Establish connection to database try: self.db = sqlite3.connect("tracker.db") except sqlite3.OperationalError as oe: logging.error(f"Error trying to connect to DB: {oe}") self.cur = self.db.cursor() # Set Fonts default_font = tkinter.font.Font(family='Helvetica', size=48, weight='bold') # Buttons self.btn_adres = Button(master, text="0", command=lambda: self.onclick_ad(), font=default_font, bg='#003B74', fg='white', relief=FLAT, bd=0, highlightthickness=0, highlightcolor="#003B74", cursor="none", activeforeground='white', activebackground='#003B74', height=1, width=3) self.btn_civ = Button(master, text="0", command=lambda: self.onclick_civ(), font=default_font, bg='#328400', fg='white', relief=FLAT, bd=0, highlightthickness=0, highlightcolor="#328400", cursor="none", activeforeground='white', activebackground='#328400', height=1, width=3) self.btn_ret = Button(master, text="0", command=lambda: self.onclick_ret(), font=default_font, bg='#A00004', fg='white', relief=FLAT, bd=0, highlightthickness=0, highlightcolor="#A00004", cursor="none", activeforeground='white', activebackground='#A00004', height=1, width=3) # Background Image self.filename = PhotoImage(file="Assets/gym9.png") self.background_label = Label(master, image=self.filename) # Menu Bar self.menubar = Menu(master) self.filemenu = Menu(self.menubar, relief=FLAT, font=default_font, tearoff=0) self.filemenu.add_command(label="Show Report", command=self.show_chart) self.filemenu.add_command(label="E-Mail", command=self.email) self.filemenu.add_command(label="Quit", command=master.destroy) self.menubar.add_cascade(label='Admin', menu=self.filemenu) # Place Widgets self.background_label.place(x=0, y=0) self.background_label.lower(self.btn_adres) master.config(menu=self.menubar) self.btn_adres.place(x=100, y=925) self.btn_civ.place(x=747, y=925) self.btn_ret.place(x=1370, y=925) # Start reset everyday at midnight self.sched = BackgroundScheduler() self.sched.add_job(self.reset, trigger='cron', hour=0, minute=0) self.sched.start() def onclick_ad(self): # Save to database self.save_db("Active Duty") # Increment ad numbers and update display self.ad_num += 1 self.btn_adres.configure(text=self.ad_num) # Load and play sound pygame.mixer.music.load("Assets/Whistle-noise.mp3") pygame.mixer.music.play(0) def onclick_civ(self): # Save to database self.save_db("Civilian") # Increment civilian numbers and update display self.civ_num += 1 self.btn_civ.configure(text=self.civ_num) # Load and play sound pygame.mixer.music.load("Assets/Retro.mp3") pygame.mixer.music.play(0) def onclick_ret(self): # Save to database self.save_db("Retired") # Increment retiree numbers and update display self.ret_num += 1 self.btn_ret.configure(text=self.ret_num) # Load and play sound pygame.mixer.music.load("Assets/Wrong-number.mp3") pygame.mixer.music.play(0) def email(self): fromaddr = '' toaddr = [''] SUBJECT = "Totals for " TEXT = f'{self.get_report(1)}\n\n{self.get_report(7)}\n\n{self.get_report(14)}' msg = 'Subject: {}\n\n{}'.format(SUBJECT, TEXT) username = '******' password = '' server = smtplib.SMTP('smtp.gmail.com:587') try: server.starttls() server.login(username, password) server.sendmail(fromaddr, toaddr, msg) showinfo("SUCCESS!", "E-mail Sent") except Exception as e: logging.error(f"Server connection error: {e}") showinfo("FAILED!", "Error Sending Email. See app.log for further details.") finally: server.quit() def save_db(self, category): # SQL Statements sql_create_table = 'create table if not exists count(dt_date DATE, dt_time DATE, category TEXT)' sql_insert = f'INSERT INTO count VALUES (DATE(\'now\', \'localtime\'), TIME(\'now\', \'localtime\'), "{category}")' # Execute the above statements try: self.cur.execute(sql_create_table) self.cur.execute(sql_insert) self.db.commit() except Exception as e: logging.error(f"Error trying to execute SQL: {e}") def get_report(self, days): # SQL Statements sql_ad = f'SELECT COUNT(*) FROM count WHERE dt_date BETWEEN DATE(\'now\', \'localtime\', \'-{days} day\'' \ f') AND DATE(\'now\', \'localtime\') AND category is \'Active Duty\'' sql_civ = f'SELECT COUNT(*) FROM count WHERE dt_date BETWEEN DATE(\'now\', \'localtime\', \'-{days} day\'' \ f') AND DATE(\'now\', \'localtime\') AND category is \'Active Duty\'' sql_ret = f'SELECT COUNT(*) FROM count WHERE dt_date BETWEEN DATE(\'now\', \'localtime\', \'-{days} day\'' \ f') AND DATE(\'now\', \'localtime\') AND category is \'Active Duty\'' # Set numbers based on days for reuse self.active_duty_numbers = self.cur.execute(sql_ad).fetchone()[0] self.civilian_numbers = self.cur.execute(sql_civ).fetchone()[0] self.retiree_numbers = self.cur.execute(sql_ret).fetchone()[0] return f'Last {days} days:\n' \ f'Active Duty: {self.active_duty_numbers}\nCivilian: {self.civilian_numbers}\nRetirees: ' \ f'{self.retiree_numbers}\nTotal: {self.active_duty_numbers + self.civilian_numbers + self.retiree_numbers}' def show_chart(self): ad_data = pd.read_sql_query( 'SELECT dt_date as Date, category as Category, count(*) as Count FROM count WHERE category IS ' '\'Active Duty\' GROUP BY dt_date, category ', self.db) civ_data = pd.read_sql_query( 'SELECT dt_date as Date, category as Category, count(*) as Count FROM count WHERE category IS ' '\'Civilian\' GROUP BY dt_date, category ', self.db) ret_data = pd.read_sql_query( 'SELECT dt_date as Date, category as Category, count(*) as Count FROM count WHERE category IS ' '\'Retired\' GROUP BY dt_date, category ', self.db) plt.plot(ad_data.Date, ad_data.Count, '-', label='Active Duty') plt.plot(civ_data.Date, civ_data.Count, '-', label='Civilian') plt.plot(ret_data.Date, ret_data.Count, '-', label='Retirees') plt.xlabel('Date') plt.ylabel('Number') plt.gcf().canvas.set_window_title('Gym Tracker (BETA)') plt.title('Patron Analysis') plt.suptitle("Gym Tracker") plt.legend() plt.show() def set_db(self): """ Future implementation: Allow user to set folder location for database :return: """ folder_location = askdirectory() def __del__(self): """ Destructor to close database connection :return: None """ self.db.close() self.sched.shutdown() def reset(self): self.ad_num = 0 self.civ_num = 0 self.ret_num = 0 self.btn_adres.configure(text=self.ad_num) self.btn_civ.configure(text=self.civ_num) self.btn_ret.configure(text=self.ret_num)
from server.k2000_prime import k2000 import atexit from flask import render_template, request from apscheduler.schedulers.background import BackgroundScheduler from server import app, socketio s = BackgroundScheduler( deamon=True, job_defaults={'apscheduler.job_defaults.max_instances': '5'}) s.start() # Shutdown your cron thread if the web process is stopped atexit.register(lambda: s.shutdown(wait=False)) inst = k2000() @app.route('/') def hello_world(): return render_template('index.html') @app.route('/inst/pause', methods=['GET']) def pause(): if s.get_job('send_read') is not None: # k.write('disp:enab 1') s.remove_job('send_read') return '' @app.route('/inst/start', methods=['GET']) def start():
user[key] = datetime.strptime(user[key], "%Y-%m-%dT%H:%M:%S") # Save current state of account db.witness.update({'_id': user['owner']}, user, upsert=True) # Create our Snapshot dict snapshot = user.copy() _id = user['owner'] + '|' + now.strftime('%Y%m%d') snapshot.update({ '_id': _id, 'created': scantime }) # Save Snapshot in Database db.witness_history.update({'_id': _id}, snapshot, upsert=True) def run(): update_witnesses() check_misses() if __name__ == '__main__': # Start job immediately run() # Schedule it to run every 1 minute scheduler = BackgroundScheduler() scheduler.add_job(run, 'interval', seconds=30, id='run') scheduler.start() # Loop try: while True: time.sleep(2) except (KeyboardInterrupt, SystemExit): scheduler.shutdown()
def main(): """Declare command line options""" parser = ArgumentParser( description='ouroboros', formatter_class=RawTextHelpFormatter, epilog= 'EXAMPLE: ouroboros -d tcp://1.2.3.4:5678 -i 20 -m container1 container2 -l warn' ) core_group = parser.add_argument_group( "Core", "Configuration of core functionality") core_group.add_argument('-v', '--version', action='version', version=VERSION) core_group.add_argument( '-d', '--docker-sockets', nargs='+', default=Config.docker_sockets, dest='DOCKER_SOCKETS', help='Sockets for docker management\n' 'DEFAULT: "unix://var/run/docker.sock"\n' 'EXAMPLE: -d unix://var/run/docker.sock tcp://192.168.1.100:2376') core_group.add_argument('-t', '--docker-tls', default=Config.docker_tls, dest='DOCKER_TLS', action='store_true', help='Enable docker TLS\n' 'REQUIRES: docker cert mount') core_group.add_argument('-T', '--docker-tls-verify', default=Config.docker_tls_verify, dest='DOCKER_TLS_VERIFY', action='store_false', help='Verify the CA Certificate mounted for TLS\n' 'DEFAULT: True') core_group.add_argument( '-i', '--interval', type=int, default=Config.interval, dest='INTERVAL', help='Interval in seconds between checking for updates\n' 'DEFAULT: 300') core_group.add_argument('-C', '--cron', default=Config.cron, dest='CRON', help='Cron formatted string for scheduling\n' 'EXAMPLE: "*/5 * * * *"') core_group.add_argument( '-l', '--log-level', choices=['debug', 'info', 'warn', 'error', 'critical'], dest='LOG_LEVEL', default=Config.log_level, help='Set logging level\n' 'DEFAULT: info') core_group.add_argument('-u', '--self-update', default=Config.self_update, dest='SELF_UPDATE', action='store_true', help='Let ouroboros update itself') core_group.add_argument('-S', '--swarm', default=Config.swarm, dest='SWARM', action='store_true', help='Put ouroboros in swarm mode') core_group.add_argument('-o', '--run-once', default=Config.run_once, action='store_true', dest='RUN_ONCE', help='Single run') core_group.add_argument( '-A', '--dry-run', default=Config.dry_run, action='store_true', dest='DRY_RUN', help='Run without making changes. Best used with run-once') core_group.add_argument( '--monitor-only', default=Config.monitor_only, action='store_true', dest='MONITOR_ONLY', help='Run without making changes. Best used with run-once') core_group.add_argument( '-N', '--notifiers', nargs='+', default=Config.notifiers, dest='NOTIFIERS', help='Apprise formatted notifiers\n' 'EXAMPLE: -N discord://1234123412341234/jasdfasdfasdfasddfasdf ' 'mailto://*****:*****@gmail.com') docker_group = parser.add_argument_group( "Docker", "Configuration of docker functionality") docker_group.add_argument('-m', '--monitor', nargs='+', default=Config.monitor, dest='MONITOR', help='Which container(s) to monitor\n' 'DEFAULT: All') docker_group.add_argument('-n', '--ignore', nargs='+', default=Config.ignore, dest='IGNORE', help='Container(s) to ignore\n' 'EXAMPLE: -n container1 container2') docker_group.add_argument( '-k', '--label-enable', default=Config.label_enable, dest='LABEL_ENABLE', action='store_true', help='Enable label monitoring for ouroboros label options\n' 'Note: labels take precedence' 'DEFAULT: False') docker_group.add_argument( '-M', '--labels-only', default=Config.labels_only, dest='LABELS_ONLY', action='store_true', help='Only watch containers that utilize labels\n' 'This allows a more strict compliance for environments' 'DEFAULT: False') docker_group.add_argument('-c', '--cleanup', default=Config.cleanup, dest='CLEANUP', action='store_true', help='Remove old images after updating') docker_group.add_argument('-r', '--repo-user', default=Config.repo_user, dest='REPO_USER', help='Private docker registry username\n' 'EXAMPLE: [email protected]') docker_group.add_argument('-R', '--repo-pass', default=Config.repo_pass, dest='REPO_PASS', help='Private docker registry password\n' 'EXAMPLE: MyPa$$w0rd') data_group = parser.add_argument_group( 'Data Export', 'Configuration of data export functionality') data_group.add_argument('-D', '--data-export', choices=['prometheus', 'influxdb'], default=Config.data_export, dest='DATA_EXPORT', help='Enable exporting of data for chosen option') data_group.add_argument('-a', '--prometheus-addr', default=Config.prometheus_addr, dest='PROMETHEUS_ADDR', help='Bind address to run Prometheus exporter on\n' 'DEFAULT: 127.0.0.1') data_group.add_argument('-p', '--prometheus-port', type=int, default=Config.prometheus_port, dest='PROMETHEUS_PORT', help='Port to run Prometheus exporter on\n' 'DEFAULT: 8000') data_group.add_argument('-I', '--influx-url', default=Config.influx_url, dest='INFLUX_URL', help='URL for influxdb\n' 'DEFAULT: 127.0.0.1') data_group.add_argument('-P', '--influx-port', type=int, default=Config.influx_port, dest='INFLUX_PORT', help='PORT for influxdb\n' 'DEFAULT: 8086') data_group.add_argument('-U', '--influx-username', default=Config.influx_username, dest='INFLUX_USERNAME', help='Username for influxdb\n' 'DEFAULT: root') data_group.add_argument('-x', '--influx-password', default=Config.influx_password, dest='INFLUX_PASSWORD', help='Password for influxdb\n' 'DEFAULT: root') data_group.add_argument( '-X', '--influx-database', default=Config.influx_database, dest='INFLUX_DATABASE', help='Influx database name. Required if using influxdb') data_group.add_argument('-s', '--influx-ssl', default=Config.influx_ssl, dest='INFLUX_SSL', action='store_true', help='Use SSL when connecting to influxdb') data_group.add_argument( '-V', '--influx-verify-ssl', default=Config.influx_verify_ssl, dest='INFLUX_VERIFY_SSL', action='store_true', help='Verify SSL certificate when connecting to influxdb') docker_group.add_argument( '--skip-startup-notifications', default=Config.skip_startup_notifications, dest='SKIP_STARTUP_NOTIFICATIONS', action='store_true', help='Do not send ouroboros notifications when starting') args = parser.parse_args() if environ.get('LOG_LEVEL'): log_level = environ.get('LOG_LEVEL') else: log_level = args.LOG_LEVEL ol = OuroborosLogger(level=log_level) ol.logger.info('Version: %s-%s', VERSION, BRANCH) config = Config(environment_vars=environ, cli_args=args) config_dict = { key: value for key, value in vars(config).items() if key.upper() in config.options } ol.logger.debug("Ouroboros configuration: %s", config_dict) data_manager = DataManager(config) notification_manager = NotificationManager(config, data_manager) scheduler = BackgroundScheduler() scheduler.start() for socket in config.docker_sockets: try: docker = Docker(socket, config, data_manager, notification_manager) if config.swarm: mode = Service(docker) else: mode = Container(docker) if config.run_once: scheduler.add_job( mode.update, name=f'Run Once container update for {socket}') else: if mode.mode == 'container': scheduler.add_job(mode.self_check, name=f'Self Check for {socket}') if config.cron: scheduler.add_job( mode.update, name=f'Cron container update for {socket}', trigger='cron', minute=config.cron[0], hour=config.cron[1], day=config.cron[2], month=config.cron[3], day_of_week=config.cron[4], misfire_grace_time=15) else: scheduler.add_job( mode.update, name= f'Initial run interval container update for {socket}') scheduler.add_job( mode.update, name=f'Interval container update for {socket}', trigger='interval', seconds=config.interval) except ConnectionError: ol.logger.error( "Could not connect to socket %s. Check your config", socket) if config.run_once: next_run = None elif config.cron: next_run = scheduler.get_jobs()[0].next_run_time else: now = datetime.now(timezone.utc).astimezone() next_run = ( now + timedelta(0, config.interval)).strftime("%Y-%m-%d %H:%M:%S") if not config.skip_startup_notifications: notification_manager.send(kind='startup', next_run=next_run) while scheduler.get_jobs(): sleep(1) scheduler.shutdown()
class SpeakReader(object): _INITIALIZED = False SIGNAL = None transcribeEngine = None HTTP_PORT = None _INPUT_DEVICE = None ################################################################################################### # Initialize SpeakReader ################################################################################################### def __init__(self, initOptions): if SpeakReader._INITIALIZED: return with INIT_LOCK: global PROG_DIR PROG_DIR = initOptions['prog_dir'] global DATA_DIR DATA_DIR = initOptions['data_dir'] global CONFIG CONFIG = initOptions['config'] assert CONFIG is not None if isinstance(initOptions['http_port'], int): self.HTTP_PORT = initOptions['http_port'] else: self.HTTP_PORT = int(CONFIG.HTTP_PORT) if self.HTTP_PORT < 21 or self.HTTP_PORT > 65535: logger.warn("HTTP_PORT out of bounds: 21 < %s < 65535", self.HTTP_PORT) self.HTTP_PORT = 8880 # Check if pyOpenSSL is installed. It is required for certificate generation # and for CherryPy. if CONFIG.ENABLE_HTTPS: try: import OpenSSL except ImportError: logger.warn("The pyOpenSSL module is missing. Install this " "module to enable HTTPS. HTTPS will be disabled.") CONFIG.ENABLE_HTTPS = False if not CONFIG.HTTPS_CERT: CONFIG.HTTPS_CERT = os.path.join(DATA_DIR, 'server.crt') if not CONFIG.HTTPS_KEY: CONFIG.HTTPS_KEY = os.path.join(DATA_DIR, 'server.key') if not (os.path.exists(CONFIG.HTTPS_CERT) and os.path.exists(CONFIG.HTTPS_KEY)): logger.warn("Disabled HTTPS because of missing certificate and key.") CONFIG.ENABLE_HTTPS = False # Check if we has a jwt_secret if CONFIG.JWT_SECRET == '' or not CONFIG.JWT_SECRET: logger.debug("Generating JWT secret...") CONFIG.JWT_SECRET = generate_uuid() CONFIG.write() ################################################################################################### # Get Version Information and check for updates ################################################################################################### self.versionInfo = Version() ################################################################################################### # Get the Input Device ################################################################################################### self.get_input_device() ################################################################################################### # Initialize the Transcribe Engine ################################################################################################### self.transcribeEngine = TranscribeEngine() if CONFIG.START_TRANSCRIBE_ON_STARTUP : self.startTranscribeEngine() ################################################################################################### # Initialize the webserver ################################################################################################### logger.info('WebServer Initializing') webServerOptions = { 'config': CONFIG, 'prog_dir': PROG_DIR, 'data_dir': DATA_DIR, 'http_port': self.HTTP_PORT, } self.webServer = webstart.initialize(webServerOptions) self.webServer.root.SR = self cherrypy.server.start() # Launch the WebBrowser if CONFIG.LAUNCH_BROWSER and not initOptions['nolaunch']: launch_browser(CONFIG.HTTP_HOST, self.HTTP_PORT, CONFIG.HTTP_ROOT + 'manage') ################################################################################################### # Run cleanup of old logs, transcripts, and recordings and start a scheduler to run every 24 hours ################################################################################################### self.cleanup_files() self.scheduler = BackgroundScheduler() self.scheduler.add_job(self.cleanup_files, 'interval', hours=24) self.scheduler.start() SpeakReader._INITIALIZED = True @property def is_initialized(self): return self._INITIALIZED ################################################################################################### # Start the Transcribe Engine ################################################################################################### def startTranscribeEngine(self): if self.transcribeEngine.is_online: logger.info("Transcribe Engine already started.") return if self.get_input_device() is None: logger.warn("No Input Devices Available. Can't start Transcribe Engine.") return if CONFIG.SPEECH_TO_TEXT_SERVICE == 'google': if CONFIG.GOOGLE_CREDENTIALS_FILE == "": logger.warn("API Credentials not available. Can't start Transcribe Engine.") return try: with open(CONFIG.GOOGLE_CREDENTIALS_FILE) as f: json.loads(f.read()) except json.decoder.JSONDecodeError: logger.warn("API Credentials does not appear to be a valid JSON file. Can't start Transcribe Engine.") return elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM': if CONFIG.IBM_CREDENTIALS_FILE == "": logger.warn("API Credentials not available. Can't start Transcribe Engine.") return APIKEY = None URL = None try: with open(CONFIG.IBM_CREDENTIALS_FILE) as f: for line in f.read().splitlines(): parm = line.split('=') if parm[0] == 'SPEECH_TO_TEXT_APIKEY': APIKEY = parm[1] if parm[0] == 'SPEECH_TO_TEXT_URL': URL = parm[1] except: pass if APIKEY is None or URL is None: logger.warn("APIKEY or URL not found in IBM credentials file. Can't start Transcribe Engine.") return elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft': if CONFIG.MICROSOFT_SERVICE_APIKEY == "" or CONFIG.MICROSOFT_SERVICE_REGION == "": logger.warn("Microsoft Azure APIKEY and Region are required. Can't start Transcribe Engine.") return else: return self.transcribeEngine.start() ################################################################################################### # Stop the Transcribe Engine ################################################################################################### def stopTranscribeEngine(self): if self.transcribeEngine.is_online: self.transcribeEngine.stop() ################################################################################################### # Shutdown SpeakReader ################################################################################################### def shutdown(self, restart=False, update=False, checkout=False): SpeakReader._INITIALIZED = False self.transcribeEngine.shutdown() self.scheduler.shutdown() CONFIG.write() if not restart and not update and not checkout: logger.info("Shutting Down SpeakReader") if update: logger.info("********************************") logger.info("* SpeakReader is updating... *") logger.info("********************************") try: self.versionInfo.update() except Exception as e: logger.warn("SpeakReader failed to update: %s. Restarting." % e) logger.info('WebServer Terminating') cherrypy.engine.exit() if checkout: logger.info("SpeakReader is switching the git branch...") try: self.versionInfo.checkout_git_branch() except Exception as e: logger.warn("SpeakReader failed to switch git branch: %s. Restarting." % e) ################################################################################################### # Get Input Device ################################################################################################### def get_input_device(self): self._INPUT_DEVICE = CONFIG.INPUT_DEVICE try: p = pyaudio.PyAudio() defaultInputDevice = p.get_default_input_device_info() if self._INPUT_DEVICE not in list(d['name'] for d in self.get_input_device_list()): CONFIG.INPUT_DEVICE = self._INPUT_DEVICE = defaultInputDevice.get('name') CONFIG.write() except: self._INPUT_DEVICE = None return self._INPUT_DEVICE ################################################################################################### # Get Input Device List ################################################################################################### def get_input_device_list(self): deviceList = [] try: p = pyaudio.PyAudio() defaultHostAPIindex = p.get_default_host_api_info().get('index') numdevices = p.get_default_host_api_info().get('deviceCount') for i in range(0, numdevices): inputDevice = p.get_device_info_by_host_api_device_index(defaultHostAPIindex, i) if inputDevice.get('maxInputChannels') > 0: device = { 'index': inputDevice.get('index'), 'name': inputDevice.get('name'), 'selected': True if inputDevice.get('name') == self._INPUT_DEVICE else False, } deviceList.append(device) except Exception: pass return deviceList ################################################################################################### # Delete any files over the retention days ################################################################################################### def cleanup_files(self): logger.info("Running File Cleanup") def delete(path, days): try: days = int(days) except ValueError: return delete_date = datetime.datetime.now() - datetime.timedelta(days=days) with os.scandir(path=path) as files: for file in files: file_info = file.stat() if datetime.datetime.fromtimestamp(file_info.st_ctime) < delete_date: filename = os.path.join(path, file.name) logger.debug("Deleting: %s" % filename) os.remove(filename) if CONFIG.LOG_RETENTION_DAYS != "": delete(CONFIG.LOG_DIR, CONFIG.LOG_RETENTION_DAYS) if CONFIG.TRANSCRIPT_RETENTION_DAYS != "": delete(CONFIG.TRANSCRIPTS_FOLDER, CONFIG.TRANSCRIPT_RETENTION_DAYS) if CONFIG.RECORDING_RETENTION_DAYS != "": delete(CONFIG.RECORDINGS_FOLDER, CONFIG.RECORDING_RETENTION_DAYS)
class JenniferBrain(object): UNSURE_TEXT = "Sorry, I can't help with that" MULTIPLE_LESSONS_APPLY = 'Which one of my lessons applies here?' def __init__(self, allow_network_plugins=False, always_allow_plugins=None): self._initialize_paths() # Lessons + Settings self.allow_network_plugins = allow_network_plugins self.always_allow_plugins = always_allow_plugins or [] self.responders = [] self.notifiers = [] self.notification_queue = Queue.PriorityQueue() self._load_profile_and_settings() # Requires self.database & self.settings self._load_lessons() # Just to save time later self.nltktagger = PerceptronTagger() self.tagset = None # Notifications self.notification_clients = [] self._initialize_background_tasks() def _initialize_paths(self): """Create the paths needed""" self.base_path = os.path.join(os.path.dirname(__file__), '..') self.profile_file = os.path.join(self.base_path, 'profile.json') self.lessons_path = os.path.join(self.base_path, 'lessons') def _load_lessons(self): """ Search the lessons/ package for lessons & store them in sorted order by priority :return: """ pkgs = [ n for _, n, _ in pkgutil.iter_modules(['lessons']) if n != 'base' ] for name in pkgs: exec 'import lessons.' + name + '.plugin' responders = [ cls(self).set_profile(self.database['profile']) for cls in JenniferResponsePlugin.__subclasses__() if self._is_lesson_allowed(cls) ] self.notifiers = [ cls(self).set_profile(self.database['profile']) for cls in JenniferNotificationPlugin.__subclasses__() if self._is_lesson_allowed(cls) ] for r in (responders + self.notifiers): r.set_settings(self._get_settings_for_lesson(r)) self.responders = sorted(responders, key=lambda l: l.PRIORITY) def _is_lesson_allowed(self, lesson_cls): if lesson_cls in self.always_allow_plugins: return True if lesson_cls.REQUIRES_NETWORK and not self.allow_network_plugins: return False return True def _load_profile_and_settings(self): """ Load the profile :return: """ try: with open(self.profile_file, 'r+') as profile_file: data = json.loads(profile_file.read(), strict=False) self.database = data if 'profile' in self.database and 'settings' in self.database: profile_file.close() return except (IOError, ValueError): self.database = {} self._init_profile() self._save_profile_to_file() def _get_settings_for_lesson(self, lesson, lesson_name=None): """ Get the settings dict for the lesson (Must be called from a lesson) :return: """ if not lesson_name: lesson_name = unicode(lesson.settings_name) try: return self.database['settings'][lesson_name] except KeyError: if self._test_if_settings_template_exists(lesson): print "--------{} SETTINGS--------".format(lesson_name) self._add_lesson_to_settings_and_write(lesson) return self._get_settings_for_lesson(lesson) return {} def _settings_template_path_for_lesson(self, lesson): """Gets a settings_template for a given lesson""" lesson_settings_name = lesson.settings_name return os.path.join(self.lessons_path, lesson_settings_name, 'settings_template.json') def _test_if_settings_template_exists(self, lesson): """Returns if a settings_template for a given lesson""" return os.path.isfile(self._settings_template_path_for_lesson(lesson)) def _add_lesson_to_settings_and_write(self, lesson): """Loads a lesson's settings_template, runs an initialization function if available, and copies into DB""" lesson_settings_name = lesson.settings_name with open(self._settings_template_path_for_lesson(lesson)) as template: try: # Try to load initial template settings_template_dict = json.loads(template.read(), strict=False) settings_template_dict = lesson.initialize_settings( settings_template_dict) # Push to DB & save self.database['settings'][ lesson_settings_name] = settings_template_dict self._save_profile_to_file() except ValueError: exit("{} has an invalid settings_template.json".format( lesson_settings_name)) def _save_profile_to_file(self): """Writes to profile.json""" with open(self.profile_file, "w+") as f: plain_text = json.dumps(self.database, indent=4, sort_keys=True) f.write(plain_text) f.close() def _init_profile(self): """Should be run if profile.json doesn't exist""" fields = [ ('first name', 'firstName'), ('last name', 'lastName'), ] location_fields = [ ('city', 'city', 'New York City'), ('region', 'region', 'NY'), ('country', 'country', 'US'), ('zip', 'zip'), ] if 'profile' not in self.database: for field in fields: self.database.update({'profile': {'location': {}}}) print "What is your {}?".format(field[0]) self.database['profile'][field[1]] = raw_input("> ") self.database['profile']['location'] = {} for field in location_fields: txt = "What is your {}?".format(field[0]) if len(location_fields) >= 3: txt += " example: ({})".format(field[2]) print txt self.database['profile']['location'][field[1]] = raw_input( "> ") while True: print "What is your timezone? example: ({})".format( random.choice(common_timezones)) tz = raw_input('> ') if timezone(tz): self.database['profile']['location']['timezone'] = tz break else: print "Invalid timezone" if 'settings' not in self.database: self.database.update( {'settings': { 'notifications': { 'quiet_hours': [] } }}) def _get_profile(self): """Get the user's profile""" return self.database['profile'] def take_input(self, text_input, client): """ Search all lessons for lessons that can respond :param text_input: :return: """ text_input = text_input.lower() tokens = nltk.word_tokenize(text_input) tags = nltk.tag._pos_tag(tokens, self.tagset, self.nltktagger) # TODO: extrap this out to a custom stopwords try: tags.remove( ('please', 'NN') ) # It's common to say 'please' when asking Jennifer something except: pass # Find the lessons that can answer respond_to = None matching_lessons = [ lesson for lesson in self.responders if lesson.can_respond( tags=tags, client=client, brain=self, plain_text=text_input) ] # No answer if len(matching_lessons) == 0: self.respond_or_unsure(None, tags, client, text_input) # Only one module can respond elif len(matching_lessons) == 1: respond_to = matching_lessons[0] # Multiple lessons can response else: priority_counts = {} for lesson in matching_lessons: key = lesson.PRIORITY priority_counts.setdefault(key, []).append(lesson) # Now we have something like {999: [TimePlugin(), LowPriorityTimePlugin()], 0: [ImportantTimePlugin()]} min_priority = min(priority_counts.keys()) if len(priority_counts[min_priority]) == 1: respond_to = priority_counts[min_priority][0] else: client.give_output_string("brain", self.MULTIPLE_LESSONS_APPLY) for lesson in priority_counts[min_priority]: if client.confirm("brain", lesson.VERBOSE_NAME + "?"): # TODO: would be nice to remember this decision.. that's v3.0 though. respond_to = lesson break return self.respond_or_unsure(respond_to, tags, client, text_input) def respond_or_unsure(self, respond_to, tags, client, text_input): try: return respond_to.respond(tags=tags, client=client, brain=self, plain_text=text_input) except Exception as e: return JenniferResponse( self, [JenniferTextResponseSegment(self.UNSURE_TEXT)]) def _initialize_background_tasks(self): self.scheduler = BackgroundScheduler(timezone="UTC", daemon=True) self.scheduler.start() self.scheduler.add_job(self._collect_notifications_from_notifiers, 'interval', seconds=10) self.scheduler.add_job(self.push_notifications_to_clients, 'interval', seconds=2) atexit.register(lambda: self.scheduler.shutdown(wait=False)) def _collect_notifications_from_notifiers(self): for notification_provider in self.notifiers: while not notification_provider.queue.empty(): self.notification_queue.put(notification_provider.queue.get()) def register_notification_client(self, client): self.notification_clients.append(client) def push_notifications_to_clients(self): while not self.notification_queue.empty(): notification = self.notification_queue.get() for client in self.notification_clients: client.give_output_string("brain", notification[1])
class Scheduler(object): def __init__(self): self._logger = logging.getLogger() config_path = os.path.join(os.path.expanduser('~'), '.shipane_sdk', 'config', 'scheduler.ini') self._logger.info('Config path: %s', config_path) self._config = configparser.RawConfigParser() self._config.readfp(codecs.open(config_path, encoding="utf_8_sig"), ) self._scheduler = BackgroundScheduler() self._client = Client(self._logger, **dict(self._config.items('ShiPanE'))) def start(self): self.__add_job(self.__create_new_stock_purchase_job()) self.__add_job(self.__create_repo_job()) self.__add_job(self.__create_batch_job()) self.__add_job(self.__create_join_quant_following_job()) self.__add_job(self.__create_rice_quant_following_job()) self.__add_job(self.__create_uqer_following_job()) self.__add_job(self.__create_guorn_sync_job()) self._scheduler.start() print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) try: while True: time.sleep(1) except (KeyboardInterrupt, SystemExit): self._scheduler.shutdown() def __add_job(self, job): if job.is_enabled: self._scheduler.add_job(job, APCronParser.parse(job.schedule), name=job.name, misfire_grace_time=None) else: self._logger.warning('{} is not enabled'.format(job.name)) def __create_new_stock_purchase_job(self): section = 'NewStocks' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) return NewStockPurchaseJob(self._client, client_aliases, '{}Job'.format(section), **options) def __create_repo_job(self): section = 'Repo' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) return RepoJob(self._client, client_aliases, '{}Job'.format(section), **options) def __create_batch_job(self): section = 'Batch' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) return BatchJob(self._client, client_aliases, '{}Job'.format(section), **options) def __create_join_quant_following_job(self): section = 'JoinQuant' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) quant_client = JoinQuantClient(**options) return OnlineQuantFollowingJob(self._client, quant_client, client_aliases, '{}FollowingJob'.format(section), **options) def __create_rice_quant_following_job(self): section = 'RiceQuant' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) quant_client = RiceQuantClient(**options) return OnlineQuantFollowingJob(self._client, quant_client, client_aliases, '{}FollowingJob'.format(section), **options) def __create_uqer_following_job(self): section = 'Uqer' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) quant_client = UqerClient(**options) return OnlineQuantFollowingJob(self._client, quant_client, client_aliases, '{}FollowingJob'.format(section), **options) def __create_guorn_sync_job(self): section = 'Guorn' options = self.__build_options(section) client_aliases = self.__filter_client_aliases(section) quant_client = GuornClient(**options) return OnlineQuantSyncJob(self._client, quant_client, client_aliases, '{}SyncJob'.format(section), **options) def __build_options(self, section): if not self._config.has_section(section): return dict() options = dict(self._config.items(section)) options['enabled'] = bool(distutils.util.strtobool(options['enabled'])) return options def __filter_client_aliases(self, section): if not self._config.has_section(section): return dict() all_client_aliases = dict(self._config.items('ClientAliases')) client_aliases = [client_alias.strip() for client_alias in filter(None, self._config.get(section, 'clients').split(','))] return collections.OrderedDict( (client_alias, all_client_aliases[client_alias]) for client_alias in client_aliases)
class HarryBotter(object): def __init__(self, debug=False, stopcb=None): oplogs("Harry Botter is rebooting") self.__init_robot(debug) self.stopcb = stopcb #self.__init_db() self.__init_jobs() oplogs("Harry Botter is online") def __del__(self): oplogs("Harry Botter is shutting down~~~") self.stop_jobs() #self.close_db() def __init_robot(self, debug): self.enable = True self.mouth = None # output callback self.ear = None # input for text and audio callback self.eye = None # input for graph callback self.brain = None # processor modules self.memory = [] # memory for what heard or seen self.support_groups = [] self.debug = debug self.sched = None self.starttime = datetime.now() def subscribe(self, msgSend): self.mouth = msgSend oplogs('Robot unmuted.') def unsubscribe(self): self.mouth = None oplogs('Robot muted.') def install_mods(self, hmod): self.stockmod = hmod #StockMod(self.debug) def __init_db(self): # 创建数据库数据库用于永久记忆 self.conn = sqlite3.connect('messages.db') self.cursor = self.conn.cursor() self.cursor.execute( 'create table if not exists msgqueue (id integer primary key autoincrement, date TEXT, groupname TEXT, user TEXT, msg TEXT)' ) self.cursor.execute( 'create table if not exists monitorstocks (id integer primary key autoincrement, groupname TEXT, code TEXT)' ) self.cursor.commit() oplogs("[%s]messages.db connected") def __close_db(self): self.conn.close() def __init_jobs(self): self.sched = BackgroundScheduler() # 添加任务作业 # 每天清理一次缓存 #self.sched.add_job(self.job_clean_cache, trigger='cron', day_of_week='*',hour=1, minute=0, second=0) # 提供给group的5分钟检测 self.sched.add_job(self.job_stock_monitor, trigger='cron', id='job_stock_monitor', minute="*/5") #, next_run_time=None) # 交易日9:30:15生成开盘报告 self.sched.add_job(self.job_open_scan, trigger='cron', day_of_week='0-4', hour=9, minute=30, second=15) # 交易日11:30--13:00之间关闭扫描 self.sched.add_job(self.job_close_scan, trigger='cron', day_of_week='0-4', hour=11, minute=30, second=15) self.sched.add_job(self.job_open_scan, trigger='cron', day_of_week='0-4', hour=13, minute=0, second=0) # 交易日15:05:00生成收盘报告 self.sched.add_job(self.job_close_scan, trigger='cron', day_of_week='0-4', hour=15, minute=5, second=0) # 启动调度器 self.sched.start() oplogs("schedulers started") def stop_jobs(self): if self.sched == None: self.sched.shutdown() oplogs("schedulers stopped") def job_stock_monitor(self, valve=3.0): if self.enable is False: return oplogs("job_stock_monitor triggerred") group_alarms = self.stockmod.get_alarms(valve) for (group, alarms) in group_alarms.items(): notice = "" for alarm in alarms: # alarm = [stockname,price,change,highlow,closeopen] alarm_content = "5分钟涨跌\n[%s %.2f%%]幅度:%.2f%%, 波动:%.2f%%\n" % ( alarm[0], alarm[2], alarm[4], alarm[3]) notice += alarm_content if len(notice) > 0: print(notice) self.mouth(group, notice) time.sleep(1) def job_open_scan(self): oplogs("job_open_scan triggerred") self.job_stock_monitor() self.sched.resume_job(job_id="job_stock_monitor") def job_close_scan(self): oplogs("job_close_scan triggerred") self.sched.pause_job(job_id="job_stock_monitor") oplogs("job_stock_monitor paused") def job_forget(self, persist=False): # 清除记忆 if persist is True: #save memory before forget them pass self.memory.clear() oplogs("memory erased") def listen(self, group, user, msg): # 添加一条记录到记忆中 record = [time.strftime("%Y-%m-%d %H:%M:%S"), group, user, msg] self.stockmod.scan_stock(group, msg) self.memory.append(record) def save_to_db(self): oplogs("save_to_db called") # 保存消息 for msg in self.memory: sql = "INSERT INTO msgqueue VALUES (%s,%s,%s,%s)" % ( msg[0], msg[1], msg[2], msg[3]) self.cursor.execute(sql) # 保存监控列表 for (key, value) in self.stockmod.monitor_queue.items(): for code in value: # date TEXT, group TEXT, code TEXT # date = time.strftime("%Y-%m-%d %H:%M:%S") sql = "INSERT INTO monitorstocks VALUES (%s,%s)" % (key, code) self.cursor.execute(sql) self.conn.commit() def load_from_db(self): pass def isCmd(self, cmd): if USER_CMD in cmd: return True for keyword in syscmd: if keyword in cmd: return True return False def action_user(self, cmd, group, user): cmds = cmd.lower().split() #"命令格式: # [添加监控]harry 股票名 # [删除监控]harry del 股票名 # [显示群推荐]harry 本群推荐 # [显示群数据统计]harry stat # [显示报告]harry report"] # user 用户命令 if len(cmds) == 1: return random.choice(auto_replys) if ('list' in cmds[1]) or ('本群推荐' in cmds[1]): oplogs("action_user:本群推荐 [%s]" % cmd) return self.stockmod.get_group_stock_price(group) elif 'del' == cmds[1]: return self.stockmod.del_from_list(group, cmds[2]) elif 'stat' in cmds[1]: boottime = datetime.now() - self.starttime boothour = int(boottime.seconds / 3600) bootmin = int(boottime.seconds % 3600 / 60) stat = "机器人已运行:{}天{}小时{}分钟\n股价扫描任务:{}".format( boottime.days, boothour, bootmin, self.sched.state) return stat elif 'report' in cmds[1]: return "建设中" elif 'help' in cmds[1]: oplogs("action_user:help called") return auto_replys[0] elif 'ver' in cmds[1]: return HR__VERSION elif '启动' in cmds[1]: return "臣在" elif '关闭' in cmds[1]: return "跪安" else: oplogs("action_user:harry [stock] called %s" % cmd) if self.stockmod.isvalid_stock(cmds[1]): self.add_stock(group, cmds[1]) return self.show_stock(cmds[1]) return "" def action(self, cmd): cmds = cmd.split() # 管理员后台控制命令 if not self.isCmd(cmds[0]): return "" # 去除'/',提取控制命令函数名 _cmd = cmds[0].strip('/') if len(cmds) > 1: return eval("self.{}({})".format(_cmd, cmds[1:])) else: return eval("self.{}()".format(_cmd)) def ping(self, cmds=[]): #self.mouth('量化技术讨论','test') return 'pong' def help(self, cmds=[]): # 显示命令帮助 return ("Help\n" + ",".join(syscmd)) def show(self, cmds=[]): # 显示在线信息和数据 # show gmsg/stock [stockname]/monitor/stats print("/show called") if cmds[0] == 'memory': return "Robot has %d messages in memory" % (len(self.memory)) if cmds[0] == 'stock': return self.show_stock(cmds[1]) if cmds[0] == 'groups': # 显示所有支持的群 return ",".join(self.support_groups) if cmds[0] == 'group': # 显示指定群支持的股票 if len(cmds) > 1 and cmds[1].isdigit(): groupid = int(cmds[1]) else: groupid = 0 return self.action_user("harry 本群推荐", self.support_groups[groupid], "robot") if cmds[0] == 'mq': return str(self.stockmod.monitor_queue) if cmds[0] == 'stats': return "no supported cmd" return "no supported cmd" def show_stock(self, stock): if '机器人' in stock or '300024' in stock: return "harry拒绝关注其他机器人" if stock.isdigit(): return self.stockmod.get_stock_price(stock) else: return self.stockmod.get_stock_price_by_name(stock) def cfg(self): # 修改在线配置 return ("/cfg called") def add(self, cmds=[]): if len(cmds) >= 2: # /add group [groupname], 添加支持的group if cmds[0] is "group": self.support_groups.append(cmds[1]) return "group added" return "not supprt command" def add_stock(self, group, stock): self.stockmod.add_to_monitor_group(group, stock) def delete(self, cmds=[]): if len(cmds) > 0: # 删除支持的group if cmds[0] is "group": self.support_groups.remove(cmds[1]) return "group deleted" return "not supprt command" def is_support_group(self, group): return group in self.support_groups def test(self, cmd): return "test" def restart(self): # 重启机器人 return "restarted" def start(self): # 启动机器人 self.enable = True return "臣在" def pause(self): self.enable = False return "跪安" def stop(self): print("stop called") # 离线机器人 self.enable = False self.stopcb() def use(self): # 加载模块控制 # module management to be implemented here return "All mods loaded" def add_msg(self, msg): # save message self.memory.append(msg) def add_groups(self, groups): for group in groups: if group not in self.support_groups: self.support_groups.append(group)
class Scheduler: def __init__(self, timezone='UTC'): """Initialize scheduler.""" self._scheduler = BackgroundScheduler(timezone=timezone) self._scheduler.add_listener(self._event_job_executed, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) self._scheduler.add_listener(self._event_job_added, EVENT_JOB_ADDED) self._scheduler.add_listener(self._event_job_removed, EVENT_JOB_REMOVED) self._status, self._data = {}, {} def _event_job_executed(self, event): """Update job data after execution.""" t = arrow.utcnow().to(self._scheduler.timezone).format() d = str(event.exception) if event.exception else event.retval self._data[event.job_id].append([t, d]) def _event_job_added(self, event): """Update job status and data after adding.""" self._status[event.job_id] = 'started' self._data.setdefault(event.job_id, []) def _event_job_removed(self, event): """Update job status after removing.""" self._status[event.job_id] = 'nonexistent' def start(self): """Start scheduler.""" self._scheduler.start() def shutdown(self, wait=False): """Shutdown scheduler.""" self._scheduler.shutdown(wait=wait) def add(self, id, func, trigger, **kwargs): """Add job.""" self.remove(id) self._scheduler.add_job(id=id, func=func, trigger=trigger, **kwargs) def remove(self, id): """Remove job.""" if self._status.get(id) in ('started', 'paused'): self._scheduler.remove_job(id) def job_result(self, id, latest=True): """Get job results.""" r = self._data.get(id) return r if not latest else r[0] if r else None def job_status(self, id): """Get job status. :return 'started', 'stopped', 'nonexistent'. """ if self._status.get(id, None) is None: return 'nonexistent' return self._status.get(id) def job_next_run_time(self, id, fmt='YYYY-MM-DD HH:mm:ssZZ'): """Get job next run time as string.""" if self._status.get(id): t = self._scheduler.get_job(id).next_run_time return arrow.get(t).format(fmt=fmt) return None def print(self): """Print jobs information.""" self._scheduler.print_jobs()
def DScheduler(cls, action, start_date=None, execute_date=None, end_date=None, execute_interval=3, tz=None, **kwargs): """ 一个依托于时间驱动的实时任务,action所挂载的任务由相应的时间驱动,这跟run方法由K线更新驱动不一样 :param action: :param start_date:like '09:30:00' :param execute_date:like '09:30:00-11:30:00' or '09:30:00-11:30:00 13:00:00-15:00:00' :param end_date:like '15:00:00' :param execute_interval:连续任务的执行时间间隔,以秒计 :param tz:时区 :return: """ fmt = '%Y-%m-%d %H:%M:%S' if start_date is not None: try: sdt = dt.datetime.strptime('2000-01-01 ' + start_date, fmt) except Exception: raise TypeError( 'this start_date param like a "09:30:00" string') if execute_date is not None: try: xdt = [] dts = execute_date.split(' ') for et in dts: t = et.split('-') s = dt.datetime.strptime('2000-01-01 ' + t[0], fmt) e = dt.datetime.strptime('2000-01-01 ' + t[1], fmt) # if s > e: # 如果execute的start大于end说明是当天的end到第二天的start # raise TypeError('execute start datetime must less than end') xdt.append([s, e]) del s, e, t del dts except Exception: raise TypeError( 'this start_date param like a "09:30:00-11:30:00" or' ' "09:30:00-11:30:00 13:00:00-15:00:00"') if end_date is not None: try: edt = dt.datetime.strptime('2000-01-01 ' + end_date, fmt) except Exception: raise TypeError( 'this start_date param like a "15:30:00" string') if tz is not None: if tz not in pytz.all_timezones: raise ValueError('this tz: %s not in pytz time zones' % tz) else: tz = pytz.timezone(tz) from apscheduler.triggers.date import DateTrigger from apscheduler.triggers.interval import IntervalTrigger from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor while 1: # scheduler = BlockingScheduler(daemonic=False) # crt = CalfDateTime.now(tz, offset) crt = dt.datetime.now() if tz is None else dt.datetime.now( tz=tz).replace(tzinfo=None) tdy = dt.datetime(crt.year, crt.month, crt.day) # 非交易日 if not action.is_trade_day(tdy): print(fontcolor.F_RED + '-' * 80) print('Note:Non-transaction date;Datetime:' + str(crt)) print('-' * 80 + fontcolor.END) delta = (tdy + dt.timedelta(days=1) - crt).seconds delta = 1 if delta < 1 else delta time.sleep(delta) # sleep to next day continue # 交易日 else: try: nsds = list() executors = { 'default': ThreadPoolExecutor(4), 'processpool': ProcessPoolExecutor(4) } job_defaults = {'coalesce': True, 'max_instances': 1} scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults, daemonic=False, timezone=tz) if start_date is not None: d = tdy + dt.timedelta(hours=sdt.hour, minutes=sdt.minute, seconds=sdt.second) nsds.append(d + dt.timedelta(days=1)) def action_start(**args): print(fontcolor.F_GREEN + '-' * 80) print('Calf-Note:start task running on ', dt.datetime.now(tz=tz)) print('-' * 80 + fontcolor.END) try: action.start(**args) except Exception as ep: ExceptionInfo(ep) scheduler.add_job(func=action_start, trigger=DateTrigger(d), id='action_start', args=[kwargs]) if execute_date is not None: def action_execute(**args): print(fontcolor.F_GREEN + '-' * 80) print('Calf-Note:execute task running on ', dt.datetime.now(tz=tz)) print('-' * 80 + fontcolor.END) try: action.execute(**args) except Exception as ep: ExceptionInfo(ep) for x in xdt: sd = tdy + dt.timedelta(hours=x[0].hour, minutes=x[0].minute, seconds=x[0].second) ed = tdy + dt.timedelta(hours=x[1].hour, minutes=x[1].minute, seconds=x[1].second) if sd > ed: sd = sd + dt.timedelta(days=1) scheduler.add_job(func=action_execute, trigger=IntervalTrigger( seconds=execute_interval, start_date=ed, end_date=sd), args=[kwargs]) else: scheduler.add_job(func=action_execute, trigger=IntervalTrigger( seconds=execute_interval, start_date=sd, end_date=ed), args=[kwargs]) nsds.append(sd + dt.timedelta(days=1)) if end_date is not None: def action_end(**args): print(fontcolor.F_GREEN + '-' * 80) print('Calf-Note:end task running on ', dt.datetime.now(tz=tz)) print('-' * 80 + fontcolor.END) try: action.end(**args) except Exception as ep: ExceptionInfo(ep) d = tdy + dt.timedelta(hours=edt.hour, minutes=edt.minute, seconds=edt.second) nsds.append(d + dt.timedelta(days=1)) scheduler.add_job(func=action_end, trigger=DateTrigger(d), id='action_end', args=[kwargs]) print(fontcolor.F_GREEN + '-' * 80) print('Note:enter Calf real task and mount these tasks:') scheduler.print_jobs() print('Datetime:' + str(crt)) print('-' * 80 + fontcolor.END) scheduler.start() # 计算距离下一次启动应该休眠多久 if len(nsds) == 0: break # ed = CalfDateTime.now(tz, offset) nd = dt.datetime.now() if tz is None else dt.datetime.now( tz=tz).replace(tzinfo=None) delta = (min(nsds) - nd) delta = delta.seconds + delta.days * 86400 print(fontcolor.F_YELLOW + '-' * 80) print( 'Note:Calf will sleep {0} seconds and restart on {1}:'. format(delta, min(nsds))) print('Datetime:', str(crt)) print('-' * 80 + fontcolor.END) delta = 1 if delta < 1 else delta time.sleep(delta) scheduler.shutdown(wait=False) del scheduler except Exception as e: ExceptionInfo(e) pass
class JobQueue: """This class allows you to periodically perform tasks with the bot. It is a convenience wrapper for the APScheduler library. Attributes: scheduler (:class:`apscheduler.schedulers.background.BackgroundScheduler`): The APScheduler bot (:class:`telegram.Bot`): The bot instance that should be passed to the jobs. DEPRECATED: Use :attr:`set_dispatcher` instead. """ def __init__(self) -> None: self._dispatcher: 'Dispatcher' = None # type: ignore[assignment] self.logger = logging.getLogger(self.__class__.__name__) self.scheduler = BackgroundScheduler(timezone=pytz.utc) self.scheduler.add_listener(self._update_persistence, mask=EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) # Dispatch errors and don't log them in the APS logger def aps_log_filter(record): # type: ignore return 'raised an exception' not in record.msg logging.getLogger('apscheduler.executors.default').addFilter( aps_log_filter) self.scheduler.add_listener(self._dispatch_error, EVENT_JOB_ERROR) def _build_args(self, job: 'Job') -> List[Union[CallbackContext, 'Bot', 'Job']]: if self._dispatcher.use_context: return [CallbackContext.from_job(job, self._dispatcher)] return [self._dispatcher.bot, job] def _tz_now(self) -> datetime.datetime: return datetime.datetime.now(self.scheduler.timezone) def _update_persistence(self, event: JobEvent) -> None: # pylint: disable=W0613 self._dispatcher.update_persistence() def _dispatch_error(self, event: JobEvent) -> None: try: self._dispatcher.dispatch_error(None, event.exception) # Errors should not stop the thread. except Exception: self.logger.exception( 'An error was raised while processing the job and an ' 'uncaught error was raised while handling the error ' 'with an error_handler.') @overload def _parse_time_input(self, time: None, shift_day: bool = False) -> None: ... @overload def _parse_time_input( self, time: Union[float, int, datetime.timedelta, datetime.datetime, datetime.time], shift_day: bool = False, ) -> datetime.datetime: ... def _parse_time_input( self, time: Union[float, int, datetime.timedelta, datetime.datetime, datetime.time, None], shift_day: bool = False, ) -> Optional[datetime.datetime]: if time is None: return None if isinstance(time, (int, float)): return self._tz_now() + datetime.timedelta(seconds=time) if isinstance(time, datetime.timedelta): return self._tz_now() + time if isinstance(time, datetime.time): date_time = datetime.datetime.combine( datetime.datetime.now( tz=time.tzinfo or self.scheduler.timezone).date(), time) if date_time.tzinfo is None: date_time = self.scheduler.timezone.localize(date_time) if shift_day and date_time <= datetime.datetime.now(pytz.utc): date_time += datetime.timedelta(days=1) return date_time # isinstance(time, datetime.datetime): return time def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: """Set the dispatcher to be used by this JobQueue. Use this instead of passing a :class:`telegram.Bot` to the JobQueue, which is deprecated. Args: dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher. """ self._dispatcher = dispatcher if dispatcher.bot.defaults: if dispatcher.bot.defaults: self.scheduler.configure( timezone=dispatcher.bot.defaults.tzinfo or pytz.utc) def run_once( self, callback: Callable[['CallbackContext'], None], when: Union[float, datetime.timedelta, datetime.datetime, datetime.time], context: object = None, name: str = None, job_kwargs: JSONDict = None, ) -> 'Job': """Creates a new ``Job`` that runs once and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. when (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \ :obj:`datetime.datetime` | :obj:`datetime.time`): Time in or at which the job should run. This parameter will be interpreted depending on its type. * :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the job should run. * :obj:`datetime.timedelta` will be interpreted as "time from now" in which the job should run. * :obj:`datetime.datetime` will be interpreted as a specific date and time at which the job should run. If the timezone (``datetime.tzinfo``) is :obj:`None`, the default timezone of the bot will be used. * :obj:`datetime.time` will be interpreted as a specific time of day at which the job should run. This could be either today or, if the time has already passed, tomorrow. If the timezone (``time.tzinfo``) is :obj:`None`, the default timezone of the bot will be used. context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the ``scheduler.add_job()``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. """ if not job_kwargs: job_kwargs = {} name = name or callback.__name__ job = Job(callback, context, name, self) date_time = self._parse_time_input(when, shift_day=True) j = self.scheduler.add_job( callback, name=name, trigger='date', run_date=date_time, args=self._build_args(job), timezone=date_time.tzinfo or self.scheduler.timezone, **job_kwargs, ) job.job = j return job def run_repeating( self, callback: Callable[['CallbackContext'], None], interval: Union[float, datetime.timedelta], first: Union[float, datetime.timedelta, datetime.datetime, datetime.time] = None, last: Union[float, datetime.timedelta, datetime.datetime, datetime.time] = None, context: object = None, name: str = None, job_kwargs: JSONDict = None, ) -> 'Job': """Creates a new ``Job`` that runs at specified intervals and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. interval (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta`): The interval in which the job will run. If it is an :obj:`int` or a :obj:`float`, it will be interpreted as seconds. first (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \ :obj:`datetime.datetime` | :obj:`datetime.time`, optional): Time in or at which the job should run. This parameter will be interpreted depending on its type. * :obj:`int` or :obj:`float` will be interpreted as "seconds from now" in which the job should run. * :obj:`datetime.timedelta` will be interpreted as "time from now" in which the job should run. * :obj:`datetime.datetime` will be interpreted as a specific date and time at which the job should run. If the timezone (``datetime.tzinfo``) is :obj:`None`, the default timezone of the bot will be used. * :obj:`datetime.time` will be interpreted as a specific time of day at which the job should run. This could be either today or, if the time has already passed, tomorrow. If the timezone (``time.tzinfo``) is :obj:`None`, the default timezone of the bot will be used. Defaults to ``interval`` last (:obj:`int` | :obj:`float` | :obj:`datetime.timedelta` | \ :obj:`datetime.datetime` | :obj:`datetime.time`, optional): Latest possible time for the job to run. This parameter will be interpreted depending on its type. See ``first`` for details. If ``last`` is :obj:`datetime.datetime` or :obj:`datetime.time` type and ``last.tzinfo`` is :obj:`None`, the default timezone of the bot will be assumed. Defaults to :obj:`None`. context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the ``scheduler.add_job()``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. Note: `interval` is always respected "as-is". That means that if DST changes during that interval, the job might not run at the time one would expect. It is always recommended to pin servers to UTC time, then time related behaviour can always be expected. """ if not job_kwargs: job_kwargs = {} name = name or callback.__name__ job = Job(callback, context, name, self) dt_first = self._parse_time_input(first) dt_last = self._parse_time_input(last) if dt_last and dt_first and dt_last < dt_first: raise ValueError("'last' must not be before 'first'!") if isinstance(interval, datetime.timedelta): interval = interval.total_seconds() j = self.scheduler.add_job( callback, trigger='interval', args=self._build_args(job), start_date=dt_first, end_date=dt_last, seconds=interval, name=name, **job_kwargs, ) job.job = j return job def run_monthly( self, callback: Callable[['CallbackContext'], None], when: datetime.time, day: int, context: object = None, name: str = None, day_is_strict: bool = True, job_kwargs: JSONDict = None, ) -> 'Job': """Creates a new ``Job`` that runs on a monthly basis and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. when (:obj:`datetime.time`): Time of day at which the job should run. If the timezone (``when.tzinfo``) is :obj:`None`, the default timezone of the bot will be used. day (:obj:`int`): Defines the day of the month whereby the job would run. It should be within the range of 1 and 31, inclusive. context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. day_is_strict (:obj:`bool`, optional): If :obj:`False` and day > month.days, will pick the last day in the month. Defaults to :obj:`True`. job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the ``scheduler.add_job()``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. """ if not job_kwargs: job_kwargs = {} name = name or callback.__name__ job = Job(callback, context, name, self) if day_is_strict: j = self.scheduler.add_job( callback, trigger='cron', args=self._build_args(job), name=name, day=day, hour=when.hour, minute=when.minute, second=when.second, timezone=when.tzinfo or self.scheduler.timezone, **job_kwargs, ) else: trigger = OrTrigger([ CronTrigger( day=day, hour=when.hour, minute=when.minute, second=when.second, timezone=when.tzinfo, **job_kwargs, ), CronTrigger( day='last', hour=when.hour, minute=when.minute, second=when.second, timezone=when.tzinfo or self.scheduler.timezone, **job_kwargs, ), ]) j = self.scheduler.add_job(callback, trigger=trigger, args=self._build_args(job), name=name, **job_kwargs) job.job = j return job def run_daily( self, callback: Callable[['CallbackContext'], None], time: datetime.time, days: Tuple[int, ...] = Days.EVERY_DAY, context: object = None, name: str = None, job_kwargs: JSONDict = None, ) -> 'Job': """Creates a new ``Job`` that runs on a daily basis and adds it to the queue. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. time (:obj:`datetime.time`): Time of day at which the job should run. If the timezone (``time.tzinfo``) is :obj:`None`, the default timezone of the bot will be used. days (Tuple[:obj:`int`], optional): Defines on which days of the week the job should run (where ``0-6`` correspond to monday - sunday). Defaults to ``EVERY_DAY`` context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to :obj:`None`. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. job_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to pass to the ``scheduler.add_job()``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. Note: For a note about DST, please see the documentation of `APScheduler`_. .. _`APScheduler`: https://apscheduler.readthedocs.io/en/stable/modules/triggers/cron.html #daylight-saving-time-behavior """ if not job_kwargs: job_kwargs = {} name = name or callback.__name__ job = Job(callback, context, name, self) j = self.scheduler.add_job( callback, name=name, args=self._build_args(job), trigger='cron', day_of_week=','.join([str(d) for d in days]), hour=time.hour, minute=time.minute, second=time.second, timezone=time.tzinfo or self.scheduler.timezone, **job_kwargs, ) job.job = j return job def run_custom( self, callback: Callable[['CallbackContext'], None], job_kwargs: JSONDict, context: object = None, name: str = None, ) -> 'Job': """Creates a new customly defined ``Job``. Args: callback (:obj:`callable`): The callback function that should be executed by the new job. Callback signature for context based API: ``def callback(CallbackContext)`` ``context.job`` is the :class:`telegram.ext.Job` instance. It can be used to access its ``job.context`` or change it to a repeating job. job_kwargs (:obj:`dict`): Arbitrary keyword arguments. Used as arguments for ``scheduler.add_job``. context (:obj:`object`, optional): Additional data needed for the callback function. Can be accessed through ``job.context`` in the callback. Defaults to ``None``. name (:obj:`str`, optional): The name of the new job. Defaults to ``callback.__name__``. Returns: :class:`telegram.ext.Job`: The new ``Job`` instance that has been added to the job queue. """ name = name or callback.__name__ job = Job(callback, context, name, self) j = self.scheduler.add_job(callback, args=self._build_args(job), name=name, **job_kwargs) job.job = j return job def start(self) -> None: """Starts the job_queue thread.""" if not self.scheduler.running: self.scheduler.start() def stop(self) -> None: """Stops the thread.""" if self.scheduler.running: self.scheduler.shutdown() def jobs(self) -> Tuple['Job', ...]: """ Returns a tuple of all *pending/scheduled* jobs that are currently in the ``JobQueue``. """ return tuple( Job.from_aps_job(job, self) for job in self.scheduler.get_jobs()) def get_jobs_by_name(self, name: str) -> Tuple['Job', ...]: """Returns a tuple of all *pending/scheduled* jobs with the given name that are currently in the ``JobQueue``""" return tuple(job for job in self.jobs() if job.name == name)
# fig.set_facecolor('#222222') # ax.set_facecolor('#222222') if (download_plot == True): plt.savefig("data.png", bbox_inches='tight', pad_inches=0.05, dpi=150) # close the plot plt.close('all') return plot_vaccine(merged_totals, dates, True) scheduler = BackgroundScheduler() scheduler.add_job(scheduled_update, 'interval', seconds=30) scheduler.start() atexit.register(lambda: scheduler.shutdown(wait=False)) def hello(): return jsonify("hello").headers.add("Access-Control-Allow-Origin", "*") # Shut down the scheduler when exiting the app web_site.run(host='0.0.0.0', port=PORT) # web_site.run(host='0.0.0.0', port=8080) # DEPRECIATED FUNCTIONS FROM THE V1 WEBSITE
def Assign(request,pk): homework_course = get_object_or_404(course, pk=pk) user = User.objects.get(name=request.session.get('user_name')) print(homework_course.course_name) #email_title = 'test' #email_body = '你该交作业啦!' #email = '*****@*****.**' # 对方的邮箱 #email_from = user.email #send_status = send_mail(email_title, email_body, email_from, [email]) if request.method == "POST": print("343434") form = AssignForm(request.POST,request.FILES) if form.is_valid(): print("ccccc") deadline = form.cleaned_data['deadline'] name = form.cleaned_data['name'] #new_homework = models.Homework.objects.create() # new_homework.name = name # new_homework.content = content # new_homework.course = homework_course # # new_homework.deadline = deadline # new_homework.save() c = datetime.datetime.now() d = datetime.datetime.strptime(deadline, "%Y-%m-%d %H:%M:%S") if d < c: message = "deadline不能设置过去的时间" return redirect('new_homework', pk=pk) else: print("dddd") homework=form.save(commit=False) homework.course = homework_course homework.save() try: sched = BackgroundScheduler() @sched.scheduled_job('interval', seconds=1) def timed_job(): a=datetime.datetime.now() b=datetime.datetime.strptime(deadline,"%Y-%m-%d %H:%M:%S") for user in homework.course.users.all(): if user.identity == "student": flag = 0 for submit in homework.submit.all(): if submit.author.name == user.name: flag = 1 if flag == 0: if b>a : if (b-a).seconds == 86399 : email_title = '请尽快提交作业——作业提醒' email_body = '点击此处提交作业http://127.0.0.1:8000/course/'+pk+'/homework/' email = user.email # 对方的邮箱 send_status = send_mail(email_title, email_body, EMAIL_FROM, [email]) sched.start() except Exception as e: print(e) sched.shutdown() return redirect('homework_list',pk=pk) print("Nononono") print(form.errors) else: print("ttt") assign_form = AssignForm() # print(models.Homework.objects.all()) return render(request,'login/assign.html',locals())
def worker_loop(queue, parsed_args, setup_pyenv=False, single_experiment=False, fetch_artifacts=False, timeout=0): logger = logging.getLogger('worker_loop') hold_period = 4 while queue.has_next(): first_exp, ack_key = queue.dequeue(acknowledge=False) experiment_key = json.loads(first_exp)['experiment']['key'] config = json.loads(first_exp)['config'] parsed_args.config = config verbose = model.parse_verbosity(config.get('verbose')) logger.setLevel(verbose) logger.debug( 'Received experiment {} with config {} from the queue'.format( experiment_key, config)) executor = LocalExecutor(parsed_args) experiment = executor.db.get_experiment(experiment_key) if allocate_resources(experiment, config, verbose=verbose): def hold_job(): queue.hold(ack_key, hold_period) hold_job() sched = BackgroundScheduler() sched.add_job(hold_job, 'interval', minutes=hold_period / 2) sched.start() try: if setup_pyenv: logger.info('Setting up python packages for experiment') pipp = subprocess.Popen(['pip', 'install'] + experiment.pythonenv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pipout, _ = pipp.communicate() logger.info("pip output: \n" + pipout) # pip.main(['install'] + experiment.pythonenv) for tag, art in experiment.artifacts.iteritems(): if fetch_artifacts or 'local' not in art.keys(): logger.info('Fetching artifact ' + tag) if tag == 'workspace': # art['local'] = executor.db.store.get_artifact( # art, '.', only_newer=False) art['local'] = executor.db.store.get_artifact( art, only_newer=False) else: art['local'] = executor.db.store.get_artifact(art) executor.run(experiment) finally: sched.shutdown() queue.acknowledge(ack_key) if single_experiment: logger.info('single_experiment is True, quitting') return else: logger.info('Cannot run experiment ' + experiment.key + ' due lack of resources. Will retry') time.sleep(config['sleep_time']) wait_for_messages(queue, timeout, logger) # queue = glob.glob(fs_tracker.get_queue_directory() + "/*") logger.info("Queue in {} is empty, quitting".format( fs_tracker.get_queue_directory()))
class ListenerService(MashService): """ Base class for MASH services that live in the image listener. """ def post_init(self): """Initialize base service class and job scheduler.""" self.listener_queue = 'listener' self.service_queue = 'service' self.job_document_key = 'job_document' self.listener_msg_key = 'listener_msg' self.jobs = {} # setup service job directory self.job_directory = self.config.get_job_directory( self.service_exchange) os.makedirs(self.job_directory, exist_ok=True) self.prev_service = self._get_previous_service() if not self.custom_args: self.custom_args = {} if 'job_factory' not in self.custom_args: raise MashListenerServiceException( 'Job factory is required as a custom arg in listener service.') else: self.job_factory = self.custom_args['job_factory'] logfile_handler = setup_logfile( self.config.get_log_file(self.service_exchange)) self.log.addHandler(logfile_handler) self.bind_queue(self.service_exchange, self.job_document_key, self.service_queue) self.bind_queue(self.prev_service, self.listener_msg_key, self.listener_queue) thread_pool_count = self.custom_args.get( 'thread_pool_count', self.config.get_base_thread_pool_count()) executors = {'default': ThreadPoolExecutor(thread_pool_count)} self.scheduler = BackgroundScheduler(executors=executors, timezone=utc) self.scheduler.add_listener( self._process_job_result, events.EVENT_JOB_EXECUTED | events.EVENT_JOB_ERROR) self.scheduler.add_listener(self._process_job_missed, events.EVENT_JOB_MISSED) signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop) restart_jobs(self.job_directory, self._add_job) self.start() def _add_job(self, job_config): """ Create job using job factory if job id does not already exist. Job config is persisted to disk if not already done. """ job_id = job_config['id'] if job_id not in self.jobs: try: job = self.job_factory.create_job(job_config, self.config) except Exception as error: self.log.error('Invalid job: {0}.'.format(error)) else: self.jobs[job.id] = job job.log_callback = self.log if 'job_file' not in job_config: job_config['job_file'] = '{0}job-{1}.json'.format( self.job_directory, job_id) persist_json(job_config['job_file'], job_config) job.job_file = job_config['job_file'] self.log.info('Job queued, awaiting listener message.', extra=job.get_job_id()) else: self.log.warning('Job already queued.', extra={'job_id': job_id}) def _cleanup_job(self, job_id): """ Job failed upstream. Delete job and notify the next service. """ job = self.jobs[job_id] self.log.warning('Failed upstream.', extra=job.get_job_id()) self._delete_job(job.id) message = self._get_status_message(job) self._publish_message(message, job.id) def _delete_job(self, job_id): """ Remove job from file store and delete from listener queue. Also attempt to remove any running instances of the job. """ if job_id in self.jobs: job = self.jobs[job_id] self.log.info('Deleting job.', extra=job.get_job_id()) del self.jobs[job_id] remove_file(job.job_file) else: self.log.warning('Job deletion failed, job is not queued.', extra={'job_id': job_id}) def _get_previous_service(self): """ Return the previous service based on the current exchange. """ services = self.config.get_service_names() try: index = services.index(self.service_exchange) - 1 except ValueError: return None if index < 0: return None return services[index] def _get_status_message(self, job): """ Build and return json message. Message contains completion status to post to next service exchange. """ key = '{0}_result'.format(self.service_exchange) return JsonFormat.json_message({key: job.get_status_message()}) def _handle_listener_message(self, message): """ Callback for listener messages. """ listener_msg = self._get_listener_msg( message.body, '{0}_result'.format(self.prev_service)) job_id = None if listener_msg: status = listener_msg['status'] job_id = listener_msg['id'] if job_id and job_id in self.jobs: job = self.jobs[listener_msg['id']] job.listener_msg = message job.set_status_message(listener_msg) if status == SUCCESS: self._schedule_job(job.id) return # Don't ack message until job finishes else: self._cleanup_job(job_id) message.ack() def _handle_service_message(self, message): """ Callback for events from jobcreator. """ job_key = '{0}_job'.format(self.service_exchange) try: job_desc = json.loads(message.body) self._add_job(job_desc[job_key]) except Exception as e: self.log.error('Error adding job: {0}.'.format(e)) message.ack() def _process_job_result(self, event): """ Callback when job background process finishes. Handle exceptions and errors that occur and logs info to job log. """ job_id = event.job_id job = self.jobs[job_id] metadata = job.get_job_id() self._delete_job(job_id) if event.exception: job.status = EXCEPTION msg = 'Exception in {0}: {1}'.format(self.service_exchange, event.exception) job.add_error_msg(msg) self.log.error(msg, extra=metadata) elif job.status == SUCCESS: self.log.info('{0} successful.'.format(self.service_exchange), extra=metadata) else: self.log.error('Error occurred in {0}.'.format( self.service_exchange), extra=metadata) message = self._get_status_message(job) self._publish_message(message, job.id) job.listener_msg.ack() def _process_job_missed(self, event): """ Callback when job background process misses execution. This should not happen as no jobs are scheduled, log any occurrences. """ job_id = event.job_id job = self.jobs[job_id] metadata = job.get_job_id() self.log.warning('Job missed during {0}.'.format( self.service_exchange), extra=metadata) def _publish_message(self, message, job_id): """ Publish message to next service exchange. """ try: self.publish_job_result(self.service_exchange, message) except AMQPError: self.log.warning('Message not received: {0}'.format(message), extra={'job_id': job_id}) def _schedule_job(self, job_id): """ Schedule new job in background scheduler for job based on id. """ try: self.scheduler.add_job(self._start_job, args=(job_id, ), id=job_id, max_instances=1, misfire_grace_time=None, coalesce=True) except ConflictingIdError: self.log.warning( 'Job already running. Received multiple ' 'listener messages.', extra={'job_id': job_id}) def _start_job(self, job_id): """ Process job based on job id. """ job = self.jobs[job_id] job.process_job() def _get_listener_msg(self, message, key): """Load json and attempt to get message by key.""" try: listener_msg = json.loads(message)[key] except Exception: self.log.error('Invalid listener message: {0}, ' 'missing key: {1}'.format(message, key)) listener_msg = None return listener_msg def publish_job_result(self, exchange, message): """ Publish the result message to the listener queue on given exchange. """ self._publish(exchange, self.listener_msg_key, message) def start(self): """ Start listener service. """ self.scheduler.start() self.consume_queue(self._handle_service_message, self.service_queue, self.service_exchange) self.consume_queue(self._handle_listener_message, self.listener_queue, self.prev_service) try: self.channel.start_consuming() except Exception: self.stop() raise def stop(self, signum=None, frame=None): """ Gracefully stop the service. Shutdown scheduler and wait for running jobs to finish. Close AMQP connection. """ if signum: self.log.info( 'Got a TERM/INTERRUPT signal, shutting down gracefully.') else: self.log.info('An unhandled Exception occurred in event loop, ' 'shutting down gracefully.') self.scheduler.shutdown() self.close_connection()
time.sleep(600) pi.write(station[5], 1) time.sleep(1) pi.write(station[6], 0) time.sleep(600) pi.write(station[6], 1) running = False if __name__ == '__main__': # Maker sure that all stations are turned off initially all_off() scheduler = BackgroundScheduler() # Change this to manage what days and time you want to run your job scheduler.add_job(job1, 'cron', day_of_week='mon,wed,fri,sun', hour=6, minute=30) scheduler.start() print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) server = ThreadedTCPServer(('', 5555), ThreadedServer) server.serve_forever() timer = ThreadedTimer() all_off() client.close() scheduler.shutdown(wait=False)
def runtimeReport(): print('*' * 15 + 'Time Report! The time is: %s' % datetime.now()) robot_timereport.sendTextEmail('Running report', 'program is running', 'timereport') print('*' * 15 + 'Time Report over! The time is: %s' % datetime.now()) def Start(): runtimeStart() run360() runWooYun() runFreeBuf() if __name__ == '__main__': Start() scheduler = BackgroundScheduler() scheduler.add_job(run360, 'interval', seconds=1800) scheduler.add_job(runWooYun, 'interval', seconds=300) scheduler.add_job(runFreeBuf, 'interval', seconds=3600) scheduler.add_job(runtimeReport, 'interval', seconds=43200) scheduler.start() try: # This is here to simulate application activity (which keeps the main thread alive). while True: time.sleep(2) except (KeyboardInterrupt, SystemExit): scheduler.shutdown( ) # Not strictly necessary if daemonic mode is enabled but should be done if possible
class WebApp(Flask): """ A web application. """ def __init__(self, name, config): """ Create a new web application. :param name: the application name. :param config: the configuration. """ Flask.__init__(self, name) self.config.from_object(config) self.api = Api(self) # JSON Response self.json_encoder = SimpleJSONEncoder self.api.representations = OrderedDict(REPRESENTATIONS) # Error Handling for exc in default_exceptions: self.register_error_handler(exc, error_ctrl.handle_exception) self.register_error_handler(Exception, error_ctrl.handle_exception) # Logging level = logging._nameToLevel[self.config["LOG_LEVEL"]] logging.basicConfig(level=level, handlers=[ConsoleHandler(level, FORMATTER)]) logging.getLogger("apscheduler.scheduler").setLevel(logging.ERROR) logging.getLogger("apscheduler.executors.default").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) logging.getLogger("redis_lock").setLevel(logging.ERROR) self.logger.setLevel(level) for hdlr in self.logger.handlers: hdlr.setFormatter(FORMATTER) # Scheduler self.scheduler = None def start(self, host="0.0.0.0", port=None): """ Start the application. :param host: the server address. :param port: the server port number. :return: None """ if self.scheduler is not None: self.scheduler.start() Flask.run(self, host, port, threaded=True, use_reloader=False) def shutdown(self): """ Shutdown the application. :return: None """ fn_shutdown = request.environ.get("werkzeug.server.shutdown") if fn_shutdown is None: raise RuntimeError("Not running with the Werkzeug Server") fn_shutdown() def add_rest_api(self, res, url): """ Register a REST interface. :param res: the resource. :param url: the url. :return: None """ self.api.add_resource(res, url) def add_teardown_hook(self, func, *args, **kwargs): """ Register a teardown hook. :param func: the function. :param args: optional arguments to pass to func. :param kwargs: optional keyword arguments to pass to func :return: None """ self.teardown_appcontext(func) def add_shutdown_hook(self, func, *args, **kwargs): """ Register a shutdown hook. :param func: the function. :param args: optional arguments to pass to func. :param kwargs: optional keyword arguments to pass to func :return: None """ if args is None and kwargs is None: atexit.register(func) else: atexit.register(func) def add_scheduled_job(self, job): """ Add a scheduler. :param job: a scheduler job :return: None """ if self.scheduler is None: self.scheduler = BackgroundScheduler() self.add_shutdown_hook(self._shutdown_scheduler) self.scheduler.add_job(id=job.name, func=job.func, kwargs=job.kwargs, trigger=job.trigger, max_instances=1, coalesce=True) def _start_scheduler(self): """ Start the app scheduler. :return: None """ self.scheduler.start() def _shutdown_scheduler(self): """ Shut down the app scheduler. :return: None """ if self.scheduler is not None and self.scheduler.state is not STATE_STOPPED: self.scheduler.shutdown()
class ChromecastDriver(object): class Listener(object): def __init__(self, expected_url_prefix, callback_another_cast_started): self.expected_url_prefix = expected_url_prefix self.callback_another_cast_started = callback_another_cast_started self.last_media = None def new_media_status(self, status): # Sometimes empty status appear. Should be safe to ignore since if # content == null then no one must be casting. I hope. if status.content_id is None: return self.last_media = status.content_id # If content_id doesn't have our server prefix someone else started # casting and we should seppuku if self.last_media.find(self.expected_url_prefix) != 0: self.callback_another_cast_started() def __init__(self, logger, target_chromecast_name, img_url_provider, interval_seconds): """ Run a background task: every $interval seconds to load a new url in a chromecast, as provided by $img_url_provider Constructor will look for all chromcasts in the network (ie: slow) target_chromecast: Name of CC to use $img_url_provider should provide a unique URL each time it's called """ self.cleanup_on_exit = True self.logger = logger self.target_chromecast_name = target_chromecast_name self.img_url_provider = img_url_provider self.interval_seconds = interval_seconds # Try to find the right chromecast self.logger.info('Looking for all Chromecasts in the network') all_casts = pychromecast.get_chromecasts() try: self.cast = next(cc for cc in all_casts if cc.device.friendly_name == target_chromecast_name) except StopIteration: all_casts_names = [cc.device.friendly_name for cc in all_casts] self.logger.error('Chromecast {} not found. These are available: {}'.format( target_chromecast, all_casts_names)) raise pychromecast.NoChromecastFoundError() self.logger.info('Found {}, connecting...'.format(self.target_chromecast_name)) self.cast.wait() self.cast.quit_app() self.cast.wait() # Register callback for status changes self.cc_listener = ChromecastDriver.Listener(img_url_provider.get_url_prefix(), self.on_another_cast_started) self.cast.media_controller.register_status_listener(self.cc_listener) # Call show_image once to load the first one (otherwise we need to # wait for the first interval trigger) self.show_image() # Call self every $interval_seconds to reload image self.scheduler = BackgroundScheduler() self.sched_job_obj = self.scheduler.add_job(func=self.show_image, trigger="interval", seconds=interval_seconds) self.scheduler.start() # TODO: Move from atexit to main obj? atexit.register(self.disconnect) def on_another_cast_started(self): self.logger.info('Someone else started casting to {}! Will shutdown...'.\ format(self.target_chromecast_name)) # pychromecast doesn't like shutting down while on a listener thread, so # instead we remove our 'show new image' job and schedule a disconnect self.sched_job_obj.remove() self.cleanup_on_exit = False print("RQ CLEAN SHUTDOWN") self.scheduler.add_job(func=self.disconnect, trigger="interval", seconds=1) def disconnect(self): self.logger.info('Shutdown: disconnecting from Chromecast') self.scheduler.shutdown() if self.cleanup_on_exit: self.cast.quit_app() self.cast.wait() self.cast.disconnect() self.cast.join() def show_image(self): url = self.img_url_provider.get_random_image_url() self.logger.info('Asking CC {} to load image {}'.format(self.target_chromecast_name, url)) # TODO: Hardcoded mime type might break self.cast.play_media(url=url, content_type='image/jpeg') self.cast.wait() # TODO: Configure timeout count timeout_count = 5 while timeout_count > 0: if self.cc_listener.last_media == url: self.logger.debug('Image should be shown') break timeout_count -= 1 try: time.sleep(1) except Exception as ex: raise ex if timeout_count == 0: self.logger.error('Image display seems to have failed')
class Client(object): """The propsd client Keyword Args: propsd_server (str): The server hostname/ip address (default localhost) propsd_port (int): The server port (default 9100) """ def __init__(self, propsd_server='localhost', propsd_port=9100): self.propsd_server = propsd_server self.propsd_port = propsd_port self.__update_callbacks = [] self.__update_properties_previous = {} self.__update_scheduler = BackgroundScheduler({ 'apscheduler.jobstores.default': { 'class': 'apscheduler.jobstores.memory:MemoryJobStore', }, 'apscheduler.executors.default': { 'class': 'apscheduler.executors.pool:ThreadPoolExecutor', 'max_workers': '1' }, 'apscheduler.job_defaults.coalesce': 'true', 'apscheduler.job_defaults.max_instances': '1', 'apscheduler.timezone': 'UTC', }) self.__update_job = self.__update_scheduler.add_job( self.__update_properties, 'interval', seconds=1, id='update-check-job') self.__update_scheduler.start() def get(self, key): """Gets a specific property Args: key (str): The key to retrieve Returns: str: The property value, or None. """ try: response = requests.get( "http://%s:%d/v1/conqueso/api/roles/default/properties/%s" % (self.propsd_server, self.propsd_port, key)) return response.text except: log.warn("Could not retrieve property value") def properties(self): """Gets all propsd properties Returns: dict: The complete propsd property set """ try: response = requests.get("http://%s:%d/v1/properties" % (self.propsd_server, self.propsd_port)) return json.loads(response.text) except: log.warn("Could not retrieve property value") def status(self): """Gets the status of the propsd service Returns: dict: A dictionary containing the status parameters. """ response = requests.get("http://%s:%d/v1/status" % (self.propsd_server, self.propsd_port)) return json.loads(response.text) def health(self): """Gets the health of the propsd service Returns: dict: A dictionary containing the health parameters. """ response = requests.get("http://%s:%d/v1/health" % (self.propsd_server, self.propsd_port)) return json.loads(response.text) def subscribe(self, search, callback): """Subscribe to document changes Args: search (str): The objectpatch search string callback (object): The function to call """ self.__update_callbacks.append({ 'search': search, 'callback': callback }) def shutdown(self): """Shuts down the propsd client """ self.__update_scheduler.shutdown() def __update_properties(self): properties = self.properties() for item in self.__update_callbacks: search = item['search'] thistree = objectpath.Tree(properties) thisresult = thistree.execute(search) thattree = objectpath.Tree(self.__update_properties_previous) thatresult = thattree.execute(search) if thisresult != thatresult: item['callback'](search, properties, thisresult) self.__update_properties_previous = properties