class UpdateSuccessRate: """ Author: Marten Bolin Date: 2017-11-22 Last update: Purpose: This will be ran continously to check for any changes made and rearrange the model """ def __init__(self, daemon=False): """ Author: Marten Bolin Date: 2017-11-22 Last update: Purpose: The instantiation of the class, make sure to catch it in a variable so it can be terminated properly. Start the scheduler :param daemon : Sets daemon, (Optional) Default is False """ # Set up and start the scheduler self.scheduled = BackgroundScheduler() if not daemon: # TODO Should use a good method that doesnt change a protected member # TODO of a class directly self.scheduled._daemon = False self.scheduled.add_job(self._run, 'interval', days=1, id="3") self.scheduled.start() self.scheduled.modify_job(job_id="3", next_run_time=datetime.now()) @staticmethod def _run(): """ Author: Marten Bolin, John Andree Lidquist Date:2017-11-22 Last update: 2017-11-23 Purpose: The actual process to be ran. Adds the success rate to the database. """ print("Updating success rate") InsertSuccessRate().insert_success_rate() def terminate(self): """ Author: Marten Bolin Date: 2017-11-22 Last update: Purpose: Terminates the process """ print("Shutting down update_success_rate..") self.scheduled.shutdown() print("Update_success_rate has been shut down.")
class Scheduler(object): def __init__(self): self._scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults) self._scheduler.add_jobstore('redis', jobs_key='crontpy.jobs', run_times_key='crontpy.run_times') @property def running(self): return self._scheduler.running def start(self): self._scheduler.start() def shutdown(self, wait=True): self._scheduler.shutdown(wait) def pause(self): self._scheduler.pause() def resume(self): self._scheduler.resume() def get_jobs(self): return self._scheduler.get_jobs() def get_job(self, jid): return self._scheduler.get_job(job_id=jid) def run_job(self, jid): job = self.get_job(jid) if not job: raise Exception('job id:{0} not found'.format(jid)) job.func(*job.args, **job.kwargs) def resume_job(self, jid): self._scheduler.resume_job(job_id=jid) def pause_job(self, jid): self._scheduler.pause_job(job_id=jid) def modify_job(self, jid, **changes): return self._scheduler.modify_job(job_id=jid, **changes) def delete_job(self, jid): self._scheduler.remove_job(job_id=jid)
class Trigger: def __init__(self, app): self.scheduler = None self.app = app def setup(self): self.scheduler = BackgroundScheduler({ 'apscheduler.jobstores.default': { 'type': 'sqlalchemy', 'url': self.app.config[ "TRIGGER_DATABASE_URL"] #os.environ.get('TRIGGER_DATABASE_URL') }, 'apscheduler.executors.processpool': { 'type': 'processpool', 'max_workers': '30' }, 'apscheduler.job_defaults.coalesce': 'false', 'apscheduler.job_defaults.max_instances': '20', 'apscheduler.timezone': 'UTC', }) def start(self): self.scheduler.start() def is_running(self): return self.scheduler.running() def shutdown(self): self.scheduler.shutdown() def load_job_list(self): with self.app.app_context(): projects = AutoProject.query.all() # key_list = ("minute", "hour", "day", "month", "day_of_week") for p in projects: if self.scheduler.get_job(p.id) is None: cron = p.cron.replace("\n", "").strip().split(" ") #print(cron) if len(cron) < 5: continue j = self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, replace_existing=True, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % p.id, args=(p.id, )) def update_job(self, id): with self.app.app_context(): p = AutoProject.query.filter_by(id=id).first() cron = p.cron.replace("\n", "").strip().split(" ") if len(cron) < 5: return False if self.scheduler.get_job(id) is None: self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, eplace_existing=True, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], id="%s" % id, args=(id, )) else: self.scheduler.modify_job(job_id="%s" % id, name=p.name, eplace_existing=True, minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4], args=(id, )) return True def remove_job(self, id): if self.scheduler.get_job(id) is not None: self.scheduler.remove_job(id) def pause_job(self, id): pass def resume_job(self, id): pass def get_jobs(self): to_zone = tz.gettz("CST") jobs = self.scheduler.get_jobs() data = {"total": len(jobs), "rows": []} urls = { "pass": "******", "fail": "fail.png", "running": "run.gif" } for job in jobs: status = "running" task = AutoTask.query.filter_by(project_id=job.id).order_by( AutoTask.build_no.desc()).first() output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no) if os.path.exists(output_dir + "/report.html"): tree = ET.parse(output_dir + "/output.xml") root = tree.getroot() #passed = root.find("./statistics/suite/stat").attrib["pass"] fail = root.find("./statistics/suite/stat").attrib["fail"] if int(fail) != 0: status = 'fail' else: status = 'pass' data["rows"].append({ "id": job.id, "name": job.name, "status": status, "url": url_for('static', filename='images/%s' % urls[status]), "cron": AutoProject.query.filter_by(id=job.id).first().cron, "next_run_time": job.next_run_time.astimezone(to_zone).strftime( "%Y-%m-%d %H:%M:%S") }) return data def print_jobs(self): pass
class BileanScheduler(object): """Billing scheduler based on apscheduler""" job_types = ( NOTIFY, DAILY, FREEZE, ) = ( 'notify', 'daily', 'freeze', ) trigger_types = (DATE, CRON) = ('date', 'cron') def __init__(self, **kwargs): super(BileanScheduler, self).__init__() self._scheduler = BackgroundScheduler() self.notifier = notifier.Notifier() self.engine_id = kwargs.get('engine_id', None) self.context = kwargs.get('context', None) if not self.context: self.context = bilean_context.get_admin_context() if cfg.CONF.bilean_task.store_ap_job: self._scheduler.add_jobstore(cfg.CONF.bilean_task.backend, url=cfg.CONF.bilean_task.connection) def init_scheduler(self): """Init all jobs related to the engine from db.""" jobs = db_api.job_get_all(self.context, engine_id=self.engine_id) if not jobs: LOG.info(_LI("No job found from db")) return True for job in jobs: if self.bilean_scheduler.is_exist(job.id): continue task_name = "_%s_task" % (job.job_type) task = getattr(self, task_name) self.bilean_task.add_job(task, job.id, job_type=job.job_type, params=job.parameters) def add_job(self, task, job_id, trigger_type='date', **kwargs): """Add a job to scheduler by given data. :param str|unicode user_id: used as job_id :param datetime alarm_time: when to first run the job """ mg_time = cfg.CONF.bilean_task.misfire_grace_time job_time_zone = cfg.CONF.bilean_task.time_zone user_id = job_id.split('-')[1] if trigger_type == 'date': run_date = kwargs.get('run_date') if run_date is None: msg = "Param run_date cannot be None for trigger type 'date'." raise exception.InvalidInput(reason=msg) self._scheduler.add_job(task, 'date', timezone=job_time_zone, run_date=run_date, args=[user_id], id=job_id, misfire_grace_time=mg_time) return True # Add a cron type job hour = kwargs.get('hour', None) minute = kwargs.get('minute', None) if not hour or not minute: hour, minute = self._generate_timer() self._scheduler.add_job(task, 'cron', timezone=job_time_zone, hour=hour, minute=minute, args=[user_id], id=job_id, misfire_grace_time=mg_time) return True def modify_job(self, job_id, **changes): """Modifies the properties of a single job. Modifications are passed to this method as extra keyword arguments. :param str|unicode job_id: the identifier of the job """ self._scheduler.modify_job(job_id, **changes) def remove_job(self, job_id): """Removes a job, preventing it from being run any more. :param str|unicode job_id: the identifier of the job """ self._scheduler.remove_job(job_id) def start(self): LOG.info(_('Starting Billing scheduler')) self._scheduler.start() def stop(self): LOG.info(_('Stopping Billing scheduler')) self._scheduler.shutdown() def is_exist(self, job_id): """Returns if the Job exists that matches the given ``job_id``. :param str|unicode job_id: the identifier of the job :return: True|False """ job = self._scheduler.get_job(job_id) return job is not None def _notify_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) msg = {'user': user.id, 'notification': 'The balance is almost use up'} self.notifier.info('billing.notify', msg) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'notify')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) self._add_freeze_job(user) def _daily_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'daily')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def _freeze_task(self, user_id): user = user_mod.User.load(self.context, user_id=user_id) if user.status != user.FREEZE and user.rate > 0: user.do_bill(self.context) try: db_api.job_delete( self.context, self._generate_job_id(user.id, 'freeze')) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def _add_notify_job(self, user): if not user.rate: return False total_seconds = user['balance'] / user['rate'] prior_notify_time = cfg.CONF.bilean_task.prior_notify_time * 3600 notify_seconds = total_seconds - prior_notify_time notify_seconds = notify_seconds if notify_seconds > 0 else 0 run_date = timeutils.utcnow() + timedelta(seconds=notify_seconds) job_params = {'run_date': run_date} job_id = self._generate_job_id(user['id'], self.NOTIFY) self.add_job(self._notify_task, job_id, params=job_params) # Save job to database job = {'id': job_id, 'job_type': self.NOTIFY, 'engine_id': self.engine_id, 'parameters': {'run_date': run_date}} db_api.job_create(self.context, job) def _add_freeze_job(self, user): if not user.rate: return False total_seconds = user.balance / user.rate run_date = timeutils.utcnow() + timedelta(seconds=total_seconds) job_params = {'run_date': run_date} job_id = self._generate_job_id(user.id, self.FREEZE) self.add_job(self._freeze_task, job_id, params=job_params) # Save job to database job = {'id': job_id, 'job_type': self.FREEZE, 'engine_id': self.engine_id, 'parameters': {'run_date': run_date}} db_api.job_create(self.context, job) return True def _add_daily_job(self, user): job_id = self._generate_job_id(user.id, self.DAILY) params = {'hour': random.randint(0, 23), 'minute': random.randint(0, 59)} self.add_job(self._daily_task, job_id, trigger_type='cron', params=params) # Save job to database job = {'id': job_id, 'job_type': self.DAILY, 'engine_id': self.engine_id, 'parameters': params} db_api.job_create(self.context, job) return True def _delete_all_job(self, user): for job_type in self.job_types: job_id = self._generate_job_id(user.id, job_type) if self.is_exist(job_id): self.remove_job(job_id) try: db_api.job_delete(self.context, job_id) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) def update_user_job(self, user): """Update user's billing job""" if user.status not in [user.ACTIVE, user.WARNING]: self._delete_all_job(user.id) return for job_type in self.NOTIFY, self.FREEZE: job_id = self._generate_job_id(user.id, job_type) if self.is_exist(job_id): self.remove_job(job_id) try: db_api.job_delete(self.context, job_id) except exception.NotFound as e: LOG.warn(_("Failed in deleting job: %s") % six.text_type(e)) daily_job_id = self._generate_job_id(user.id, self.DAILY) if not self.is_exist(daily_job_id): self._add_daily_job(user) if user.status == user.ACTIVE: self._add_notify_job(user) else: self._add_freeze_job(user) def _generate_timer(self): """Generate a random timer include hour and minute.""" hour = random.randint(0, 23) minute = random.randint(0, 59) return hour, minute def _generate_job_id(self, user_id, job_type): """Generate job id by given user_id and job type""" return "%s-%s" % (job_type, user_id)
class CronManager: def __init__(self, use_mongo_db=True): self.scheduler = BackgroundScheduler(timezone=shanghai_tz) self.scheduler.configure() if use_mongo_db: self.job_store = MongoDBJobStore(database='apscheduler', collection='cronTab', client=db) self.scheduler.add_jobstore(self.job_store) self.is_replace_existing = True else: self.is_replace_existing = False def add_cron(self, cron_instance): if not isinstance(cron_instance, Cron): raise TypeError('please add correct cron!') if cron_instance.trigger_type == 'interval': seconds = cron_instance.trigger_args.get('seconds') if not isinstance(seconds, int) and not common.can_convert_to_int(seconds): raise TypeError('请输入合法的时间间隔!') seconds = int(seconds) if seconds <= 0: raise TypeError('请输入大于0的时间间隔!') job = self.scheduler.add_job( func=cron_instance.cron_mission, trigger=cron_instance.trigger_type, seconds=seconds, replace_existing=self.is_replace_existing, coalesce=True, id=cron_instance.get_id(), max_instances=5, jitter=0) # 玄学,新增job的时候不用加args,直接加对象调用的func elif cron_instance.trigger_type == 'date': run_date = cron_instance.trigger_args.get('run_date') # TODO 判断run_date类型 job = self.scheduler.add_job( func=cron_instance.cron_mission, trigger=cron_instance.trigger_type, run_date=run_date, replace_existing=self.is_replace_existing, coalesce=True, id=cron_instance.get_id()) # 玄学,新增job的时候不用加args,直接加对象调用的func elif cron_instance.trigger_type == 'cron': raise TypeError('暂时不支持 trigger_type 等于 \'cron\'') return cron_instance.get_id() def start(self, paused=False): self.scheduler.start(paused=paused) def pause_cron(self, cron_id=None, pause_all=False): if pause_all: self.scheduler.pause() elif cron_id: self.scheduler.pause_job(job_id=cron_id) def resume_cron(self, cron_id=None, resume_all=False): if resume_all: self.scheduler.resume() elif cron_id: self.scheduler.resume_job(job_id=cron_id) def del_cron(self, cron_id=None, del_all=False): if del_all: self.scheduler.remove_all_jobs() elif cron_id: self.scheduler.remove_job(job_id=cron_id) def update_cron(self, cron_id, cron_info): if not isinstance(cron_id, str): raise TypeError('cron_id must be str') if not isinstance(cron_info, dict): raise TypeError('cron_info must be dict') trigger_type = cron_info.get('triggerType') interval = cron_info.get('interval') run_date = cron_info.get('runDate') test_case_suite_id_list = cron_info.get('testCaseSuiteIdList') is_execute_forbiddened_case = cron_info.get('isExecuteForbiddenedCase') test_case_id_list = cron_info.get('testCaseIdList') test_domain = cron_info.get('testDomain') global_vars_id = cron_info.get('globalVarsId') alarm_mail_list = cron_info.get('alarmMailList') is_ding_ding_notify = cron_info.get('isDingDingNotify') ding_ding_access_token = cron_info.get('dingdingAccessToken') ding_ding_notify_strategy = cron_info.get('dingdingNotifyStrategy') is_enterprise_wechat_notify = cron_info.get('isEnterpriseWechatNotify') enterprise_wechat_access_token = cron_info.get( 'enterpriseWechatAccessToken') enterprise_wechat_notify_strategy = cron_info.get( 'enterpriseWechatNotifyStrategy') cron_name = cron_info.get('name') try: if trigger_type == 'interval' and int(interval) > 0: self.scheduler.modify_job( job_id=cron_id, trigger=IntervalTrigger(seconds=interval)) elif trigger_type == 'date': # TODO 判断run_date类型 self.scheduler.modify_job( job_id=cron_id, trigger=DateTrigger(run_date=run_date)) else: raise TypeError('更新定时任务触发器失败!') if run_date: cron = Cron( test_case_suite_id_list=test_case_suite_id_list, is_execute_forbiddened_case=is_execute_forbiddened_case, test_domain=test_domain, global_vars_id=global_vars_id, alarm_mail_list=alarm_mail_list, is_ding_ding_notify=is_ding_ding_notify, ding_ding_access_token=ding_ding_access_token, ding_ding_notify_strategy=ding_ding_notify_strategy, is_enterprise_wechat_notify=is_enterprise_wechat_notify, enterprise_wechat_access_token= enterprise_wechat_access_token, enterprise_wechat_notify_strategy= enterprise_wechat_notify_strategy, trigger_type=trigger_type, # 更新定时器时,此参数并没有真正起到作用, 仅修改展示字段 test_case_id_list=test_case_id_list, run_date=run_date, cron_name=cron_name) # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 else: cron = Cron( test_case_suite_id_list=test_case_suite_id_list, is_execute_forbiddened_case=is_execute_forbiddened_case, test_domain=test_domain, global_vars_id=global_vars_id, alarm_mail_list=alarm_mail_list, is_ding_ding_notify=is_ding_ding_notify, ding_ding_access_token=ding_ding_access_token, ding_ding_notify_strategy=ding_ding_notify_strategy, is_enterprise_wechat_notify=is_enterprise_wechat_notify, enterprise_wechat_access_token= enterprise_wechat_access_token, enterprise_wechat_notify_strategy= enterprise_wechat_notify_strategy, trigger_type=trigger_type, # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 test_case_id_list=test_case_id_list, seconds=interval, # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 cron_name=cron_name) # 玄学,更改job的时候必须改args,不能改func self.scheduler.modify_job(job_id=cron_id, coalesce=True, args=[cron]) except BaseException as e: raise TypeError('更新定时任务失败: %s' % e) def shutdown(self, force_shutdown=False): if force_shutdown: self.scheduler.shutdown(wait=False) else: self.scheduler.shutdown(wait=True) def get_crons(self): return self.scheduler.get_jobs()
class MainRunner(object): workers = {} dirpath = '.' defaultOutputPath = 'output' def __init__(self, dirpath='.'): ''' local path for load config ''' logger.info("Initialing Main Runner for Hasal agent") if os.path.isdir(dirpath): self.dirpath = dirpath logger.info("loading runner config folder: " + dirpath) else: logger.info(dirpath + " is invalid, use default path instead") self.scheduler = BackgroundScheduler() self.scheduler.start() self.load_dir(self.dirpath) event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True) event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove) observer = Observer() observer.schedule(event_handler, self.dirpath, recursive=True) observer.start() def load_dir(self, folder): (dirpath, dirnames, filenames) = os.walk(folder).next() for fname in filenames: if 'json' in fname[-4:]: self.load(os.path.join(dirpath, fname)) def load(self, fp): ''' given a json file, load and create a task run regularly ''' logger.info(fp + " was loaded!") with open(fp) as in_data: try: data = json.load(in_data) data['name'] = "Jenkins Job" data['path'] = fp except ValueError as e: logger.warning(fp + " loaded failed: " + e.message) return None interval = 30 if 'interval' in data: interval = int(data['interval']) if 'output' in data: if 'defaultOutputPath' in data['output']: self.defaultOutputPath = data['output']['defaultOutputPath'] if 'dirpath' in data['output']: data['output']['outputPath'] = os.path.join(self.defaultOutputPath, data['output']['dirpath']) else: data['output'] = {'outputPath': self.defaultOutputPath} if fp in self.workers: # existing runner found logger.info("Update exisitng runner [%s]" % fp) runner = self.workers[fp] runner.update(**data) # //memo: Interval can't be modified self.scheduler.modify_job(job_id=fp, func=runner.run, name=runner.name ) else: # Create new logger.info("Create new runner [%s]" % fp) module_path = "hasalTask" object_name = "HasalTask" try: runner_module = getattr(importlib.import_module( module_path), object_name) except Exception as e: logger.exception(e) return None runner = runner_module(**data) self.workers[fp] = runner self.scheduler.add_job(runner.run, 'interval', id=fp, name=runner.name, seconds=interval ) return runner def list(self): ''' to list all configs loaded format: [squence number] [minion name] [config_path] [status] ''' for (fp, worker) in self.workers: logger.info("path=" + fp + "," + str(worker) + ";") def remove(self, fp): ''' given file path, stop running instance if possible ''' if fp in self.workers: self.workers[fp].onstop() self.scheduler.remove_job(job_id=fp) del self.workers[fp] return True return False def remove_advanced(self): ''' TODO: 1. remove by start, end 2. by directory(?) ''' pass def unload_all(self): ''' stop all running instances ''' self.scheduler.shutdown() def pause(self, fp): ''' simply stop running instance but not remove config TODO: should have timeout if stop failed ''' self.scheduler.pause(job_id=fp) def resume(self, sn): # not sure we can do this pass def __del__(self): self.unload_all() def get_config(self): conf = {} return conf def _wake(self): ''' For periodical minions, waking them according to timing ''' pass
class MainRunner(object): workers = {} dirpath = '.' defaultOutputPath = 'output' def __init__(self, dirpath='.'): ''' local path for load config ''' logger.info("Initialing Main Runner for Hasal agent") if os.path.isdir(dirpath): self.dirpath = dirpath logger.info("loading runner config folder: " + dirpath) else: logger.info(dirpath + " is invalid, use default path instead") self.scheduler = BackgroundScheduler() self.scheduler.start() self.load_dir(self.dirpath) event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True) event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove) observer = Observer() observer.schedule(event_handler, self.dirpath, recursive=True) observer.start() def load_dir(self, folder): (dirpath, dirnames, filenames) = os.walk(folder).next() for fname in filenames: if 'json' in fname[-4:]: self.load(os.path.join(dirpath, fname)) def load(self, fp): ''' given a json file, load and create a task run regularly ''' logger.info(fp + " was loaded!") with open("agent.log", 'w+') as f: f.write(fp + " was loaded!") data = {} with open(fp) as in_data: try: data = json.load(in_data) data['name'] = "Jenkins Job" data['path'] = fp except ValueError as e: logger.warning(fp + " loaded failed: " + e.message) return None interval = 30 if 'interval' in data: interval = int(data['interval']) if 'output' in data: if 'defaultOutputPath' in data['output']: self.defaultOutputPath = data['output']['defaultOutputPath'] if 'dirpath' in data['output']: data['output']['outputPath'] = os.path.join( self.defaultOutputPath, data['output']['dirpath']) else: data['output'] = {'outputPath': self.defaultOutputPath} if fp in self.workers: # existing runner found logger.info("Update exisitng runner [%s]" % fp) runner = self.workers[fp] runner.update(**data) # //memo: Interval can't be modified self.scheduler.modify_job(job_id=fp, func=runner.run, name=runner.name) else: # Create new logger.info("Create new runner [%s]" % fp) module_path = "hasalTask" object_name = "HasalTask" try: runner_module = getattr(importlib.import_module(module_path), object_name) except Exception as e: logger.exception(e) return None runner = runner_module(**data) self.workers[fp] = runner self.scheduler.add_job(runner.run, 'interval', id=fp, name=runner.name, seconds=interval) return runner def list(self): ''' to list all configs loaded format: [squence number] [minion name] [config_path] [status] ''' for (fp, worker) in self.workers: logger.info("path=" + fp + "," + str(worker) + ";") def remove(self, fp): ''' given file path, stop running instance if possible ''' if fp in self.workers: self.workers[fp].onstop() self.scheduler.remove_job(job_id=fp) del self.workers[fp] return True return False def remove_advanced(self): ''' TODO: 1. remove by start, end 2. by directory(?) ''' pass def unload_all(self): ''' stop all running instances ''' self.scheduler.shutdown() def pause(self, fp): ''' simply stop running instance but not remove config TODO: should have timeout if stop failed ''' self.scheduler.pause(job_id=fp) def resume(self, sn): # not sure we can do this pass def __del__(self): self.unload_all() def get_config(self): conf = {} return conf def _wake(self): ''' For periodical minions, waking them according to timing ''' pass
class JobScheduler(ISingleton): @inject def __init__( self, database_config: DatabaseConfig, database_session_manager: DatabaseSessionManager, ): self.database_session_manager: DatabaseSessionManager = database_session_manager self.database_config: DatabaseConfig = database_config self.scheduler: BackgroundScheduler = None def run(self): self.run_scheduler() print("job_process started") def run_scheduler(self): jobstores = { 'default': SQLAlchemyJobStore(url=self.database_config.connection_string, tablename='ApSchedulerJobsTable', engine=self.database_session_manager.engine, metadata=IocManager.Base.metadata, tableschema='Aps') } executors = { 'default': ThreadPoolExecutor(20), 'processpool': ProcessPoolExecutor(5) } job_defaults = {'coalesce': False, 'max_instances': 5} self.scheduler = BackgroundScheduler(daemon=True, jobstores=jobstores, executors=executors, job_defaults=job_defaults) JobSchedulerEvent.job_scheduler_type = JobScheduler self.scheduler.add_listener(JobSchedulerEvent.listener_finish, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) self.scheduler.add_listener(JobSchedulerEvent.listener_job_added, EVENT_JOB_ADDED) self.scheduler.add_listener(JobSchedulerEvent.listener_job_submitted, EVENT_JOB_SUBMITTED) self.scheduler.add_listener(JobSchedulerEvent.listener_job_removed, EVENT_JOB_REMOVED) self.scheduler.add_listener( JobSchedulerEvent.listener_all_jobs_removed, EVENT_ALL_JOBS_REMOVED) self.scheduler.add_listener( JobSchedulerEvent.listener_job_others, EVENT_JOB_MODIFIED | EVENT_JOB_MISSED | EVENT_JOB_MAX_INSTANCES) self.scheduler.add_listener( JobSchedulerEvent.listener_scheduler_other_events, EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN | EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED | EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED) self.scheduler.start() print('To clear the alarms, delete the example.sqlite file.') print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C')) def add_job_with_date(self, job_function, run_date, args=None, kwargs=None) -> Job: # if run_date is None: # run_date = datetime.now() + timedelta(seconds=10) job: Job = self.scheduler.add_job(job_function, 'date', run_date=run_date, misfire_grace_time=30000, args=args, kwargs=kwargs) return job def add_job_with_cron(self, job_function, cron: CronTrigger, args=None, kwargs=None) -> Job: # if cron.start_date is not None and cron.start_date < datetime.now().astimezone(get_localzone()): # cron.start_date = None # if cron.end_date is not None and cron.end_date < datetime.now().astimezone(get_localzone()): # cron.end_date = None job: Job = self.scheduler.add_job(job_function, cron, misfire_grace_time=15, args=args, kwargs=kwargs) return job def remove_job(self, job_id): self.scheduler.remove_job(job_id) def modify_job(self, job_id, jobstore=None, **changes): return self.scheduler.modify_job(job_id, jobstore, **changes) def reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args): return self.scheduler.reschedule_job(job_id, jobstore, trigger, **trigger_args) def pause_job(self, job_id, jobstore=None): return self.scheduler.pause_job(job_id, jobstore) def resume_job(self, job_id, jobstore=None): return self.scheduler.resume_job(job_id, jobstore) def remove_job(self, job_id, jobstore=None): self.scheduler.remove_job(job_id, jobstore) def get_job(self, job_id): return self.scheduler.get_job(job_id) def get_jobs(self, jobstore=None): return self.scheduler.get_jobs(jobstore)
}, 'apscheduler.executors.default': { 'class': 'apscheduler.executors.pool:ThreadPoolExecutor', }, }) @scheduler.scheduled_job('interval', id='pull_latest_image', seconds=pull_interval, timezone=utc) def pull_latest_image(): # Try to pull a new image if interactions.pull(image_name): # If we got a new image, trigger a rebalance balance_containers.trigger() pull_latest_image.trigger = lambda: scheduler.modify_job( 'pull_latest_image', next_run_time=datetime.now()) @scheduler.scheduled_job('interval', id='balance_containers', seconds=rebalance_interval, timezone=utc) def balance_containers(): try: # Clean up before interacting with the pool: # - checks in containers that are checked out longer than the max lifetime # - stops containers that aren't running if their image is out of date # - stops containers from the pool not running selenium for container in interactions.containers(): if container.checked_out: if ((datetime.utcnow() - container.checked_out).total_seconds() > max_checkout_time): logger.info('Container %s checked out longer than %d seconds, forcing stop',
class SchedulerManager(object): # __metaclass__ = ABCMeta global _mongoclient def __init__(self): self.jobstores = { 'mongo': MongoDBJobStore(collection='job1', database='saasjob', client=_mongoclient), 'default': MemoryJobStore() } self.executors = { 'default': ThreadPoolExecutor(1), 'processpool': ProcessPoolExecutor(1) } self.job_defaults = { 'coalesce': False, 'misfire_grace_time': 1, 'max_instances': 1 } self._sched = BackgroundScheduler(jobstores=self.jobstores, executors=self.executors, job_defaults=self.job_defaults) # 添加 任务提交 事件监听 self._sched.add_listener(self.when_job_submitted, EVENT_JOB_SUBMITTED) # 添加 任务执行完成 事件监听 self._sched.add_listener(self.when_job_executed, EVENT_JOB_EXECUTED) # 添加 任务异常退出 事件监听 self._sched.add_listener(self.when_job_crashed, EVENT_JOB_ERROR) self._jobs = {} self._jobhandlers = {} # format, key: jobid, value: jobhandler self._jobs_key = ["name", "func", "args", "kwargs"] self.start() def cmd_valid(self, cmd): cmd = cmd.strip() if cmd.startswith("python"): return True else: return False def get_job_trigger(self, _job): # ('trigger', <CronTrigger (second='4', timezone='Asia/Shanghai')>) _trigger = self._get_job_attr(_job, "trigger") # options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default] if _trigger: return dict([(f.name, f.__str__()) for f in _trigger.fields if not f.is_default]) else: return {} # 获取job属性 def _get_job_attr(self, _job, attr): try: result = eval("_job.%s" % attr) return result except: import traceback print(traceback.print_exc()) return None def when_job_submitted(self, event): try: job_id = event.job_id if job_id not in self._jobhandlers and job_id in self._jobhandlers: self._jobhandlers.setdefault(job_id, JobHandler(self._jobs[job_id])) jobhandler = self._jobhandlers[event.job_id] jobhandler.when_job_submitted() print("%s submitted at %s" % (event.job_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))) except: import traceback print(traceback.print_exc()) def when_job_executed(self, event): try: job_id = event.job_id if job_id not in self._jobhandlers: self._jobhandlers.setdefault(job_id, JobHandler(self._jobs[job_id])) jobhandler = self._jobhandlers[event.job_id] jobhandler.when_job_executed() print("%s executed at %s" % (event.job_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))) except: import traceback print(traceback.print_exc()) def when_job_crashed(self, event): try: if event.exception: job_id = event.job_id if job_id not in self._jobhandlers: self._jobhandlers.setdefault( job_id, JobHandler(self._jobs[job_id])) jobhandler = self._jobhandlers[event.job_id] jobhandler.when_job_crashed() print("%s crashed at %s" % (event.job_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())))) except: import traceback print(traceback.print_exc()) # 添加例行任务,crontab 格式 def addCron(self, cmd, **params): try: create_jobid = uuid.uuid4().hex if not self.cmd_valid(cmd): return {"errinfo": "wrong cmd"} jobcmdobj = JobCmd(cmd) data = params.get("data", {}) jobcmdobj.set_jobid(create_jobid) s = params.get("second", None) if params.get("second", None) != "*" else None m = params.get("minute", None) if params.get("minute", None) != "*" else None h = params.get("hour", None) if params.get("hour", None) != "*" else None d = params.get("day", None) if params.get("day", None) != "*" else None dw = params.get( "day_of_week", None) if params.get("day_of_week", None) != "*" else None mnth = params.get( "month", None) if params.get("month", None) != "*" else None y = params.get("year", None) if params.get("year", None) != "*" else None _job = self._sched.add_job(jobcmdcallable, 'cron', year=y, month=mnth, day=d, day_of_week=dw, hour=h, minute=m, second=s, args=[jobcmdobj, data], executor="processpool", jobstore="mongo", id=create_jobid) self._jobhandlers.setdefault(create_jobid, JobHandler(_job)) # 保存 job 属性 return {"job_id": create_jobid} except: import traceback print(traceback.print_exc(), cmd, params) return False # 修改 job 属性 def modifyJobAttr(self, job_id, **changes): try: _job = self._sched.modify_job(job_id=job_id, **changes) self._jobs[job_id] = _job if job_id in self._jobhandlers: self._jobhandlers[job_id].job = _job else: self._jobhandlers.setdefault(job_id, JobHandler(_job)) return True except: import traceback print(traceback.print_exc(), job_id, changes) return False def modifyJobData(self, job_id, data): try: args = self._get_job_attr(self._jobhandlers[job_id].job, "args") # args_copy = [item for item in args] for key in data: args[1][key] = data[key] _job = self._sched.modify_job(job_id, args=args) self._jobs[job_id] = _job if job_id in self._jobhandlers: self._jobhandlers[job_id].job = _job else: self._jobhandlers.setdefault(job_id, JobHandler(_job)) return True except: import traceback print(traceback.print_exc(), job_id, data) return False # 修改执行时间,crontab 格式 def modifyJobFreq(self, job_id, cronargs): try: _job = self._sched.reschedule_job(job_id, trigger='cron', **cronargs) self._jobs[job_id] = _job if job_id in self._jobhandlers: self._jobhandlers[job_id].job = _job else: self._jobhandlers.setdefault(job_id, JobHandler(_job)) return True except: import traceback print(traceback.print_exc(), job_id, cronargs) return False # 删除 job def removeFromCron(self, job_id): try: self._sched.remove_job(job_id) if job_id in self._jobhandlers: self._jobhandlers.pop(job_id) if job_id in self._jobs: self._jobs.pop(job_id) return True except: import traceback print(traceback.print_exc(), job_id) return False def job_exists(self, job_id): if job_id in self._jobhandlers or job_id in self._jobs: if job_id not in self._jobhandlers and job_id in self._jobs: self._jobhandlers[job_id] = JobHandler(self._jobs[job_id]) elif job_id in self._jobhandlers and job_id not in self._jobs: self._jobs[job_id] = self._jobhandlers[job_id].job return True else: return False # 根据 job id 查询任务信息 def findCronJob(self, job_ids): result = [] _keys = [ "cmd", "create_stamp", "is_running", "start_stamp", "hope_runtime", "is_success", "is_pause", "status", "name", "desc", "allowmodify" ] for job_id in job_ids: print("job_exists", self.job_exists(job_id)) if self.job_exists(job_id): _jobhander = self._jobhandlers[job_id] job_info = _jobhander.jobhandlerattr cron_trigger = {} # cron_trigger = self.get_cron_trigger(_jobhander.job) tmp = {} tmp["job_id"] = job_id if job_info["is_running"]: execute_time = time.time() - job_info["start_stamp"] tmp["running_time"] = round(execute_time, 3) else: tmp["running_time"] = round(job_info["hope_runtime"], 3) for key in _keys: v = job_info.get(key, None) if key == "is_running": tmp["finished"] = False if job_info[ "is_running"] else True else: tmp[key] = v if tmp["finished"]: tmp["completed_per"] = 1.0 else: tmp["completed_per"] = round( tmp["running_time"] / max([tmp["running_time"], tmp["hope_runtime"]]), 3) # del tmp["hope_runtime"] # del tmp["is_success"] # del tmp["is_pause"] tmp.pop("hope_runtime") tmp.pop("is_success") tmp.pop("is_pause") _result = dict(tmp, **cron_trigger) print("_result", _result) if _result["status"] == 3: _result["completed_per"] = 0 _result["running_time"] = 0 _result["start_stamp"] = None result.append(_result) else: result.append({"job_id": job_id, "errinfo": "no exists"}) return result def getAllJobInfo(self): try: result = self.findCronJob( set(self._jobhandlers.keys()) | set(self._jobs.keys())) return result except: import traceback print(traceback.print_exc()) return False def start_addition(self): for _job in self._sched.get_jobs(): job_id = self._get_job_attr(_job, "id") self._jobs.setdefault(job_id, _job) def start(self): try: self._sched.start() self._sched.pause() self.start_addition() self._sched.resume() return True except: import traceback print(traceback.print_exc()) return False def stop(self, iswait=True): try: self._sched.shutdown(wait=iswait) self._jobhandlers.clear() return True except: import traceback print(traceback.print_exc()) return False def pause_job(self, job_id): try: self._sched.pause_job(job_id=job_id) self._jobhandlers[job_id].ispause = True self._jobhandlers[job_id].status = 3 self._jobhandlers[job_id].isrunning = False return True except: import traceback print(traceback.print_exc()) return False def resume_job(self, job_id): try: self._sched.resume_job(job_id=job_id) self._jobhandlers[job_id].ispause = False self._jobhandlers[job_id].status = 1 return True except: import traceback print(traceback.print_exc()) return False
class RecreateModel: """ Author: Marten Bolin Date: 2017-11-22 Last update: Purpose: This will be ran continously to check for any changes made and rearrange the model """ def __init__(self, daemon=False): """ Author: Marten Bolin Date: 2017-11-22 Last update: Purpose: The instantiation of the class, make sure to catch it in a variable so it can be terminated properly. Start the scheduler :param daemon : Sets daemon, (Optional) Default is False """ # Set the variables to start state self.number_of_users = 0 self.number_of_movies = 0 self.number_of_ratings = 0 # Set up and start the scheduler self.scheduled = BackgroundScheduler() if not daemon: self.scheduled._daemon = False self.scheduled.add_job(self._run, 'interval', seconds=2, id="2") self.scheduled.start() self.scheduled.modify_job(job_id="2", next_run_time=datetime.now()) def _run(self): """ Author: Marten Bolin Date:2017-11-22 Last update: Purpose: The process that is ran, checks for updates and updates model """ # Get current state current_number_of_users = len(RetrieveUser().retrieve_all_users()) current_number_of_movies = len(RetrieveMovie().retrieve_movie()) current_number_of_ratings = len(RetrieveRating().retrieve_ratings()) # Checks rating first due to most likely to change if (self.number_of_ratings == current_number_of_ratings and self.number_of_users == current_number_of_users and self.number_of_movies == current_number_of_movies): print("Nothing new, no changes made.") else: print("Changes detected, adjusting model") CreateNewModel.create_new_model() self.number_of_users = len(RetrieveUser().retrieve_all_users()) self.number_of_movies = len(RetrieveMovie().retrieve_movie()) self.number_of_ratings = len(RetrieveRating().retrieve_ratings()) def terminate(self): """ Author: Marten Bolin Date: 2017-11-22 Last update: Purpose: Terminates the process """ print("Shutting down update_recommendations..") self.scheduled.shutdown() print("Recreatemodel has been shut down.")
class CronManager: def __init__(self, use_mongo_db=True): self.scheduler = BackgroundScheduler( timezone=timezone('Asia/Shanghai')) self.scheduler.configure() if use_mongo_db: self.job_store = MongoDBJobStore(database='apscheduler', collection='cronJob', client=db) self.scheduler.add_jobstore(self.job_store) self.is_replace_existing = True else: self.is_replace_existing = False def add_cron(self, cron_instance): if not isinstance(cron_instance, Cron): raise TypeError('please add correct cron!') if cron_instance.trigger_type == 'interval': seconds = cron_instance.trigger_args.get('seconds') if not isinstance(seconds, int) and not common.can_convert_to_int(seconds): raise TypeError('please set correct time interval') seconds = int(seconds) if seconds <= 0: raise ValueError('please set interval > 0') job = self.scheduler.add_job( func=cron_instance.cron_mission, trigger=cron_instance.trigger_type, seconds=seconds, replace_existing=self.is_replace_existing, coalesce=True, id=cron_instance.get_cron_job_id(), max_instances=5, jitter=0) # 玄学,新增job的时候不用加args,直接加对象调用的func elif cron_instance.trigger_type == 'date': run_date = cron_instance.trigger_args.get('run_date') # TODO 判断run_date类型 job = self.scheduler.add_job( func=cron_instance.cron_mission, trigger=cron_instance.trigger_type, run_date=run_date, replace_existing=self.is_replace_existing, coalesce=True, id=cron_instance.get_cron_job_id( )) # 玄学,新增job的时候不用加args,直接加对象调用的func elif cron_instance.trigger_type == 'cron': raise TypeError('暂时不支持 trigger_type 等于 \'cron\'') return cron_instance.get_cron_job_id() def start(self, paused=False): self.scheduler.start(paused=paused) def pause_cron(self, cron_id=None, pause_all=False): if pause_all: self.scheduler.pause() elif cron_id: self.scheduler.pause_job(job_id=cron_id) def resume_cron(self, cron_id=None, resume_all=False): if resume_all: self.scheduler.resume() elif cron_id: self.scheduler.resume_job(job_id=cron_id) def del_cron(self, cron_id=None, del_all=False): if del_all: self.scheduler.remove_all_jobs() elif cron_id: self.scheduler.remove_job(job_id=cron_id) def update_cron(self, cron_job_id, project_id, cron_info): if not isinstance(cron_job_id, str): raise TypeError('cron_id must be str') if not isinstance(project_id, str): raise TypeError('project_id must be str') if not isinstance(cron_info, dict): raise TypeError('cron_info must be dict') trigger_type = cron_info.get('triggerType') interval = cron_info.get('interval') run_date = cron_info.get('runDate') test_suite_id_list = cron_info.get('testSuiteIdList') include_forbidden = cron_info.get('includeForbidden') test_env_id = cron_info.get('testEnvId') alarm_mail_list = cron_info.get('alarmMailList') try: if trigger_type == 'interval' and int(interval) > 0: self.scheduler.modify_job( job_id=cron_job_id, trigger=IntervalTrigger(seconds=interval)) elif trigger_type == 'date': # TODO 判断run_date类型 self.scheduler.modify_job( job_id=cron_job_id, trigger=DateTrigger(run_date=run_date)) else: raise TypeError('更新定时任务触发器失败!') if run_date: cron = Cron( test_suite_id_list=test_suite_id_list, project_id=project_id, include_forbidden=include_forbidden, test_env_id=test_env_id, alarm_mail_list=alarm_mail_list, trigger_type=trigger_type, # 更新定时器时,此参数并没有真正起到作用, 仅修改展示字段 run_date=run_date) # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 else: cron = Cron( test_suite_id_list=test_suite_id_list, project_id=project_id, include_forbidden=include_forbidden, test_env_id=test_env_id, alarm_mail_list=alarm_mail_list, trigger_type=trigger_type, # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 seconds=interval) # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 # 玄学,更改job的时候必须改args,不能改func self.scheduler.modify_job(job_id=cron_job_id, coalesce=True, args=[cron]) except BaseException as e: raise TypeError('更新定时任务失败: %s' % e) def shutdown(self, force_shutdown=False): if force_shutdown: self.scheduler.shutdown(wait=False) else: self.scheduler.shutdown(wait=True) def get_jobs(self): return self.scheduler.get_jobs()
def cook_report_uuid(scheduler: BackgroundScheduler, report_uuid: str, action="add") -> None: """ :param scheduler: :param report_uuid: :param action: add/mod/del :return: """ print( f"cook report uuid, scheduler: {scheduler}, report_uuid: {report_uuid}" f"action: {action}") if action == "del": scheduler.remove_job(job_id=report_uuid) ok, report_info = storage.get_report_info(report_uuid) print(f"report_info: {report_info}") if not ok: print(f"Fail to get report info for uuid: {report_uuid}" f"errmsg: {report_info}") return ok, page_info = storage.get_report_detail_info_by_id(report_uuid) if not ok: print(f"Fail to get report detail info for uuid: {report_uuid}, " f"errmsg: {page_info}") return # week/month sensor_group_ids = report_info["sensor_group_ids"] # 获取探针、探针组信息 ok, sensor_id_group_mapping = storage.get_group_sensor_map( sensor_group_ids) if not ok: print(f"fail to get sensor_groups by id: {sensor_group_ids}") return sensors = list(sensor_id_group_mapping.keys()) cron = CronTrigger.from_crontab(report_info["send_date"]) _report_params = { "report_name": report_info["report_name"], "report_format": report_info["report_format"], "timed": report_info["timed"], "data_scope": report_info["data_scope"], "sensor_ids": sensors, "sensor_id_group_mapping": sensor_id_group_mapping, "page_info": page_info, "send_mail": report_info["send_email"], } if action == "mod": scheduler.modify_job(job_id=report_uuid, trigger=cron, kwargs=_report_params) elif action == "add": gen_report(**_report_params) scheduler.add_job(func=gen_report, trigger=cron, kwargs=_report_params, id=report_uuid) else: raise ValueError("action must in ('add', 'del', 'mod')")
class MainRunner(object): workers = {} dirpath = "." defaultOutputPath = "output" def __init__(self, dirpath="."): """ local path for load config """ logger.info("Initialing Main Runner for Fuzzy Testing") if os.path.isdir(dirpath): self.dirpath = dirpath logger.info("loading runner config folder: " + dirpath) else: logger.info(dirpath + " is invalid, use default path instead") self.scheduler = BackgroundScheduler() self.scheduler.start() self.load_dir(dirpath) event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True) event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove) observer = Observer() observer.schedule(event_handler, self.dirpath, recursive=True) observer.start() def load_dir(self, folder): (dirpath, dirnames, filenames) = os.walk(folder).next() for fname in filenames: if "json" in fname[-4:]: self.load(os.path.join(dirpath, fname)) def load(self, fp): """ given a json file, load and create a task run regularly """ logger.info(fp + " was loaded!") with open(fp) as in_data: try: data = json.load(in_data) data["path"] = fp except ValueError as e: logger.warning(fp + " loaded failed: " + e.message) return None interval = 30 if "interval" in data: interval = int(data["interval"]) if "output" in data: if "defaultOutputPath" in data["output"]: self.defaultOutputPath = data["output"]["defaultOutputPath"] if "dirpath" in data["output"]: data["output"]["outputPath"] = os.path.join(self.defaultOutputPath, data["output"]["dirpath"]) else: data["output"] = {"outputPath": self.defaultOutputPath} if "type" not in data: logger.error( "Missing type attribute in \ your configuration file [%s]" % fp ) return None if fp in self.workers: # existing runner found logger.info("Update exisitng runner [%s]" % fp) runner = self.workers[fp] runner.update(**data) # //memo: Interval can't be modified self.scheduler.modify_job(job_id=fp, func=runner.run, name=runner.name) else: # Create new logger.info("Create new runner [%s]" % fp) module_path = data["type"][: data["type"].rfind(".")] object_name = data["type"][data["type"].rfind(".") + 1 :] try: runner_module = getattr(importlib.import_module(module_path), object_name) except Exception as e: logger.exception(e) return None runner = runner_module(**data) self.workers[fp] = runner self.scheduler.add_job(runner.run, "interval", id=fp, name=runner.name, seconds=interval) return runner return None def list(self): """ to list all configs loaded format: [squence number] [minion name] [config_path] [status] """ for (fp, worker) in self.workers: logger.info("path=" + fp + "," + str(worker) + ";") def remove(self, fp): """ given file path, stop running instance if possible """ if fp in self.workers: self.workers[fp].onstop() self.scheduler.remove_job(job_id=fp) del self.workers[fp] return True return False def remove_advanced(self): """ TODO: 1. remove by start, end 2. by directory(?) """ pass def unload_all(self): """ stop all running instances """ self.scheduler.shutdown() def pause(self, fp): """ simply stop running instance but not remove config TODO: should have timeout if stop failed """ self.scheduler.pause(job_id=fp) def resume(self, sn): # not sure we can do this pass def __del__(self): self.unload_all() def get_config(self): conf = {} return conf def _wake(self): """ For periodical minions, waking them according to timing """ pass
class Boss(object): workers = {} dirpath = '.' output = None def __init__(self, dirpath='.', output='output'): ''' local path for load config ''' logger.info("Initialing BOSS") if os.path.isdir(dirpath): self.dirpath = dirpath logger.info("loading job config folder: " + dirpath) else: logger.info(dirpath + " is invalid, use default path instead") self.output = output logger.info("Setup output folder: " + output) if not os.path.isdir(output): logger.info("target directory " + output + " doesn't exist, creating..") os.makedirs(output) self.scheduler = BackgroundScheduler() self.scheduler.start() self.load_dir(dirpath) event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True) event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove) observer = Observer() observer.schedule(event_handler, self.dirpath, recursive=True) observer.start() def load_dir(self, folder): (dirpath, dirnames, filenames) = os.walk(folder).next() for fname in filenames: if 'json' in fname[-4:]: self.load(os.path.join(dirpath, fname)) def load(self, fp): ''' given a json file, load and create a task run regularly ''' logger.info(fp + " was loaded!") with open(fp) as in_data: try: data = json.load(in_data) data['path'] = fp except ValueError as e: logger.warning(fp + " loaded failed: " + e.message) return None interval = 30 if 'interval' in data: interval = int(data['interval']) if self.output: # TODO: test case for no 'output' key in data if 'output' not in data: data['output'] = {} output = data['output'] if 'dirpath' in output: output['dirpath'] = os.path.join(self.output, output['dirpath']) else: output['dirpath'] = self.output if 'type' not in data: logger.error("Missing type attribute in \ your configruation file [%s]" % fp) return None if fp in self.workers: # existing minion found logger.info("Update exisitng minion [%s]" % fp) minion = self.workers[fp] minion.update(**data) # //memo: Interval can't be modified self.scheduler.modify_job(job_id=fp, func=minion.collect, name=minion.name+'_'+minion.serial ) else: # Create new logger.info("Create new minion [%s]" % fp) module_path = data['type'][:data['type'].rfind(".")] object_name = data['type'][data['type'].rfind(".") + 1:] try: minion_module = getattr(importlib.import_module( module_path), object_name) except Exception as e: logger.exception(e) return None minion = minion_module(**data) self.workers[fp] = minion self.scheduler.add_job(minion.collect, 'interval', id=fp, name=minion.name + '_' + minion.serial, seconds=interval ) return minion return None def list(self): ''' to list all configs loaded format: [squence number] [minion name] [config_path] [status] ''' for (fp, worker) in self.workers: logger.info("path=" + fp + "," + str(worker) + ";") def remove(self, fp): ''' given file path, stop running instance if possible ''' if fp in self.workers: self.workers[fp].onstop() self.scheduler.remove_job(job_id=fp) del self.workers[fp] return True return False def remove_advanced(self): ''' TODO: 1. remove by start, end 2. by directory(?) ''' pass def unload_all(self): ''' stop all running instances ''' self.scheduler.shutdown() def pause(self, fp): ''' simply stop running instance but not remove config TODO: should have timeout if stop failed ''' self.scheduler.pause(job_id=fp) def resume(self, sn): # not sure we can do this pass def __del__(self): self.unload_all() def get_config(self): conf = {} return conf def _wake(self): ''' For periodical minions, waking them according to timing ''' pass
class SchedulerService(rpyc.Service): def __init__(self, **config): self._scheduler = BackgroundScheduler() self._scheduler.configure(**config) self._scheduler.start() self.logger = logging.getLogger("Heartbeat.core") self.logger.info("Heartbeat Core Initalized") def on_connect(self, conn): # code that runs when a connection is created # (to init the service, if needed) self.logger.info("----------Begin New Client----------") self.logger.info(conn) self.logger.info("----------End New Client----------") def on_disconnect(self, conn): # code that runs after the connection has already closed # (to finalize the service, if needed) self.logger.info("----------Begin Goodbye Client----------") self.logger.info(conn) self.logger.info("----------End Goodbye Client----------") def exposed_add_job(self, func, *args, **kwargs): self.logger.info("----------Begin New Job----------") self.logger.info("Function: %s", str(func)) self.logger.info("*args: %s", str(args)) self.logger.info("**kwargs: %s", str(dict(kwargs))) self.logger.info("----------Eng New Job----------") return self._scheduler.add_job(func, *args, **kwargs) def exposed_modify_job(self, job_id, jobstore=None, **changes): return self._scheduler.modify_job(job_id, jobstore, **changes) def exposed_reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args): return self._scheduler.reschedule_job(job_id, jobstore, trigger, **trigger_args) def exposed_pause_job(self, job_id, jobstore=None): return self._scheduler.pause_job(job_id, jobstore) def exposed_resume_job(self, job_id, jobstore=None): return self._scheduler.resume_job(job_id, jobstore) def exposed_remove_job(self, job_id, jobstore=None): self._scheduler.remove_job(job_id, jobstore) def exposed_get_job(self, job_id, jobstore=None): return self._scheduler.get_job(job_id, jobstore=jobstore) def exposed_get_jobs(self, jobstore=None): results = self._scheduler.get_jobs(jobstore) return results def exposed_get_tasks(self): """Return a list of schedule-able function""" tasks = [] for module_file in os.listdir( os.path.join(os.path.dirname(__file__), "task")): if module_file == "__init__.py" or module_file[-3:] != ".py": continue module_name = "server.task.{}".format(module_file[:-3]) module = importlib.import_module(module_name) if not hasattr(module, "__all__"): continue for function_name in module.__all__: function = getattr(module, function_name) if not callable(function): continue parameters = inspect.signature(function).parameters parameters_str = ", ".join( [str(val) for key, val in parameters.items()]) tasks.append("{}:{}({})".format(module_name, function_name, parameters_str)) return tasks
class CronManager: def __init__(self, use_mongo_db=True): self.scheduler = BackgroundScheduler( timezone=timezone('Asia/Shanghai')) self.scheduler.configure() if use_mongo_db: self.job_store = MongoDBJobStore(database='apscheduler', collection='cronJob', client=db) self.scheduler.add_jobstore(self.job_store) self.is_replace_existing = True else: self.is_replace_existing = False def add_cron(self, cron_instance): try: if not isinstance(cron_instance, Cron): raise TypeError('please add correct cron!') if cron_instance.trigger_type == 'interval': seconds = cron_instance.trigger_args.get('seconds') if not isinstance( seconds, int) and not common.can_convert_to_int(seconds): raise TypeError('please set correct time interval') seconds = int(seconds) if seconds <= 0: raise ValueError('please set interval > 0') job = self.scheduler.add_job( func=cron_instance.cron_mission, trigger=cron_instance.trigger_type, seconds=seconds, replace_existing=self.is_replace_existing, coalesce=True, id=cron_instance.get_cron_job_id(), max_instances=5, jitter=0) elif cron_instance.trigger_type == 'date': run_date = cron_instance.trigger_args.get('run_date') # TODO 判断run_date类型 job = self.scheduler.add_job( func=cron_instance.cron_mission, trigger=cron_instance.trigger_type, run_date=run_date, replace_existing=self.is_replace_existing, coalesce=True, id=cron_instance.get_cron_job_id()) elif cron_instance.trigger_type == 'cron': raise TypeError('暂时不支持 trigger_type 等于 \'cron\'') return cron_instance.get_cron_job_id() except BaseException as e: with app.app_context(): current_app.logger.error("add_cron failed. - %s" % str(e)) def start(self, paused=False): self.scheduler.start(paused=paused) def pause_cron(self, cron_id=None, pause_all=False): if pause_all: self.scheduler.pause() elif cron_id: self.scheduler.pause_job(job_id=cron_id) def resume_cron(self, cron_id=None, resume_all=False): if resume_all: self.scheduler.resume() elif cron_id: self.scheduler.resume_job(job_id=cron_id) def del_cron(self, cron_id=None, del_all=False): if del_all: self.scheduler.remove_all_jobs() elif cron_id: self.scheduler.remove_job(job_id=cron_id) def update_cron(self, cron_job_id, project_id, cron_info): try: if not isinstance(cron_job_id, str): raise TypeError('cron_id must be str') if not isinstance(project_id, str): raise TypeError('project_id must be str') if not isinstance(cron_info, dict): raise TypeError('cron_info must be dict') trigger_type = cron_info.get('triggerType') interval = cron_info.get('interval') run_date = cron_info.get('runDate') test_suite_id_list = cron_info.get('testSuiteIdList') include_forbidden = cron_info.get('includeForbidden') test_env_id = cron_info.get('testEnvId') always_send_mail = cron_info.get('alwaysSendMail') alarm_mail_group_list = cron_info.get('alarmMailGroupList') enable_wxwork_notify = cron_info.get('enableWXWorkNotify') wxwork_api_key = cron_info.get('WXWorkAPIKey') wxwork_mention_mobile_list = cron_info.get( 'WXWorkMentionMobileList') always_wxwork_notify = cron_info.get('alwaysWXWorkNotify') enable_ding_talk_notify = cron_info.get('enableDingTalkNotify') ding_talk_access_token = cron_info.get('DingTalkAccessToken') ding_talk_at_mobiles = cron_info.get('DingTalkAtMobiles') ding_talk_secret = cron_info.get('DingTalkSecret') always_ding_talk_notify = cron_info.get('alwaysDingTalkNotify') if trigger_type == 'interval' and int(interval) > 0: self.scheduler.modify_job( job_id=cron_job_id, trigger=IntervalTrigger(seconds=interval)) elif trigger_type == 'date': self.scheduler.modify_job( job_id=cron_job_id, trigger=DateTrigger(run_date=run_date)) else: raise TypeError('更新定时任务触发器失败!') if run_date: cron = Cron( cron_job_id=cron_job_id, test_suite_id_list=test_suite_id_list, project_id=project_id, test_env_id=test_env_id, include_forbidden=include_forbidden, enable_wxwork_notify=enable_wxwork_notify, wxwork_api_key=wxwork_api_key, wxwork_mention_mobile_list=wxwork_mention_mobile_list, always_wxwork_notify=always_wxwork_notify, enable_ding_talk_notify=enable_ding_talk_notify, ding_talk_access_token=ding_talk_access_token, ding_talk_at_mobiles=ding_talk_at_mobiles, ding_talk_secret=ding_talk_secret, always_ding_talk_notify=always_ding_talk_notify, always_send_mail=always_send_mail, alarm_mail_group_list=alarm_mail_group_list, trigger_type=trigger_type, # 更新定时器时,此参数并没有真正起到作用, 仅修改展示字段 run_date=run_date) # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 else: cron = Cron( cron_job_id=cron_job_id, test_suite_id_list=test_suite_id_list, project_id=project_id, include_forbidden=include_forbidden, enable_wxwork_notify=enable_wxwork_notify, wxwork_api_key=wxwork_api_key, wxwork_mention_mobile_list=wxwork_mention_mobile_list, always_wxwork_notify=always_wxwork_notify, enable_ding_talk_notify=enable_ding_talk_notify, ding_talk_access_token=ding_talk_access_token, ding_talk_at_mobiles=ding_talk_at_mobiles, ding_talk_secret=ding_talk_secret, always_ding_talk_notify=always_ding_talk_notify, test_env_id=test_env_id, always_send_mail=always_send_mail, alarm_mail_group_list=alarm_mail_group_list, trigger_type=trigger_type, # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 seconds=interval) # 更新定时器时,此参数并没有起到作用, 仅修改展示字段 # 玄学,更改job的时候必须改args,不能改func self.scheduler.modify_job(job_id=cron_job_id, coalesce=True, args=[cron]) except BaseException as e: with app.app_context(): current_app.logger.error("update_cron failed. - %s" % str(e)) def shutdown(self, force_shutdown=False): if force_shutdown: self.scheduler.shutdown(wait=False) else: self.scheduler.shutdown(wait=True) def get_jobs(self): return self.scheduler.get_jobs()
class ExScheduler(object): def __init__(self): self.__scheduler = BackgroundScheduler() self.__jobstatuslist = {} def start_scheduler(self, **options): self.__scheduler.configure(**options) self.__scheduler.start() def shutdownscheduler(self): self.__scheduler.shutdown(wait=False) def getjob(self, jobid): return self.__scheduler.get_job(jobid) def scheduledjob(self, *args, **kw): return self.__scheduler.scheduled_job(*args, **kw) def addjob(self, *args, **kw): job = self.__scheduler.add_job(*args, **kw) self.__scheduler.wakeup() return job def removejob(self, job_id, jobstore=None): self.__scheduler.remove_job(job_id=job_id, jobstore=jobstore) def pausejob(self, job_id, jobstore=None): self.__scheduler.pause_job(job_id=job_id, jobstore=jobstore) self.setjobstatus(job_id, JobStatus.PAUSED) def resumejob(self, job_id, jobstore=None): self.__scheduler.resume_job(job_id=job_id, jobstore=jobstore) self.setjobstatus(job_id, JobStatus.SCHEDULING) def modifyjob(self, job_id, jobstore=None, **kw): job = self.__scheduler.modify_job(job_id=job_id, jobstore=jobstore, **kw) return job def getjoblist(self): joblist = self.__scheduler.get_jobs() return map(self._getjobinfo, joblist) def addlistener(self, callback, mask=EVENT_ALL): self.__scheduler.add_listener(callback, mask) def setjobstatus(self, jobid, jobstatus): self.__jobstatuslist[jobid] = jobstatus def jobstatusinitial(self, job_id, jobstore=None): job = self.__scheduler.get_job(job_id=job_id, jobstore=jobstore) if job_id not in self.__jobstatuslist: status = (JobStatus.SCHEDULING if job.next_run_time else JobStatus.PAUSED) if hasattr( job, 'next_run_time') else JobStatus.PENDING self.setjobstatus(job_id, status) def _getjobinfo(self, job): self.jobstatusinitial(job.id) return { "id": str(job.id), "name": str(job.name), "kwargs": job.kwargs, "trigger": trigger_str_to_dict(str(job.trigger)), "next_run_time": datetime_repr(job.next_run_time) if self.__jobstatuslist[str(job.id)] == JobStatus.SCHEDULING or self.__jobstatuslist[str(job.id)] == JobStatus.RUNNING else "--", "status": self.__jobstatuslist[str(job.id)] }
class MainRunner(object): workers = {} dirpath = '.' defaultOutputPath = 'output' class NoRunningFilter(logging.Filter): def filter(self, record): return not record.msg.startswith('Execution') def __init__(self, dirpath='.'): ''' local path for load config ''' logger.info("Initialing Main Runner for Hasal agent") if os.path.isdir(dirpath): self.dirpath = dirpath logger.info("loading runner config folder: " + dirpath) else: logger.info(dirpath + " is invalid, use default path instead") self.scheduler = BackgroundScheduler() self.scheduler.start() self.load_dir(self.dirpath) event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True) event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove) observer = Observer() observer.schedule(event_handler, self.dirpath, recursive=True) observer.start() my_filter = self.NoRunningFilter() logging.getLogger("apscheduler.scheduler").addFilter(my_filter) def load_dir(self, folder): (dirpath, dirnames, filenames) = os.walk(folder).next() for fname in filenames: if 'json' in fname[-4:]: self.load(os.path.join(dirpath, fname)) def load(self, fp): ''' given a json file, load and create a task run regularly ''' logger.info(fp + " was loaded!") with open("agent.log", 'w+') as f: f.write(fp + " was loaded!") data = {} loaded = False for _ in range(10): try: with open(fp) as in_data: data = json.load(in_data) # default will load JOB_NAME parameter in Jenkins created json file data['name'] = data.get('JOB_NAME', "Jenkins Job") data['path'] = fp loaded = True except ValueError as e: logger.warning(fp + " loaded failed: " + e.message) return None except Exception as e: logger.warning("File is not ready. Wait 1 second for another try.") time.sleep(1) if not loaded: logger.warning(fp + " is not ready for 10 seconds.") return None # load interval value from Jenkins created json file (default : 30 ) interval = int(data.get('interval', 30)) # load outputpath and defaultoutputpath from Jenkins created json file if 'output' in data: if 'defaultOutputPath' in data['output']: self.defaultOutputPath = data['output']['defaultOutputPath'] if 'dirpath' in data['output']: data['output']['outputPath'] = os.path.join(self.defaultOutputPath, data['output']['dirpath']) else: data['output'] = {'outputPath': self.defaultOutputPath} if fp in self.workers: # existing runner found logger.info("Update exisitng runner [%s]" % fp) runner = self.workers[fp] runner.update(**data) # //memo: Interval can't be modified self.scheduler.modify_job(job_id=fp, func=runner.run, name=runner.name ) else: # Create new logger.info("Create new runner [%s]" % fp) module_path = data.get('AGENT_MODULE_PATH', "hasalTask") object_name = data.get('AGENT_OBJECT_NAME', "HasalTask") try: runner_module = getattr(importlib.import_module( module_path), object_name) except Exception as e: logger.exception(e) return None runner = runner_module(**data) self.workers[fp] = runner self.scheduler.add_job(runner.run, 'interval', id=fp, name=runner.name, seconds=interval ) return runner def list(self): ''' to list all configs loaded format: [squence number] [minion name] [config_path] [status] ''' for (fp, worker) in self.workers: logger.info("path=" + fp + "," + str(worker) + ";") def remove(self, fp): ''' given file path, stop running instance if possible ''' if fp in self.workers: self.workers[fp].onstop() self.scheduler.remove_job(job_id=fp) del self.workers[fp] return True return False def remove_advanced(self): ''' TODO: 1. remove by start, end 2. by directory(?) ''' pass def unload_all(self): ''' stop all running instances ''' self.scheduler.shutdown() def pause(self, fp): ''' simply stop running instance but not remove config TODO: should have timeout if stop failed ''' self.scheduler.pause(job_id=fp) def resume(self, sn): # not sure we can do this pass def __del__(self): self.unload_all() def get_config(self): conf = {} return conf def _wake(self): ''' For periodical minions, waking them according to timing ''' pass
class Scheduler: TRIGGERS = { "trig_5minutes": { "id": "trig_5minutes", "name": "Every five minutes", "options": [], "schema": {}, "trigger_args": lambda args: dict(minute="*/5"), "from_trigger": lambda trig: [] }, "trig_hourly": { "id": "trig_hourly", "name": "Each hour", "options": [], "schema": {}, "trigger_args": lambda args: dict(hour="*"), "from_trigger": lambda trig: [] }, "trig_daily": { "id": "trig_daily", "name": "Each day", "options": [], "schema": {}, "trigger_args": lambda args: dict(day="*"), "from_trigger": lambda trig: [] }, "trig_weekday": { "id": "trig_weekday", "name": "Each weekday", "options": [{ "id": i, "name": el, "active": True } for i, el in enumerate("Mon Tue Wed Thu Fri Sat Sun".split())], "schema": { "id": { "type": "integer", "coerce": int, "min": 0, "max": 6 }, "name": { "type": "string", "required": False }, "active": { "type": "boolean", "coerce": utility.coerce_bool, "required": True } }, "trigger_args": lambda args: dict(day_of_week=",".join(str(a) for a in args)), "from_trigger": lambda trig: [int(d) for d in str(trig.fields[4]).split(",")] }, "trig_monthly": { "id": "trig_monthly", "name": "Each month", "options": [{ "id": i + 1, "name": el, "active": True } for i, el in enumerate(("Jan Feb Mar Apr May Jun " "Jul Aug Sep Oct Nov Dec").split())], "schema": { "id": { "type": "integer", "coerce": int, "min": 0, "max": 12 }, "name": { "type": "string", "required": False }, "active": { "type": "boolean", "coerce": utility.coerce_bool, "required": True } }, "trigger_args": lambda args: dict(month=",".join(str(a) for a in args)), "from_trigger": lambda trig: [int(d) for d in str(trig.fields[1]).split(",")] }, } """Predefined triggers and their argument checks.""" def __init__(self, elastic, crawler_dir="crawlers", crawler_args={}, **cron_defaults): """Initializes the scheduler by binding it to it's elasticsearch db. Args: elastic (elasticsearch.Elasticsearh): The es-client to save the crawling jobs in. crawler_dir (str): the directory, where the crawlers will be found. Defaults to "crawlers". job_defaults (dict): a dictionary of keyword arguments for the schedulers job_defaults. **cron_defaults (dict): a dictionary of keyword arguments for the schedulers job_defaults. Returns: Scheduler: a fresh Scheduler instance. """ jobstores = { "default": { "type": "memory" }, "elastic": InjectorJobStore(kwargs=crawler_args, client=elastic) } executors = { "default": ThreadPoolExecutor(10), "processpool": ProcessPoolExecutor(10) } job_defaults = { "misfire_grace_time": 5 * 60, # 5min "coalesce": True, } self.cron_defaults = utility.DefaultDict( { # standard is every day at 00:00:00 "hour": 0, "minute": 0, "second": 0 }, **cron_defaults) self.scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc) self.crawlers = _detect_crawlers() # set up the validator schema. self.job_validator = cerberus.Validator(SCHEMATA["job"]({ "trigger_ids": list(self.TRIGGERS) }), allow_unknown=True) self.scheduler.start() def upsert_job(self, job_dict, **runtime_args): """Adds or updates a job using the provided user_input. If an id field is present in the dict, the job is updated, otherwise a new one is created. Args: job_dict (dict): user input for a job, as defined in `SCHEMATA`. **runtime_args (dict): additional runtime arguments for the crawler. Returns: apscheduler.job.Job: a new job Object. """ if not self.job_validator.validate(job_dict): raise (AssertionError(str(self.job_validator.errors))) doc = utility.SDA(job_dict) job = self.crawlers.get(doc["crawler.id"], None) # default to the SearchPlugin, and give the search name as argument. if job is None: inst = { "args": ("SearchPlugin", runtime_args), "kwargs": dict(search_id=doc["crawler.id"]) } else: inst = {"args": (doc["crawler.id"], runtime_args), "kwargs": {}} trigger = self._make_trigger(doc["schedule"]) if doc["id"]: self.scheduler.modify_job(doc["id"], jobstore="elastic", func=_run_plugin, name=doc["name.name"], **inst) new_job = self.scheduler.reschedule_job(doc["id"], jobstore="elastic", trigger=trigger) else: # use the crawler id as name, when the job is created. new_job = self.scheduler.add_job(_run_plugin, jobstore="elastic", trigger=trigger, name=doc["crawler.id"], **inst) return new_job def get_triggers(self): """Returns a list of triggers, that are predefined in the system. Returns: list: a list of tuples, holding id and name for each trigger. """ return [{ "id": v["id"], "name": v["name"], "options": v["options"] } for v in self.TRIGGERS.values()] def sync_jobs(self, joblist): """Synchronize the current jobs with a given list of jobs. This means, that all jobs not included in the list will be removed, existing ones will be updated and new ones will be added to the scheduler. Args: joblist (list): a list of jobs in the format of the schema. Returns: bool: whether this operation was successful or not. """ logger.debug("Syncing job lists ...") current_jobs = self.get_jobs() jobs_to_keep = {j["id"] for j in joblist if j.get("id")} # remove old jobs for job in current_jobs: if job["id"] not in jobs_to_keep: self.scheduler.remove_job(job["id"], jobstore="elastic") # update and add jobs for job in joblist: self.upsert_job(job) return True def _make_trigger(self, trigger_doc): """Creates a trigger from a given dictionary of user input.""" # we can assume, that an id for the trigger is given in the input. cur_trigger = self.TRIGGERS[trigger_doc["id"]] option_validator = cerberus.Validator(cur_trigger["schema"]) args = [ o["id"] for o in trigger_doc["options"] if option_validator(o) and o["active"] ] trigger_args = cur_trigger["trigger_args"](args) return CronTrigger(**trigger_args) def _serialize_trigger(self, trigger): """Serializes a trigger into a json array, as defined in TRIGGERS.""" # since we only have a defined set of triggers, the following is # possible. mapping = [(v["trigger_args"]([]).keys(), k) for k, v in self.TRIGGERS.items()] trigger_doc = None result = {} for keys, name in mapping: # all keys for the mapping need to be defined. def_keys = [f.name for f in trigger.fields if not f.is_default] if all([(key in def_keys) for key in keys]): trigger_doc = self.TRIGGERS[name] break if not trigger_doc: return result result["name"] = trigger_doc["name"] result["id"] = trigger_doc["id"] args = set(trigger_doc["from_trigger"](trigger)) # copy the list of options (otherwise this leads to nasty side effects) options = [dict(**item) for item in trigger_doc["options"]] for option in options: option["active"] = option["id"] in args result["options"] = options return result def get_jobs(self): """Returns a list of jobs that are scheduled in the system. Returns: list: a list of job-dicts, holding the id and the runtimes. """ jobs = self.scheduler.get_jobs() joblist = [] for job in jobs: joblist.append({ "id": job.id, "name": { "name": job.name }, "crawler": { "id": job.args[0] }, "schedule": self._serialize_trigger(job.trigger), "next_run": { "name": job.next_run_time } }) logger.debug(f"Retrieved {len(joblist)} jobs from the jobstore.") return joblist def run_job(self, job_id): """Runs the job with the specified id immediately. Args: job_id: the id of the job that should be run. Returns: bool: whether running the job succeeded or not. """ logger.debug(f"Running job '{job_id}' directly.") cur_job = self.scheduler.get_job(job_id, jobstore="elastic") if cur_job is None: return False cur_job.func(*cur_job.args, **cur_job.kwargs) return True
}, }) @scheduler.scheduled_job('interval', id='pull_latest_image', seconds=pull_interval, timezone=utc) def pull_latest_image(): # Try to pull a new image if interactions.pull(image_name): # If we got a new image, trigger a rebalance balance_containers.trigger() pull_latest_image.trigger = lambda: scheduler.modify_job( 'pull_latest_image', next_run_time=datetime.now()) @scheduler.scheduled_job('interval', id='balance_containers', seconds=rebalance_interval, timezone=utc) def balance_containers(): try: # Clean up before interacting with the pool: # - checks in containers that are checked out longer than the max lifetime # - stops containers that aren't running if their image is out of date # - stops containers from the pool not running selenium for container in interactions.containers(): if container.checked_out: if ((datetime.utcnow() -
class Boss(object): workers = {} dirpath = '.' output = None def __init__(self, dirpath='.', output='output'): ''' local path for load config ''' logger.info("Initialing BOSS") if os.path.isdir(dirpath): self.dirpath = dirpath logger.info("loading job config folder: " + dirpath) else: logger.info(dirpath + " is invalid, use default path instead") self.output = output logger.info("Setup output folder: " + output) if not os.path.isdir(output): logger.info("target directory " + output + " doesn't exist, creating..") os.makedirs(output) self.scheduler = BackgroundScheduler() self.scheduler.start() self.load_dir(dirpath) event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True) event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove) observer = Observer() observer.schedule(event_handler, self.dirpath, recursive=True) observer.start() def load_dir(self, folder): (dirpath, dirnames, filenames) = os.walk(folder).next() for fname in filenames: if 'json' in fname[-4:]: self.load(os.path.join(dirpath, fname)) def load(self, fp): ''' given a json file, load and create a task run regularly ''' logger.info(fp + " was loaded!") with open(fp) as in_data: try: data = json.load(in_data) data['path'] = fp except ValueError as e: logger.warning(fp + " loaded failed: " + e.message) return None interval = 30 if 'interval' in data: interval = int(data['interval']) if self.output: # TODO: test case for no 'output' key in data if 'output' not in data: data['output'] = {} output = data['output'] if 'dirpath' in output: output['dirpath'] = os.path.join(self.output, output['dirpath']) else: output['dirpath'] = self.output if 'type' not in data: logger.error("Missing type attribute in \ your configruation file [%s]" % fp) return None if fp in self.workers: # existing minion found logger.info("Update exisitng minion [%s]" % fp) minion = self.workers[fp] minion.update(**data) # //memo: Interval can't be modified self.scheduler.modify_job(job_id=fp, func=minion.collect, name=minion.name + '_' + minion.serial) else: # Create new logger.info("Create new minion [%s]" % fp) module_path = data['type'][:data['type'].rfind(".")] object_name = data['type'][data['type'].rfind(".") + 1:] try: minion_module = getattr( importlib.import_module(module_path), object_name) except Exception as e: logger.exception(e) return None minion = minion_module(**data) self.workers[fp] = minion self.scheduler.add_job(minion.collect, 'interval', id=fp, name=minion.name + '_' + minion.serial, seconds=interval) return minion return None def list(self): ''' to list all configs loaded format: [squence number] [minion name] [config_path] [status] ''' for (fp, worker) in self.workers: logger.info("path=" + fp + "," + str(worker) + ";") def remove(self, fp): ''' given file path, stop running instance if possible ''' if fp in self.workers: self.workers[fp].onstop() self.scheduler.remove_job(job_id=fp) del self.workers[fp] return True return False def remove_advanced(self): ''' TODO: 1. remove by start, end 2. by directory(?) ''' pass def unload_all(self): ''' stop all running instances ''' self.scheduler.shutdown() def pause(self, fp): ''' simply stop running instance but not remove config TODO: should have timeout if stop failed ''' self.scheduler.pause(job_id=fp) def resume(self, sn): # not sure we can do this pass def __del__(self): self.unload_all() def get_config(self): conf = {} return conf def _wake(self): ''' For periodical minions, waking them according to timing ''' pass
class TestSchedulerListener(unittest.TestCase): def setUp(self): self.scheduler = BackgroundScheduler() self.scheduler.add_jobstore(MemoryJobStore(), alias='in_memory') self.scheduler.add_executor(ThreadPoolExecutor(1), alias='secondary_executor') self.scheduler.start() def tearDown(self): self.scheduler.shutdown() def test_watcher_injection(self): watcher = SchedulerWatcher(self.scheduler) self.assertEqual(watcher.scheduler, self.scheduler, 'Watcher should keep a reference to the scheduler') self.assertEqual(1, len(self.scheduler._listeners), 'Watcher should inject itself as a scheduler listener') self.assertEqual( self.scheduler._listeners[0][1], EVENT_ALL, 'Watcher should register iself to watch all events' ) def test_scheduler_inspection(self): self.scheduler.add_job(lambda: 0, jobstore='in_memory', trigger='interval', minutes=60, id='test_job') watcher = SchedulerWatcher(self.scheduler) self.assertEqual('running', watcher.scheduler_info['state'], 'Watcher should inspect scheduler status') self.assertEqual( str(self.scheduler.timezone), watcher.scheduler_info['timezone'], 'Watcher should inspect scheduler timezone' ) self.assertEqual( 'BackgroundScheduler', watcher.scheduler_info['class'], 'Watcher should inspect scheduler class' ) self.assertEqual(2, len(watcher.jobstores), 'Watcher should inspect all scheduler jobstores') self.assertIn('in_memory', watcher.jobstores, 'Watcher should have inspected the in_memory jobstore') self.assertEqual(2, len(watcher.executors), 'Watcher should inspect all scheduler executors') self.assertIn('secondary_executor', watcher.executors, 'Watcher should have inspected the secondary_executor') self.assertEqual(1, len(watcher.jobs), 'Watcher should inspect all jobs in scheduler on init') self.assertIn('test_job', watcher.jobs, 'Watcher should index jobs by id') def test_job_properties_on_add(self): watcher = SchedulerWatcher(self.scheduler) self.scheduler.add_job( lambda x, y: x + y, id='added_job', name='Added job', jobstore='in_memory', trigger='interval', minutes=60, args=(1,), kwargs={'y': 2} ) self.assertIn('added_job', watcher.jobs) job_properties = watcher.jobs['added_job']['properties'] self.assertEqual('added_job', job_properties['id'], 'Job properties should have the job id') self.assertEqual('Added job', job_properties['name'], 'Job properties should have the job name') self.assertIn('trigger', job_properties, 'Job properties should have a representation of the trigger') self.assertEqual('in_memory', job_properties['jobstore'], 'Job properties should have the jobstore name') self.assertEqual('default', job_properties['executor'], 'Job properties should have the executor name') self.assertIn('lambda', job_properties['func'], 'Job properties should have the function string repr') self.assertIn('func_ref', job_properties, 'Job properties should have the function reference') self.assertEqual('(1,)', job_properties['args'], 'Job properties should have the job arguments') self.assertEqual("{'y': 2}", job_properties['kwargs'], 'Job properties should have the job keyword arguments') self.assertIn('pending', job_properties, 'Job properties should have the job pending status') self.assertFalse(job_properties['pending'], 'Job status should not be pending') self.assertIn('coalesce', job_properties, 'Job properties should have the job coalesce configuration') self.assertIn('next_run_time', job_properties, 'Job properties should have the next run time calculated') self.assertIn('misfire_grace_time', job_properties, 'Job properties should have the misfire grace time') self.assertIn('max_instances', job_properties, 'Job properties should have the max instances configuration') def test_job_inspection_matches_job_added_event(self): # We're going to add two jobs that should have the exact same properties, except for the id, in two different # stages of the usage: before the watcher is created and after we start watching for events. def job_function(x, y): return x + y next_run_time = datetime.now() + timedelta(hours=1) # Job that is added before the user calls us. self.scheduler.add_job( job_function, id='job_1', name='Added job', jobstore='in_memory', trigger='interval', minutes=60, args=(1,), kwargs={'y': 2}, next_run_time=next_run_time ) watcher = SchedulerWatcher(self.scheduler) # Job that gets added after we start watching. self.scheduler.add_job( job_function, id='job_2', name='Added job', jobstore='in_memory', trigger='interval', minutes=60, args=(1,), kwargs={'y': 2}, next_run_time=next_run_time ) self.assertEqual(2, len(watcher.jobs)) job_1 = watcher.jobs['job_1'] job_2 = watcher.jobs['job_2'] for property_name in job_1['properties'].keys(): # All properties, except the id, should match. if property_name == 'id': continue self.assertEqual(job_1['properties'][property_name], job_2['properties'][property_name]) def test_all_events_have_a_processing_method(self): for event_name in list(SchedulerWatcher.apscheduler_events.values()): self.assertIn(event_name, dir(SchedulerWatcher)) def test_job_execution_monitoring(self): watcher = SchedulerWatcher(self.scheduler) self.scheduler.add_job( lambda: time.sleep(0.02), id='waiting_job', name='Waiting job', jobstore='in_memory', trigger='interval', seconds=0.2, next_run_time=datetime.now() ) job_events = watcher.jobs['waiting_job']['events'] self.assertEqual(1, len(job_events)) self.assertEqual('job_added', job_events[0]['event_name']) time.sleep(0.05) self.assertEqual(3, len(job_events), 'Job execution needs to be tracked in job events') self.assertEqual( 'job_submitted', job_events[1]['event_name'], 'Job submision needs to be tracked in job events' ) self.assertEqual('job_executed', job_events[2]['event_name'], 'Job execution needs to be tracked in job events') time.sleep(0.2) self.assertEqual(5, len(job_events), 'Subsequent executions get tracked') def test_job_failure_monitoring(self): watcher = SchedulerWatcher(self.scheduler) def fail(): time.sleep(0.02) return 0 / 0 self.scheduler.add_job( fail, id='failing_job', name='Failing job', jobstore='in_memory', trigger='interval', next_run_time=datetime.now(), minutes=60 ) failing_job_events = watcher.jobs['failing_job']['events'] time.sleep(0.05) self.assertEqual(3, len(failing_job_events)) self.assertEqual('job_error', failing_job_events[2]['event_name']) def test_scheduler_summary(self): watcher = SchedulerWatcher(self.scheduler) summary = watcher.scheduler_summary() self.assertEqual(sorted(['scheduler', 'jobs', 'executors', 'jobstores']), sorted(summary.keys())) self.assertEqual('running', summary['scheduler']['state'], 'scheduler_summary should have the scheduler status') self.assertEqual(2, len(summary['executors']), 'scheduler_summaru should have the two added executors') self.assertEqual(2, len(summary['jobstores']), 'scheduler_summary should have the two executors') self.assertEqual(0, len(summary['jobs']), 'scheduler_summary should have no jobs') self.scheduler.add_job(lambda: 0, id='job_1') summary = watcher.scheduler_summary() self.assertIn('job_1', summary['jobs'], 'scheduler_summary should have the added jobs in it') self.scheduler.remove_job('job_1') summary = watcher.scheduler_summary() self.assertIn('job_1', summary['jobs'], 'scheduler_summary should have all jobs in it, even if job was removed') def test_removed_jobs_are_only_flagged_as_removed(self): self.scheduler.add_job(lambda: 0, id='a_job') watcher = SchedulerWatcher(self.scheduler) self.assertIn('a_job', watcher.jobs) self.assertIsNone(watcher.jobs['a_job']['removed_time']) self.scheduler.remove_job('a_job') self.assertIn('a_job', watcher.jobs, 'removed jobs should be still tracked in the scheduler watcher') self.assertIsNotNone(watcher.jobs['a_job']['removed_time'], 'removed_time should be set') def test_modified_job_properties_are_tracked(self): self.scheduler.add_job( lambda x, y: x + y, id='a_job', name='A job', jobstore='in_memory', trigger='interval', minutes=60, args=(1,), kwargs={'y': 2} ) watcher = SchedulerWatcher(self.scheduler) self.assertEqual(watcher.jobs['a_job']['modified_time'], watcher.jobs['a_job']['added_time']) next_run_time = watcher.jobs['a_job']['properties']['next_run_time'][0] self.scheduler.modify_job('a_job', name='A modified job', next_run_time=datetime.now() + timedelta(days=1)) self.assertGreater(watcher.jobs['a_job']['modified_time'], watcher.jobs['a_job']['added_time']) self.assertEqual('A modified job', watcher.jobs['a_job']['properties']['name']) self.assertGreater(watcher.jobs['a_job']['properties']['next_run_time'][0], next_run_time) @patch('apschedulerui.watcher.SchedulerWatcher.notify_jobstore_event') def test_removing_a_jobstore_removes_all_jobs(self, mock_notify_jobstore_event): watcher = SchedulerWatcher(self.scheduler) self.scheduler.add_job(lambda: 0, id='job_1', jobstore='in_memory', trigger='interval', minutes=60) self.scheduler.add_job(lambda: 0, id='job_2', jobstore='in_memory', trigger='interval', minutes=60) self.assertEqual(2, len(watcher.jobs)) self.assertIsNone(watcher.jobs['job_1']['removed_time'], 'job_1 removed time should be None') self.assertEqual('in_memory', watcher.jobs['job_1']['properties']['jobstore']) self.scheduler.remove_jobstore('in_memory') mock_notify_jobstore_event.assert_called() self.assertEqual(2, len(watcher.jobs), 'The amount of jobs after removing a jobstore should not change') self.assertIsNotNone(watcher.jobs['job_1']['removed_time'], 'job_1 removed time should be set') self.assertIsNotNone(watcher.jobs['job_2']['removed_time'], 'job_2 removed time should be set') @patch('apschedulerui.watcher.SchedulerWatcher._repr_job') @patch('apschedulerui.watcher.SchedulerWatcher.notify_job_event') @patch('apschedulerui.watcher.SchedulerWatcher.notify_jobstore_event') def test_adding_a_jobstore_adds_all_jobs_in_it(self, mock_notify_jobstore_event, mock_notify_job_event, _): watcher = SchedulerWatcher(self.scheduler) jobstore = MemoryJobStore() jobstore.add_job(Job(scheduler=self.scheduler, id='job_1', next_run_time=datetime.now() + timedelta(days=1))) jobstore.add_job(Job(scheduler=self.scheduler, id='job_2', next_run_time=datetime.now() + timedelta(days=2))) self.assertEqual(0, len(watcher.jobs)) self.scheduler.add_jobstore(jobstore, alias='in_memory_2') self.assertIn('in_memory_2', watcher.jobstores, 'Watcher should have the new jobstore tracked') self.assertEqual(2, len(watcher.jobs), 'Watcher should add all jobs in the newly added jobstore') self.assertTrue(all([job_id in watcher.jobs for job_id in ['job_1', 'job_2']])) self.assertEqual(2, mock_notify_job_event.call_count) mock_notify_jobstore_event.assert_called_once() @patch('apschedulerui.watcher.SchedulerWatcher.notify_job_event') def test_removing_all_jobs_flags_all_as_removed(self, mock_notify_job_event): watcher = SchedulerWatcher(self.scheduler) self.scheduler.add_job(lambda: 0, id='job_1', jobstore='default', trigger='interval', minutes=60) self.scheduler.add_job(lambda: 0, id='job_2', jobstore='in_memory', trigger='interval', minutes=60) self.assertEqual(2, len(watcher.jobs)) self.assertEqual(2, mock_notify_job_event.call_count) mock_notify_job_event.reset_mock() self.scheduler.remove_all_jobs() self.assertEqual(2, len(watcher.jobs), 'job count should not change after removing all jobs') self.assertEqual(2, mock_notify_job_event.call_count) @patch('apschedulerui.watcher.SchedulerWatcher.notify_executor_event') def test_adding_and_removing_executors(self, mock_notify_executor_event): watcher = SchedulerWatcher(self.scheduler) self.scheduler.add_executor(ThreadPoolExecutor(), alias='new_executor') self.assertIn('new_executor', watcher.executors) mock_notify_executor_event.assert_called() mock_notify_executor_event.reset_mock() self.scheduler.remove_executor('new_executor') self.assertNotIn('new_executor', watcher.executors) mock_notify_executor_event.assert_called() def test_job_event_history_is_limited(self): watcher = SchedulerWatcher(self.scheduler, max_events_per_job=4) self.scheduler.add_job(lambda: 0, trigger='interval', seconds=0.01, id='recurrent_job') time.sleep(0.1) # recurrent_job should have been executed ~10 times now, generating ~20 events (submission + execution). self.assertEqual( watcher.max_events_per_job, len(watcher.jobs['recurrent_job']['events']), 'job event history should be limited' )
class ApplicationController(object): def __init__(self): self.thread_scheduler = BackgroundScheduler() # Load or set up the config. self.yaml_path = self._find_config_home() try: self.yaml_config = yaml.safe_load(open(self._find_config_home())) if self.yaml_config is None: logging.getLogger(__name__).warning( "YAML found but empty, remaking.") self.yaml_config = [] else: logging.getLogger(__name__).debug("YAML found and loaded.") except FileNotFoundError: logging.getLogger(__name__).warning( "No YAML found. Making a new one.") open(self.yaml_path, 'a').close() self.yaml_config = [] except yaml.YAMLError: logging.getLogger(__name__).warning( "Something's wrong with the YAML... remaking.") for job in self.yaml_config: self.thread_scheduler.add_job( lambda: self._check_for_processes(list(job.keys())[0]), id=list(job.keys())[0], trigger=IntervalTrigger(minutes=list(job.values())[0])) logging.getLogger(__name__).debug("Initialized jobs.") def _find_config_home(self): platform = sys.platform if 'linux' in platform: return os.path.expanduser('~/.local/share/tabber.cfg') elif 'win32' in platform: return os.path.expanduser('~/AppData/Roaming/tabber.cfg') elif 'mac' in platform: return os.path.expanduser( '~/Library/Application Support/tabber.cfg') else: print("Unknown OS. Only Windows, Mac, and Linux are supported.") return None def _save_config(self): with open(self.yaml_path, 'w') as yp: yaml.dump(self.yaml_config, yp) print("Saved config.") def _update_processes(self): self.current_processes.delete(0, 'end') try: proc_to_add = [ str(["{:<20} {:<3}".format(p, i) for p, i in proc.items()]) for proc in self.yaml_config ] for i, proc in enumerate(self.yaml_config): self.current_processes.insert( i, "{:<15} {:<3}".format( list(proc.keys())[0], list(proc.values())[0])) except AttributeError: raise self.current_processes.grid(row=3, column=0) def _add_proc(self, procname, interval): # IDs for the current jobs are equal to the procname if procname in [j.id for j in self.thread_scheduler.get_jobs()]: # it already exists, modify it self.thread_scheduler.modify_job( job_id=procname, trigger=IntervalTrigger(minutes=interval)) # go through the list of current procname/interval k/v's and edit the combo for i, proc in enumerate(self.yaml_config): if proc == procname: self.yaml_config[i][procname] = interval else: self.thread_scheduler.add_job( lambda: self._check_for_processes(procname), id=procname, trigger=IntervalTrigger(minutes=interval)) self.yaml_config.append({procname: interval}) self._save_config() self._update_processes() def _delete_proc_cursor(self, selected): procname = self.current_processes.get(selected[0])[:-3].strip() interval = int(self.current_processes.get(selected[0])[-3:]) for i, x in enumerate(self.yaml_config): if procname == list(x.keys())[0] and interval == list( x.values())[0]: del self.yaml_config[i] self._save_config() self.thread_scheduler.remove_job( self.current_processes.get(selected[0])[:-3].strip()) self.current_processes.delete(selected[0]) self._update_processes() def _check_for_processes(self, procname): do_alt_tab = False for proc in psutil.process_iter(): if proc.name() in procname: do_alt_tab = True if do_alt_tab: self.alt_tab() def alt_tab(self): if "mac" in sys.platform: pyautogui.hotkey('command', 'tab') else: pyautogui.hotkey('alt', 'tab') def _init_gui(self): oriR = 1 oriC = 0 # Create the root window root = tk.Tk() root.geometry("500x400") root.title("Pro Gamer Switcher") menu = tk.Menu(root) root.config(menu=menu) filemenu = tk.Menu(menu) menu.add_cascade(label='File', menu=filemenu) filemenu.add_command(label='New') filemenu.add_command(label='Open...') filemenu.add_separator() filemenu.add_command(label='Exit', command=root.quit) helpmenu = tk.Menu(menu) menu.add_cascade(label='Help', menu=helpmenu) helpmenu.add_command(label='About') self.current_processes = tk.Listbox(root) try: proc_to_add = [ str(["{:<20} {:<3}".format(p, i) for p, i in proc.items()]) for proc in self.yaml_config ] for i, proc in enumerate(self.yaml_config): self.current_processes.insert( i, "{:<15} {:<3}".format( list(proc.keys())[0], list(proc.values())[0])) except AttributeError: raise self.current_processes.grid(row=oriR + 2, column=oriC) name_desc = tk.Label(root, text="Process:") name_desc.grid(row=oriR + 1, column=oriC) name_entry = tk.Entry(root, width=10) name_entry.grid(row=oriR + 1, column=oriC + 1) interval_desc = tk.Label(root, text="Interval (min):") interval_desc.grid(row=oriR + 1, column=oriC + 5) interval_entry = tk.Entry(root, width=3) interval_entry.grid(row=oriR + 1, column=oriC + 6) add_btn = tk.Button(root, text="Add", command=lambda: self._add_proc( name_entry.get(), int(interval_entry.get()))) add_btn.grid(row=oriR + 1, column=oriC + 7) del_btn = tk.Button(root, text="Delete", command=lambda: self._delete_proc_cursor( self.current_processes.curselection())) del_btn.grid(row=oriR + 2, column=oriC + 7) root.mainloop()
class Scheduler: def __init__(self): self.__running_tasks = [] self.aps_scheduler = BackgroundScheduler() # task listener def task_listener_add(event): if event.job_id not in self.__running_tasks: self.__running_tasks.append(event.job_id) event_stream(type='task', task=event.job_id) def task_listener_remove(event): if event.job_id in self.__running_tasks: self.__running_tasks.remove(event.job_id) event_stream(type='task', task=event.job_id) self.aps_scheduler.add_listener(task_listener_add, EVENT_JOB_SUBMITTED) self.aps_scheduler.add_listener(task_listener_remove, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR) # configure all tasks self.__cache_cleanup_task() self.update_configurable_tasks() self.aps_scheduler.start() def update_configurable_tasks(self): self.__sonarr_update_task() self.__radarr_update_task() self.__sonarr_full_update_task() self.__radarr_full_update_task() self.__update_bazarr_task() self.__search_wanted_subtitles_task() self.__upgrade_subtitles_task() self.__randomize_interval_task() if args.no_tasks: self.__no_task() def add_job(self, job, name=None, max_instances=1, coalesce=True, args=None): self.aps_scheduler.add_job(job, DateTrigger(run_date=datetime.now()), name=name, id=name, max_instances=max_instances, coalesce=coalesce, args=args) def execute_job_now(self, taskid): self.aps_scheduler.modify_job(taskid, next_run_time=datetime.now()) def get_running_tasks(self): return self.__running_tasks def get_task_list(self): def get_time_from_interval(td_object): seconds = int(td_object.total_seconds()) periods = [('year', 60 * 60 * 24 * 365), ('month', 60 * 60 * 24 * 30), ('day', 60 * 60 * 24), ('hour', 60 * 60), ('minute', 60), ('second', 1)] strings = [] for period_name, period_seconds in periods: if seconds > period_seconds: period_value, seconds = divmod(seconds, period_seconds) has_s = 's' if period_value > 1 else '' strings.append("%s %s%s" % (period_value, period_name, has_s)) return ", ".join(strings) def get_time_from_cron(cron): year = str(cron[0]) if year == "2100": return "Never" day = str(cron[4]) hour = str(cron[5]) if day == "*": text = "everyday" else: text = "every " + day_name[int(day)] if hour != "*": text += " at " + hour + ":00" return text task_list = [] for job in self.aps_scheduler.get_jobs(): next_run = 'Never' if job.next_run_time: next_run = pretty.date(job.next_run_time.replace(tzinfo=None)) if isinstance(job.trigger, CronTrigger): if job.next_run_time and str( job.trigger.__getstate__()['fields'][0]) != "2100": next_run = pretty.date( job.next_run_time.replace(tzinfo=None)) if job.id in self.__running_tasks: running = True else: running = False if isinstance(job.trigger, IntervalTrigger): interval = "every " + get_time_from_interval( job.trigger.__getstate__()['interval']) task_list.append({ 'name': job.name, 'interval': interval, 'next_run_in': next_run, 'next_run_time': next_run, 'job_id': job.id, 'job_running': running }) elif isinstance(job.trigger, CronTrigger): task_list.append({ 'name': job.name, 'interval': get_time_from_cron(job.trigger.fields), 'next_run_in': next_run, 'next_run_time': next_run, 'job_id': job.id, 'job_running': running }) return task_list def __sonarr_update_task(self): if settings.general.getboolean('use_sonarr'): self.aps_scheduler.add_job( update_series, IntervalTrigger(minutes=int(settings.sonarr.series_sync)), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_series', name='Update Series list from Sonarr', replace_existing=True) self.aps_scheduler.add_job( sync_episodes, IntervalTrigger(minutes=int(settings.sonarr.episodes_sync)), max_instances=1, coalesce=True, misfire_grace_time=15, id='sync_episodes', name='Sync episodes with Sonarr', replace_existing=True) def __radarr_update_task(self): if settings.general.getboolean('use_radarr'): self.aps_scheduler.add_job( update_movies, IntervalTrigger(minutes=int(settings.radarr.movies_sync)), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_movies', name='Update Movie list from Radarr', replace_existing=True) def __cache_cleanup_task(self): self.aps_scheduler.add_job(cache_maintenance, IntervalTrigger(hours=24), max_instances=1, coalesce=True, misfire_grace_time=15, id='cache_cleanup', name='Cache maintenance') def __sonarr_full_update_task(self): if settings.general.getboolean('use_sonarr'): full_update = settings.sonarr.full_update if full_update == "Daily": self.aps_scheduler.add_job( update_all_episodes, CronTrigger(hour=settings.sonarr.full_update_hour), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_episodes', name='Update all Episode Subtitles from disk', replace_existing=True) elif full_update == "Weekly": self.aps_scheduler.add_job( update_all_episodes, CronTrigger(day_of_week=settings.sonarr.full_update_day, hour=settings.sonarr.full_update_hour), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_episodes', name='Update all Episode Subtitles from disk', replace_existing=True) elif full_update == "Manually": self.aps_scheduler.add_job( update_all_episodes, CronTrigger(year='2100'), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_episodes', name='Update all Episode Subtitles from disk', replace_existing=True) def __radarr_full_update_task(self): if settings.general.getboolean('use_radarr'): full_update = settings.radarr.full_update if full_update == "Daily": self.aps_scheduler.add_job( update_all_movies, CronTrigger(hour=settings.radarr.full_update_hour), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_movies', name='Update all Movie Subtitles from disk', replace_existing=True) elif full_update == "Weekly": self.aps_scheduler.add_job( update_all_movies, CronTrigger(day_of_week=settings.radarr.full_update_day, hour=settings.radarr.full_update_hour), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_movies', name='Update all Movie Subtitles from disk', replace_existing=True) elif full_update == "Manually": self.aps_scheduler.add_job( update_all_movies, CronTrigger(year='2100'), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_movies', name='Update all Movie Subtitles from disk', replace_existing=True) def __update_bazarr_task(self): if not args.no_update: task_name = 'Update Bazarr from source on Github' if args.release_update: task_name = 'Update Bazarr from release on Github' if settings.general.getboolean('auto_update'): self.aps_scheduler.add_job(check_and_apply_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_bazarr', name=task_name, replace_existing=True) else: self.aps_scheduler.add_job(check_and_apply_update, CronTrigger(year='2100'), hour=4, id='update_bazarr', name=task_name, replace_existing=True) self.aps_scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update Release Info', replace_existing=True) else: self.aps_scheduler.add_job(check_releases, IntervalTrigger(hours=6), max_instances=1, coalesce=True, misfire_grace_time=15, id='update_release', name='Update Release Info', replace_existing=True) def __search_wanted_subtitles_task(self): if settings.general.getboolean('use_sonarr'): self.aps_scheduler.add_job( wanted_search_missing_subtitles_series, IntervalTrigger( hours=int(settings.general.wanted_search_frequency)), max_instances=1, coalesce=True, misfire_grace_time=15, id='wanted_search_missing_subtitles_series', name='Search for wanted Series Subtitles', replace_existing=True) if settings.general.getboolean('use_radarr'): self.aps_scheduler.add_job( wanted_search_missing_subtitles_movies, IntervalTrigger( hours=int(settings.general.wanted_search_frequency_movie)), max_instances=1, coalesce=True, misfire_grace_time=15, id='wanted_search_missing_subtitles_movies', name='Search for wanted Movies Subtitles', replace_existing=True) def __upgrade_subtitles_task(self): if settings.general.getboolean('upgrade_subs') and \ (settings.general.getboolean('use_sonarr') or settings.general.getboolean('use_radarr')): self.aps_scheduler.add_job( upgrade_subtitles, IntervalTrigger(hours=int(settings.general.upgrade_frequency)), max_instances=1, coalesce=True, misfire_grace_time=15, id='upgrade_subtitles', name='Upgrade previously downloaded Subtitles', replace_existing=True) def __randomize_interval_task(self): for job in self.aps_scheduler.get_jobs(): if isinstance(job.trigger, IntervalTrigger): self.aps_scheduler.modify_job( job.id, next_run_time=datetime.now() + timedelta(seconds=randrange( job.trigger.interval.total_seconds() * 0.75, job.trigger.interval.total_seconds()))) def __no_task(self): for job in self.aps_scheduler.get_jobs(): self.aps_scheduler.modify_job(job.id, next_run_time=None)
class ScheduleDisplay(Surface): def __init__(self, width, height): Surface.__init__(self, (width, height)) self.width = width self.height = height # You need this if you intend to display any text font.init() self.scheduler = BackgroundScheduler() self.scheduler.start() # These are the folders we will use for images and fonts self.dir_path = path.dirname(path.realpath(__file__)) self.assets_path = path.join(self.dir_path, 'Assets') # notice the 'Fonts' folder is located in the 'Assets' self.fonts_path = path.join(self.assets_path, 'Fonts') # Change this when done testing ---- # Information specific to semester, building, and campus. # These are most often used by the BCScheduleCreator module to fill in web forms. # The same forms you would see if you went to http://searchclasses.butte.edu/ self.location = 'Main Campus' self.building = 'MC' self.term = '' # The relative location of the files containing the raw class data. self.scheduleFile = 'scheduleData.json' # This file is a compilation of Subjects in the 'MC' building. The greatly shortens # the amount of time required to update the json file. self.compiledSubjectsFile = 'subjectsIn_MC.txt' self.backgroundImageFile = 'MapBG.png' self.backgroundImage = image.load( path.join(self.assets_path, self.backgroundImageFile)) self.backgroundImage = transform.smoothscale(self.backgroundImage, (self.width, self.height)) self.backgroundImage.convert_alpha() # This defines the time after a class starts that it will still be displayed. # Ex: TimeSlot ends at 10:00, it will be visable until 10:00 + however miniutes. self.timeSlotDisplayBuffer = 8 # Holds All Surfaces that are displayed, and normally the very next one to be displayed after the # next update. self.classesSurfacesAndTimes = [] # Flag tells object to no longer update screen because nothing has changed. self.noMoreClassesFlag = False # Called by LoadTodaysClasses() self.todaysClasses = [] # Called by Load TodaysTimeSlots() self.todaysTimeSlots = [] self.timeSlotFont = (path.join(self.fonts_path, 'OpenSans-CondBold.ttf'), int(height * .03425)) # Max number of classes that should be displayed at once. self.scheduleDisplay_numberOfClasses = 20 # Shot letter-number phrase before the class title. self.classSurface_classCodeFont = (path.join(self.fonts_path, 'OpenSans-Bold.ttf'), int(height * .02877)) # Distance between classCode and roomNumber in pixels. self.classSurface_classCodeLeftBuffer = int(width * .01580) self.classSurface_classTitleFont = (path.join(self.fonts_path, 'OpenSans-Regular.ttf'), int(height * .02740)) # Distance between classTitle and classCode in pixels. self.classSurface_classTitleLeftBuffer = int(width * .18167) self.classSurface_classInstructorFont = (path.join( self.fonts_path, 'OpenSans-Regular.ttf'), int(height * .01370)) # These can be removed, then we can just put (int(width/height * 1111)) where ever they end up in the code. self.classSurface_widthBuffer = int(width * .0158) self.classSurface_heightBuffer = int(height * .00411) self.classSurface_bgColors = ((242, 242, 242), (255, 255, 255)) self.classSurface_roomNumberFont = (path.join(self.fonts_path, 'OpenSans-CondBold.ttf'), int(height * .04110)) self.classSurface_floorSurface_widthRatio = .15 self.classSurface_floorSurface_buffer = (0, 0) self.scheduler.add_job(CompileSubjectsInBuilding, 'cron', id='CompileSubjects01', day_of_week='mon-fri', hour=1, args=[ self.building, self.term, self.location, path.join(self.dir_path, self.compiledSubjectsFile) ]) self.scheduler.add_job(self.UpdateJson, 'cron', id='UpdateJson01', day_of_week='mon-fri', hour=2) self.scheduler.add_job(self.LoadTodaysClasses, 'cron', id='LoadTodaysClasses01', day_of_week='mon-fri', hour=2, minute=30) self.scheduler.add_job(self.LoadTodaysTimeSlots, 'cron', id='LoadTodaysTimeSlots01', day_of_week='mon-fri', hour=2, minute=35) self.InitializeJsonData() self.LoadTodaysClasses() self.LoadTodaysTimeSlots() def Update(self): needsUpdate = False maxHeight = self.height currentTime = datetime.now() if self.classesSurfacesAndTimes: latestTimeSlot = self.classesSurfacesAndTimes[0][1] if latestTimeSlot < currentTime: self.classesSurfacesAndTimes.pop(0) if not self.classesSurfacesAndTimes: return self needsUpdate = True self.fill((252, 252, 252)) self.blit(self.backgroundImage, (0, 0)) for classesSurfaceAndTime in self.classesSurfacesAndTimes[:-1]: self.blit(classesSurfaceAndTime[0], ((self.classSurface_widthBuffer), (self.height - maxHeight))) maxHeight -= classesSurfaceAndTime[0].get_rect().height newestTimeSlotHeight = self.classesSurfacesAndTimes[-1][ 0].get_rect().height if newestTimeSlotHeight < maxHeight: self.blit(self.classesSurfacesAndTimes[-1][0], ((self.classSurface_widthBuffer), (self.height - maxHeight))) maxHeight -= newestTimeSlotHeight for i in range(len(self.todaysTimeSlots)): nextClasses = self.GetNextTimeSlotClasses() nextClassesSurface = self.CreateClassesSurface( nextClasses) if nextClassesSurface.get_rect().height > maxHeight: break self.blit(nextClassesSurface, ((self.classSurface_widthBuffer), (self.height - maxHeight))) maxHeight -= nextClassesSurface.get_rect().height else: if self.todaysClasses: self.noMoreClassesFlag = False needsUpdate = True self.fill((252, 252, 252)) self.blit(self.backgroundImage, (0, 0)) for i in range(len(self.todaysTimeSlots)): nextClasses = self.GetNextTimeSlotClasses() nextClassesSurface = self.CreateClassesSurface(nextClasses) if nextClassesSurface.get_rect().height > maxHeight: break self.blit(nextClassesSurface, ((self.classSurface_widthBuffer), (self.height - maxHeight))) maxHeight -= nextClassesSurface.get_rect().height else: if not self.noMoreClassesFlag: self.noMoreClassesFlag = True needsUpdate = True noClassesSurface = self.CreateClassesSurface([]) self.fill((252, 252, 252)) self.blit(self.backgroundImage, (0, 0)) self.blit(noClassesSurface, (0, 0)) return needsUpdate # Should be called as the clock striked midnight. def LoadTodaysClasses(self): print('LoadTodaysClasses') currentTime = datetime.now() self.todaysClasses = [] # Returns number from 0-6 today = datetime.today().weekday() daysOfWeek = ['M', 'T', 'W', 'Th', 'F', 'Sat', 'S'] data = LoadJsonToList(path.join(self.dir_path, self.scheduleFile)) for meeting in data[1:]: if DoesClassMeet( daysOfWeek[today], meeting, 'LEC' ): # <<<< """''Artificially''""" made to "T" for testing, replace with daysOfWeek[today] meetingStart = ConvertMilitaryToStd(meeting['LEC']['Start']) timeSlot = datetime.combine( currentTime.date(), datetime.strptime(meetingStart, '%I:%M %p').time()) timeSlot = timeSlot + timedelta( minutes=self.timeSlotDisplayBuffer) if timeSlot >= currentTime: self.todaysClasses.append(meeting) self.todaysClasses = sorted(self.todaysClasses, key=lambda k: k['LEC']['Start']) # Should be called as the clock striked midnight. def LoadTodaysTimeSlots(self): print('LoadTodaysTimeSlots') today = datetime.today().weekday() daysOfWeek = ['M', 'T', 'W', 'Th', 'F', 'Sat', 'S'] self.todaysTimeSlots = [] for meeting in self.todaysClasses: nextTimeSlot = ConvertMilitaryToStd(meeting['LEC']['Start']) if not self.todaysTimeSlots or self.todaysTimeSlots[ -1] != nextTimeSlot: self.todaysTimeSlots.append(nextTimeSlot) def GetNextTimeSlotClasses(self): print('GetNextTimeSlotClasses') nextTimeSlotClasses = [] if self.todaysTimeSlots: for meeting in list(self.todaysClasses): if ConvertMilitaryToStd( meeting['LEC']['Start']) == self.todaysTimeSlots[0]: nextTimeSlotClasses.append(meeting) self.todaysClasses.pop(0) else: break self.todaysTimeSlots.pop(0) nextTimeSlotClasses = sorted(nextTimeSlotClasses, key=lambda k: k['LEC']['Room']) print(nextTimeSlotClasses) return nextTimeSlotClasses def CreateClassSurface(self, meeting, bg, width, height): room = meeting['LEC']['Room'] instructor = meeting['Instructor'] title = meeting['Title'] classCodePatter = re.compile(r'^\w{2,5}-\d{1,3}') classCode = classCodePatter.search(title).group() title = title.replace(classCode + ' ', '') # Chooses which png to load based on number roomSurface = image.load( path.join(self.assets_path, '2ndFloor2.png')) if int(room) < 200 else image.load( path.join(self.assets_path, '1stFloor2.png')) roomSurface.convert_alpha() floorSurfaceDimensions = ( int(width * self.classSurface_floorSurface_widthRatio), int(height - (2 * self.classSurface_floorSurface_buffer[1]))) roomSurface = transform.smoothscale(roomSurface, floorSurfaceDimensions) roomSurfaceFont = font.Font(*self.classSurface_roomNumberFont) roomSurfaceText = roomSurfaceFont.render(room, True, (255, 255, 255)) roomSurface.blit(roomSurfaceText, (roomSurface.get_rect().centerx - roomSurfaceText.get_rect().centerx, roomSurface.get_rect().centery - roomSurfaceText.get_rect().centery)) classCodeFont = font.Font(*self.classSurface_classCodeFont) classCodeText = classCodeFont.render(classCode, True, (0, 0, 0)) classTitleFont = font.Font(*self.classSurface_classTitleFont) classTitleText = classTitleFont.render(title, True, (0, 0, 0)) classInstructorFont = font.Font(*self.classSurface_classInstructorFont) classInstructorText = classInstructorFont.render( instructor, True, (0, 0, 0)) classSurface = Surface((width, height)) classSurface.fill(bg) classSurface.blit(roomSurface, self.classSurface_floorSurface_buffer) classSurface.blit(classCodeText, (roomSurface.get_rect().right + self.classSurface_classCodeLeftBuffer, classSurface.get_rect().centery - classCodeText.get_rect().centery)) classSurface.blit(classTitleText, (roomSurface.get_rect().right + self.classSurface_classCodeLeftBuffer + self.classSurface_classTitleLeftBuffer, classSurface.get_rect().centery - classTitleText.get_rect().centery)) classSurface.blit(classInstructorText, (classSurface.get_rect().width - classInstructorText.get_rect().width - 10, classSurface.get_rect().centery + classTitleText.get_rect().centery - classInstructorText.get_rect().height - 4)) return classSurface def CreateClassesSurface(self, classes): print('CreateClassesSurface') if classes: timeSurfaceText = ConvertMilitaryToStd(classes[0]['LEC']['Start']) else: timeSurfaceText = 'No More Classes Today' timeSurfaceFont = font.Font(*self.timeSlotFont) timeSurface = timeSurfaceFont.render(timeSurfaceText, True, (51, 51, 51)) classSurfaceHeight = (self.height - (timeSurface.get_rect().height * 2) - ((self.scheduleDisplay_numberOfClasses - 1) * self.classSurface_heightBuffer) ) / self.scheduleDisplay_numberOfClasses classesSurfaceHeight = timeSurface.get_rect().height + ( len(classes) * (classSurfaceHeight + self.classSurface_heightBuffer)) classesSurface = Surface( (self.width - (self.classSurface_widthBuffer * 2), classesSurfaceHeight), SRCALPHA, 32) classesSurface.blit(timeSurface, (self.classSurface_widthBuffer, 0)) for i, meeting in enumerate(classes): nextClass = self.CreateClassSurface( meeting, self.classSurface_bgColors[i % 2], self.width - (2 * self.classSurface_widthBuffer), classSurfaceHeight) classesSurface.blit(nextClass, (0, timeSurface.get_rect().height + (nextClass.get_rect().height + self.classSurface_heightBuffer) * i)) classesSurface.convert_alpha() if timeSurfaceText != 'No More Classes Today': currentTime = datetime.now() timeSlot = datetime.combine( currentTime.date(), datetime.strptime(timeSurfaceText, '%I:%M %p').time()) timeSlot = timeSlot + timedelta(minutes=self.timeSlotDisplayBuffer) self.classesSurfacesAndTimes.append((classesSurface, timeSlot)) return classesSurface def UpdateJson(self): print('UpdateJson') try: currentTermDict = GetCurrentTerm() except TimeoutError: print("TIMEOUT CURRENTTERM") return schedulePath = path.join(self.dir_path, self.scheduleFile) if currentTermDict['Term']: self.term = currentTermDict['Term'] self.scheduler.modify_job('CompileSubjects01', args=[ self.building, self.term, self.location, path.join(self.dir_path, self.compiledSubjectsFile) ]) subjectsPath = path.join(self.dir_path, self.compiledSubjectsFile) try: newClassesList = CreateClassesList(self.building, self.term, self.location, subjectsPath) except TimeoutError: print("TIMEOUT NEWCLASSLIST") return newClassesList.insert(0, currentTermDict) if path.isfile(schedulePath) and path.getsize(schedulePath) > 0: currentClassesList = LoadJsonToList(schedulePath) if newClassesList != currentClassesList: DumpListToJson(newClassesList, schedulePath) else: open(path.join(self.dir_path, self.scheduleFile), 'w').close() DumpListToJson(newClassesList, schedulePath) else: emptyList = [currentTermDict] DumpListToJson(emptyList, self.scheduleFile) def InitializeJsonData(self): schedulePath = path.join(self.dir_path, self.scheduleFile) if not path.isfile(path.join(self.dir_path, self.compiledSubjectsFile)): CompileSubjectsInBuilding( self.building, self.term, self.location, path.join(self.dir_path, self.compiledSubjectsFile)) if path.isfile(schedulePath) and path.getsize(schedulePath) > 0: termDict = LoadJsonToList(schedulePath)[0] if termDict['Term']: self.term = termDict['Term'] self.scheduler.modify_job( 'CompileSubjects01', args=[ self.building, self.term, self.location, path.join(self.dir_path, self.compiledSubjectsFile) ]) currentDate = datetime.now() startDate = datetime.strptime(termDict['Start'], '%m/%d/%Y') endDate = datetime.strptime(termDict['End'], '%m/%d/%Y') if not (startDate <= currentDate and currentDate <= endDate): self.UpdateJson() print(1) if len(LoadJsonToList(schedulePath)) <= 1: self.UpdateJson() print(2) else: self.UpdateJson() print(3) else: self.UpdateJson() print(4)