def main():
config.parse_args(sys.argv)
logging.setup(CONF, "billing")
executors = {
'default': ThreadPoolExecutor(10),
'processpool': ProcessPoolExecutor(3)
}
job_defaults = {
'coalesce': True,
'max_instances': 2,
'misfire_grace_time': 3600
}
scheduler = BlockingScheduler(executors=executors,
job_defaults=job_defaults,
timezone=CONF.timezone_local)
# scheduler.add_jobstore('sqlalchemy', url=CONF.database.connection)
# scheduler.add_job(tick, 'interval', seconds=10,id="abcdefg")
# scheduler.add_job(tick, 'cron', second='5,10,15,20,25,30,35,40,45,50,55',id="bill_generation")
scheduler.get_job("bill_generation") or scheduler.add_job(
tick,
'cron',
month=CONF.scheduler_time.bill_generation_month,
day=CONF.scheduler_time.bill_generation_day,
hour=CONF.scheduler_time.bill_generation_hour,
minute=CONF.scheduler_time.bill_generation_minute,
id="bill_generation")
scheduler.start()
class App:
def __init__(self):
self.logger = logging.getLogger(App.__name__)
self.logger.info('launched Karakteraz')
try:
with open('app.yml', 'r') as stream:
self.configuration = yaml.load(stream)['karakteraz']
except yaml.YAMLError:
self.logger.exception('configuration format error')
sys.exit(1)
except FileNotFoundError:
self.logger.exception('configuration not found')
sys.exit(1)
self.configuration['notification']['email'] = None if sum(
[1 for _, v in self.configuration['notification']['email'].items() if v is None]) > 0 else \
self.configuration['notification']['email']
self.configuration['notification']['telegram'] = None if sum(
[1 for _, v in self.configuration['notification']['telegram'].items() if v is None]) > 0 else \
self.configuration['notification']['telegram']
self.configuration['watch-list'] = [x.upper() for x in self.configuration['watch-list']]
self.configuration['results-page'] = 'https://fsweb.no/studentweb/resultater.jsf'
self.logger.info(
'installed configuration:\n{}'.format(yaml.dump(self.configuration, default_flow_style=False)))
self.interval = durationpy.from_str(self.configuration['frequency']).total_seconds()
self.driver = webdriver.Firefox()
self.driver.wait = WebDriverWait(self.driver, 10)
self.scheduler = BlockingScheduler()
def start(self):
self.scheduler.add_job(self.trigger_schedule, trigger='cron', hour='6,23')
job = self.scheduler.add_job(fetch_grades, id='fetch', trigger='interval', seconds=self.interval,
args=[self.driver, self.logger, self.configuration], coalesce=True)
time = datetime.now()
if time.hour >= 23 or time.hour < 6:
job.pause()
atexit.register(lambda: self.scheduler.shutdown(wait=False))
atexit.register(lambda: self.driver.quit())
self.scheduler.start()
def trigger_schedule(self):
job = self.scheduler.get_job('fetch')
if datetime.now().hour == 6:
job.resume()
self.logger.info('ready for a new days work. Resuming scheduled tasks')
else:
job.pause()
self.logger.info('sleeping for the night.')
class Updater:
def __init__(self):
logging.info('Reading configuration')
self.configuration = Configuration()
if len(self.configuration['no-ip-authorization']) is 0:
username = input("Enter the no-ip username: "******"Enter the password: "******"{}:{}".format(username, password).encode('utf-8')).decode()
self.configuration['no-ip-hostname'] = input(
"Enter the hostname (ex. mytest.testdomain.com): ")
self.configuration['no-ip-update-interval-minutes'] = \
input("Enter the interval to no-ip update in minutes: ")
self.configuration.save_configuration()
print("Configuration was saved. Install application on /opt/No-IP-Updater/ and " + \
"use 'systemctl start noip-updater.service'")
exit(0)
logging.info('Creating scheduler')
jobstores = {
'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')
}
self.scheduler = BlockingScheduler(jobstores)
if self.scheduler.get_job(job_id='no-ip-update-task') is None:
logging.info("Registering job 'no-ip-update-task'")
self.scheduler.add_job(
noip_update,
'interval', [
self.configuration['no-ip-authorization'],
self.configuration['no-ip-hostname']
],
id='no-ip-update-task',
max_instances=1,
replace_existing=True,
minutes=int(
self.configuration['no-ip-update-interval-minutes']))
# Register listener
logging.info('Registering listener')
self.scheduler.add_listener(task_listener,
EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
logging.info('Registering quit task')
atexit.register(self.exit_handler)
logging.info('Starting scheduler')
self.scheduler.start()
def exit_handler(self):
logging.info('Quitting scheduler')
self.scheduler.shutdown()
logging.shutdown()
class abstract_schedule(metaclass=ABCMeta):
def __init__(self):
self.sched = BlockingScheduler()
def lstCronJob(self, job_id=None):
result = {}
if not job_id:
jobs = self.sched.get_jobs()
for j in jobs:
result[j.id] = j
else:
jobs = self.sched.get_job(job_id)
result[job_id] = jobs
return result
def delCronJob(self, job_id):
jobs = self.lstCronJob(job_id)
if not jobs:
sys.stdout.write("Job %s not found" %job_id)
else:
self.sched.remove_job(job_id)
sys.stdout.write("Job %s 删除成功!"%job_id)
return True
def addCronJob(self, job_id, func, policy, args):
cron = CronTrigger(**policy)
self.sched.add_job(func, cron, args=args, id=job_id)
def start(self):
print("123123")
self.sched.add_job(self.autoAddJob, IntervalTrigger(seconds=5), id="autoAddJob")
self.sched.start()
def autoAddJob(self):
history_jobs = self.lstCronJob()
print(history_jobs, 'history_jobs')
current_jobs = self.getBackupPolicy()
print(current_jobs, 'current_jobs')
only_current_jobs = set(current_jobs.keys()).difference(set(history_jobs.keys()))
print(only_current_jobs, 'only_current_jobs')
# 当前任务调度列表中有的 历史任务列表中没有的
only_history_jobs = set(history_jobs.keys()).difference(set(current_jobs.keys()))
print(only_history_jobs, 'only_history_jobs')
#历史任务中有的当前任务列表中没有的任务
#
for j in only_history_jobs:
if j == 'autoAddJob':
continue
self.delCronJob(job_id=j)
for j in only_current_jobs:
func = current_jobs[j].pop('func')
args = current_jobs[j].pop('args')
policy = current_jobs[j]
self.addCronJob(job_id=j, func=func, policy=policy, args=args)
@abstractmethod
def getBackupPolicy(self):
pass
'redis': RedisJobStore(host='172.16.0.121',port=6379,db=0), #用redis作backend
}
# ZRANGE apscheduler.run_times 0 1
# HGETALL apscheduler.jobs
executors = {
'default': ThreadPoolExecutor(10), #默认线程数
'processpool': ProcessPoolExecutor(3) #默认进程
}
sched = BlockingScheduler(jobstores=jobstores, executors=executors)
def aps_test():
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'H')
# @scheduler.scheduled_job('interval', seconds=3,id='job003')
# def test03():
# print('test03......')
#添加任务
sched.add_job(func=aps_test, trigger='cron', second='*/5',id='job001',jobstore='redis',replace_existing=True)
#查看任务状态
print(sched.get_job(job_id='job001'))
#移除任务
# scheduler.remove_job('job001')
# print(scheduler.get_job(job_id='job001'))
sched.start()
del resource_dict['updated_at']
del resource_dict['deleted_at']
'''append item from using table;'''
resource_dict['using_id'] = each['using_id']
resource_dict['started_at'] = each['started_at']
resource_dict['ended_at'] = each['ended_at']
#print "DDD %s"%resource_dict;
send_mq_message_pconn(connection, exchange, resource_dict)
if __name__ == "__main__":
from apscheduler.schedulers import *
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
executors = {
'default': ThreadPoolExecutor(10),
'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 60}
schedudler = BlockingScheduler(executors=executors,
job_defaults=job_defaults)
schedudler.add_job(resend_mq,
'cron',
minute='5,10,15,20,25,30,35,40,45,50,55',
id="five_minute")
print schedudler.get_job("five_minute")
schedudler.start()
class FreezerScheduler(object):
def __init__(self, apiclient, interval, job_path):
# config_manager
self.client = apiclient
self.freezerc_executable = spawn.find_executable('freezerc')
if self.freezerc_executable is None:
# Needed in the case of a non-activated virtualenv
self.freezerc_executable = spawn.find_executable(
'freezerc', path=':'.join(sys.path))
self.job_path = job_path
self._client = None
self.lock = threading.Lock()
self.execution_lock = threading.Lock()
job_defaults = {
'coalesce': True,
'max_instances': 1
}
self.scheduler = BlockingScheduler(job_defaults=job_defaults)
if self.client:
self.scheduler.add_job(self.poll, 'interval',
seconds=interval, id='api_poll')
self.add_job = self.scheduler.add_job
self.remove_job = self.scheduler.remove_job
self.jobs = {}
def get_jobs(self):
if self.client:
job_doc_list = utils.get_active_jobs_from_api(self.client)
try:
utils.save_jobs_to_disk(job_doc_list, self.job_path)
except Exception as e:
logging.error('Unable to save jobs to {0}. '
'{1}'.format(self.job_path, e))
return job_doc_list
else:
return utils.get_jobs_from_disk(self.job_path)
def start_session(self, session_id, job_id, session_tag):
if self.client:
return self.client.sessions.start_session(session_id,
job_id,
session_tag)
else:
raise Exception("Unable to start session: api not in use.")
def end_session(self, session_id, job_id, session_tag, result):
if self.client:
return self.client.sessions.end_session(session_id,
job_id,
session_tag,
result)
else:
raise Exception("Unable to end session: api not in use.")
def upload_metadata(self, metadata_doc):
if self.client:
self.client.backups.create(metadata_doc)
def start(self):
utils.do_register(self.client)
self.poll()
self.scheduler.start()
def update_job(self, job_id, job_doc):
if self.client:
try:
return self.client.jobs.update(job_id, job_doc)
except Exception as e:
logging.error("[*] Job update error: {0}".format(e))
def update_job_status(self, job_id, status):
doc = {'job_schedule': {'status': status}}
self.update_job(job_id, doc)
def is_scheduled(self, job_id):
return self.scheduler.get_job(job_id) is not None
def create_job(self, job_doc):
job = Job.create(self, self.freezerc_executable, job_doc)
if job:
self.jobs[job.id] = job
logging.info("Created job {0}".format(job.id))
return job
def poll(self):
try:
work_job_doc_list = self.get_jobs()
except Exception as e:
logging.error("[*] Unable to get jobs: {0}".format(e))
return
work_job_id_list = []
# create job if necessary, then let it process its events
for job_doc in work_job_doc_list:
job_id = job_doc['job_id']
work_job_id_list.append(job_id)
job = self.jobs.get(job_id, None) or self.create_job(job_doc)
if job:
job.process_event(job_doc)
# request removal of any job that has been removed in the api
for job_id, job in self.jobs.iteritems():
if job_id not in work_job_id_list:
job.remove()
remove_list = [job_id for job_id, job in self.jobs.items()
if job.can_be_removed()]
for k in remove_list:
self.jobs.pop(k)
def stop(self):
try:
self.scheduler.shutdown(wait=False)
except:
pass
def reload(self):
logging.warning("reload not supported")
if __name__ == "__main__":
executors = {
'default': ThreadPoolExecutor(10),
'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 60}
log_init()
schedudler = BlockingScheduler(executors=executors,
job_defaults=job_defaults)
#schedudler.add_job(collect_res, 'cron', second='0',id="one_miniuts")
#print schedudler.get_job("one_miniuts")
schedudler.add_job(collect_res, 'cron', hour='0-23', id="hourly")
print schedudler.get_job("hourly")
schedudler.start()
"""
#help function
#schedudler.add_job(tick, 'cron', second='5,10,15,20,25,30,35,40,45,50,55',id="five_second")
#print schedudler.get_job("five_second")
#print help(schedudler)
#@schedudler.cron_schedule(second='*', day_of_week='0-7', hour='0-23')
#def quote_send_sh_job():
# print 'a simple cron job start at', datetime.datetime.now()
"""
# 精确时间执行
# scheduler.add_job(job1, 'date', run_date=datetime(2020, 8, 8, 16, 30, 5), id='job1')
# scheduler.add_job(job1, 'date', run_date=‘2020-8-8 16:30:5’, id='job1')
# 每5秒执行一次
scheduler.add_job(job1, 'interval', seconds=5, id='job1')
scheduler.add_job(job1, 'interval', seconds=5, id='job2')
# 移除任务，得在start()之前
# scheduler.remove_job('job1')
# 暂停作业
# scheduler.pause_job('job1')
# 恢复作业
# scheduler.resume_job('job1')
# 获取作业列表
jobs = scheduler.get_jobs()
print(jobs)
# 根据id获取某个作业
job = scheduler.get_job('job1')
print(job)
scheduler.start()
# 关闭调度器,加上false表示为不想等待
scheduler.shutdown()
scheduler.shutdown(wait=False)
send_mq_message_pconn(connection, exchange, each)
save_to_resource_mq_buffer_pconn(session, each)
save_to_using_mq_buffer_pconn(session, each['mq_uuid'],
each['resource_id'])
if count == 1024:
session.commit()
count = 0
session.close()
print('end of collecting resource at %s' % datetime.now())
if __name__ == "__main__":
executors = {
'default': ThreadPoolExecutor(10),
'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 60}
log_init()
schedudler = BlockingScheduler(executors=executors,
job_defaults=job_defaults)
schedudler.add_job(collect_cdn_res, 'cron', second='0', id="one_miniuts")
print schedudler.get_job("one_miniuts")
schedudler.add_job(collect_cdn_res, 'cron', day='0-7', id="dailiy")
print schedudler.get_job("dailiy")
schedudler.start()
class FreezerScheduler(object):
def __init__(self, apiclient, interval, job_path):
# config_manager
self.client = apiclient
self.freezerc_executable = spawn.find_executable('freezer-agent')
if self.freezerc_executable is None:
# Needed in the case of a non-activated virtualenv
self.freezerc_executable = spawn.find_executable('freezer-agent',
path=':'.join(
sys.path))
LOG.debug('Freezer-agent found at {0}'.format(
self.freezerc_executable))
self.job_path = job_path
self._client = None
self.lock = threading.Lock()
self.execution_lock = threading.Lock()
job_defaults = {'coalesce': True, 'max_instances': 1}
self.scheduler = BlockingScheduler(job_defaults=job_defaults)
if self.client:
self.scheduler.add_job(self.poll,
'interval',
seconds=interval,
id='api_poll')
self.add_job = self.scheduler.add_job
self.remove_job = self.scheduler.remove_job
self.jobs = {}
def get_jobs(self):
if self.client:
job_doc_list = utils.get_active_jobs_from_api(self.client)
try:
utils.save_jobs_to_disk(job_doc_list, self.job_path)
except Exception as e:
LOG.error('Unable to save jobs to {0}. '
'{1}'.format(self.job_path, e))
return job_doc_list
else:
return utils.get_jobs_from_disk(self.job_path)
def start_session(self, session_id, job_id, session_tag):
if self.client:
return self.client.sessions.start_session(session_id, job_id,
session_tag)
else:
raise Exception("Unable to start session: api not in use.")
def end_session(self, session_id, job_id, session_tag, result):
if self.client:
return self.client.sessions.end_session(session_id, job_id,
session_tag, result)
else:
raise Exception("Unable to end session: api not in use.")
def upload_metadata(self, metadata_doc):
if self.client:
self.client.backups.create(metadata_doc)
def start(self):
utils.do_register(self.client)
self.poll()
self.scheduler.start()
def update_job(self, job_id, job_doc):
if self.client:
try:
return self.client.jobs.update(job_id, job_doc)
except Exception as e:
LOG.error("[*] Job update error: {0}".format(e))
def update_job_status(self, job_id, status):
doc = {'job_schedule': {'status': status}}
self.update_job(job_id, doc)
def is_scheduled(self, job_id):
return self.scheduler.get_job(job_id) is not None
def create_job(self, job_doc):
job = scheduler_job.Job.create(self, self.freezerc_executable, job_doc)
if job:
self.jobs[job.id] = job
LOG.info("Created job {0}".format(job.id))
return job
def poll(self):
try:
work_job_doc_list = self.get_jobs()
except Exception as e:
LOG.error("[*] Unable to get jobs: {0}".format(e))
return
work_job_id_list = []
# create job if necessary, then let it process its events
for job_doc in work_job_doc_list:
job_id = job_doc['job_id']
work_job_id_list.append(job_id)
job = self.jobs.get(job_id, None) or self.create_job(job_doc)
if job:
job.process_event(job_doc)
# request removal of any job that has been removed in the api
for job_id, job in six.iteritems(self.jobs):
if job_id not in work_job_id_list:
job.remove()
remove_list = [
job_id for job_id, job in self.jobs.items()
if job.can_be_removed()
]
for k in remove_list:
self.jobs.pop(k)
def stop(self):
try:
self.scheduler.shutdown(wait=False)
except Exception:
pass
def reload(self):
LOG.warning("reload not supported")
__date__ = "2017-12-26"
'''
python3使用apscheduler判断是否存在job
'''
from apscheduler.schedulers.blocking import BlockingScheduler
from datetime import datetime
def job():
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
sched.print_jobs()
# 定义BlockingScheduler
sched = BlockingScheduler()
sched.add_job(job, 'interval', seconds=3, id='test_job1')
sched.add_job(job, 'interval', seconds=50, id='test_job2')
print(sched.get_job('test_job1'))
print(sched.get_job('test_job1111'))
print(type(sched.get_job('test_job1111')))
if sched.get_job('test_job1111') is None:
print("haha")
if sched.get_job('test_job2') is not None:
print("lala")
# apscheduler 让job立即运行
sched.get_job('test_job2').func()
print("lala")
sched.start()
print('hi')
# sched.modify_job('test_job1', seconds=6)
class SchedUtility(object, metaclass=Singleton):
def __init__(self):
try:
self.Global = Global()
self.Utility = Utility()
self.InfraUtil = InfraUtility()
self.db = DBMySql('Scheduler')
self.myModulePyFile = os.path.abspath(__file__)
self.myClass = self.__class__.__name__
#Setting the infrastructure
self.Infra = self.InfraUtil.setInfra(self.Global.SchedulerInfraKey)
if not self.Infra:
raise InfraInitializationError('Could not initialize {cls}'.format(cls=(self.myModulePyFile,self.myClass)))
# we need to get the proper logger for a given module
self.logger = self.Infra.getInfraLogger(self.Global.SchedulerInfraKey)
# loading Schduler config and starting scheduler
self.__startScheduler__()
except Exception as err:
raise err
def __startScheduler__(self):
try:
mySchedulerType = self.Global.DefaultSchedulerType
mySchedulerMode = self.Global.DefaultSchedulerMode
if mySchedulerMode == 'Run':
myArgPaused = False
else:
myArgPaused = True
#fi
mySchedulerConfig = self.Utility.getACopy(self.Infra.schedulerConfigData)
if mySchedulerType == 'Background':
self.Scheduler = BackgroundScheduler(mySchedulerConfig)
else:
self.Scheduler = BlockingScheduler(mySchedulerConfig)
#fi
if not self.Scheduler.running:
self.Scheduler.start(paused = myArgPaused)
except Exception as err:
raise err
def getAllJobDetail(self):
'''
Description: Returns all jobs as stored in scheduler
'''
myJobDetail = []
for job in self.Scheduler.get_jobs():
myJobDetail.append(self.getAJobDetail(job.id))
return myJobDetail
def getAJobDetail(self, jobIdArg):
'''
Description: Print all jobs as stored in scheduler
'''
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
myJobDetail = job.__getstate__()
return myJobDetail
def suspendJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.pause()
def resumeJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.resume()
def getCurrentlyExecutingJob(self):
return len(self.Scheduler.get_jobs())
def removeJob(self, jobId):
try:
self.Scheduler.remove_job(jobId)
except JobLookupError as err:
print('Invalid Job !!')
def removeAllJobs(self):
try:
self.Scheduler.remove_all_jobs()
except Exception as err:
raise err
def getAllJobsFromRep(self):
for job in self.Scheduler.get_jobs():
myJobDetail = self.Scheduler.get_job(job.id)
print(job,myJobDetail)
def getNewJob(self,prefixArg):
# random number between 10 and 99 to ensure we always get 2 digit
if isinstance(prefixArg,str) and prefixArg is not None:
return prefixArg + '_' + str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99)))
else:
return datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99))
def getJobInfoFromDb(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.processDbRequest(operation = self.Global.fetch, container = 'ScheduledJobs', contents = ['*'], criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
return myResponse
def getNextSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) + 1
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getCurrentSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getElapsedStatsForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def processJobStartEvent(self, jobIdArg):
'''
1. Mark job started in ScheduledJobs
2. Create new entry for this job in ScheduledJobsRunLog
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
# building data for SchedulerJobsRunLog
myJobCriteria = ' JobId = %s' %repr(myJobId)
myNextSeqForJob = self.getNextSeqForJob(myJobId)
# will mark the job started and creat the run log for this run
self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={'Status': 'Executing'}, criteria = myJobCriteria, commitWork=True )
# creating run information
self.db.processDbRequest(operation='create', container='ScheduledJobsRunLog', \
dataDict={'JobId':myJobId, 'Seq' : myNextSeqForJob, 'ExecutionStarted': self.Utility.getCurrentTime()}, commitWork=True )
self.Utility.buildResponse(myResponse, self.Global.Success, self.Global.Success, {'Seq':myNextSeqForJob})
else:
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, 'Cound not find job details for job {job}'.format(job = myJobId))
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
#raise err # will raise the error so this can be logged by scheduler as an error occurred in processing job
return myResponse
def processJobFinishEvent(self, jobIdArg, execDetailsArg):
'''
1. Mark job completed (update failure cnt and total count and consc fail count, lastrunstatus) in ScheduledJobs
2. Update ScheduledJobsRunlog container
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
myExecDetails = execDetailsArg
myJobStatus = self.Global.NextJobRun
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
self.logger.debug('Job details found, proceeding with finish event')
myJobCriteria = 'JobId = %s' %repr(myJobId)
myCurrentSeqForJob = self.getCurrentSeqForJob(myJobId)
myJobRunCriteria = ' JobId = %s and Seq = %s ' %(repr(myJobId), myCurrentSeqForJob)
self.logger.debug('Job criteria {criteria}'.format(criteria = myJobCriteria))
self.logger.debug('Job criteria with seq {criteria}'.format(criteria = myJobRunCriteria))
myJobDetailsFromSched = self.getAJobDetail(myJobId)
# Updating execution details in ScheduledJobsRunLog
self.logger.debug('udating statistics of this run')
myDbResult = self.db.processDbRequest(operation = 'change', container = 'ScheduledJobsRunLog', \
dataDict={
'Status': myExecDetails['Status'], 'ElapsedSeconds':myExecDetails['Data']['ElapsedSecs'],
'ExecutionCompleted': self.Utility.getCurrentTime(), 'ExecutionDetail': json.dumps(myExecDetails['Data'])
}, criteria = myJobRunCriteria, commitWork=True )
self.logger.debug('ScheduledJobsRunLog: db results >> {results}'.format(results = myDbResult))
# Updating execution details in ScheduledJobs
#if myExecDetails['Status'] == self.Global.Success:
# if success, reset consecfailcnt to 0, increment totalrun by 1 and update next run
myElapsedStats = self.db.executeDynamicSql(\
operation = 'fetch', \
sql_text = 'select min(ElapsedSeconds) "Min", max(ElapsedSeconds) "Max", avg(ElapsedSeconds) "Avg" from ScheduledJobsRunLog')
self.logger.debug('Elapsed Stats: {stats}'.format(stats = myElapsedStats))
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'NextRun' : myJobDetailsFromSched['next_run_time'].strftime('%Y-%m-%d% %H:%M:%S'), 'LatConsecFailCnt' : 0,
'MinElapsedSecs' : myElapsedStats['Data'][0]['Min'], 'MaxElapsedSecs' : myElapsedStats['Data'][0]['Min'] ,
'AvgElapsedSecs' : myElapsedStats['Data'][0]['Avg']
}, criteria = myJobCriteria, commitWork=True )
self.logger.debug('ScheduledJobs: last stats update >> {result}'.format(result = myDbResult))
#self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
'''
else:
# process job was unsuccessful
if myJobDetailsFromDb[0]['LatConsecFailCnt'] >= self.Global.SchedConsecFailCntThreshold:
myJobStatus = self.Global.SuspendMode
self.logger.info('suspending job {job}'.format(job=myJobId))
self.suspendJob(myJobId)
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'next_run' : myJobDetailsFromSched['next_run_time'], 'LatConsecFailCnt' : myJobDetailsFromDb[0]['LatConsecFailCnt'] + 1,
'TotalFailure' : myJobDetailsFromDb[0]['TotalFailure' + 1]
}, criteria = myJobCriteria, commitWork=True )
# will suspend the job if total failure count has been reached beyond Total consecutive failure threshold
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,self.Global.UnSuccess)
raise processJobError(myExecDetails['Message'])
'''
self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, myErrorMsg)
return myResponse
def test_EndpointStatuses(mocker):
"""Tests the main workflow of endpointStatuses.
We mock away at two places,
gordo_components.watchman.endpoints_status.watch_for_model_server_service
which listens for kubernetes events is mocked away, and we construct the events
manually.
And the job-scheduler is never actually started, so we dont proceed to do any of
the jobs, we just check that they are added/removed as desired.
"""
# We will never call start on the scheduler, so we wont actually do any of the
# scheduled jobs|
scheduler = BlockingScheduler()
project_name = "super_project"
project_version = "101"
namespace = "somenamespace"
host = "localhost"
target_names = ["target 1", "target 2"]
mocked_watch = mocker.patch(
"gordo_components.watchman.endpoints_status.watch_for_model_server_service"
)
eps = EndpointStatuses(
scheduler=scheduler,
project_name=project_name,
ambassador_host=host,
model_names=target_names,
project_version=project_version,
namespace=namespace,
)
assert namespace == mocked_watch.call_args[1]["namespace"]
assert project_name == mocked_watch.call_args[1]["project_name"]
assert project_version == mocked_watch.call_args[1]["project_version"]
event_handler = mocked_watch.call_args[1]["event_handler"]
# Before receiving any events we only have the targets in `target_names`
cur_status = eps.statuses()
assert set([ep["target"] for ep in cur_status]) == set(target_names)
# And none of them are healthy
assert all([ep["healthy"] is False for ep in cur_status])
# Lets start adding some events.
# Target 1 is online!
# We make a fake event
mock_event_obj = MagicMock()
mock_event_obj.metadata = MagicMock()
mock_event_obj.metadata.labels = {
"applications.gordo.equinor.com/model-name": "target 1"
}
# And we let the caller know about it
event_handler({"type": "ADDED", "object": mock_event_obj})
# The job to update target 1 is added to the joblist
jobs = scheduler.get_jobs()
assert len(jobs) == 1
assert scheduler.get_job("update_model_metadata_target 1") is not None
# Target 2 is up as well
mock_event_obj.metadata.labels[
"applications.gordo.equinor.com/model-name"] = "target 2"
event_handler({"type": "ADDED", "object": mock_event_obj})
jobs = scheduler.get_jobs()
assert len(jobs) == 2
assert scheduler.get_job("update_model_metadata_target 2") is not None
# Oida, target 1 seems to be removed!
mock_event_obj.metadata.labels = {
"applications.gordo.equinor.com/model-name": "target 1"
}
event_handler({"type": "DELETED", "object": mock_event_obj})
jobs = scheduler.get_jobs()
assert len(jobs) == 1
assert scheduler.get_job("update_model_metadata_target 1") is None
assert scheduler.get_job("update_model_metadata_target 2") is not None
def main():
config.parse_args()
executors = {
'default': ThreadPoolExecutor(10),
'processpool': ProcessPoolExecutor(3)
}
job_defaults = {
'coalesce': True,
'max_instances': 2,
'misfire_grace_time': 3600
}
scheduler = BlockingScheduler(executors=executors,
job_defaults=job_defaults,
timezone="UTC")
# scheduler.add_executor('processpool')
scheduler.add_jobstore('sqlalchemy', url=CONF.database.connection)
print CONF.database.connection
scheduler.add_job(tick, 'interval', seconds=10, id="abcdefg")
# scheduler.add_job(tick, 'cron', day='2,7,10,15',id="bill_generation")
scheduler.get_job("get_instance_by_hour") or scheduler.add_job(
collectInstance, 'cron', hour='*', id="get_instance_by_hour")
scheduler.get_job("get_disk_by_hour") or scheduler.add_job(
collectDisk, 'cron', hour='*', id="get_disk_by_hour")
scheduler.get_job("get_snapshot_by_hour") or scheduler.add_job(
collectSnapshot, 'cron', hour='*', id="get_snapshot_by_hour")
scheduler.get_job("get_router_by_hour") or scheduler.add_job(
collectRouter, 'cron', hour='*', id="get_router_by_hour")
scheduler.get_job("get_ip_by_hour") or scheduler.add_job(
collectIp, 'cron', hour='*', id="get_ip_by_hour")
scheduler.get_job("get_image_by_hour") or scheduler.add_job(
collectImage, 'cron', hour='*', id="get_image_by_hour")
scheduler.get_job("get_vpn_by_hour") or scheduler.add_job(
collectVpn, 'cron', hour='*', id="get_vpn_by_hour")
scheduler.get_job("send_data_msg") or scheduler.add_job(
send_data_msg, 'cron', minute='*/2', id="send_data_msg")
# print help(scheduler)
scheduler.start()
class ColorScheduler:
mqttc: mqtt.Client
homie_device: HomieDevice
def __init__(self):
self.scheduler = BlockingScheduler()
# self.color_finder = ColorFinder(color_filter_hue)
self.color_finder = ColorFinder(color_filter_hue_brightness)
self.current_track = None
self.init_mqtt()
self.init_homie_device()
self.scheduler.add_job(self.update_job,
"interval", (),
id="job_updater",
seconds=5)
def on_connect(self, client, userdata, flags, rc, properties=None):
self.homie_device.publish_config()
def init_mqtt(self):
self.mqttc = mqtt.Client()
if Config.MQTT_USER is not None:
self.mqttc.username_pw_set(Config.MQTT_USER, Config.MQTT_PASSWORD)
self.mqttc.on_connect = self.on_connect
self.mqttc.connect(Config.MQTT_HOST)
self.mqttc.loop_start()
def init_homie_device(self):
homie_device = HomieDevice("spotibridge", self.mqttc)
homie_device.name = "Spotibridge"
homie_device.implementation = "SpotiBridge"
homie_device.version = Version("4.0.0")
homie_device.extensions = set()
node = HomieNode("player", homie_device, True)
node.name = "Player"
node.type = "player"
is_playing_property = HomieProperty("is-playing", node, True)
is_playing_property.name = "Is Playing"
is_playing_property.datatype = HomieDataType.BOOLEAN
is_playing_property.value = False
current_track_property = HomieProperty("track", node, True)
current_track_property.name = "Track"
current_track_property.datatype = HomieDataType.STRING
current_track_property.value = ""
dominant_album_color_property = HomieProperty("dominant-album-color",
node, True)
dominant_album_color_property.name = "Dominant album color"
dominant_album_color_property.datatype = HomieDataType.COLOR
dominant_album_color_property.format = "rgb"
dominant_album_color_property.value = (0, 0, 0)
album_cover_palette_property = HomieProperty("album-cover-palette",
node, True)
album_cover_palette_property.name = "Album cover color palette"
album_cover_palette_property.datatype = HomieDataType.STRING
album_cover_palette_property.value = "[]"
self.homie_device = homie_device
def start(self):
self.scheduler.start()
def get_color_palette(self, image: Image) -> List[Tuple[int, int, int]]:
color_thief = ColorThief(image)
palette = color_thief.get_palette(5, 1)
return palette
def set_color_palette(self, palette: List[Tuple[int, int, int]]):
color_palette_property = self.homie_device.nodes["player"].properties[
"album-cover-palette"]
current_value = color_palette_property.value
color_palette_property.value = json.dumps(palette)
if current_value != color_palette_property.value:
color_palette_property.publish_value()
def update_job(self):
def update_color_caller():
self.set_color((0, 0, 0))
self.set_color_palette([])
token = util.prompt_for_user_token(
Config.SPOTIFY_USERNAME,
"user-read-playback-state",
client_id=Config.SPOTIFY_CLIENT_ID,
client_secret=Config.SPOTIFY_CLIENT_SECRET,
redirect_uri=Config.SPOTIFY_REDIRECT_URI,
)
sp = Spotify(auth=token)
current_track = sp.current_user_playing_track()
if (current_track is None or not current_track["is_playing"]
or current_track["item"] is None):
job = self.scheduler.get_job("color_updater")
if job is not None:
job.remove()
if job is not None or self.current_track is not None:
self.set_color((0, 0, 0))
self.set_color_palette([])
self.set_is_playing(False)
self.set_current_track_title("")
return
track_id = current_track["item"]["id"]
if self.current_track is None:
self.set_is_playing(True)
if self.current_track != track_id:
self.current_track = track_id
cover_urls = current_track["item"]["album"]["images"]
cover_url = cover_urls[0]["url"]
response = requests.get(cover_url)
response.raise_for_status()
image = Image.open(BytesIO(response.content))
self.set_color(self.color_finder.get_most_prominent_color(image))
self.set_color_palette(self.get_color_palette(image))
self.set_current_track_title(current_track["item"]["name"])
# One cannot use this as this is not correct
# now = datetime.fromtimestamp(current_track['timestamp'] / 1000)
now = datetime.now()
start_of_track = now - timedelta(
milliseconds=current_track["progress_ms"])
next_change = start_of_track + timedelta(
milliseconds=current_track["item"]["duration_ms"])
self.scheduler.add_job(
update_color_caller,
"date",
(),
id="color_updater",
run_date=next_change,
replace_existing=True,
)
def set_color(self, color: Tuple[int, int, int]) -> None:
color_property = self.homie_device.nodes["player"].properties[
"dominant-album-color"]
if color_property.value != color:
color_property.value = color
color_property.publish_value()
def set_is_playing(self, is_playing: bool):
is_playing_property = self.homie_device.nodes["player"].properties[
"is-playing"]
if is_playing_property.value != is_playing:
is_playing_property.value = is_playing
is_playing_property.publish_value()
def set_current_track_title(self, current_track_title: str):
current_track_property = self.homie_device.nodes["player"].properties[
"track"]
if current_track_property.value != current_track_title:
current_track_property.value = current_track_title
current_track_property.publish_value()
