def beginJobService(rHelper):
logger.info("Starting up")
if rHelper.debug:
dbName = 'jobs.sqlite'
else:
dbName = os.path.join(prodconf.DIR_PATH, "jobs.sqlite")
logger.debug("DBPATH: %s" % dbName)
jobstores = {'default': SQLAlchemyJobStore(url='sqlite:///%s' % dbName)}
firstTrigger = CronTrigger(hour=22, minute=5)
secondTrigger = CronTrigger(hour=10, minute=5)
sched = BlockingScheduler(jobstores=jobstores)
sched.add_job(rHelper.postDaily, trigger=firstTrigger)
sched.add_job(rHelper.postDaily, trigger=secondTrigger)
for w in weeklies:
sched.add_job(rHelper.postWeekly, trigger=w.trigger, args=[w])
try:
logger.info("Starting blocking scheduler")
sched.start()
except (KeyboardInterrupt, SystemExit):
logger.info("Removing all jobs in end exception")
sched.remove_all_jobs()
finally:
logger.info("Removing all jobs")
sched.remove_all_jobs()
class ProxyScheduler(object):
# @my_log(log_name='schedule.log')
def __init__(self, refresh_freq):
self.manager = ProxyManager()
self.scheduler = BlockingScheduler()
self.lock = threading.Lock()
refresh_run_time = datetime.datetime.now() + datetime.timedelta(
minutes=1)
clear_run_time = datetime.datetime.now() + datetime.timedelta(hours=12)
# 每隔6个小时刷新一次代理池，每隔24小时清空可用代理并重新验证
self.scheduler.add_job(self.refresh_proxy_pool,
'interval',
hours=refresh_freq,
next_run_time=refresh_run_time)
self.scheduler.add_job(self.clear_proxy_pool,
'interval',
hours=24,
next_run_time=clear_run_time)
# @my_log(log_name='schedule.log')
def refresh_proxy_pool(self):
self.lock.acquire()
print('Refresh proxy pool: start.')
print(datetime.datetime.now())
unchecked_list = self.manager.get_unchecked_proxy()
self.manager.insert_proxy_list(ProxyManager.all_list, unchecked_list)
valid_list = self.manager.check_proxies()
self.manager.insert_proxy_list(ProxyManager.valid_list, valid_list)
print('Refresh proxy pool: done.')
self.lock.release()
# @my_log(log_name='schedule.log')
def clear_proxy_pool(self):
self.lock.acquire()
print('Clear proxy pool: start.')
print(datetime.datetime.now())
self.manager.clear_list(ProxyManager.all_list)
overdue_list = self.manager.get_all_proxies()
self.manager.insert_proxy_list(ProxyManager.all_list, overdue_list)
self.manager.clear_list(ProxyManager.valid_list)
print('Clear proxy pool: done.')
self.lock.release()
# @my_log(log_name='schedule.log')
def start_scheduler(self):
self.scheduler.start()
# @my_log(log_name='schedule.log')
def __del__(self):
self.scheduler.remove_all_jobs()
print('ProxyScheduler: remove all jobs.')
jobstores = {
'mongo': MongoDBJobStore(),
'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')
}
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
scheduler.remove_all_jobs()
scheduler.add_job(p, 'cron', second='*/5', minute='*', hour='*', day='*', month='*', day_of_week='0-6', year='*')
# scheduler.reschedule_job('my_job_id', trigger='cron', minute='*/5')
# scheduler.reschedule_job('my_job_id', trigger='cron', minute='*/5')
print("befor")
t = threading.Thread(target=scheduler.start)
t.start()
print("after")
def load(packname):
exec("from libs.apscheduler.dynamic_load import "+packname)
p = eval(packname+".run")
# p()
scheduler.add_job(p, 'cron', second='*/5', minute='*', hour='*', day='*', month='*', day_of_week='0-6', year='*')
from datetime import datetime, timedelta
import sys
import os
from apscheduler.schedulers.blocking import BlockingScheduler
def alarm(time):
print('Alarm! This alarm was scheduled at %s.' % time)
if __name__ == '__main__':
scheduler = BlockingScheduler()
scheduler.add_jobstore('mongodb', collection='example_jobs')
if len(sys.argv) > 1 and sys.argv[1] == '--clear':
scheduler.remove_all_jobs()
alarm_time = datetime.now() + timedelta(seconds=10)
scheduler.add_job(alarm,
'date',
run_date=alarm_time,
args=[datetime.now()])
print('To clear the alarms, run this example with the --clear argument.')
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
pass
class eagle_eye_bot(object):
def __init__(self, stock_list=[]):
self._scheduler = BlockingScheduler()
self.stock_list = stock_list
self._money_flows = {}
self._start_time = {}
for stock in stock_list:
self._money_flows[stock] = money_flow_level()
self._start_time[stock] = '00:00:00'
def start(self):
self._scheduler.add_job(self._start_monitor_job,
'cron',
day_of_week='mon-fri',
hour=9,
minute=29,
second=58) # 启动监控9:29:58,先获取集合竞价的数据
self._scheduler.add_job(self._end_monitor_job,
'cron',
day_of_week='mon-fri',
hour=11,
minute=30,
second=5) # 结束上午的监控11:30:05
self._scheduler.add_job(self._start_monitor_job,
'cron',
day_of_week='mon-fri',
hour=13,
minute=0,
second=0) # 启动下午的监控
self._scheduler.add_job(self._end_monitor_job,
'cron',
day_of_week='mon-fri',
hour=15,
minute=0,
second=5) # 结束下午的监控15:00:05
try:
self._scheduler.start()
print('start to monitor tick data...')
except (KeyboardInterrupt, SystemExit):
self._scheduler.remove_all_jobs()
def _start_monitor_job(self):
self._scheduler.add_job(self.__stock_monitor_on_second,
'interval',
seconds=3,
id='monitor_tick')
def _end_monitor_job(self):
self._scheduler.remove_job(job_id='monitor_tick')
def add_stock(self, stock_code, levels=[]):
"""
add code to be monitored
:param stock_code:
:return:
"""
levels = fit_levels(levels)
self.stock_list.append(stock_code)
self._money_flows[stock_code] = money_flow_level(
levels[0], levels[1], levels[2], levels[3])
self._start_time[stock_code] = '00:00:00'
def remove_stock(self, stock_code):
"""
:param stock_code:
:return:
"""
self.stock_list.remove(stock_code)
del self._money_flows[stock_code]
del self._start_time[stock_code]
def get_stock_money_flow(self, stock_code):
"""
:param stock_code:
:return:
"""
return self._money_flows[stock_code]
def __stock_monitor_on_second(self): # 定时获取tick数据处理
data = ts.get_realtime_quotes(self.stock_list)
for _, item in data.iterrows():
if item['time'] != self._start_time[item['code']]: # 是否是新成交的数据
self.__add_tick(item)
self._start_time[item['code']] = item['time']
def __add_tick(self, tick_data):
price = float(tick_data['price'])
if price <= float(tick_data['ask']): # 小于等于买一价，卖盘
tick_type = -1
elif price >= float(tick_data['bid']): # 大于等于卖一价，买盘
tick_type = 1
else:
tick_type = 0
self._money_flows[tick_data['code']].add_tick(
float(tick_data['amount']), float(tick_data['volume']), tick_type)
import os
import sys
from datetime import datetime, timedelta
from apscheduler.schedulers.blocking import BlockingScheduler
def alarm(time):
print('Alarm! This is alarm was scheduled at %s' % time)
if __name__ == '__main__':
sched = BlockingScheduler()
sched.add_jobstore('redis')
if len(sys.argv) > 1 and sys.argv[1] == '--clear':
sched.remove_all_jobs()
alarm_time = datetime.now() + timedelta(seconds=1)
sched.add_job(alarm, 'date', run_date=alarm_time, args=[datetime.now()])
sched.add_job(alarm, 'interval', seconds=1)
print('To clear the alarms, run this example with the --clear argument.')
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
pass
class Quant:
def __init__(self):
# 启动就登录baostock系统
bao_stock.login()
self.today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
# 调度器
self.scheduler = BlockingScheduler()
"""
持仓
明细 证券代码 证券名称 股票余额 可用余额 冻结数量 盈亏 成本价 盈亏比(%) 市价 市值 市场代码 交易市场 股东帐户 实际数量 资讯 买入 冻结 卖出冻结 股份实时余额
0 000859 国风塑业 100 100 0 -35.100 5.701 -6.16 5.350 535.000 1 深Ａ 0272058802 100 0 0 100
1 600313 农发种业 100 100 0 -1.110 2.711 -0.41 2.700 270.000 2 沪Ａ A570522883 100 0 0 100
"""
self.positions = pd.DataFrame()
"""
账户余额信息
可用金额 总市值 总资产 资金余额 资金帐户
1028.44 805 1833.44 1028.44 210400045648
"""
self.sub_accounts = pd.DataFrame()
""" 前期的数据，一般是前一天的 """
self.before_data = pd.DataFrame()
""" 止损，滑点等 """
self.pos_amount = 4
self.max_loss = -0.03
self.slippage = 0.001
self.commission = 0.0005
self.wait_stock = []
def job(self, msg=''):
print('标记信息：', msg)
"""
任务调度
判断是否是交易日，是则执行自动交易
:return:
"""
print('检查交易日：', self.today)
self.today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
# self.today = '2019-11-29'
trade_dates = bao_stock.get_trade_dates(start_date=self.today,
end_date=self.today)
if len(trade_dates) == 0 or trade_dates.at[0, 'is_trading_day'] == 0:
print('今天不是交易日', self.today)
else:
# with open('../logo.txt', encoding='utf-8') as f:
# # data = f.read()
# # print(data)
print('》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》')
self.before_open()
print('正常交易日', self.today)
self.scheduler = BlockingScheduler()
""" 开盘前运行一次 """
before_open_trigger = DateTrigger(
run_date=f'{self.today} 09:15:00')
self.scheduler.add_job(func=self.before_open,
trigger=before_open_trigger,
misfire_grace_time=1000,
id='before_open_job')
""" 开盘运行一次 """
open_trigger = DateTrigger(run_date=f'{self.today} 09:30:00')
self.scheduler.add_job(func=self.open,
trigger=open_trigger,
misfire_grace_time=1000,
id='open_job')
""" 止盈止损 """
# 在 10:30至11:30期间，每隔1分 运行一次 stop_loss 方法
am_stop_loss_trigger = IntervalTrigger(
minutes=1,
start_date=f'{self.today} 10:30:00',
end_date=f'{self.today} 11:30:00')
self.scheduler.add_job(func=self.stop_loss,
trigger=am_stop_loss_trigger,
misfire_grace_time=1000,
id='am_stop_loss_job')
# 在 13:00至15:00期间，每隔1分 运行一次 stop_loss 方法
pm_stop_loss_trigger = IntervalTrigger(
minutes=1,
start_date=f'{self.today} 13:00:00',
end_date=f'{self.today} 15:00:00')
self.scheduler.add_job(func=self.stop_loss,
trigger=pm_stop_loss_trigger,
misfire_grace_time=1000,
id='pm_stop_loss_job')
""" 收盘前30分钟，运行一次 """
before_close_trigger = DateTrigger(
run_date=f'{self.today} 14:30:00')
self.scheduler.add_job(func=self.close,
trigger=before_close_trigger,
misfire_grace_time=1000,
id='before_close_job')
""" 收盘后10分钟，运行一次 """
close_trigger = DateTrigger(run_date=f'{self.today} 15:10:00')
self.scheduler.add_job(func=self.close,
trigger=close_trigger,
misfire_grace_time=1000,
id='close_job')
""" 17:35爬取最新数据，选股，运行一次 """
select_trigger = DateTrigger(run_date=f'{self.today} 17:35:00')
self.scheduler.add_job(func=self.select,
trigger=select_trigger,
misfire_grace_time=1000,
id='select_job')
# 开始任务
self.scheduler.start()
def before_open(self):
"""
开盘前15分钟，可以选出今天要交易的股票
:return:
"""
print('即将开盘')
# 查询资金股份
positions = client.get_positions()
print('查询资金股份', positions)
# 仓位
self.positions = positions['positions']
# 账户余额信息
self.sub_accounts = positions['sub_accounts']
# 选出要交易的股票,此操作放在收盘后会比较好
if len(self.wait_stock) == 0:
# 如果还没有待交易的股票，就重新选一次
self.select()
# 此处可以查看集合竞价的情况
def open(self):
print(f'{self.today}开盘', datetime.datetime.now())
def stop_loss(self):
print('止损监控', datetime.datetime.now())
for pos in self.positions:
# 查询个股当前的价位，如果达到止损点，进行止损
print(pos)
pass
# 交易后，查询持仓情况
self.positions = []
def before_close(self):
"""
收盘前30分钟
:return:
"""
print('还有半小时收盘', self.today)
def close(self):
print('已收盘', datetime.datetime.now())
# 查询持仓情况
self.positions = []
# 统计盈亏
# 更新个股的目标价，止损将按照这个价进行
for pos in self.positions:
# 更新个股的目标价，止损将按照这个价进行
print(pos)
pass
def select(self):
print('选股', self.today)
# 更新数据并选出第二天要交易的票
self.wait_stock = ['000001', '000004']
# 删除所有调度任务
print('关闭所有调度任务')
self.scheduler.remove_all_jobs()
def buy(self, code, price, amount):
"""
买入
:param code:买入的股票代码
:return:
"""
re = client.buy(client='title:网上股票交易系统5.0',
timeout=5,
symbol=code,
type='LIMIT',
priceType=0,
price=price,
amount=amount)
print(re)
pass
def sell(self, code, price, amount):
try:
sell = client.sell(client='title:网上股票交易系统5.0',
timeout=5,
symbol=code,
type='LIMIT',
priceType=0,
price=price,
amount=amount)
print(type(sell))
print(sell)
except requests.exceptions.HTTPError as e:
print(type(e))
print(11, e)
def init(self):
"""
总调度
:return:
"""
print('自动交易启动。。。')
# 设置调度器,每天9:00运行一次
run = BlockingScheduler()
trigger = CronTrigger(day_of_week='*',
hour=8,
minute=30,
end_date='2100-01-01')
# trigger = CronTrigger(day_of_week='*', hour='9-18', minute='0-59', second=30, end_date='2100-01-01')
run.add_job(self.job,
trigger,
max_instances=10000,
misfire_grace_time=1000,
args=['正式调度'])
run.start()
def run(self):
"""
程序运行入口
使用线程，让其先运行一次
:return:
"""
threading.Thread(target=self.init).start()
threading.Thread(target=self.job, args=("先执行一次", )).start()
'Test'
#print('-------------')
#for i in ArrayCourse:
# print(i)
print('-------------')
print("课程:" + Class)
print("教室:" + Location)
print("时间:" + Time)
print('-------------')
#调用阿里云接口
print("---用阿里云接口---")
AliSend.send(Class, Location, Time)
if __name__ == '__main__':
try:
print("---初始化定时任务列队---")
scheduler = BlockingScheduler()
print("注册任务：每天13点45分提醒")
#每天13点10分提醒
scheduler.add_job(main, 'cron', hour=13, minute=10)
print("注册任务：每天22点25分提醒 提醒内容为明天早上")
#每天22点25分提醒 提醒内容为明天早上
scheduler.add_job(main, 'cron', hour=22, minute=40)
print("执行中")
scheduler.start()
except (KeyboardInterrupt, SystemExit):
print("\n---删除所有作业并退出---")
scheduler.remove_all_jobs(jobstore=None) #删除所有作业
class ZhiHuiShuCourseWorkerBlocking:
def __init__(self,
course_id: str,
hour=21,
minute=0,
second=0,
study_count: int = 3):
self.course_recruit_id = course_id
self.hour = hour
self.minute = minute
self.second = second
self.study_count = study_count
self.scheduler = BlockingScheduler()
def job_start(self):
logger.info('开始学习')
def before_qr(self):
logger.info('请扫码')
def after_qrcode(self):
logger.info('扫码成功')
def job_finish(self):
logger.info('全部学完了')
def lesson_finish(self):
logger.info('学完这一课了')
def course_start(self):
logger.info('开始学习这一门课程啦')
def course_end(self):
logger.info('这一门课程学完啦')
def get_zhs(self, cookies: str = None) -> ZhiHuiShu:
zhs = ZhiHuiShu()
if not cookies:
zhs.login(True, self.before_qr, self.after_qrcode)
else:
zhs.set_cookies(cookies)
return zhs
def job(self, cookies: str = None):
# sleep(random.randrange(0, 1000))
self.job_start()
flag = False
while not flag:
try:
zhs = self.get_zhs(cookies)
except RuntimeError as e:
logger.exception(e)
logger.info('job尝试重新登录')
else:
flag = True
# noinspection PyUnboundLocalVariable
vl = zhs.video_list(self.course_recruit_id)
si = zhs.query_study_info(vl)
sc = self.study_count
flag = False
for k, v in si['lv'].items():
if v['watchState'] == 0 and 'learnTime' not in v:
flag = True
pln = zhs.pre_learning_note(int(k), vl)
logger.info(f'开始学习: {int(k)}')
zhs.start_watch_blocking(int(k), vl, si, pln)
self.lesson_finish()
sc -= 1
if sc <= 0:
break
if not flag:
# 没有一节课了
self.scheduler.remove_all_jobs()
self.scheduler.shutdown(wait=False)
self.job_finish()
zhs.close()
def start(self):
self.scheduler.add_job(
self.job, 'cron', **{
'hour': self.hour if self.hour else None,
'minute': self.minute if self.minute else None,
'second': self.second if self.second else None
})
self.scheduler.start()
def start_with_cookies(self, cookies: str):
self.scheduler.add_job(
partial(self.job, cookies), 'cron', **{
'hour': self.hour if self.hour else None,
'minute': self.minute if self.minute else None,
'second': self.second if self.second else None
})
self.scheduler.start()
class CustomScheduler():
def __init__(self, bot: CustomBot, database: SchedulerDatabase):
self.bot = bot
self.database = database
self.scheduler = BlockingScheduler()
self.scheduler.add_jobstore(MemoryJobStore(), alias='scheduled')
self.logger = getLogger(__name__)
self.actions = [
['send_and_delete_message', self.send_and_delete_message],
['open_link_with_delay', self.open_link_with_delay]
]
def reload_database(self):
self.scheduler.remove_all_jobs(jobstore='default')
self.database.load_database()
self.create_jobs()
self.database.load_database()
def disable_apscheduler_logger(self):
""" Optional """
getLogger('apscheduler.scheduler').setLevel(WARNING)
getLogger('apscheduler.executors.default').setLevel(WARNING)
def add_delay_job_run_once(self, func, delta, kwargs):
run_date = datetime.now() + timedelta(**delta)
self.scheduler.add_job(func=func, trigger="date",
run_date=run_date, kwargs=kwargs, jobstore='scheduled', misfire_grace_time=None)
def send_message(self, payload):
payload.get('message').update(
text='\n'.join(payload.get('message').get('text')))
return self.bot.send_message_thank(**payload.get('message'))
def delete_message(self, payload, message_id):
kwargs = {
'func': self.bot.delete_message,
'delta': payload.get('timedelta'),
"kwargs": {
'chat_id': payload.get('message').get('chat_id'),
'message_id': message_id
}
}
self.add_delay_job_run_once(**kwargs)
def send_and_delete_message(self, payload: dict):
message_id = self.send_message(payload)
self.delete_message(payload, message_id)
def open_link(self, payload: dict):
kwargs = {
'func': startfile,
'delta': payload.get('timedelta'),
'kwargs': {
'filepath': payload.get('url')
}
}
self.add_delay_job_run_once(**kwargs)
def open_zoom_link(self, payload: dict):
# TODO:
# parse link and open through build in app
# not though browser it's annoying
self.open_link(payload)
def open_link_with_delay(self, payload: dict):
url = payload.get('url')
if 'zoom.us' in url:
self.open_zoom_link(payload)
else:
self.open_link(payload)
def timeout_job_manager(self, payload: list):
for income_action in payload:
action = income_action.get('action')
for action_name, func in self.actions:
if action == action_name:
func(income_action)
def create_job(self, job: dict):
self.scheduler.add_job(func=self.timeout_job_manager,
kwargs={'payload': job.get('payload')}, jobstore='default', misfire_grace_time=None, **job.get('time'))
def create_jobs(self):
for job in self.database.get_all_jobs():
self.logger.info(f'Processing job with id: {job.get("id")}')
self.create_job(job)
def start(self):
self.logger.info('Custom Scheduler started')
self.scheduler.start()
def stop(self):
self.logger.info('Custom Scheduler has been shut down')
self.scheduler.shutdown()
class SchedUtility(object, metaclass=Singleton):
def __init__(self):
try:
self.Global = Global()
self.Utility = Utility()
self.InfraUtil = InfraUtility()
self.db = DBMySql('Scheduler')
self.myModulePyFile = os.path.abspath(__file__)
self.myClass = self.__class__.__name__
#Setting the infrastructure
self.Infra = self.InfraUtil.setInfra(self.Global.SchedulerInfraKey)
if not self.Infra:
raise InfraInitializationError('Could not initialize {cls}'.format(cls=(self.myModulePyFile,self.myClass)))
# we need to get the proper logger for a given module
self.logger = self.Infra.getInfraLogger(self.Global.SchedulerInfraKey)
# loading Schduler config and starting scheduler
self.__startScheduler__()
except Exception as err:
raise err
def __startScheduler__(self):
try:
mySchedulerType = self.Global.DefaultSchedulerType
mySchedulerMode = self.Global.DefaultSchedulerMode
if mySchedulerMode == 'Run':
myArgPaused = False
else:
myArgPaused = True
#fi
mySchedulerConfig = self.Utility.getACopy(self.Infra.schedulerConfigData)
if mySchedulerType == 'Background':
self.Scheduler = BackgroundScheduler(mySchedulerConfig)
else:
self.Scheduler = BlockingScheduler(mySchedulerConfig)
#fi
if not self.Scheduler.running:
self.Scheduler.start(paused = myArgPaused)
except Exception as err:
raise err
def getAllJobDetail(self):
'''
Description: Returns all jobs as stored in scheduler
'''
myJobDetail = []
for job in self.Scheduler.get_jobs():
myJobDetail.append(self.getAJobDetail(job.id))
return myJobDetail
def getAJobDetail(self, jobIdArg):
'''
Description: Print all jobs as stored in scheduler
'''
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
myJobDetail = job.__getstate__()
return myJobDetail
def suspendJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.pause()
def resumeJob(self, jobIdArg):
myJobId = jobIdArg
job = self.Scheduler.get_job(myJobId)
job.resume()
def getCurrentlyExecutingJob(self):
return len(self.Scheduler.get_jobs())
def removeJob(self, jobId):
try:
self.Scheduler.remove_job(jobId)
except JobLookupError as err:
print('Invalid Job !!')
def removeAllJobs(self):
try:
self.Scheduler.remove_all_jobs()
except Exception as err:
raise err
def getAllJobsFromRep(self):
for job in self.Scheduler.get_jobs():
myJobDetail = self.Scheduler.get_job(job.id)
print(job,myJobDetail)
def getNewJob(self,prefixArg):
# random number between 10 and 99 to ensure we always get 2 digit
if isinstance(prefixArg,str) and prefixArg is not None:
return prefixArg + '_' + str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99)))
else:
return datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d_%-H%M%S_') + str(random.randrange(10,99))
def getJobInfoFromDb(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.processDbRequest(operation = self.Global.fetch, container = 'ScheduledJobs', contents = ['*'], criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
return myResponse
def getNextSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria) + 1
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getCurrentSeqForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def getElapsedStatsForJob(self, jobIdArg):
try:
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg = myJobId))
myJobCriteria = 'JobId = %s ' %repr(myJobId)
return self.db.getTotalRowCount(container = 'ScheduledJobsRunLog', criteria = myJobCriteria)
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
return myErrorMsg
def processJobStartEvent(self, jobIdArg):
'''
1. Mark job started in ScheduledJobs
2. Create new entry for this job in ScheduledJobsRunLog
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
# building data for SchedulerJobsRunLog
myJobCriteria = ' JobId = %s' %repr(myJobId)
myNextSeqForJob = self.getNextSeqForJob(myJobId)
# will mark the job started and creat the run log for this run
self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={'Status': 'Executing'}, criteria = myJobCriteria, commitWork=True )
# creating run information
self.db.processDbRequest(operation='create', container='ScheduledJobsRunLog', \
dataDict={'JobId':myJobId, 'Seq' : myNextSeqForJob, 'ExecutionStarted': self.Utility.getCurrentTime()}, commitWork=True )
self.Utility.buildResponse(myResponse, self.Global.Success, self.Global.Success, {'Seq':myNextSeqForJob})
else:
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, 'Cound not find job details for job {job}'.format(job = myJobId))
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,myErrorMsg)
#raise err # will raise the error so this can be logged by scheduler as an error occurred in processing job
return myResponse
def processJobFinishEvent(self, jobIdArg, execDetailsArg):
'''
1. Mark job completed (update failure cnt and total count and consc fail count, lastrunstatus) in ScheduledJobs
2. Update ScheduledJobsRunlog container
'''
try:
# initializing
myResponse = self.Utility.getResponseTemplate()
myJobId = self.Utility.getACopy(jobIdArg)
myExecDetails = execDetailsArg
myJobStatus = self.Global.NextJobRun
self.logger.debug('arg [{arg}] received'.format(arg=myJobId))
myJobDetailsFromDb = self.getJobInfoFromDb(myJobId)['Data']
if myJobDetailsFromDb:
self.logger.debug('Job details found, proceeding with finish event')
myJobCriteria = 'JobId = %s' %repr(myJobId)
myCurrentSeqForJob = self.getCurrentSeqForJob(myJobId)
myJobRunCriteria = ' JobId = %s and Seq = %s ' %(repr(myJobId), myCurrentSeqForJob)
self.logger.debug('Job criteria {criteria}'.format(criteria = myJobCriteria))
self.logger.debug('Job criteria with seq {criteria}'.format(criteria = myJobRunCriteria))
myJobDetailsFromSched = self.getAJobDetail(myJobId)
# Updating execution details in ScheduledJobsRunLog
self.logger.debug('udating statistics of this run')
myDbResult = self.db.processDbRequest(operation = 'change', container = 'ScheduledJobsRunLog', \
dataDict={
'Status': myExecDetails['Status'], 'ElapsedSeconds':myExecDetails['Data']['ElapsedSecs'],
'ExecutionCompleted': self.Utility.getCurrentTime(), 'ExecutionDetail': json.dumps(myExecDetails['Data'])
}, criteria = myJobRunCriteria, commitWork=True )
self.logger.debug('ScheduledJobsRunLog: db results >> {results}'.format(results = myDbResult))
# Updating execution details in ScheduledJobs
#if myExecDetails['Status'] == self.Global.Success:
# if success, reset consecfailcnt to 0, increment totalrun by 1 and update next run
myElapsedStats = self.db.executeDynamicSql(\
operation = 'fetch', \
sql_text = 'select min(ElapsedSeconds) "Min", max(ElapsedSeconds) "Max", avg(ElapsedSeconds) "Avg" from ScheduledJobsRunLog')
self.logger.debug('Elapsed Stats: {stats}'.format(stats = myElapsedStats))
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'NextRun' : myJobDetailsFromSched['next_run_time'].strftime('%Y-%m-%d% %H:%M:%S'), 'LatConsecFailCnt' : 0,
'MinElapsedSecs' : myElapsedStats['Data'][0]['Min'], 'MaxElapsedSecs' : myElapsedStats['Data'][0]['Min'] ,
'AvgElapsedSecs' : myElapsedStats['Data'][0]['Avg']
}, criteria = myJobCriteria, commitWork=True )
self.logger.debug('ScheduledJobs: last stats update >> {result}'.format(result = myDbResult))
#self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
'''
else:
# process job was unsuccessful
if myJobDetailsFromDb[0]['LatConsecFailCnt'] >= self.Global.SchedConsecFailCntThreshold:
myJobStatus = self.Global.SuspendMode
self.logger.info('suspending job {job}'.format(job=myJobId))
self.suspendJob(myJobId)
myDbResult = self.db.processDbRequest(operation='change', container='ScheduledJobs', \
dataDict={
'Status': myJobStatus, 'LastRunStatus': myExecDetails['Status'], 'TotalRun' : myJobDetailsFromDb[0]['TotalRun'] + 1,
'next_run' : myJobDetailsFromSched['next_run_time'], 'LatConsecFailCnt' : myJobDetailsFromDb[0]['LatConsecFailCnt'] + 1,
'TotalFailure' : myJobDetailsFromDb[0]['TotalFailure' + 1]
}, criteria = myJobCriteria, commitWork=True )
# will suspend the job if total failure count has been reached beyond Total consecutive failure threshold
self.Utility.buildResponse(myResponse, self.Global.UnSuccess,self.Global.UnSuccess)
raise processJobError(myExecDetails['Message'])
'''
self.Utility.buildResponse(myResponse, self.Global.Success,self.Global.Success)
return myResponse
except Exception as err:
myErrorMsg, myTraceback = self.Utility.getErrorTraceback()
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myErrorMsg)
self.logger.error(self.Global.DefPrefix4Error * self.Global.DefPrefixCount , myTraceback)
self.Utility.buildResponse(myResponse, self.Global.UnSuccess, myErrorMsg)
return myResponse
