import os, time
from datetime import datetime
from apscheduler.schedulers.background import BackgroundScheduler
def myjob():
print(datetime.now())
if __name__ == '__main__':
scheduler = BackgroundScheduler()
scheduler.start()
job = scheduler.add_job(myjob, 'interval', seconds=1, id='myjob')
print(job)
jobs = scheduler.get_jobs()
print(jobs)
try:
time.sleep(5)
print('pause job')
scheduler.pause_job('myjob')
time.sleep(5)
print('resume job')
scheduler.resume_job('myjob')
print('reschedule job ...')
scheduler.reschedule_job('myjob', trigger='cron', second='*/5')
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
class ApsJob:
def __init__(self):
self.scheduler = BackgroundScheduler()
# self.scheduler = BlockingScheduler()
def job(self):
print(datetime.datetime.now())
if random.randint(1, 10) > 5:
raise Exception
def my_listener(self, event):
if event.exception:
msg = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + ' Job function Crashed !'
print(msg)
self.scheduler.pause_job('my_job_id')
self.scheduler.remove_job('my_job_id')
self.scheduler.shutdown(wait=False)
os._exit(os.EX_OK)
# os.abort()
# sys.exit("Some exception happened. Exit!")
else:
msg = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S") + ' Job Done !'
print(msg)
class SegmentCleanup:
"""Clean up segments created by FFmpeg."""
def __init__(self, config):
self._directory = os.path.join(config.recorder.segments_folder,
config.camera.name)
# Make sure we dont delete a segment which is needed by recorder
self._max_age = config.recorder.lookback + (CAMERA_SEGMENT_DURATION *
3)
self._scheduler = BackgroundScheduler(timezone="UTC")
self._scheduler.add_job(
self.cleanup,
"interval",
seconds=CAMERA_SEGMENT_DURATION,
id="segment_cleanup",
)
self._scheduler.start()
def cleanup(self):
"""Delete all segments that are no longer needed."""
now = datetime.datetime.now().timestamp()
for segment in os.listdir(self._directory):
try:
start_time = datetime.datetime.strptime(
segment.split(".")[0], "%Y%m%d%H%M%S").timestamp()
except ValueError as error:
LOGGER.error(
f"Could not extract timestamp from segment {segment}: {error}"
)
continue
if now - start_time > self._max_age:
os.remove(os.path.join(self._directory, segment))
def start(self):
"""Start the scheduler."""
LOGGER.debug("Starting segment cleanup")
self._scheduler.start()
def pause(self):
"""Pauise the scheduler."""
LOGGER.debug("Pausing segment cleanup")
self._scheduler.pause_job("segment_cleanup")
def resume(self):
"""Resume the scheduler."""
LOGGER.debug("Resuming segment cleanup")
self._scheduler.resume_job("segment_cleanup")
class Scheduler(object):
def __init__(self):
self._scheduler = BackgroundScheduler(executors=executors,
job_defaults=job_defaults)
self._scheduler.add_jobstore('redis',
jobs_key='crontpy.jobs',
run_times_key='crontpy.run_times')
@property
def running(self):
return self._scheduler.running
def start(self):
self._scheduler.start()
def shutdown(self, wait=True):
self._scheduler.shutdown(wait)
def pause(self):
self._scheduler.pause()
def resume(self):
self._scheduler.resume()
def get_jobs(self):
return self._scheduler.get_jobs()
def get_job(self, jid):
return self._scheduler.get_job(job_id=jid)
def run_job(self, jid):
job = self.get_job(jid)
if not job:
raise Exception('job id:{0} not found'.format(jid))
job.func(*job.args, **job.kwargs)
def resume_job(self, jid):
self._scheduler.resume_job(job_id=jid)
def pause_job(self, jid):
self._scheduler.pause_job(job_id=jid)
def modify_job(self, jid, **changes):
return self._scheduler.modify_job(job_id=jid, **changes)
def delete_job(self, jid):
self._scheduler.remove_job(job_id=jid)
class diango_task():
def __init__(self):
try:
# 实例化调度器
self.scheduler = BackgroundScheduler()
# self.scheduler = BlockingScheduler()
# 调度器使用DjangoJobStore()
self.scheduler.add_jobstore(DjangoJobStore(), "default")
except Exception as e:
print(e)
# 有错误就停止运行
self.scheduler.shutdown()
def addtask(self, fun, uuid, time, user, task):
'''添加job'''
return self.scheduler.add_job(fun,
"cron",
id=uuid,
hour=time,
args=[user, task],
misfire_grace_time=3600)
def starttask(self):
'''开启job'''
register_events(self.scheduler)
self.scheduler.start()
def gettasks(self):
self.scheduler.get_jobs()
def pausejob(self, job_name):
'''暂停job'''
self.scheduler.pause_job(job_name)
def resumejob(self, job_name):
'''重启job'''
self.scheduler.resume_job(job_name)
def removejob(self, job_name):
'''删除任job'''
self.scheduler.remove_job(job_name)
class SegmentCleanup:
def __init__(self, config):
self._directory = os.path.join(
config.recorder.segments_folder, config.camera.name
)
# Make sure we dont delete a segment which is needed by recorder
self._max_age = config.recorder.lookback + (CAMERA_SEGMENT_DURATION * 3)
self._scheduler = BackgroundScheduler(timezone="UTC")
self._scheduler.add_job(
self.cleanup,
"interval",
seconds=CAMERA_SEGMENT_DURATION,
id="segment_cleanup",
)
self._scheduler.start()
def cleanup(self):
now = datetime.datetime.now().timestamp()
for segment in os.listdir(self._directory):
start_time = datetime.datetime.strptime(
segment.split(".")[0], "%Y%m%d%H%M%S"
).timestamp()
if now - start_time > self._max_age:
os.remove(os.path.join(self._directory, segment))
def start(self):
LOGGER.debug("Starting segment cleanup")
self._scheduler.start()
def pause(self):
LOGGER.debug("Pausing segment cleanup")
self._scheduler.pause_job("segment_cleanup")
def resume(self):
LOGGER.debug("Resuming segment cleanup")
self._scheduler.resume_job("segment_cleanup")
class Scheduler(object):
def __init__(self):
self._scheduler = BackgroundScheduler(executors=config.EXECUTORS,
job_defaults=config.JOB_DEFAULTS,
timezone=config.TIME_ZONE)
# self._all_jobs = {}
self._scheduler.add_listener(self._on_job_add, EVENT_JOB_ADDED)
self._scheduler.add_listener(self._on_job_remove, EVENT_JOB_REMOVED)
self._scheduler.add_listener(self._on_job_modify, EVENT_JOB_MODIFIED)
self._scheduler.add_listener(self._on_job_execute, EVENT_JOB_EXECUTED)
self._scheduler.add_listener(self._on_job_error, EVENT_JOB_ERROR)
self._scheduler.add_listener(self._on_scheduler_start,
EVENT_SCHEDULER_STARTED)
self._scheduler.add_listener(self._on_scheduler_stop,
EVENT_SCHEDULER_SHUTDOWN)
self._scheduler.add_listener(self._on_scheduler_pause,
EVENT_SCHEDULER_PAUSED)
self._scheduler.add_listener(self._on_scheduler_resume,
EVENT_SCHEDULER_RESUMED)
def start(self):
if self.status() != STOPPED:
raise ServiceException(ErrorCode.FAIL, '无法启动调度器，调度器不在停止状态')
self._scheduler.start()
def shutdown(self, waited=False):
"""
:param waited: 等待所有任务执行结束后再停止
:return:
"""
if self.status() != RUNNING:
raise ServiceException(ErrorCode.FAIL, '无法停止调度器，调度器不在运行状态')
self._scheduler.remove_all_jobs()
self._scheduler.shutdown(wait=waited)
def pause(self):
if self.status() != RUNNING:
raise ServiceException(ErrorCode.FAIL, '无法暂停调度器，调度器不在运行状态')
self._scheduler.pause()
def resume(self):
if self.status() != PAUSED:
raise ServiceException(ErrorCode.FAIL, '无法重启调度器，调度器不在暂停状态')
self._scheduler.resume()
def status(self):
"""
获取当前调度器状态
0 停止，1 运行，2暂停
:return:
"""
return self._scheduler.state
def add_job(self, task):
if self.status() != RUNNING:
raise ServiceException(ErrorCode.FAIL,
'启动任务%s失败，调度器没有运行' % task.job_id)
# 判断是否已经加入了调度器
if self._scheduler.get_job(task.job_id) is None:
# 动态导入脚本
script = importlib.import_module('job.' + task.job_id)
if script is None or not hasattr(script, 'run'):
raise ServiceException(ErrorCode.FAIL,
("%s任务没有run方法" % task.job_id))
try:
cron_dict = eval(task.cron)
except Exception as e:
raise ServiceException(ErrorCode.FAIL,
("%s任务cron规则错误" % task.job_id), str(e))
try:
if task.type == Job.Type.INTERVAL.value:
self._scheduler.add_job(script.run,
'interval',
**cron_dict,
id=task.job_id,
max_instances=task.instance_cnt)
elif task.type == Job.Type.CRON.value:
self._scheduler.add_job(script.run,
'cron',
**cron_dict,
id=task.job_id,
max_instances=task.instance_cnt)
except Exception as e:
raise ServiceException(ErrorCode.INTERNAL_ERROR,
("加入任务%s失败" % task.job_id), str(e))
else:
raise ServiceException(ErrorCode.FAIL, ("任务%s已存在" % task.job_id))
def pause_job(self, task):
current_job = self._scheduler.get_job(task.job_id)
if current_job is not None:
# 如果不在运行中
if current_job.next_run_time is None:
raise ServiceException(ErrorCode.FAIL,
'无法暂停任务%s, 该任务现在已经停止' % (task.job_id))
else:
self._scheduler.pause_job(task.job_id)
else:
raise ServiceException(ErrorCode.FAIL, ("任务%s不存在" % task.job_id))
def resume_job(self, task):
current_job = self._scheduler.get_job(task.job_id)
if current_job is not None:
# 如果在运行中
if current_job.next_run_time is not None:
raise ServiceException(ErrorCode.FAIL,
'无法重启任务%s, 该任务的现在已经在运行' % (task.job_id))
else:
self._scheduler.resume_job(task.job_id)
else:
raise ServiceException(ErrorCode.FAIL, ("任务%s不存在" % task.job_id))
# def modify_job(self, task):
# current_job = self._scheduler.get_job(task.job_id)
# if current_job is not None:
# try:
# cron_dict = eval(task.cron)
# except Exception as e:
# raise ServiceException(ErrorCode.FAIL, ("%s任务cron规则错误" % task.job_id), str(e))
#
# try:
# if task.type == Job.Type.INTERVAL.value:
# self._scheduler.reschedule_job(task.job_id, trigger='interval', **cron_dict,
# max_instances=task.instance_cnt)
# elif task.type == Job.Type.CRON.value:
# self._scheduler.reschedule_job(task.job_id, trigger='cron', **cron_dict,
# max_instances=task.instance_cnt)
# except Exception as e:
# raise ServiceException(ErrorCode.INTERNAL_ERROR, ("修改任务%s失败" % task.job_id), str(e))
# else:
# raise ServiceException(ErrorCode.FAIL, ("任务%s不存在" % task.job_id))
def remove_job(self, task):
current_job = self._scheduler.get_job(task.job_id)
if current_job is not None:
self._scheduler.remove_job(task.job_id)
else:
raise ServiceException(ErrorCode.FAIL, ("任务%s不存在" % task.job_id))
###### Listener ######
def _on_job_add(self, event):
from service.job_service import JobService
job_id = event.job_id
job = JobService.get_job(job_id)
# 有自动加入的情况
if job.status != job.Status.RUNNING.value:
JobService.change_job_status(job_id, job.Status.RUNNING)
job.status = job.Status.RUNNING.value
logger.info('定时任务%s加入了调度器，信息：%s' % (job_id, str(job)))
def _on_job_remove(self, event):
from service.job_service import JobService
job_id = event.job_id
job = JobService.get_job(job_id)
# 有自动删除的情况
if job.status != job.Status.STOPPED.value:
JobService.change_job_status(job_id, job.Status.STOPPED)
logger.info('定时任务%s被移除了调度器，信息：%s' % (job_id, str(job)))
def _on_job_modify(self, event):
from service.job_service import JobService
job_id = event.job_id
job = self._scheduler.get_job(job_id)
if job.next_run_time is None:
job = JobService.get_job(job_id)
logger.info('定时任务%s已暂停，信息：%s' % (job_id, str(job)))
if job.status != job.Status.SUSPENDED.value:
JobService.change_job_status(job_id, job.Status.SUSPENDED)
else:
logger.info('定时任务%s被修改或重启了并正在运行中，信息：%s' % (job_id, str(job)))
def _on_job_error(self, event):
from service.job_service import JobService
e = event.exception
job_id = event.job_id
logger.warn('定时任务%s发生错误，将被移除调度器' % job_id)
logger.error(str(e))
task = JobService.get_job(job_id)
self.remove_job(task)
def _on_job_execute(self, event):
from service.job_service import JobService
job_id = event.job_id
logger.info('定时任务%s开始执行' % job_id)
JobService.add_executed_times(job_id, 1)
def _on_scheduler_start(self, event):
logger.info('调度器开始执行')
def _on_scheduler_pause(self, event):
logger.info('调度器暂停')
def _on_scheduler_resume(self, event):
logger.info('调度器重启')
def _on_scheduler_stop(self, event):
logger.info('调度器停止')
def next_run_time(self, job_id):
return self._scheduler.get_job(job_id).next_run_time
class Scheduler(object):
# 클래스 생성시 스케쥴러 데몬을 생성합니다.
def __init__(self):
self.sched = BackgroundScheduler()
self.sched.start()
# 클래스가 종료될때, 모든 job들을 종료시켜줍니다.
def __del__(self):
self.shutdown()
# 모든 job들을 종료시켜주는 함수입니다.
def shutdown(self):
print("start shutdown all schedule: ")
try:
self.sched.shutdown()
except Exception as err:
print("Exception shutdown sched: %s" % err)
return
def resume_schedule(self, service_id):
print("start resume schedule: " + service_id)
try:
self.sched.resume_job(service_id)
except JobLookupError as err:
print("fail to resume scheduler: %s" % err)
return
def pause_schedule(self, service_id):
print("start pause schedule: " + service_id)
try:
self.sched.pause_job(service_id)
except JobLookupError as err:
print("fail to pause scheduler: %s" % err)
return
# 특정 job을 종료시켜줍니다.
def remove_schedule(self, service_id):
print("start remove schedule: " + service_id)
try:
self.sched.remove_job(service_id)
except JobLookupError as err:
print("fail to stop scheduler: %s" % err)
return
def load_all_schedules(self):
print("start load all schedules")
tv_channels.append("ADMIN")
for owner in tv_channels:
print("load channel: " + owner)
repeat_services = get_tv_service(owner).objects.filter(
process_type=ProcessType.REPEAT).exclude(
schedule_state=ScheduleState.FINISH)
reserve_services = get_tv_service(owner).objects.filter(
process_type=ProcessType.RESERVE).exclude(
schedule_state=ScheduleState.FINISH)
pending_quiz_answer_reserve = get_tv_service(owner).objects.filter(
Q(process_state=ProcessState.WAIT_SEND_ANSWER_RESERVE)
| Q(process_state=ProcessState.ANSWER_RESERVE))
for service_obj in repeat_services:
print("repeat_services, service_title: " +
service_obj.service_title)
if service_obj.process_state != ProcessState.FINISH:
self.set_schedule(json.loads(service_obj.process_info),
service_obj.service_id, owner,
service_obj.channel_name)
if service_obj.schedule_state == ScheduleState.INACTIVE:
print("INACTIVE repeat_services, service_id: " +
service_obj.service_id)
self.sched.pause_job(service_obj.service_id)
else:
print("ACTIVE repeat_services, service_id: " +
service_obj.service_id)
else:
print("FINISH repeat_services, service_id: " +
service_obj.service_id)
service_obj.schedule_state = ScheduleState.FINISH
service_obj.save()
for service_obj in reserve_services:
print("reserve_services, service_title: " +
service_obj.service_title)
if service_obj.process_state == ProcessState.WAIT_SEND or service_obj.process_state == ProcessState.WAIT_SEND_ANSWER_RESERVE:
process_info = json.loads(service_obj.process_info)
current_date = datetime.now()
process_date = process_info["date"]
process_time = process_info["time"]
process_datetime = datetime.strptime(
process_date + " " + process_time, "%Y-%m-%d %H:%M")
if process_datetime < current_date:
print("$$ process date time: " + process_date + " " +
process_time)
print(
"$$ process date time already passed!!!! set the service FINISH"
)
service_obj.process_state = ProcessState.FINISH
service_obj.schedule_state = ScheduleState.FINISH
service_obj.save()
else:
self.set_schedule(process_info, service_obj.service_id,
owner, service_obj.channel_name)
if service_obj.schedule_state == ScheduleState.INACTIVE:
print("INACTIVE reserve_services, service_id: " +
service_obj.service_id)
self.sched.pause_job(service_obj.service_id)
else:
print("ACTIVE reserve_services, service_id: " +
service_obj.service_id)
else:
print("FINISH reserve_services, service_id: " +
service_obj.service_id)
service_obj.schedule_state = ScheduleState.FINISH
service_obj.save()
for service_obj in pending_quiz_answer_reserve:
print("reserve quiz answer, service_title: " +
service_obj.service_title)
contents_data = json.loads(service_obj.contents)
current_date = datetime.now()
if "quiz_answer_process_info" in contents_data:
quiz_answer_process_info = contents_data[
"quiz_answer_process_info"]
process_date = quiz_answer_process_info["date"]
process_time = quiz_answer_process_info["time"]
process_datetime = datetime.strptime(
process_date + " " + process_time, "%Y-%m-%d %H:%M")
if process_datetime < current_date:
print("$$ process date time: " + process_date + " " +
process_time)
print(
"$$ process date time already passed!!!! set the service FINISH"
)
#service_obj.process_state = ProcessState.FINISH
#service_obj.schedule_state = ScheduleState.FINISH
contents_data[
"answer_schedule_state"] = ScheduleState.FINISH
service_obj.contents = json.dumps(contents_data)
service_obj.save()
else:
self.set_quiz_answer_schedule(quiz_answer_process_info,
service_obj.service_id,
owner,
service_obj.channel_name)
if "answer_schedule_state" in contents_data and contents_data[
"answer_schedule_state"] == ScheduleState.INACTIVE:
job_id = service_obj.service_id + QUIZ_ANSWER_POSTFIX
self.sched.pause_job(job_id)
print(
"INACTIVE reserve quiz answer, service_id: " +
service_obj.service_id)
else:
print("ACTIVE reserve quiz answer, service_id: " +
service_obj.service_id)
else:
contents_data[
"answer_schedule_state"] = ScheduleState.FINISH
service_obj.contents = json.dumps(contents_data)
service_obj.save()
def send_service(self, process_type, service_id, owner, room_name):
print('@@ send service, id:' + service_id + ', channel: ' + room_name +
' type: ' + str(process_type))
domain = get_server_domain()
ws = websocket.create_connection(
"wss://" + domain + "/ws/chat/" + room_name + "/",
header={"everydaytalk": "scheduler"}) # FIXME
send_data = json.dumps({
'category': Category.TV,
'owner': owner,
'channel_name': room_name,
'request_type': RequestType.SCHEDULE,
'service_id': service_id,
'schedule_target': ScheduleTarget.SERVICE,
'process_type': process_type
})
ws.send(send_data)
ws.close()
for job in self.sched.get_jobs():
print("@@ remain jobs: " + str(job.id))
def send_quiz_answer(self, process_type, service_id, owner, room_name):
print('@@ send service, id:' + service_id + ', channel: ' + room_name +
' type: ' + str(process_type))
domain = get_server_domain()
ws = websocket.create_connection(
"wss://" + domain + "/ws/chat/" + room_name + "/",
header={"everydaytalk": "scheduler"}) # FIXME
send_data = json.dumps({
'category': Category.TV,
'owner': owner,
'channel_name': room_name,
'request_type': RequestType.SCHEDULE,
'service_id': service_id,
'process_type': process_type,
'schedule_target': ScheduleTarget.QUIZ_ANSWER,
'answer_request_type': AnswerRequestType.SUBMIT
})
ws.send(send_data)
ws.close()
for job in self.sched.get_jobs():
print("@@ remain jobs: " + str(job.id))
def set_schedule(self, process_info, service_id, owner, room_name):
process_type = process_info["type"]
if process_type == ProcessType.RESERVE:
current_date = datetime.now()
process_date = process_info["date"]
process_time = process_info["time"]
process_datetime = datetime.strptime(
process_date + " " + process_time, "%Y-%m-%d %H:%M")
if process_datetime < current_date:
print("$$ process date time: " + process_date + " " +
process_time)
print(
"$$ process date time already passed!!!! set the service FINISH"
)
return
year, month, day = get_date_for_schedule(process_date)
hour, minute = get_time_for_schedule(process_time)
print('add reserve schedule, id:' + service_id + " " + year +
month + day + " " + hour + minute)
self.sched.add_job(self.send_service,
'cron',
year=year,
month=month,
day=day,
hour=hour,
minute=minute,
id=service_id,
args=(process_type, service_id, owner,
room_name))
elif process_type == ProcessType.REPEAT:
process_weekdays = process_info["weekdays"]
process_time = process_info["time"]
weekdays = get_weekdays_for_schedule(process_weekdays)
hour, minute = get_time_for_schedule(process_time)
print('add repeat schedule, id:' + service_id + " " + weekdays +
" " + hour + minute)
self.sched.add_job(self.send_service,
'cron',
day_of_week=weekdays,
hour=hour,
minute=minute,
id=service_id,
args=(process_type, service_id, owner,
room_name))
def set_quiz_answer_schedule(self, process_info, service_id, owner,
room_name):
job_id = service_id + QUIZ_ANSWER_POSTFIX
process_type = process_info["type"]
if process_type == ProcessType.RESERVE:
current_date = datetime.now()
process_date = process_info["date"]
process_time = process_info["time"]
process_datetime = datetime.strptime(
process_date + " " + process_time, "%Y-%m-%d %H:%M")
if process_datetime < current_date:
print("$$ process date time: " + process_date + " " +
process_time)
print(
"$$ process date time already passed!!!! set the service FINISH"
)
return
year, month, day = get_date_for_schedule(process_date)
hour, minute = get_time_for_schedule(process_time)
print('add quiz answer schedule, id:' + service_id + " " + year +
month + day + " " + hour + minute)
self.sched.add_job(self.send_quiz_answer,
'cron',
year=year,
month=month,
day=day,
hour=hour,
minute=minute,
id=job_id,
args=(process_type, service_id, owner,
room_name))
def activate_schedule(self, service_id, room_name):
service_obj = get_tv_service(room_name).objects.get(
service_id=service_id)
if service_obj.schedule_state == ScheduleState.INACTIVE:
print('activate schedule, id:' + service_id)
service_obj.schedule_state = ScheduleState.ACTIVE
self.sched.resume_job(service_id)
service_obj.save()
def deactivate_schedule(self, service_id, room_name):
service_obj = get_tv_service(room_name).objects.get(
service_id=service_id)
if service_obj.schedule_state == ScheduleState.ACTIVE:
print('deactivate schedule, id:' + service_id)
service_obj.schedule_state = ScheduleState.INACTIVE
self.sched.pause_job(service_id)
service_obj.save()
time.sleep(5)
print(3)
time.sleep(5)
print(4)
time.sleep(5)
print(5)
time.sleep(5)
print(6)
time.sleep(5)
if __name__ == '__main__':
sche = BackgroundScheduler()
sche.add_job(count, "interval", seconds=5, id="count")
sche.start()
time.sleep(6)
print("休眠截止")
sche.pause_job("count")
time.sleep(60)
# while True:
#
# now = datetime.now()
# if run_time is not None and now - timedelta(seconds=9) < run_time:
# job = sche.get_job("count")
# job.pause()
# print("休眠开始")
# time.sleep(30)
# print("休眠截止")
# job.resume()
class CronManager:
def __init__(self, use_mongo_db=True):
self.scheduler = BackgroundScheduler(
timezone=timezone('Asia/Shanghai'))
self.scheduler.configure()
if use_mongo_db:
self.job_store = MongoDBJobStore(database='apscheduler',
collection='cronJob',
client=db)
self.scheduler.add_jobstore(self.job_store)
self.is_replace_existing = True
else:
self.is_replace_existing = False
def add_cron(self, cron_instance):
if not isinstance(cron_instance, Cron):
raise TypeError('please add correct cron!')
if cron_instance.trigger_type == 'interval':
seconds = cron_instance.trigger_args.get('seconds')
if not isinstance(seconds,
int) and not common.can_convert_to_int(seconds):
raise TypeError('please set correct time interval')
seconds = int(seconds)
if seconds <= 0:
raise ValueError('please set interval > 0')
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
seconds=seconds,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_cron_job_id(),
max_instances=5,
jitter=0) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'date':
run_date = cron_instance.trigger_args.get('run_date')
# TODO 判断run_date类型
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
run_date=run_date,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_cron_job_id(
)) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'cron':
raise TypeError('暂时不支持 trigger_type 等于 \'cron\'')
return cron_instance.get_cron_job_id()
def start(self, paused=False):
self.scheduler.start(paused=paused)
def pause_cron(self, cron_id=None, pause_all=False):
if pause_all:
self.scheduler.pause()
elif cron_id:
self.scheduler.pause_job(job_id=cron_id)
def resume_cron(self, cron_id=None, resume_all=False):
if resume_all:
self.scheduler.resume()
elif cron_id:
self.scheduler.resume_job(job_id=cron_id)
def del_cron(self, cron_id=None, del_all=False):
if del_all:
self.scheduler.remove_all_jobs()
elif cron_id:
self.scheduler.remove_job(job_id=cron_id)
def update_cron(self, cron_job_id, project_id, cron_info):
if not isinstance(cron_job_id, str):
raise TypeError('cron_id must be str')
if not isinstance(project_id, str):
raise TypeError('project_id must be str')
if not isinstance(cron_info, dict):
raise TypeError('cron_info must be dict')
trigger_type = cron_info.get('triggerType')
interval = cron_info.get('interval')
run_date = cron_info.get('runDate')
test_suite_id_list = cron_info.get('testSuiteIdList')
include_forbidden = cron_info.get('includeForbidden')
test_env_id = cron_info.get('testEnvId')
alarm_mail_list = cron_info.get('alarmMailList')
try:
if trigger_type == 'interval' and int(interval) > 0:
self.scheduler.modify_job(
job_id=cron_job_id,
trigger=IntervalTrigger(seconds=interval))
elif trigger_type == 'date':
# TODO 判断run_date类型
self.scheduler.modify_job(
job_id=cron_job_id, trigger=DateTrigger(run_date=run_date))
else:
raise TypeError('更新定时任务触发器失败！')
if run_date:
cron = Cron(
test_suite_id_list=test_suite_id_list,
project_id=project_id,
include_forbidden=include_forbidden,
test_env_id=test_env_id,
alarm_mail_list=alarm_mail_list,
trigger_type=trigger_type, # 更新定时器时，此参数并没有真正起到作用, 仅修改展示字段
run_date=run_date) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
else:
cron = Cron(
test_suite_id_list=test_suite_id_list,
project_id=project_id,
include_forbidden=include_forbidden,
test_env_id=test_env_id,
alarm_mail_list=alarm_mail_list,
trigger_type=trigger_type, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
seconds=interval) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
# 玄学，更改job的时候必须改args，不能改func
self.scheduler.modify_job(job_id=cron_job_id,
coalesce=True,
args=[cron])
except BaseException as e:
raise TypeError('更新定时任务失败: %s' % e)
def shutdown(self, force_shutdown=False):
if force_shutdown:
self.scheduler.shutdown(wait=False)
else:
self.scheduler.shutdown(wait=True)
def get_jobs(self):
return self.scheduler.get_jobs()
class CronManager:
def __init__(self, use_mongo_db=True):
self.scheduler = BackgroundScheduler(
timezone=timezone('Asia/Shanghai'))
self.scheduler.configure()
if use_mongo_db:
self.job_store = MongoDBJobStore(database='apscheduler',
collection='cronJob',
client=db)
self.scheduler.add_jobstore(self.job_store)
self.is_replace_existing = True
else:
self.is_replace_existing = False
def add_cron(self, cron_instance):
try:
if not isinstance(cron_instance, Cron):
raise TypeError('please add correct cron!')
if cron_instance.trigger_type == 'interval':
seconds = cron_instance.trigger_args.get('seconds')
if not isinstance(
seconds,
int) and not common.can_convert_to_int(seconds):
raise TypeError('please set correct time interval')
seconds = int(seconds)
if seconds <= 0:
raise ValueError('please set interval > 0')
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
seconds=seconds,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_cron_job_id(),
max_instances=5,
jitter=0)
elif cron_instance.trigger_type == 'date':
run_date = cron_instance.trigger_args.get('run_date')
# TODO 判断run_date类型
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
run_date=run_date,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_cron_job_id())
elif cron_instance.trigger_type == 'cron':
raise TypeError('暂时不支持 trigger_type 等于 \'cron\'')
return cron_instance.get_cron_job_id()
except BaseException as e:
with app.app_context():
current_app.logger.error("add_cron failed. - %s" % str(e))
def start(self, paused=False):
self.scheduler.start(paused=paused)
def pause_cron(self, cron_id=None, pause_all=False):
if pause_all:
self.scheduler.pause()
elif cron_id:
self.scheduler.pause_job(job_id=cron_id)
def resume_cron(self, cron_id=None, resume_all=False):
if resume_all:
self.scheduler.resume()
elif cron_id:
self.scheduler.resume_job(job_id=cron_id)
def del_cron(self, cron_id=None, del_all=False):
if del_all:
self.scheduler.remove_all_jobs()
elif cron_id:
self.scheduler.remove_job(job_id=cron_id)
def update_cron(self, cron_job_id, project_id, cron_info):
try:
if not isinstance(cron_job_id, str):
raise TypeError('cron_id must be str')
if not isinstance(project_id, str):
raise TypeError('project_id must be str')
if not isinstance(cron_info, dict):
raise TypeError('cron_info must be dict')
trigger_type = cron_info.get('triggerType')
interval = cron_info.get('interval')
run_date = cron_info.get('runDate')
test_suite_id_list = cron_info.get('testSuiteIdList')
include_forbidden = cron_info.get('includeForbidden')
test_env_id = cron_info.get('testEnvId')
always_send_mail = cron_info.get('alwaysSendMail')
alarm_mail_group_list = cron_info.get('alarmMailGroupList')
enable_wxwork_notify = cron_info.get('enableWXWorkNotify')
wxwork_api_key = cron_info.get('WXWorkAPIKey')
wxwork_mention_mobile_list = cron_info.get(
'WXWorkMentionMobileList')
always_wxwork_notify = cron_info.get('alwaysWXWorkNotify')
enable_ding_talk_notify = cron_info.get('enableDingTalkNotify')
ding_talk_access_token = cron_info.get('DingTalkAccessToken')
ding_talk_at_mobiles = cron_info.get('DingTalkAtMobiles')
ding_talk_secret = cron_info.get('DingTalkSecret')
always_ding_talk_notify = cron_info.get('alwaysDingTalkNotify')
if trigger_type == 'interval' and int(interval) > 0:
self.scheduler.modify_job(
job_id=cron_job_id,
trigger=IntervalTrigger(seconds=interval))
elif trigger_type == 'date':
self.scheduler.modify_job(
job_id=cron_job_id, trigger=DateTrigger(run_date=run_date))
else:
raise TypeError('更新定时任务触发器失败！')
if run_date:
cron = Cron(
cron_job_id=cron_job_id,
test_suite_id_list=test_suite_id_list,
project_id=project_id,
test_env_id=test_env_id,
include_forbidden=include_forbidden,
enable_wxwork_notify=enable_wxwork_notify,
wxwork_api_key=wxwork_api_key,
wxwork_mention_mobile_list=wxwork_mention_mobile_list,
always_wxwork_notify=always_wxwork_notify,
enable_ding_talk_notify=enable_ding_talk_notify,
ding_talk_access_token=ding_talk_access_token,
ding_talk_at_mobiles=ding_talk_at_mobiles,
ding_talk_secret=ding_talk_secret,
always_ding_talk_notify=always_ding_talk_notify,
always_send_mail=always_send_mail,
alarm_mail_group_list=alarm_mail_group_list,
trigger_type=trigger_type, # 更新定时器时，此参数并没有真正起到作用, 仅修改展示字段
run_date=run_date) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
else:
cron = Cron(
cron_job_id=cron_job_id,
test_suite_id_list=test_suite_id_list,
project_id=project_id,
include_forbidden=include_forbidden,
enable_wxwork_notify=enable_wxwork_notify,
wxwork_api_key=wxwork_api_key,
wxwork_mention_mobile_list=wxwork_mention_mobile_list,
always_wxwork_notify=always_wxwork_notify,
enable_ding_talk_notify=enable_ding_talk_notify,
ding_talk_access_token=ding_talk_access_token,
ding_talk_at_mobiles=ding_talk_at_mobiles,
ding_talk_secret=ding_talk_secret,
always_ding_talk_notify=always_ding_talk_notify,
test_env_id=test_env_id,
always_send_mail=always_send_mail,
alarm_mail_group_list=alarm_mail_group_list,
trigger_type=trigger_type, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
seconds=interval) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
# 玄学，更改job的时候必须改args，不能改func
self.scheduler.modify_job(job_id=cron_job_id,
coalesce=True,
args=[cron])
except BaseException as e:
with app.app_context():
current_app.logger.error("update_cron failed. - %s" % str(e))
def shutdown(self, force_shutdown=False):
if force_shutdown:
self.scheduler.shutdown(wait=False)
else:
self.scheduler.shutdown(wait=True)
def get_jobs(self):
return self.scheduler.get_jobs()
class Timer_handler:
############################### constructor ###############################
def __init__(self, state_handler, logger, path):
self.boInit = True
self._open_time = None
self._close_time = None
self._auto_open = None
self._auto_close = None
self.state_handler = state_handler
self.scheduler = BackgroundScheduler()
self.observers = []
self.log = logger
self.settings_file = path + "timer_data.pkl"
#read most recent settings saved to file, if available
try:
#with open(os.environ['HOME'] +
#"/hendroid/client-raspi/timer_data.pkl", "rb") as input:
with open(self.settings_file, "rb") as input:
self.open_time = pickle.load(input)
self.log.info("internal: read <open_time> from file: "
+ str(self.open_time))
self.close_time = pickle.load(input)
self.log.info("internal: read <close_time> from file: "
+ str(self.close_time))
self.auto_open = pickle.load(input)
self.log.info("internal: read <auto_open> from file: "
+ str(self.auto_open))
self.auto_close = pickle.load(input)
self.log.info("internal: read <auto_close> from file: "
+ str(self.auto_close))
self.boInit = False
except Exception as e:
self.log.error("internal error: timer state read from file: ",
exc_info=True)
self.open_time = time(hour = 7, minute = 0)
self.close_time = time(hour = 18, minute = 30)
self.auto_open = False
self.boInit = False
self.auto_close = False
self.log.info("internal: set timer to default settings")
self.scheduler.start()
########################### property definition ###########################
#Attrinutes are defined as properties as changing them requires some effort
# (i.e. changing the Scheduler settings accordingly) rather than changing
# them directly.
@property
def open_time(self):
return self._open_time
@property
def close_time(self):
return self._close_time
@property
def auto_open(self):
return self._auto_open
@property
def auto_close(self):
return self._auto_close
@auto_open.setter
def auto_open(self, value):
if (isinstance(value, bool)
and value != None
and (self.auto_open == None or value != self.auto_open)):
self._auto_open = value
if(self.scheduler.running):
self.scheduler.pause()
if(self.auto_open):
self.scheduler.resume_job(job_id=self.open_job.id)
else:
self.scheduler.pause_job(job_id=self.open_job.id)
self.log.info("internal: set <auto_open> to " + str(self.auto_open))
if(self.scheduler.running):
self.scheduler.resume()
self.save_state()
self.notify_observers("auto-open")
@auto_close.setter
def auto_close(self, value):
if (isinstance(value, bool)
and value != None
and (self.auto_close == None or value != self.auto_close)):
self._auto_close = value
if(self.scheduler.running):
self.scheduler.pause()
if(self.auto_close):
self.scheduler.resume_job(job_id=self.close_job.id)
else:
self.scheduler.pause_job(job_id=self.close_job.id)
self.log.info("internal: set <auto_close> to " +
str(self.auto_close))
if(self.scheduler.running):
self.scheduler.resume()
self.save_state()
self.notify_observers("auto-close")
@open_time.setter
def open_time(self, value):
if (isinstance(value, time)
and value != None
and (self.open_time == None or value != self.open_time)):
self._open_time = value
if(self.scheduler.running):
self.scheduler.pause()
self.open_job = self.scheduler.add_job(self.exec_motion
,trigger="cron"
,args=["opening"]
,id="jopen"
,name="open_job"
,max_instances=1
,replace_existing=True
,hour = self.open_time.hour
,minute = self.open_time.minute
)
self.log.info("internal: set <open_time> to " + str(self.open_time))
if(self.scheduler.running):
self.scheduler.resume()
self.save_state()
self.notify_observers("time-open")
@close_time.setter
def close_time(self, value):
if (isinstance(value, time)
and value != None
and (self.close_time == None or value != self.close_time)):
self._close_time = value
if(self.scheduler.running):
self.scheduler.pause()
self.close_job = self.scheduler.add_job(self.exec_motion
,trigger="cron"
,args=["closing"]
,id="jclose"
,name="close_job"
,max_instances=1
,replace_existing=True
,hour = self.close_time.hour
,minute = self.close_time.minute
)
self.log.info("internal: set <close_time> to " +
str(self.close_time))
if(self.scheduler.running):
self.scheduler.resume()
self.save_state()
self.notify_observers("time-close")
############################## other methods ##############################
def register_observer(self, callback):
self.log.info("internal: registering timer observer")
self.observers.append(callback)
self.update_observer(callback)
#Update all observers of the object, i.e. notify them about values
# of all properties of the object.
def update_all_observers(self):
[ self.update_observer(callback) for callback in self.observers ]
#Update observer defined by callback parameter, i.e. notify it about values
# of all properties of the object.
def update_observer(self, callback):
self.notify_observers("auto-open", [callback])
self.notify_observers("auto-close", [callback])
self.notify_observers("time-open", [callback])
self.notify_observers("time-close", [callback])
# notify given observers about changes encoded in update parameter
# update parameter must have the form: "<cat>-<subcat>" with
# cat in {"auto", "time"} and subcat in {"open", "close"}.
# If observers parameter is omittet, then all registered observers of the
# object are notified.
def notify_observers(self, update, observers=None):
if(self.boInit == False):
update_vals = {"time-open": [self.open_time.hour
,self.open_time.minute]
,"time-close": [self.close_time.hour
,self.close_time.minute]
,"auto-close": [self.auto_close]
,"auto-open": [self.auto_open]
}
val_str = ""
for val in update_vals[update]:
val_str = val_str + ":" + str(val)
update = update + "-" + val_str[1:]
if(observers == None):
observers = self.observers
self.log.info("internal: Calling " + str(len(observers)) +
" timer observers")
[ callback(update) for callback in observers ]
def exec_motion(self, arg):
self.log.info("internal: scheduled event - " + arg)
self.state_handler.handle_event(arg)
def save_state(self):
if(self.boInit == False):
try:
#with open(os.environ['HOME'] + "/hendroid/client-raspi/"
# + "timer_data.pkl", "wb") as output:
with open(self.settings_file, "wb") as output:
pickle.dump(self.open_time, output)
pickle.dump(self.close_time, output)
pickle.dump(self.auto_open, output)
pickle.dump(self.auto_close, output)
self.log.info("internal: successfully saved timer setting" +
" to file")
except Exception as e:
self.log.error("internal error: write timer setting to file: ",
exec_info=True)
def __del__(self):
self.scheduler.shutdown()
class SchedulerService(Service):
def __init__(self, env, db, collection, worker_num):
super(SchedulerService, self).__init__(env)
self._db_name = db
self._collection_name = collection
self._worker_num = worker_num
self._sched = None
global _cur_sched
_cur_sched = self
def on_active(self):
super(SchedulerService, self).on_active()
self._init_scheduler()
def on_inactive(self):
if self._sched is not None:
self._sched.shutdown()
def _init_scheduler(self):
jobstores = {
'default': MongoDBJobStore(self._db_name, self._collection_name,
host=self.mongodb_service.get_connection_info()),
}
executors = {
'default': ThreadPoolExecutor(self._worker_num),
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
self._sched = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
self._sched.start()
def add_job(self, handler, dt, args=None, kwargs=None, job_id=None, replace_existing=True):
'''
增加单一任务。执行即消亡
:param handler: 回调函数。需要考虑多线程问题
def handler(env, *args, **kwargs):pass
:param dt: datetime
:param job_id: string, 自定义id
:param kwargs: handler的参数
:return: job id
'''
args = args or []
args.insert(0, handler)
job = self._sched.add_job(dispatch_expire_job, 'date', run_date=dt, id=job_id, args=args, kwargs=kwargs,
replace_existing=replace_existing)
return job.id
def add_interval_job(self, handler,
weeks=0, days=0, hours=0, minutes=0, seconds=0, args=None, kwargs=None, job_id=None,
replace_existing=True):
'''
增加定期任务
:param handler: 回调函数。需要考虑多线程问题.
def handler(env, *args, **kwargs):pass
:param weeks:
:param days:
:param hours:
:param minutes:
:param seconds:
:param job_id: string, 自定义id
:param kwargs: handler的参数
:return: job id
'''
args = args or []
args.insert(0, handler)
job = self._sched.add_job(dispatch_expire_job, 'interval', weeks=weeks, days=days, hours=hours,
minutes=minutes, seconds=seconds, id=job_id, args=args, kwargs=kwargs,
replace_existing=replace_existing)
return job.id
def remove_job(self, job_id):
try:
self._sched.remove_job(job_id)
except JobLookupError:
pass
def pause_job(self, job_id):
self._sched.pause_job(job_id)
def resume_job(self, job_id):
self._sched.resume_job(job_id)
class SchedulerService(rpyc.Service):
def __init__(self, **config):
self._scheduler = BackgroundScheduler()
self._scheduler.configure(**config)
self._scheduler.start()
self.logger = logging.getLogger("Heartbeat.core")
self.logger.info("Heartbeat Core Initalized")
def on_connect(self, conn):
# code that runs when a connection is created
# (to init the service, if needed)
self.logger.info("----------Begin New Client----------")
self.logger.info(conn)
self.logger.info("----------End New Client----------")
def on_disconnect(self, conn):
# code that runs after the connection has already closed
# (to finalize the service, if needed)
self.logger.info("----------Begin Goodbye Client----------")
self.logger.info(conn)
self.logger.info("----------End Goodbye Client----------")
def exposed_add_job(self, func, *args, **kwargs):
self.logger.info("----------Begin New Job----------")
self.logger.info("Function: %s", str(func))
self.logger.info("*args: %s", str(args))
self.logger.info("**kwargs: %s", str(dict(kwargs)))
self.logger.info("----------Eng New Job----------")
return self._scheduler.add_job(func, *args, **kwargs)
def exposed_modify_job(self, job_id, jobstore=None, **changes):
return self._scheduler.modify_job(job_id, jobstore, **changes)
def exposed_reschedule_job(self,
job_id,
jobstore=None,
trigger=None,
**trigger_args):
return self._scheduler.reschedule_job(job_id, jobstore, trigger,
**trigger_args)
def exposed_pause_job(self, job_id, jobstore=None):
return self._scheduler.pause_job(job_id, jobstore)
def exposed_resume_job(self, job_id, jobstore=None):
return self._scheduler.resume_job(job_id, jobstore)
def exposed_remove_job(self, job_id, jobstore=None):
self._scheduler.remove_job(job_id, jobstore)
def exposed_get_job(self, job_id, jobstore=None):
return self._scheduler.get_job(job_id, jobstore=jobstore)
def exposed_get_jobs(self, jobstore=None):
results = self._scheduler.get_jobs(jobstore)
return results
def exposed_get_tasks(self):
"""Return a list of schedule-able function"""
tasks = []
for module_file in os.listdir(
os.path.join(os.path.dirname(__file__), "task")):
if module_file == "__init__.py" or module_file[-3:] != ".py":
continue
module_name = "server.task.{}".format(module_file[:-3])
module = importlib.import_module(module_name)
if not hasattr(module, "__all__"):
continue
for function_name in module.__all__:
function = getattr(module, function_name)
if not callable(function):
continue
parameters = inspect.signature(function).parameters
parameters_str = ", ".join(
[str(val) for key, val in parameters.items()])
tasks.append("{}:{}({})".format(module_name, function_name,
parameters_str))
return tasks
class JobScheduler(ISingleton):
@inject
def __init__(
self,
database_config: DatabaseConfig,
database_session_manager: DatabaseSessionManager,
):
self.database_session_manager: DatabaseSessionManager = database_session_manager
self.database_config: DatabaseConfig = database_config
self.scheduler: BackgroundScheduler = None
def run(self):
self.run_scheduler()
print("job_process started")
def run_scheduler(self):
jobstores = {
'default':
SQLAlchemyJobStore(url=self.database_config.connection_string,
tablename='ApSchedulerJobsTable',
engine=self.database_session_manager.engine,
metadata=IocManager.Base.metadata,
tableschema='Aps')
}
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {'coalesce': False, 'max_instances': 5}
self.scheduler = BackgroundScheduler(daemon=True,
jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
JobSchedulerEvent.job_scheduler_type = JobScheduler
self.scheduler.add_listener(JobSchedulerEvent.listener_finish,
EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
self.scheduler.add_listener(JobSchedulerEvent.listener_job_added,
EVENT_JOB_ADDED)
self.scheduler.add_listener(JobSchedulerEvent.listener_job_submitted,
EVENT_JOB_SUBMITTED)
self.scheduler.add_listener(JobSchedulerEvent.listener_job_removed,
EVENT_JOB_REMOVED)
self.scheduler.add_listener(
JobSchedulerEvent.listener_all_jobs_removed,
EVENT_ALL_JOBS_REMOVED)
self.scheduler.add_listener(
JobSchedulerEvent.listener_job_others,
EVENT_JOB_MODIFIED | EVENT_JOB_MISSED | EVENT_JOB_MAX_INSTANCES)
self.scheduler.add_listener(
JobSchedulerEvent.listener_scheduler_other_events,
EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN
| EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED
| EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED
| EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED)
self.scheduler.start()
print('To clear the alarms, delete the example.sqlite file.')
print('Press Ctrl+{0} to exit'.format('Break' if os.name ==
'nt' else 'C'))
def add_job_with_date(self,
job_function,
run_date,
args=None,
kwargs=None) -> Job:
# if run_date is None:
# run_date = datetime.now() + timedelta(seconds=10)
job: Job = self.scheduler.add_job(job_function,
'date',
run_date=run_date,
misfire_grace_time=30000,
args=args,
kwargs=kwargs)
return job
def add_job_with_cron(self,
job_function,
cron: CronTrigger,
args=None,
kwargs=None) -> Job:
# if cron.start_date is not None and cron.start_date < datetime.now().astimezone(get_localzone()):
# cron.start_date = None
# if cron.end_date is not None and cron.end_date < datetime.now().astimezone(get_localzone()):
# cron.end_date = None
job: Job = self.scheduler.add_job(job_function,
cron,
misfire_grace_time=15,
args=args,
kwargs=kwargs)
return job
def remove_job(self, job_id):
self.scheduler.remove_job(job_id)
def modify_job(self, job_id, jobstore=None, **changes):
return self.scheduler.modify_job(job_id, jobstore, **changes)
def reschedule_job(self,
job_id,
jobstore=None,
trigger=None,
**trigger_args):
return self.scheduler.reschedule_job(job_id, jobstore, trigger,
**trigger_args)
def pause_job(self, job_id, jobstore=None):
return self.scheduler.pause_job(job_id, jobstore)
def resume_job(self, job_id, jobstore=None):
return self.scheduler.resume_job(job_id, jobstore)
def remove_job(self, job_id, jobstore=None):
self.scheduler.remove_job(job_id, jobstore)
def get_job(self, job_id):
return self.scheduler.get_job(job_id)
def get_jobs(self, jobstore=None):
return self.scheduler.get_jobs(jobstore)
class SchedulerManager(object):
# __metaclass__ = ABCMeta
global _mongoclient
def __init__(self):
self.jobstores = {
'mongo':
MongoDBJobStore(collection='job1',
database='saasjob',
client=_mongoclient),
'default':
MemoryJobStore()
}
self.executors = {
'default': ThreadPoolExecutor(1),
'processpool': ProcessPoolExecutor(1)
}
self.job_defaults = {
'coalesce': False,
'misfire_grace_time': 1,
'max_instances': 1
}
self._sched = BackgroundScheduler(jobstores=self.jobstores,
executors=self.executors,
job_defaults=self.job_defaults)
# 添加 任务提交 事件监听
self._sched.add_listener(self.when_job_submitted, EVENT_JOB_SUBMITTED)
# 添加 任务执行完成 事件监听
self._sched.add_listener(self.when_job_executed, EVENT_JOB_EXECUTED)
# 添加 任务异常退出 事件监听
self._sched.add_listener(self.when_job_crashed, EVENT_JOB_ERROR)
self._jobs = {}
self._jobhandlers = {} # format, key: jobid, value: jobhandler
self._jobs_key = ["name", "func", "args", "kwargs"]
self.start()
def cmd_valid(self, cmd):
cmd = cmd.strip()
if cmd.startswith("python"):
return True
else:
return False
def get_job_trigger(self, _job):
# ('trigger', <CronTrigger (second='4', timezone='Asia/Shanghai')>)
_trigger = self._get_job_attr(_job, "trigger")
# options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
if _trigger:
return dict([(f.name, f.__str__()) for f in _trigger.fields
if not f.is_default])
else:
return {}
# 获取job属性
def _get_job_attr(self, _job, attr):
try:
result = eval("_job.%s" % attr)
return result
except:
import traceback
print(traceback.print_exc())
return None
def when_job_submitted(self, event):
try:
job_id = event.job_id
if job_id not in self._jobhandlers and job_id in self._jobhandlers:
self._jobhandlers.setdefault(job_id,
JobHandler(self._jobs[job_id]))
jobhandler = self._jobhandlers[event.job_id]
jobhandler.when_job_submitted()
print("%s submitted at %s" %
(event.job_id,
time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))))
except:
import traceback
print(traceback.print_exc())
def when_job_executed(self, event):
try:
job_id = event.job_id
if job_id not in self._jobhandlers:
self._jobhandlers.setdefault(job_id,
JobHandler(self._jobs[job_id]))
jobhandler = self._jobhandlers[event.job_id]
jobhandler.when_job_executed()
print("%s executed at %s" %
(event.job_id,
time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))))
except:
import traceback
print(traceback.print_exc())
def when_job_crashed(self, event):
try:
if event.exception:
job_id = event.job_id
if job_id not in self._jobhandlers:
self._jobhandlers.setdefault(
job_id, JobHandler(self._jobs[job_id]))
jobhandler = self._jobhandlers[event.job_id]
jobhandler.when_job_crashed()
print("%s crashed at %s" %
(event.job_id,
time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))))
except:
import traceback
print(traceback.print_exc())
# 添加例行任务，crontab 格式
def addCron(self, cmd, **params):
try:
create_jobid = uuid.uuid4().hex
if not self.cmd_valid(cmd):
return {"errinfo": "wrong cmd"}
jobcmdobj = JobCmd(cmd)
data = params.get("data", {})
jobcmdobj.set_jobid(create_jobid)
s = params.get("second",
None) if params.get("second", None) != "*" else None
m = params.get("minute",
None) if params.get("minute", None) != "*" else None
h = params.get("hour",
None) if params.get("hour", None) != "*" else None
d = params.get("day",
None) if params.get("day", None) != "*" else None
dw = params.get(
"day_of_week",
None) if params.get("day_of_week", None) != "*" else None
mnth = params.get(
"month", None) if params.get("month", None) != "*" else None
y = params.get("year",
None) if params.get("year", None) != "*" else None
_job = self._sched.add_job(jobcmdcallable,
'cron',
year=y,
month=mnth,
day=d,
day_of_week=dw,
hour=h,
minute=m,
second=s,
args=[jobcmdobj, data],
executor="processpool",
jobstore="mongo",
id=create_jobid)
self._jobhandlers.setdefault(create_jobid, JobHandler(_job))
# 保存 job 属性
return {"job_id": create_jobid}
except:
import traceback
print(traceback.print_exc(), cmd, params)
return False
# 修改 job 属性
def modifyJobAttr(self, job_id, **changes):
try:
_job = self._sched.modify_job(job_id=job_id, **changes)
self._jobs[job_id] = _job
if job_id in self._jobhandlers:
self._jobhandlers[job_id].job = _job
else:
self._jobhandlers.setdefault(job_id, JobHandler(_job))
return True
except:
import traceback
print(traceback.print_exc(), job_id, changes)
return False
def modifyJobData(self, job_id, data):
try:
args = self._get_job_attr(self._jobhandlers[job_id].job, "args")
# args_copy = [item for item in args]
for key in data:
args[1][key] = data[key]
_job = self._sched.modify_job(job_id, args=args)
self._jobs[job_id] = _job
if job_id in self._jobhandlers:
self._jobhandlers[job_id].job = _job
else:
self._jobhandlers.setdefault(job_id, JobHandler(_job))
return True
except:
import traceback
print(traceback.print_exc(), job_id, data)
return False
# 修改执行时间，crontab 格式
def modifyJobFreq(self, job_id, cronargs):
try:
_job = self._sched.reschedule_job(job_id,
trigger='cron',
**cronargs)
self._jobs[job_id] = _job
if job_id in self._jobhandlers:
self._jobhandlers[job_id].job = _job
else:
self._jobhandlers.setdefault(job_id, JobHandler(_job))
return True
except:
import traceback
print(traceback.print_exc(), job_id, cronargs)
return False
# 删除 job
def removeFromCron(self, job_id):
try:
self._sched.remove_job(job_id)
if job_id in self._jobhandlers:
self._jobhandlers.pop(job_id)
if job_id in self._jobs:
self._jobs.pop(job_id)
return True
except:
import traceback
print(traceback.print_exc(), job_id)
return False
def job_exists(self, job_id):
if job_id in self._jobhandlers or job_id in self._jobs:
if job_id not in self._jobhandlers and job_id in self._jobs:
self._jobhandlers[job_id] = JobHandler(self._jobs[job_id])
elif job_id in self._jobhandlers and job_id not in self._jobs:
self._jobs[job_id] = self._jobhandlers[job_id].job
return True
else:
return False
# 根据 job id 查询任务信息
def findCronJob(self, job_ids):
result = []
_keys = [
"cmd", "create_stamp", "is_running", "start_stamp", "hope_runtime",
"is_success", "is_pause", "status", "name", "desc", "allowmodify"
]
for job_id in job_ids:
print("job_exists", self.job_exists(job_id))
if self.job_exists(job_id):
_jobhander = self._jobhandlers[job_id]
job_info = _jobhander.jobhandlerattr
cron_trigger = {}
# cron_trigger = self.get_cron_trigger(_jobhander.job)
tmp = {}
tmp["job_id"] = job_id
if job_info["is_running"]:
execute_time = time.time() - job_info["start_stamp"]
tmp["running_time"] = round(execute_time, 3)
else:
tmp["running_time"] = round(job_info["hope_runtime"], 3)
for key in _keys:
v = job_info.get(key, None)
if key == "is_running":
tmp["finished"] = False if job_info[
"is_running"] else True
else:
tmp[key] = v
if tmp["finished"]:
tmp["completed_per"] = 1.0
else:
tmp["completed_per"] = round(
tmp["running_time"] /
max([tmp["running_time"], tmp["hope_runtime"]]), 3)
# del tmp["hope_runtime"]
# del tmp["is_success"]
# del tmp["is_pause"]
tmp.pop("hope_runtime")
tmp.pop("is_success")
tmp.pop("is_pause")
_result = dict(tmp, **cron_trigger)
print("_result", _result)
if _result["status"] == 3:
_result["completed_per"] = 0
_result["running_time"] = 0
_result["start_stamp"] = None
result.append(_result)
else:
result.append({"job_id": job_id, "errinfo": "no exists"})
return result
def getAllJobInfo(self):
try:
result = self.findCronJob(
set(self._jobhandlers.keys()) | set(self._jobs.keys()))
return result
except:
import traceback
print(traceback.print_exc())
return False
def start_addition(self):
for _job in self._sched.get_jobs():
job_id = self._get_job_attr(_job, "id")
self._jobs.setdefault(job_id, _job)
def start(self):
try:
self._sched.start()
self._sched.pause()
self.start_addition()
self._sched.resume()
return True
except:
import traceback
print(traceback.print_exc())
return False
def stop(self, iswait=True):
try:
self._sched.shutdown(wait=iswait)
self._jobhandlers.clear()
return True
except:
import traceback
print(traceback.print_exc())
return False
def pause_job(self, job_id):
try:
self._sched.pause_job(job_id=job_id)
self._jobhandlers[job_id].ispause = True
self._jobhandlers[job_id].status = 3
self._jobhandlers[job_id].isrunning = False
return True
except:
import traceback
print(traceback.print_exc())
return False
def resume_job(self, job_id):
try:
self._sched.resume_job(job_id=job_id)
self._jobhandlers[job_id].ispause = False
self._jobhandlers[job_id].status = 1
return True
except:
import traceback
print(traceback.print_exc())
return False
class Scheduler(Flask):
days = {
"0": "sun",
"1": "mon",
"2": "tue",
"3": "wed",
"4": "thu",
"5": "fri",
"6": "sat",
"7": "sun",
"*": "*",
}
seconds = {"seconds": 1, "minutes": 60, "hours": 3600, "days": 86400}
def __init__(self):
super().__init__(__name__)
with open(Path.cwd().parent / "setup" / "scheduler.json", "r") as file:
self.settings = load(file)
dictConfig(self.settings["logging"])
self.configure_scheduler()
self.register_routes()
@staticmethod
def aps_date(date):
if not date:
return
date = datetime.strptime(date, "%d/%m/%Y %H:%M:%S")
return datetime.strftime(date, "%Y-%m-%d %H:%M:%S")
def configure_scheduler(self):
self.scheduler = BackgroundScheduler(self.settings["config"])
self.scheduler.start()
def register_routes(self):
@self.route("/delete_job/<job_id>", methods=["POST"])
def delete_job(job_id):
if self.scheduler.get_job(job_id):
self.scheduler.remove_job(job_id)
return jsonify(True)
@self.route("/next_runtime/<task_id>")
def next_runtime(task_id):
job = self.scheduler.get_job(task_id)
if job and job.next_run_time:
return jsonify(job.next_run_time.strftime("%Y-%m-%d %H:%M:%S"))
return jsonify("Not Scheduled")
@self.route("/schedule", methods=["POST"])
def schedule():
if request.json["mode"] in ("resume", "schedule"):
result = self.schedule_task(request.json["task"])
if not result:
return jsonify({"alert": "Cannot schedule in the past."})
else:
return jsonify({
"response": "Task resumed.",
"active": True
})
else:
try:
self.scheduler.pause_job(request.json["task"]["id"])
return jsonify({"response": "Task paused."})
except JobLookupError:
return jsonify(
{"alert": "There is no such job scheduled."})
@self.route("/time_left/<task_id>")
def time_left(task_id):
job = self.scheduler.get_job(task_id)
if job and job.next_run_time:
delta = job.next_run_time.replace(tzinfo=None) - datetime.now()
hours, remainder = divmod(delta.seconds, 3600)
minutes, seconds = divmod(remainder, 60)
days = f"{delta.days} days, " if delta.days else ""
return jsonify(f"{days}{hours}h:{minutes}m:{seconds}s")
return jsonify("Not Scheduled")
@staticmethod
def run_service(task_id):
post(
f"{getenv('ENMS_ADDR')}/rest/run_task",
json=task_id,
auth=HTTPBasicAuth(getenv("ENMS_USER"), getenv("ENMS_PASSWORD")),
verify=int(getenv("VERIFY_CERTIFICATE", 1)),
)
def schedule_task(self, task):
if task["scheduling_mode"] == "cron":
crontab = task["crontab_expression"].split()
crontab[-1] = ",".join(self.days[day]
for day in crontab[-1].split(","))
trigger = {"trigger": CronTrigger.from_crontab(" ".join(crontab))}
elif task["frequency"]:
trigger = {
"trigger":
"interval",
"start_date":
self.aps_date(task["start_date"]),
"end_date":
self.aps_date(task["end_date"]),
"seconds":
int(task["frequency"]) * self.seconds[task["frequency_unit"]],
}
else:
trigger = {
"trigger": "date",
"run_date": self.aps_date(task["start_date"])
}
if not self.scheduler.get_job(task["id"]):
job = self.scheduler.add_job(
id=str(task["id"]),
replace_existing=True,
func=self.run_service,
args=[task["id"]],
**trigger,
)
else:
job = self.scheduler.reschedule_job(str(task["id"]), **trigger)
return job.next_run_time > datetime.now(job.next_run_time.tzinfo)
class ScheduleManager(IScheduleManager):
def __init__(self, config=None, event_listener=None):
self.__config = config
self.__event_listener = event_listener
jobstores, executors, job_defaults, timezone = self.__get_apscheduler_settings(
)
# initial apscheduler
self.__scheduler = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=timezone)
if self.__event_listener:
self.__scheduler.add_listener(self.__event_listener.event_start,
EVENT_SCHEDULER_STARTED)
self.__scheduler.add_listener(self.__event_listener.event_shutdown,
EVENT_SCHEDULER_SHUTDOWN)
self.__scheduler.add_listener(self.__event_listener.event_pause,
EVENT_SCHEDULER_PAUSED)
self.__scheduler.add_listener(self.__event_listener.event_resume,
EVENT_SCHEDULER_RESUMED)
self.__scheduler.add_listener(
self.__event_listener.event_job_submit, EVENT_JOB_SUBMITTED)
self.__scheduler.add_listener(
self.__event_listener.event_job_max_instances,
EVENT_JOB_MAX_INSTANCES)
self.__scheduler.add_listener(
self.__event_listener.event_job_execute, EVENT_JOB_EXECUTED)
self.__scheduler.add_listener(
self.__event_listener.event_job_error, EVENT_JOB_ERROR)
self.__scheduler.add_listener(self.__event_listener.event_job_miss,
EVENT_JOB_MISSED)
def __get_apscheduler_settings(self):
try:
jobstore_url = "oracle+cx_oracle://{username}:{password}${host}:{port}/{dbname}".format(
username=self.__config.db_user,
password=self.__config.db_pwd,
host=self.__config.db_host,
port=self.__config.db_port,
dbname=self.__config.db_name,
)
jobstores = {
"default":
SQLAlchemyJobStore(url=jobstore_url,
tablename=self.__config.tablename)
}
executors = {
"default": ThreadPoolExecutor(self.__config.max_workers),
"processpool": ProcessPoolExecutor(2)
}
job_defaults = {
"coalesce": True,
"max_instances": 10,
"misfire_grace_time": 30
}
timezone = self.__config.timezone
return jobstores, executors, job_defaults, timezone
except Exception as e:
raise e
def start(self, paused=False) -> bool:
try:
if self.__scheduler.state == STATE_RUNNING:
return True
self.__scheduler.start(paused=paused)
except Exception as e:
logging.error(f"scheduler start error...... {str(e)}")
raise e
def shutdown(self, wait=False) -> bool:
try:
self.__scheduler.shutdown(wait=wait)
if self.__scheduler.state == STATE_STOPPED:
return True
return False
except Exception as e:
logging.error(f"scheduler shutdown error...... {str(e)}")
raise e
def pause(self) -> bool:
try:
self.__scheduler.pause()
if self.__scheduler.state == STATE_PAUSED:
return True
return False
except Exception as e:
logging.error(f"scheduler pause error...... {str(e)}")
raise e
def resume(self) -> bool:
try:
self.__scheduler.resume()
if self.__scheduler.state == STATE_RUNNING:
return True
return False
except Exception as e:
logging.error(f"scheduler resume error...... {str(e)}")
raise e
def get_state(self) -> str:
try:
if self.__scheduler.state == STATE_RUNNING:
return "RUNNING"
if self.__scheduler.state == STATE_PAUSED:
return "PAUSED"
return "STOPPED"
except Exception as e:
logging.error(f"scheduler get_state error...... {str(e)}")
raise e
def get_job(self, id="") -> object:
try:
job = self.__scheduler.get_job(id)
return job
except Exception as e:
logging.error(f"scheduler get_job error...... {str(e)}")
raise e
def add_job(self, task_id="", trigger=None, task=None) -> bool:
try:
settings = {}
settings["id"] = task_id if task_id else str(uuid4())
settings["func"] = task.run
settings["coalesce"] = True
settings["replace_existing"] = True
settings = {**settings, **(trigger.get_settings())}
job = self.__scheduler.add_job(**settings)
return True if self.get_job(job.id) else False
except Exception as e:
logging.error(f"scheduler add_job error...... {str(e)}")
raise e
def remove_job(self, id: str) -> bool:
try:
self.__scheduler.remove_job(id)
return True if not self.get_job(id) else False
except JobLookupError as e:
return False
except Exception as e:
logging.error(f"scheduler remove_job error...... {str(e)}")
raise e
def pause_job(self, id: str) -> bool:
try:
job = self.__scheduler.pause_job(id)
return True if job else False
except Exception as e:
logging.error(f"scheduler pause_job error...... {str(e)}")
raise e
def resume_job(self, id: str) -> bool:
try:
job = self.__scheduler.resume_job(id)
return True if job else False
except Exception as e:
logging.error(f"scheduler resume_job error...... {str(e)}")
raise e
class ExScheduler(object):
def __init__(self):
self.__scheduler = BackgroundScheduler()
self.__jobstatuslist = {}
def start_scheduler(self, **options):
self.__scheduler.configure(**options)
self.__scheduler.start()
def shutdownscheduler(self):
self.__scheduler.shutdown(wait=False)
def getjob(self, jobid):
return self.__scheduler.get_job(jobid)
def scheduledjob(self, *args, **kw):
return self.__scheduler.scheduled_job(*args, **kw)
def addjob(self, *args, **kw):
job = self.__scheduler.add_job(*args, **kw)
self.__scheduler.wakeup()
return job
def removejob(self, job_id, jobstore=None):
self.__scheduler.remove_job(job_id=job_id, jobstore=jobstore)
def pausejob(self, job_id, jobstore=None):
self.__scheduler.pause_job(job_id=job_id, jobstore=jobstore)
self.setjobstatus(job_id, JobStatus.PAUSED)
def resumejob(self, job_id, jobstore=None):
self.__scheduler.resume_job(job_id=job_id, jobstore=jobstore)
self.setjobstatus(job_id, JobStatus.SCHEDULING)
def modifyjob(self, job_id, jobstore=None, **kw):
job = self.__scheduler.modify_job(job_id=job_id,
jobstore=jobstore,
**kw)
return job
def getjoblist(self):
joblist = self.__scheduler.get_jobs()
return map(self._getjobinfo, joblist)
def addlistener(self, callback, mask=EVENT_ALL):
self.__scheduler.add_listener(callback, mask)
def setjobstatus(self, jobid, jobstatus):
self.__jobstatuslist[jobid] = jobstatus
def jobstatusinitial(self, job_id, jobstore=None):
job = self.__scheduler.get_job(job_id=job_id, jobstore=jobstore)
if job_id not in self.__jobstatuslist:
status = (JobStatus.SCHEDULING
if job.next_run_time else JobStatus.PAUSED) if hasattr(
job, 'next_run_time') else JobStatus.PENDING
self.setjobstatus(job_id, status)
def _getjobinfo(self, job):
self.jobstatusinitial(job.id)
return {
"id":
str(job.id),
"name":
str(job.name),
"kwargs":
job.kwargs,
"trigger":
trigger_str_to_dict(str(job.trigger)),
"next_run_time":
datetime_repr(job.next_run_time)
if self.__jobstatuslist[str(job.id)] == JobStatus.SCHEDULING or
self.__jobstatuslist[str(job.id)] == JobStatus.RUNNING else "--",
"status":
self.__jobstatuslist[str(job.id)]
}
keep_running = True
while keep_running:
option = input(
"Please select: 1.get data 2.delete tables 3.pause project 4.exit :"
)
if option == "1":
scheduler.start()
if option == "2":
for theClass in tables.AllClasses:
theClass.__table__.drop(engine)
logger.info("%s - Delete all the tables --- %s", model,
tables.Base.metadata.tables.keys())
if option == "3":
logger.info("%s - Pause the project.", model)
scheduler.pause_job('job_timetable')
scheduler.pause_job('job_trip_update')
scheduler.pause_job('job_vehicle_position')
scheduler.pause_job('job_alert')
if option == "4":
scheduler.remove_all_jobs()
if scheduler.state is not 0:
scheduler.shutdown()
session.close_all()
logger.info("Stop running project ---- model %s", model)
keep_running = False
exit(0)
class JobManager(object):
def __init__(self):
self.scheduler = BackgroundScheduler(executors=EXECUTORS,
job_defaults=JOB_DEFAULTS,
timezone='Asia/Shanghai')
self.jobs = {}
self.scheduler.start()
def add_job_store(self):
pass
def add_job(self, method, jobtype, trigger, jobid, args=None, kwargs=None):
job = None
if jobtype == 'interval':
job = self.scheduler.add_job(method,
'interval',
seconds=trigger,
id=jobid,
args=args,
kwargs=kwargs)
if jobtype == 'cron':
Trigger = CronTrigger(**trigger)
job = self.scheduler.add_job(method,
Trigger,
id=jobid,
args=args,
kwargs=kwargs)
if jobtype == 'once':
job = self.scheduler.add_job(method,
'date',
run_date=trigger,
id=jobid,
args=args,
kwargs=kwargs)
LOG.debug("add job: %s", jobid)
if job is not None:
self.jobs[jobid] = job
def delete_job(self, jobid):
if jobid in self.jobs:
self.scheduler.remove_job(jobid)
del self.jobs[jobid]
def disable_job(self, jobid):
if jobid in self.jobs:
self.scheduler.pause_job(jobid)
def enable_job(self, jobid):
if jobid in self.jobs:
self.scheduler.resume_job(jobid)
def get_job_next_run_time(self, jobid):
job = self.scheduler.get_job(jobid)
if not job:
return None
next_run_time = job.next_run_time
return next_run_time.strftime("%Y-%m-%d %H:%M:%S")
class DSEGeneratorApp(wx.Frame):
"""DSEGeneratorApp
Generates DSE Documents based on Checklists in Word format.
To Parse Checklists a specific template in XML format is necessary.
Generation of DSE Document is done based on XML Template for mapping
Checklist values to text blocks
Arguments:
wx {[Frame]} -- [description]
"""
def __init__(self, *args, **kwargs):
"""init function
"""
super(DSEGeneratorApp, self).__init__(*args, **kwargs)
# Read Config for UI
self.display_log_size = configProvider.getConfigEntryOrDefault(
'UI Setup', 'DISPLAY_LOG_SIZE', -500)
#Scheduler
self.log_scheduler = BackgroundScheduler()
self.generator = DSEGenerator()
self.doc_generator = None
self.log_scheduler.add_job(self.log_update,
'interval',
seconds=10,
id='log_job')
self.log_scheduler.start()
self.view_show_log_item = None
self.ui_width = 880
self.ui_height_min = 500
self.ui_height_max = 700
self.init_ui()
self.SetBackgroundColour("white")
def init_ui(self):
"""[summary]
Generates the UI of the Application
"""
self.SetSize((self.ui_width, self.ui_height_max))
self.SetTitle("DSEGenerator Application")
self.Centre()
self.panel = wx.Panel(self)
self.sizer = wx.GridBagSizer(3, 16)
self.border = 10
self.line = wx.StaticLine(self.panel, -1, style=wx.LI_VERTICAL)
self.line.SetSize((100, 30))
#Header Graphic
imageSizer = wx.BoxSizer(wx.HORIZONTAL)
headerImage = wx.Image(Resources.getHeaderImage(),
wx.BITMAP_TYPE_ANY).ConvertToBitmap()
img = wx.StaticBitmap(
self.panel, -1, headerImage, (0, 0),
(headerImage.GetWidth(), headerImage.GetHeight()))
imageSizer.Add(img, flag=wx.LEFT)
self.sizer.Add(imageSizer,
pos=(0, 0),
span=(1, 3),
flag=wx.TOP | wx.LEFT | wx.RIGHT)
#Checklist Label
checklist_label = wx.StaticText(self.panel,
label="Checklist Document:")
font = checklist_label.GetFont()
font.PointSize += 2
font = font.Bold()
checklist_label.SetFont(font)
self.sizer.Add(checklist_label,
pos=(2, 0),
flag=wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL,
border=self.border)
#FileDialog Button
file_picker = wx.FilePickerCtrl(
self.panel,
message="Please select a Checklist Document in *.docx Format:",
wildcard="*.docx",
style=wx.FLP_USE_TEXTCTRL)
file_picker.SetTextCtrlGrowable(True)
file_picker.SetTextCtrlProportion(10)
self.sizer.Add(file_picker,
pos=(2, 1),
span=(1, 2),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
#Version of Document selected
version_label = wx.StaticText(self.panel,
label="Checklist Word-Document Version:")
version_label.SetFont(font)
self.sizer.Add(version_label,
pos=(3, 0),
flag=wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL,
border=self.border)
self.version_text = wx.StaticText(self.panel)
self.version_text.SetFont(font)
self.sizer.Add(self.version_text,
pos=(3, 1),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
version_label_xml = wx.StaticText(
self.panel, label="Checklist XMLTemplate Version:")
version_label_xml.SetFont(font)
self.sizer.Add(version_label_xml,
pos=(4, 0),
flag=wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL,
border=self.border)
self.version_text_xml = wx.StaticText(self.panel)
self.version_text_xml.SetFont(font)
self.sizer.Add(self.version_text_xml,
pos=(4, 1),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
self.sizer.Add(self.line,
pos=(6, 0),
span=(1, 3),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
#Status
status_label = wx.StaticText(self.panel, label="Processing Status:")
status_label.SetFont(font)
self.sizer.Add(status_label,
pos=(7, 0),
flag=wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL,
border=self.border)
self.status_text = wx.StaticText(
self.panel, label="Please select Checklist to read!")
self.status_text.SetFont(font)
self.sizer.Add(self.status_text,
pos=(7, 1),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
#Buttons
self.generate_button = wx.Button(self.panel,
label="Generate DSE document!",
name="generate")
self.generate_button.Disable()
self.sizer.Add(self.generate_button,
pos=(9, 0),
flag=wx.LEFT | wx.ALIGN_CENTER_VERTICAL,
border=self.border)
self.Bind(wx.EVT_BUTTON, self.on_generate, self.generate_button)
self.save_button = wx.Button(self.panel,
label="Save DSE Document!",
name="save")
self.save_button.Disable()
self.sizer.Add(self.save_button,
pos=(9, 1),
flag=wx.LEFT | wx.ALIGN_CENTER_VERTICAL,
border=self.border)
self.Bind(wx.EVT_BUTTON, self.on_save, self.save_button)
#Log
self.log_label = wx.StaticText(self.panel, label="Log:")
self.sizer.Add(self.log_label,
pos=(15, 0),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
self.log_view = wx.TextCtrl(self.panel,
size=(200, 200),
style=wx.TE_MULTILINE | wx.HSCROLL
| wx.TE_READONLY | wx.TE_BESTWRAP
| wx.TE_RICH2,
pos=wx.DefaultPosition)
self.log_view.SetEditable(False)
self.sizer.Add(self.log_view,
pos=(16, 0),
span=(3, 3),
flag=wx.EXPAND | wx.LEFT | wx.RIGHT,
border=self.border)
self.sizer.AddGrowableCol(2)
self.panel.SetSizer(self.sizer)
self.init_menu()
#Events
self.Bind(wx.EVT_FILEPICKER_CHANGED, self.on_pick_file, file_picker)
def init_menu(self):
""" Initialize Menu
"""
#Menubar
menubar = wx.MenuBar()
file_menu = wx.Menu()
view_menu = wx.Menu()
file_item = file_menu.Append(wx.ID_EXIT, 'Exit', 'Exit application')
self.view_show_log_item = view_menu.Append(wx.ID_ANY,
'Show Log',
'Show Log View',
kind=wx.ITEM_CHECK)
view_menu.Check(self.view_show_log_item.GetId(), False)
self.hide_log()
menubar.Append(file_menu, '&File')
menubar.Append(view_menu, '&View')
self.SetMenuBar(menubar)
#Events
self.Bind(wx.EVT_MENU, self.on_exit, file_item)
self.Bind(wx.EVT_MENU, self.on_show_log, self.view_show_log_item)
#--- EVENT HANDLER FUNCTIONS
def on_pick_file(self, evt):
"""Takes care of loading Checklist file
Arguments:
e {[type]} -- [description]
"""
log = logger.getLogger()
self.reset()
if evt.GetPath() != None:
self.generator.checklistFile = evt.GetPath()
checklist_doc = parseChecklist(self.generator.checklistFile)
self.generator.checklistObject = checklist_doc
if checklist_doc.wordVersion != None:
self.version_text.SetLabelText(checklist_doc.wordVersion)
self.version_text_xml.SetLabelText(checklist_doc.xmlVersion)
self.status_text.SetLabelText(
"Checklist Document successfully processed!")
self.generate_button.Enable()
else:
self.status_text.SetLabelText(
"Error during processing! Please refer to log for details!"
)
else:
wx.MessageBox(
"Warning! No file has been selected! Please select a valid file in order to proceed!"
)
log.warning("No file has been selected!")
def on_generate(self, evt):
"""[summary]
Arguments:
e {[type]} -- [description]
"""
log = logger.getLogger()
log.info("Button pushed")
if self.generator.checklistObject != None:
self.generate_dse_document()
else:
wx.MessageBox(
"Warning! DSE Document can't be generated because of missing or incomplete parsed Checklist Document!",
caption="Warning!")
log.warning(
"No checklist has been parsed! Please select a valid checklist document!"
)
def on_exit(self, evt):
"""[summary]
Arguments:
e {[type]} -- [description]
"""
dlg = wx.MessageDialog(
self, "Do you really want to close this application?",
"Confirm Exit", wx.OK | wx.CANCEL | wx.ICON_QUESTION)
result = dlg.ShowModal()
dlg.Destroy()
if result == wx.ID_OK:
self.Close()
def on_show_log(self, evt):
"""Shows/Hides the Log View
Arguments:
evt {[type]} -- [description]
"""
if self.view_show_log_item.IsChecked():
self.show_log()
else:
self.hide_log()
def on_save(self, evt):
"""Saves DSE Document to File system
Arguments:
evt {[type]} -- [description]
"""
if self.doc_generator.dseDocument is not None:
dlg = wx.FileDialog(
self, "Please select destination to save generated document!",
".", "", "Word Document (*.docx)|*.docx",
wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT)
if dlg.ShowModal() == wx.ID_OK:
path = dlg.GetPath()
if self.doc_generator.saveDocument(
self.version_text_xml.GetLabelText(), path):
self.status_text.SetLabelText(
"Checklist Document saved successfully!")
else:
wx.MessageBox("Warning! DSE Document hasn't been saved!",
caption="Warning!")
else:
wx.MessageBox(
"Warning! DSE Document has not yet generated! Please Generate DSE Document when Checklist has been read successfully!",
caption="Warning!")
#--- End of Event Handler
def hide_log(self):
"""Hides log view and reduces size of frame
"""
self.log_view.Hide()
self.log_label.Hide()
self.log_scheduler.pause_job("log_job")
self.SetSize((self.ui_width, self.ui_height_min))
def show_log(self):
"""show Logview
"""
self.log_view.Show()
self.log_label.Show()
self.log_scheduler.resume_job("log_job")
self.SetSize((self.ui_width, self.ui_height_max))
def log_update(self):
"""
Updates Logview with content of log file
"""
file_handle = open(Resources.getLogFile(), "r")
self.log_view.SetInsertionPoint(0)
self.log_view.SetValue(file_handle.read())
self.log_view.AppendText("")
self.log_view.Refresh()
file_handle.close()
def generate_dse_document(self):
""" Generates DSE Document by
"""
self.status_text.SetLabelText("DSE Document is being processed...")
self.doc_generator = DocGenerator(
self.generator.checklistObject) #Implement concurrency!!
self.doc_generator.parseTemplate(self.version_text_xml.GetLabelText())
self.status_text.SetLabelText("DSE Document successfully processed!")
self.save_button.Enable()
def reset(self):
"""Reset status of GUI
"""
self.generate_button.Disable()
self.save_button.Disable()
class NotificationsScheduler:
def __init__(self):
self.scheduler = BackgroundScheduler()
self.notifications_id = 'notifications_job'
self.stock_tracking_id = 'tracking_job'
self.analyst_rec_id = 'analyst_rec_job'
def start(self):
print('Started running the jobs')
self.scheduler.add_job(self.set_active_notifications,
'interval',
seconds=30,
id=self.notifications_id)
self.scheduler.add_job(self.check_tracking_model,
'interval',
hours=12,
id=self.stock_tracking_id)
self.scheduler.add_job(self.check_notifications_analyst,
'interval',
days=1,
id=self.analyst_rec_id)
# self.scheduler.add_listener(self.job_listener, EVENT_ALL)
self.scheduler.start()
def stop(self, job_id):
self.scheduler.remove_job(job_id)
def pause(self, job_id):
self.scheduler.pause_job(job_id)
def resume(self, job_id):
self.scheduler.resume_job(job_id)
@staticmethod
def get_value_of(symbol, operand):
cached = cache.get(symbol + operand) # ex: AAPLHigh or None
if cached:
print('cached')
return cached
else:
data = stock_api.get_stock_info_notification(symbol, operand)
cache.set(symbol + operand, data, (30 * 60)) # cache for 30 mins
return data
@staticmethod
def get_analyst_record_of(symbol):
cached = cache.get(symbol + 'analystRec')
if cached:
print('cached')
return cached
else:
data = stock_api.get_analyst_recommendations(symbol)
if data:
rec = data[0]
rec = rec.get('ratingScaleMark')
cache.set(symbol + 'analystRec', rec,
(30 * 60)) # cache for 30 mins
return rec
else:
return 0
@staticmethod
def is_bigger(value, api_value):
return api_value > value
@staticmethod
def is_lower(value, api_value):
return api_value < value
@staticmethod
def is_equal(value, api_value):
return api_value == value
# loops over Notification table and check if any notification must be triggered
# if yes add it to ReadyNotification table
def set_active_notifications(self):
notifications = Notification.objects.all()
operators = {
'bigger': self.is_bigger,
'lower': self.is_lower,
'equal': self.is_equal
}
for notification in notifications:
time_now = now()
should_check = (
((time_now - notification.last_checked).seconds // 60 % 60) >=
0) # check if 0 mins passed
if should_check:
user = notification.user
key = notification.operand
symbol = notification.company_symbol
api_value = self.get_value_of(symbol, key)
operator = notification.operator
value = notification.value
if api_value[key] is None:
api_value = 0
else:
api_value = api_value[key]
should_activate = operators[operator](value, api_value)
if should_activate:
description = key + ' is ' + operator + ' than ' + str(
value)
rn = ReadyNotification(user=user,
description=description,
company_symbol=symbol)
rn.save()
notification.delete()
send_to_socket(user, rn.company_symbol, rn.description,
str(rn.time.replace(microsecond=0)))
else:
notification.last_checked = now()
notification.save()
def cut_data(self, data, creation_time):
today = str((now() + relativedelta(weekday=FR(-1))).date())
creation_time = str(creation_time)
idx1 = 0
idx2 = 0
for idx, i in enumerate(data):
if i['date'] == creation_time:
idx1 = idx
if i['date'] == today:
idx2 = idx
return data[idx1:idx2]
def is_increasing(self, data, operand):
temp = 0
for i in data:
value = i[operand]
print('value', value)
if value < temp:
return False
else:
temp = value
return True
def is_decreasing(self, data, operand):
temp = 0
for i in data:
value = i[operand]
if value > temp:
return False
else:
temp = value
return True
def should_trigger(self, operand, state, symbol, weeks, creation_time):
if operand == 1:
operand = 'high'
elif operand == -1:
operand = 'low'
else:
return False
if weeks < 4:
time_range = '1m'
elif weeks < 8:
time_range = '3m'
else:
time_range = '6m'
data = stock_api.get_stock_historic_prices(symbol, time_range,
'date,' + operand)
data = self.cut_data(data, creation_time)
if not data:
return False
else:
if state == 1:
return self.is_increasing(data, operand)
else:
return self.is_decreasing(data, operand)
def check_tracking_model(self):
today = datetime.datetime.today().weekday()
if not (today == 6): # if today is not Saturday or Sunday
return # Don't check
tracking = TrackStock.objects.all()
for track in tracking:
creation_time = track.creation_time
weeks = track.weeks
date_now = now().date()
should_check = ((date_now - creation_time).days >=
((weeks * 7) - (weeks * 2)))
if should_check:
user = track.user
operand = track.operand
state = track.state
company_symbol = track.company_symbol
should_trigger = self.should_trigger(operand, state,
company_symbol, weeks,
creation_time)
if should_trigger:
if operand == 1:
operand = 'High'
else:
operand = 'Low'
if state == 1:
description = operand + ' kept increasing for the last ' + str(
weeks) + ' week/s.'
else:
description = operand + ' kept decreasing for the last ' + str(
weeks) + ' week/s.'
rn = ReadyNotification(user=user,
description=description,
company_symbol=company_symbol)
rn.save()
track.delete()
send_to_socket(user, rn.company_symbol, rn.description,
str(rn.time))
else:
if operand == 1:
operand = 'High'
else:
operand = 'Low'
if state == 1:
description = operand + ' didn\'t keep increasing for the last ' + str(
weeks) + ' week/s.'
else:
description = operand + ' didn\'t keep decreasing for the last ' + str(
weeks) + ' week/s.'
rn = ReadyNotification(user=user,
description=description,
company_symbol=company_symbol)
rn.save()
track.delete()
send_to_socket(user, rn.company_symbol, rn.description,
str(rn.time))
def check_notifications_analyst(self):
notifications = NotificationAnalystRec.objects.all()
operators = {
'bigger': self.is_bigger,
'lower': self.is_lower,
'equal': self.is_equal
}
for notification in notifications:
time_now = now()
should_check = (
((time_now - notification.last_checked).seconds // 60 % 60) >=
0) # check if 10 mins passed
if should_check:
user = notification.user
symbol = notification.company_symbol
api_value = self.get_analyst_record_of(symbol)
operator = notification.operator
value = notification.value
should_activate = operators[operator](value, api_value)
if should_activate:
description = 'Analyst Scale Rate ' + str(
operator) + ' than ' + str(value)
rn = ReadyNotification(user=user,
description=description,
company_symbol=symbol)
rn.save()
notification.delete()
send_to_socket(user, rn.company_symbol, rn.description,
str(rn.time))
else:
notification.last_checked = now()
notification.save()
class Scheduler():
"""
NERD scheduler - allows modules to register functions (callables) to be run
at specified times or intervals (like cron does).
"""
def __init__(self):
self.log = logging.getLogger("Scheduler")
#self.log.setLevel("DEBUG")
logging.getLogger("apscheduler.scheduler").setLevel("WARNING")
logging.getLogger("apscheduler.executors.default").setLevel("WARNING")
self.sched = BackgroundScheduler(timezone="UTC")
self.last_job_id = 0
def start(self):
self.log.debug("Scheduler start")
self.sched.start()
def stop(self):
self.log.debug("Scheduler stop")
self.sched.shutdown()
def register(self,
func,
year=None,
month=None,
day=None,
week=None,
day_of_week=None,
hour=None,
minute=None,
second=None,
timezone="UTC",
args=None,
kwargs=None):
"""
Register a function to be run at specified times.
func - function or method to be called
year,month,day,week,day_of_week,hour,minute,second -
cron-like specification of when the function should be called,
see docs of apscheduler.triggers.cron for details
https://apscheduler.readthedocs.io/en/latest/modules/triggers/cron.html
timezone - Timezone for time specification (default is UTC).
args, kwargs - arguments passed to func
Return job ID (integer).
"""
self.last_job_id += 1
trigger = CronTrigger(year,
month,
day,
week,
day_of_week,
hour,
minute,
second,
timezone=timezone)
self.sched.add_job(func,
trigger,
args,
kwargs,
coalesce=True,
max_instances=1,
id=str(self.last_job_id))
self.log.debug("Registered function {0} to be called at {1}".format(
func.__qualname__, trigger))
return self.last_job_id
def pause_job(self, id):
"""Pause job with given ID"""
self.sched.pause_job(str(id))
def resume_job(self, id):
"""Resume previously paused job with given ID"""
self.sched.resume_job(str(id))
class HarryBotter(object):
def __init__(self, debug=False, stopcb=None):
oplogs("Harry Botter is rebooting")
self.__init_robot(debug)
self.stopcb = stopcb
#self.__init_db()
self.__init_jobs()
oplogs("Harry Botter is online")
def __del__(self):
oplogs("Harry Botter is shutting down~~~")
self.stop_jobs()
#self.close_db()
def __init_robot(self, debug):
self.enable = True
self.mouth = None # output callback
self.ear = None # input for text and audio callback
self.eye = None # input for graph callback
self.brain = None # processor modules
self.memory = [] # memory for what heard or seen
self.support_groups = []
self.debug = debug
self.sched = None
self.starttime = datetime.now()
def subscribe(self, msgSend):
self.mouth = msgSend
oplogs('Robot unmuted.')
def unsubscribe(self):
self.mouth = None
oplogs('Robot muted.')
def install_mods(self, hmod):
self.stockmod = hmod #StockMod(self.debug)
def __init_db(self):
# 创建数据库数据库用于永久记忆
self.conn = sqlite3.connect('messages.db')
self.cursor = self.conn.cursor()
self.cursor.execute(
'create table if not exists msgqueue (id integer primary key autoincrement, date TEXT, groupname TEXT, user TEXT, msg TEXT)'
)
self.cursor.execute(
'create table if not exists monitorstocks (id integer primary key autoincrement, groupname TEXT, code TEXT)'
)
self.cursor.commit()
oplogs("[%s]messages.db connected")
def __close_db(self):
self.conn.close()
def __init_jobs(self):
self.sched = BackgroundScheduler()
# 添加任务作业
# 每天清理一次缓存
#self.sched.add_job(self.job_clean_cache, trigger='cron', day_of_week='*',hour=1, minute=0, second=0)
# 提供给group的5分钟检测
self.sched.add_job(self.job_stock_monitor,
trigger='cron',
id='job_stock_monitor',
minute="*/5") #, next_run_time=None)
# 交易日9:30:15生成开盘报告
self.sched.add_job(self.job_open_scan,
trigger='cron',
day_of_week='0-4',
hour=9,
minute=30,
second=15)
# 交易日11:30--13:00之间关闭扫描
self.sched.add_job(self.job_close_scan,
trigger='cron',
day_of_week='0-4',
hour=11,
minute=30,
second=15)
self.sched.add_job(self.job_open_scan,
trigger='cron',
day_of_week='0-4',
hour=13,
minute=0,
second=0)
# 交易日15:05:00生成收盘报告
self.sched.add_job(self.job_close_scan,
trigger='cron',
day_of_week='0-4',
hour=15,
minute=5,
second=0)
# 启动调度器
self.sched.start()
oplogs("schedulers started")
def stop_jobs(self):
if self.sched == None:
self.sched.shutdown()
oplogs("schedulers stopped")
def job_stock_monitor(self, valve=3.0):
if self.enable is False:
return
oplogs("job_stock_monitor triggerred")
group_alarms = self.stockmod.get_alarms(valve)
for (group, alarms) in group_alarms.items():
notice = ""
for alarm in alarms:
# alarm = [stockname,price,change,highlow,closeopen]
alarm_content = "5分钟涨跌\n[%s %.2f%%]幅度：%.2f%%, 波动:%.2f%%\n" % (
alarm[0], alarm[2], alarm[4], alarm[3])
notice += alarm_content
if len(notice) > 0:
print(notice)
self.mouth(group, notice)
time.sleep(1)
def job_open_scan(self):
oplogs("job_open_scan triggerred")
self.job_stock_monitor()
self.sched.resume_job(job_id="job_stock_monitor")
def job_close_scan(self):
oplogs("job_close_scan triggerred")
self.sched.pause_job(job_id="job_stock_monitor")
oplogs("job_stock_monitor paused")
def job_forget(self, persist=False):
# 清除记忆
if persist is True:
#save memory before forget them
pass
self.memory.clear()
oplogs("memory erased")
def listen(self, group, user, msg):
# 添加一条记录到记忆中
record = [time.strftime("%Y-%m-%d %H:%M:%S"), group, user, msg]
self.stockmod.scan_stock(group, msg)
self.memory.append(record)
def save_to_db(self):
oplogs("save_to_db called")
# 保存消息
for msg in self.memory:
sql = "INSERT INTO msgqueue VALUES (%s,%s,%s,%s)" % (
msg[0], msg[1], msg[2], msg[3])
self.cursor.execute(sql)
# 保存监控列表
for (key, value) in self.stockmod.monitor_queue.items():
for code in value:
# date TEXT, group TEXT, code TEXT
# date = time.strftime("%Y-%m-%d %H:%M:%S")
sql = "INSERT INTO monitorstocks VALUES (%s,%s)" % (key, code)
self.cursor.execute(sql)
self.conn.commit()
def load_from_db(self):
pass
def isCmd(self, cmd):
if USER_CMD in cmd:
return True
for keyword in syscmd:
if keyword in cmd:
return True
return False
def action_user(self, cmd, group, user):
cmds = cmd.lower().split()
#"命令格式：
# [添加监控]harry 股票名
# [删除监控]harry del 股票名
# [显示群推荐]harry 本群推荐
# [显示群数据统计]harry stat
# [显示报告]harry report"]
# user 用户命令
if len(cmds) == 1:
return random.choice(auto_replys)
if ('list' in cmds[1]) or ('本群推荐' in cmds[1]):
oplogs("action_user:本群推荐 [%s]" % cmd)
return self.stockmod.get_group_stock_price(group)
elif 'del' == cmds[1]:
return self.stockmod.del_from_list(group, cmds[2])
elif 'stat' in cmds[1]:
boottime = datetime.now() - self.starttime
boothour = int(boottime.seconds / 3600)
bootmin = int(boottime.seconds % 3600 / 60)
stat = "机器人已运行:{}天{}小时{}分钟\n股价扫描任务:{}".format(
boottime.days, boothour, bootmin, self.sched.state)
return stat
elif 'report' in cmds[1]:
return "建设中"
elif 'help' in cmds[1]:
oplogs("action_user:help called")
return auto_replys[0]
elif 'ver' in cmds[1]:
return HR__VERSION
elif '启动' in cmds[1]:
return "臣在"
elif '关闭' in cmds[1]:
return "跪安"
else:
oplogs("action_user:harry [stock] called %s" % cmd)
if self.stockmod.isvalid_stock(cmds[1]):
self.add_stock(group, cmds[1])
return self.show_stock(cmds[1])
return ""
def action(self, cmd):
cmds = cmd.split()
# 管理员后台控制命令
if not self.isCmd(cmds[0]):
return ""
# 去除'/',提取控制命令函数名
_cmd = cmds[0].strip('/')
if len(cmds) > 1:
return eval("self.{}({})".format(_cmd, cmds[1:]))
else:
return eval("self.{}()".format(_cmd))
def ping(self, cmds=[]):
#self.mouth('量化技术讨论','test')
return 'pong'
def help(self, cmds=[]):
# 显示命令帮助
return ("Help\n" + ",".join(syscmd))
def show(self, cmds=[]):
# 显示在线信息和数据
# show gmsg/stock [stockname]/monitor/stats
print("/show called")
if cmds[0] == 'memory':
return "Robot has %d messages in memory" % (len(self.memory))
if cmds[0] == 'stock':
return self.show_stock(cmds[1])
if cmds[0] == 'groups': # 显示所有支持的群
return ",".join(self.support_groups)
if cmds[0] == 'group': # 显示指定群支持的股票
if len(cmds) > 1 and cmds[1].isdigit():
groupid = int(cmds[1])
else:
groupid = 0
return self.action_user("harry 本群推荐", self.support_groups[groupid],
"robot")
if cmds[0] == 'mq':
return str(self.stockmod.monitor_queue)
if cmds[0] == 'stats':
return "no supported cmd"
return "no supported cmd"
def show_stock(self, stock):
if '机器人' in stock or '300024' in stock:
return "harry拒绝关注其他机器人"
if stock.isdigit():
return self.stockmod.get_stock_price(stock)
else:
return self.stockmod.get_stock_price_by_name(stock)
def cfg(self):
# 修改在线配置
return ("/cfg called")
def add(self, cmds=[]):
if len(cmds) >= 2:
# /add group [groupname], 添加支持的group
if cmds[0] is "group":
self.support_groups.append(cmds[1])
return "group added"
return "not supprt command"
def add_stock(self, group, stock):
self.stockmod.add_to_monitor_group(group, stock)
def delete(self, cmds=[]):
if len(cmds) > 0:
# 删除支持的group
if cmds[0] is "group":
self.support_groups.remove(cmds[1])
return "group deleted"
return "not supprt command"
def is_support_group(self, group):
return group in self.support_groups
def test(self, cmd):
return "test"
def restart(self):
# 重启机器人
return "restarted"
def start(self):
# 启动机器人
self.enable = True
return "臣在"
def pause(self):
self.enable = False
return "跪安"
def stop(self):
print("stop called")
# 离线机器人
self.enable = False
self.stopcb()
def use(self):
# 加载模块控制
# module management to be implemented here
return "All mods loaded"
def add_msg(self, msg):
# save message
self.memory.append(msg)
def add_groups(self, groups):
for group in groups:
if group not in self.support_groups:
self.support_groups.append(group)
class GzhScrapy:
def __init__(self):
self.cur_disease_id = ''
self.cur_count = 0
self.cur_company = 1
self.max_len = 0
self.cur_user_name = ''
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(self.send_disease_name, 'interval', id='main_schedule', seconds=60, args=[self])
self.scheduler.add_job(self.send_company_num, 'interval', id='sub_schedule', seconds=5, args=[self])
self.scheduler.start()
self.scheduler.pause_job('main_schedule')
self.scheduler.pause_job('sub_schedule')
self.reply = {}
self.start_collect_reply = 0
self.db = self.connect_db()
self.wechat_init(self)
# self.init_database(self)
# 初始化数据库
@staticmethod
def init_database(self):
print('Initial database started!')
# 初始化疾病表
disease_file = open('./disease.txt', 'r', encoding='UTF-8')
try:
for line in disease_file:
tmp_line = line.strip().strip('\n')
self.db.disease.insert_one({'name': tmp_line, 'reply': {}, 'finished': 0 })
print('Initial disease: ', tmp_line)
finally:
print('Initial database finished!')
disease_file.close()
@staticmethod
def send_company_num(self):
print('Start to send company number ---> ', self.cur_company)
itchat.send(str(self.cur_company), toUserName=self.cur_user_name)
self.cur_company = self.cur_company + 1
@staticmethod
def send_disease_name(self):
if self.cur_count > self.max_len:
self.scheduler.pause_job('main_schedule')
print('Congratulation! you have finished all task!')
return
print('\n\nCurrent time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
cur_disease = self.db.disease.find_one({ 'finished': 0 }, skip=self.cur_count)
self.cur_disease_id = cur_disease['_id']
self.cur_count = self.cur_count + 1
self.cur_company = 1
self.reply = {}
itchat.send(cur_disease['name'], toUserName=self.cur_user_name)
print('Start to send disease name ---> ', cur_disease['name'], ', cur_count: ', self.cur_count)
# 爬取每家公司关于当前疾病的回复
self.scheduler.resume_job('sub_schedule')
@staticmethod
def get_data(self):
mps = itchat.get_mps()
mps = itchat.search_mps(name='公众号名称')
if len(mps) > 0:
userName = mps[0]['UserName']
self.cur_user_name = userName
max_len = self.db.disease.count_documents({ 'finished': 0 })
print('There are still ', max_len, ' diseases needing to be scrapied')
# 周期性爬取疾病
# self.scheduler.resume_job('main_schedule')
self.send_disease_name(self)
else:
print('Can\'t find MPS 公众号名称')
@staticmethod
def connect_db():
instance = pymongo.MongoClient('127.0.0.1', 27017)
db = instance.hebaochacha
return db
@staticmethod
def wechat_init(self):
@itchat.msg_register(itchat.content.TEXT, isMpChat=True)
def reply_msg(msg):
print("Recive a message from MPS: ", msg['Content'].strip().strip('\n'))
if msg['FromUserName'] == self.cur_user_name:
if msg['Content'].find('请问您是问哪家公司的核保建议') >= 0:
self.start_collect_reply = 1
else:
# 疾病答案不合法，停止询问公司详情
if self.cur_company == 1:
self.db.disease.update_one({ '_id': self.cur_disease_id }, { '$set': { 'finished': 1 } })
self.scheduler.pause_job('sub_schedule')
print('No answer, has paused the sub_schedule!')
return
if self.cur_company == 10:
self.db.disease.update_one({ '_id': self.cur_disease_id }, { '$set': { 'reply': self.reply, 'finished': 1 } })
self.scheduler.pause_job('sub_schedule')
print(self.cur_disease_id, 'Save data successfully!')
return
if self.start_collect_reply == 1:
print(self.reply)
print('reply\'s length: ', str(len(self.reply)), ', cur_count: ' + str(self.cur_count) + ', cur_company: ' + str(self.cur_company))
self.reply[str(self.cur_company)] = msg['Content'].strip()
def after_login():
print('Login success!')
# self.init_database(self)
self.get_data(self)
pass
def after_logout():
# 关闭定时任务
self.scheduler.shutdown()
print('Has shutdown the scheduler!')
pass
itchat.auto_login(hotReload=True, loginCallback=after_login, exitCallback=after_logout)
itchat.run()
class Controller(object):
def __init__(self, name="defaultController"):
self.name = name
self.workflows = {}
self.instances = {}
self.tree = None
self.eventlog = []
self.schedulerStatusListener = SchedulerStatusListener(self.eventlog)
self.jobStatusListener = JobStatusListener(self.eventlog)
self.jobExecutionListener = JobExecutionListener(self.eventlog)
self.scheduler = BackgroundScheduler()
self.scheduler.add_listener(
self.schedulerStatusListener.callback(self),
EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
| EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED)
self.scheduler.add_listener(self.jobStatusListener.callback(self),
EVENT_JOB_ADDED | EVENT_JOB_REMOVED)
self.scheduler.add_listener(self.jobExecutionListener.callback(self),
EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
self.ancestry = [self.name]
def loadWorkflowsFromFile(self, path):
self.tree = et.ElementTree(file=path)
for workflow in self.tree.iter(tag="workflow"):
name = workflow.get("name")
self.workflows[name] = wf.Workflow(name=name,
workflowConfig=workflow,
parent_name=self.name)
self.addChildWorkflows()
self.addWorkflowScheduledJobs()
def addChildWorkflows(self):
for workflow in self.workflows:
children = self.workflows[workflow].options.children
for child in children:
if child in self.workflows:
children[child] = self.workflows[child]
def addWorkflowScheduledJobs(self):
for workflow in self.workflows:
if (self.workflows[workflow].options.enabled
and self.workflows[workflow].options.scheduler["autorun"]
== "true"):
schedule_type = self.workflows[workflow].options.scheduler[
"type"]
schedule = self.workflows[workflow].options.scheduler["args"]
self.scheduler.add_job(self.workflows[workflow].execute,
trigger=schedule_type,
replace_existing=True,
**schedule)
def createWorkflowFromTemplate(self, name="emptyWorkflow"):
self.loadWorkflowsFromFile(path=config.templatesPath + sep + name +
".workflow")
def removeWorkflow(self, name=""):
if name in self.workflows:
del self.workflows[name]
return True
return False
def updateWorkflowName(self, oldName="", newName=""):
self.workflows[newName] = self.workflows.pop(oldName)
self.workflows[newName].name = newName
def executeWorkflow(self, name, start="start", data=None):
steps, instances = self.workflows[name].execute(start=start, data=data)
self.jobExecutionListener.execute_event_code(self, 'JobExecuted')
return steps, instances
# Starts active execution
def start(self):
self.scheduler.start()
# Stops active execution
def stop(self, wait=True):
self.scheduler.shutdown(wait=wait)
# Pauses active execution
def pause(self):
self.scheduler.pause()
# Resumes active execution
def resume(self):
self.scheduler.resume()
# Pauses active execution of specific job
def pauseJob(self, job_id):
self.scheduler.pause_job(job_id=job_id)
# Resumes active execution of specific job
def resumeJob(self, job_id):
self.scheduler.resume_job(job_id=job_id)
# Returns jobs scheduled for active execution
def getScheduledJobs(self):
self.scheduler.get_jobs()
class CronManager:
def __init__(self, use_mongo_db=True):
self.scheduler = BackgroundScheduler(timezone=shanghai_tz)
self.scheduler.configure()
if use_mongo_db:
self.job_store = MongoDBJobStore(database='apscheduler',
collection='cronTab',
client=db)
self.scheduler.add_jobstore(self.job_store)
self.is_replace_existing = True
else:
self.is_replace_existing = False
def add_cron(self, cron_instance):
if not isinstance(cron_instance, Cron):
raise TypeError('please add correct cron！')
if cron_instance.trigger_type == 'interval':
seconds = cron_instance.trigger_args.get('seconds')
if not isinstance(seconds,
int) and not common.can_convert_to_int(seconds):
raise TypeError('请输入合法的时间间隔！')
seconds = int(seconds)
if seconds <= 0:
raise TypeError('请输入大于0的时间间隔！')
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
seconds=seconds,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_id(),
max_instances=5,
jitter=0) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'date':
run_date = cron_instance.trigger_args.get('run_date')
# TODO 判断run_date类型
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
run_date=run_date,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_id()) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'cron':
raise TypeError('暂时不支持 trigger_type 等于 \'cron\'')
return cron_instance.get_id()
def start(self, paused=False):
self.scheduler.start(paused=paused)
def pause_cron(self, cron_id=None, pause_all=False):
if pause_all:
self.scheduler.pause()
elif cron_id:
self.scheduler.pause_job(job_id=cron_id)
def resume_cron(self, cron_id=None, resume_all=False):
if resume_all:
self.scheduler.resume()
elif cron_id:
self.scheduler.resume_job(job_id=cron_id)
def del_cron(self, cron_id=None, del_all=False):
if del_all:
self.scheduler.remove_all_jobs()
elif cron_id:
self.scheduler.remove_job(job_id=cron_id)
def update_cron(self, cron_id, cron_info):
if not isinstance(cron_id, str):
raise TypeError('cron_id must be str')
if not isinstance(cron_info, dict):
raise TypeError('cron_info must be dict')
trigger_type = cron_info.get('triggerType')
interval = cron_info.get('interval')
run_date = cron_info.get('runDate')
test_case_suite_id_list = cron_info.get('testCaseSuiteIdList')
is_execute_forbiddened_case = cron_info.get('isExecuteForbiddenedCase')
test_case_id_list = cron_info.get('testCaseIdList')
test_domain = cron_info.get('testDomain')
global_vars_id = cron_info.get('globalVarsId')
alarm_mail_list = cron_info.get('alarmMailList')
is_ding_ding_notify = cron_info.get('isDingDingNotify')
ding_ding_access_token = cron_info.get('dingdingAccessToken')
ding_ding_notify_strategy = cron_info.get('dingdingNotifyStrategy')
is_enterprise_wechat_notify = cron_info.get('isEnterpriseWechatNotify')
enterprise_wechat_access_token = cron_info.get(
'enterpriseWechatAccessToken')
enterprise_wechat_notify_strategy = cron_info.get(
'enterpriseWechatNotifyStrategy')
cron_name = cron_info.get('name')
try:
if trigger_type == 'interval' and int(interval) > 0:
self.scheduler.modify_job(
job_id=cron_id, trigger=IntervalTrigger(seconds=interval))
elif trigger_type == 'date':
# TODO 判断run_date类型
self.scheduler.modify_job(
job_id=cron_id, trigger=DateTrigger(run_date=run_date))
else:
raise TypeError('更新定时任务触发器失败！')
if run_date:
cron = Cron(
test_case_suite_id_list=test_case_suite_id_list,
is_execute_forbiddened_case=is_execute_forbiddened_case,
test_domain=test_domain,
global_vars_id=global_vars_id,
alarm_mail_list=alarm_mail_list,
is_ding_ding_notify=is_ding_ding_notify,
ding_ding_access_token=ding_ding_access_token,
ding_ding_notify_strategy=ding_ding_notify_strategy,
is_enterprise_wechat_notify=is_enterprise_wechat_notify,
enterprise_wechat_access_token=
enterprise_wechat_access_token,
enterprise_wechat_notify_strategy=
enterprise_wechat_notify_strategy,
trigger_type=trigger_type, # 更新定时器时，此参数并没有真正起到作用, 仅修改展示字段
test_case_id_list=test_case_id_list,
run_date=run_date,
cron_name=cron_name) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
else:
cron = Cron(
test_case_suite_id_list=test_case_suite_id_list,
is_execute_forbiddened_case=is_execute_forbiddened_case,
test_domain=test_domain,
global_vars_id=global_vars_id,
alarm_mail_list=alarm_mail_list,
is_ding_ding_notify=is_ding_ding_notify,
ding_ding_access_token=ding_ding_access_token,
ding_ding_notify_strategy=ding_ding_notify_strategy,
is_enterprise_wechat_notify=is_enterprise_wechat_notify,
enterprise_wechat_access_token=
enterprise_wechat_access_token,
enterprise_wechat_notify_strategy=
enterprise_wechat_notify_strategy,
trigger_type=trigger_type, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
test_case_id_list=test_case_id_list,
seconds=interval, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
cron_name=cron_name)
# 玄学，更改job的时候必须改args，不能改func
self.scheduler.modify_job(job_id=cron_id,
coalesce=True,
args=[cron])
except BaseException as e:
raise TypeError('更新定时任务失败: %s' % e)
def shutdown(self, force_shutdown=False):
if force_shutdown:
self.scheduler.shutdown(wait=False)
else:
self.scheduler.shutdown(wait=True)
def get_crons(self):
return self.scheduler.get_jobs()
cloudData_job = scheduler.add_job(publishData,
'interval',
seconds=cloudInterval)
cloudDiag_job = scheduler.add_job(publishDiag,
'interval',
seconds=cloudInterval)
signal.signal(signal.SIGINT, signal_handler)
GPIO.output(auxpump1, GPIO.LOW)
print('Running')
while True:
if sysError:
scheduler.pause_job(pump_job)
else:
scheduler.resume_job(pump_job)
msgOut = (msg1 + msg2 + "\n" + msg3 + msg4)
#-------------------------------------------
# Read Ultrasonic
#-------------------------------------------
#-------------------------------------------
# Read PH
#-------------------------------------------
#-------------------------------------------
# Read EC
