class LightManager(object):
def __init__(self):
self.setterZones = [LightSetter(zone.mode, zone.pinout, description=zone.description) for zone in zones]
self.defaultSetterZone = self.setterZones[0]
self.scheduler = BackgroundScheduler()
self.scheduler.start()
def setLights(self, color, zone=None):
if not zone:
self.defaultSetterZone.setLights(color)
elif int(zone) in range(len(self.setterZones)):
self.setterZones[int(zone)].setLights(color)
else:
raise InvalidZoneException
def setEvent(self, secondsUntilEvent, color, zone=None):
eventTime = datetime.now()+timedelta(seconds=secondsUntilEvent)
eventId = str(hash((eventTime, color)))
self.scheduler.add_job(self.setLights, args=(color,zone), next_run_time=eventTime, id=eventId)
return eventId
def cancelEvent(self, eventId):
self.scheduler.remove_job(eventId)
def getZoneInfo(self):
return {str(i): {'type': zone.mode, 'description': zone.description} for i, zone in zip(range(len(self.setterZones)), self.setterZones)}
def post(self, action, position = ''):
global scheduler
self.checkStartup()
if action == 'play':
runCommand('mpc play ' + position)
#Settings.set('radio', 'state', 'play')
if scheduler is None:
scheduler = BackgroundScheduler()
scheduler.add_job(self.checkStatus, 'interval', seconds=30, id='checkStatus', replace_existing=True)
scheduler.start()
elif action == 'stop':
runCommand('mpc stop')
#Settings.set('radio', 'state', 'stop')
if scheduler is not None:
scheduler.remove_job('checkStatus')
scheduler.shutdown()
scheduler = None
return {'playMode': 'stopped'}
elif action =='pause':
runCommand('mpc pause')
elif action =='next':
runCommand('mpc next')
elif action =='previous':
runCommand('mpc prev')
else:
return {'playMode': 'invalid'}
(out, err) = runCommand('mpc status')
if err:
return {'error', err}, 500
return {'playMode': Parser.parsePlayMode(out)}
class timer:
'''
process = timer(function, [para1,para2,...], intervalseconds)
process.run()
'''
def __init__(self, func, paras, seconds, id):
self.func = func
self.paras = paras
self.time = seconds
self.id = id
self.scheduler = None
self.setTimer()
def setTimer(self):
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(self.func, 'interval', seconds=self.time, id=self.id)
def add_job(self, func, seconds, id):
self.scheduler.add_job(func, 'interval', seconds, id)
def remove_job(self, id):
self.scheduler.remove_job(id)
def run(self):
self.scheduler.start()
class schedulecontrol:
def __init__(self):
self.scheduler = BackgroundScheduler()
self.oncescheduler=BlockingScheduler()
self.scheduler.start()
def start(self):
self.oncescheduler.start()
def addschedule(self,event=None, day_of_week='0-7', hour='11',minute='57' ,second='0',id='',type='cron',run_date='',args=None):
if id=='':
id=str(time.strftime("%Y-%m-%d %X", time.localtime()));
if type=='date':
if run_date=='':
self.oncescheduler.add_job(event, args=args)
else:
self.oncescheduler.add_job(event, 'date', run_date=run_date, args=args)
elif type=='back':
self.oncescheduler.add_job(event,type, day_of_week=day_of_week, hour=hour,minute=minute ,second=second,id=id)
else:
self.scheduler.add_job(event, type, day_of_week=day_of_week, hour=hour, minute=minute, second=second, id=id)
def removeschedule(self,id):
self.scheduler.remove_job(id)
def build(cube_name_list, buildType, start_time=None, end_time=None):
run_cube_job_id = '1'
check_cube_job_id = '2'
scheduler = BackgroundScheduler()
CubeWorker.job_instance_dict = {}
for cube_name in cube_name_list:
CubeWorker.job_instance_dict[cube_name] = None
CubeWorker.scheduler = scheduler
CubeWorker.run_cube_job_id = run_cube_job_id
CubeWorker.check_cube_job_id = check_cube_job_id
# start the run cube job immediately
CubeWorker.run_cube_job(buildType, start_time, end_time)
scheduler.add_job(CubeWorker.run_cube_job, 'interval', seconds=30, id=run_cube_job_id, args=[buildType, start_time, end_time])
scheduler.add_job(CubeWorker.check_cube_job, 'interval', seconds=30, id=check_cube_job_id)
scheduler.start()
while True:
if CubeWorker.all_finished():
print "all cube jobs are finished"
scheduler.remove_job(check_cube_job_id)
scheduler.remove_job(run_cube_job_id)
scheduler.shutdown()
status = CubeWorker.get_status()
print 'Build exited with status %s' % status
return status == CubeWorkerStatus.SUCCESS
time.sleep(settings.KYLIN_CHECK_STATUS_INTERVAL)
def initialize_scheduler():
"""Initialize the task scheduler. This method configures the global
scheduler, checks the loaded tasks, and ensures they are all scheduled.
"""
global SCHEDULER
# If the scheduler is not enabled, clear it and exit. This prevents any
# unexpected database session issues.
if not CONF.scheduler.enable:
if SCHEDULER:
SCHEDULER.remove_all_jobs()
SCHEDULER = None
LOG.info("Scheduler is not enabled.")
return
# Use SQLAlchemy as a Job store.
jobstores = {
'default': SQLAlchemyJobStore(engine=get_engine())
}
# Two executors: The default is for all plugins. The second one is for
# the scheduler manager, which makes sure this scheduler instance is
# aware of all of our plugins.
executors = {
'default': ThreadPoolExecutor(10),
'manager': ThreadPoolExecutor(1),
}
# Allow executions to coalesce. See https://apscheduler.readthedocs.org/en
# /latest/userguide.html#missed-job-executions-and-coalescing
job_defaults = {
'coalesce': True,
'max_instances': 1,
'replace_existing': True
}
# This will automatically create the table.
SCHEDULER = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
SCHEDULER.start()
atexit.register(shutdown_scheduler)
# Make sure we load in the update_scheduler job. If it exists,
# we remove/update it to make sure any code changes get propagated.
if SCHEDULER.get_job(SCHEDULE_MANAGER_ID):
SCHEDULER.remove_job(SCHEDULE_MANAGER_ID)
SCHEDULER.add_job(
update_scheduler,
id=SCHEDULE_MANAGER_ID,
trigger=IntervalTrigger(minutes=1),
executor='manager',
replace_existing=True
)
class schedulecontrol:
def __init__(self):
self.scheduler = BackgroundScheduler()
self.scheduler.start()
def addschedule(self, event, day_of_week="0-7", hour="11", minute="57", second="0", id=""):
if id == "":
id = str(time.strftime("%Y-%m-%d %X", time.localtime()))
self.scheduler.add_job(event, "cron", day_of_week=day_of_week, hour=hour, minute=minute, second=second, id=id)
def removeschedule(self, id):
self.scheduler.remove_job(id)
class schedulecontrol:
def __init__(self):
self.scheduler = BackgroundScheduler()
self.scheduler.start()
def addschedule(self,event, day_of_week='0-7', hour='11',minute='57' ,second='0',id=''):
if id=='':
id=str(time.strftime("%Y-%m-%d %X", time.localtime()));
self.scheduler.add_job(event,'cron', day_of_week=day_of_week, hour=hour,minute=minute ,second=second,id=id)
def removeschedule(self,id):
self.scheduler.remove_job(id)
class Scheduler(object):
def __init__(self, bot):
self.sched = BackgroundScheduler()
self.sched.start()
self.bot = bot
self.job_id = []
def __del__(self):
self.shutdown()
def shutdown(self):
self.sched.shutdown()
def kill_scheduler(self, job_id):
try:
self.sched.remove_job(job_id)
if self.job_id.count != 0:
self.job_id.remove(job_id)
except JobLookupError as err:
return
def scheduler(self, type, job_id, myFunc, *args):
# 스케줄러 중복 방지
check=True
for ch in self.job_id:
if ch == job_id:
check=False
else:
pass
if check:
if type == 'interval':
self.sched.add_job(myFunc, type, seconds=10, id=job_id, args=args)
elif type == 'cron':
self.sched.add_job(myFunc, type, day_of_week='mon-sat',
hour='7-20', minute='*/10',
id=job_id, args=args)
self.job_id.append(job_id)
else:
pass
class schedulecontrol:
def __init__(self):
self.scheduler = BackgroundScheduler()
self.oncescheduler = BlockingScheduler()
self.scheduler.start()
def start(self):
self.oncescheduler.start()
def addschedule(
self,
event=None,
day_of_week="0-7",
hour="11",
minute="57",
second="0",
id="",
type="cron",
run_date="",
args=None,
):
if id == "":
id = str(time.strftime("%Y-%m-%d %X", time.localtime()))
if type == "date":
if run_date == "":
self.oncescheduler.add_job(event, args=args)
else:
self.oncescheduler.add_job(event, "date", run_date=run_date, args=args)
else:
self.scheduler.add_job(event, type, day_of_week=day_of_week, hour=hour, minute=minute, second=second, id=id)
def removeschedule(self, id):
self.scheduler.remove_job(id)
class WorkersManager:
class Command:
def __init__(self, callback, timeout, args=(), options=dict()):
self._callback = callback
self._timeout = timeout
self._args = args
self._options = options
self._source = '{}.{}'.format(
callback.__self__.__class__.__name__ if hasattr(
callback, '__self__') else callback.__module__,
callback.__name__)
def execute(self):
messages = []
with timeout(
self._timeout,
exception=TimeoutError(
'Execution of command {} timed out after {} seconds'.
format(self._source, self._timeout))):
messages = self._callback(*self._args)
_LOGGER.debug('Execution result of command %s: %s', self._source,
messages)
return messages
def __init__(self, config):
self._mqtt_callbacks = []
self._update_commands = []
self._scheduler = BackgroundScheduler(timezone=utc)
self._daemons = []
self._config = config
self._command_timeout = config.get('command_timeout', 35)
def register_workers(self):
for (worker_name, worker_config) in self._config['workers'].items():
#print("WorkerManager --> register_workers : start to import workers")
module_obj = importlib.import_module("workers.%s" % worker_name)
klass = getattr(module_obj, "%sWorker" % worker_name.title())
# if module_obj.REQUIREMENTS is not None:
# self._pip_install_helper(module_obj.REQUIREMENTS)
command_timeout = worker_config.get('command_timeout',
self._command_timeout)
# constructor
worker_obj = klass(command_timeout, **worker_config['args'])
if not hasattr(worker_obj, 'status_update') and not hasattr(
worker_obj, 'run'):
raise "%s cannot be initialized, it has to define run or status_update method" % worker_name
if hasattr(worker_obj, 'status_update'):
_LOGGER.debug(
"Added %s worker with %d seconds interval and a %d seconds timeout",
repr(worker_obj), worker_config['update_interval'],
worker_obj.command_timeout)
command = self.Command(worker_obj.status_update,
worker_obj.command_timeout, [])
self._update_commands.append(command)
if 'update_interval' in worker_config:
job_id = '{}_interval_job'.format(worker_name)
interval_job = self._scheduler.add_job(
partial(self._queue_command, command),
'interval',
seconds=worker_config['update_interval'],
id=job_id)
self._mqtt_callbacks.append(
(worker_obj.format_topic('update_interval'),
partial(self._update_interval_wrapper, command,
job_id)))
if hasattr(worker_obj, 'run'):
print("has run attr")
_LOGGER.debug("Registered %s as daemon", repr(worker_obj))
self._daemons.append(worker_obj)
if 'topic_subscription' in worker_config:
self._mqtt_callbacks.append(
(worker_config['topic_subscription'],
partial(self._on_command_wrapper, worker_obj)))
print("WorkerManager --> register_workers: _mqtt_callbacks = ",
self._mqtt_callbacks)
if 'topic_subscription' in self._config:
for (callback_name,
options) in self._config['topic_subscription'].items():
self._mqtt_callbacks.append(
(options['topic'],
lambda client, _, c: self._queue_if_matching_payload(
self.Command(getattr(self, callback_name), self.
_command_timeout), c.payload, options[
'payload'])))
return self
def start(self, mqtt):
mqtt.callbacks_subscription(self._mqtt_callbacks)
self._scheduler.start()
self.update_all()
for daemon in self._daemons:
threading.Thread(target=daemon.run, args=[mqtt],
daemon=True).start()
def _queue_if_matching_payload(self, command, payload, expected_payload):
if payload.decode('utf-8') == expected_payload:
self._queue_command(command)
def update_all(self):
_LOGGER.debug("Updating all workers")
for command in self._update_commands:
self._queue_command(command)
@staticmethod
def _queue_command(command):
_WORKERS_QUEUE.put(command)
@staticmethod
def _pip_install_helper(package_names):
for package in package_names:
pip_main(['install', '-q', package])
logger.reset()
def _update_interval_wrapper(self, command, job_id, client, userdata, c):
_LOGGER.info("Recieved updated interval for %s with: %s", c.topic,
c.payload)
try:
new_interval = int(c.payload)
self._scheduler.remove_job(job_id)
self._scheduler.add_job(partial(self._queue_command, command),
'interval',
seconds=new_interval,
id=job_id)
except ValueError:
logger.log_exception(_LOGGER, 'Ignoring invalid new interval: %s',
c.payload)
def _on_command_wrapper(self, worker_obj, client, userdata, c):
_LOGGER.debug("Received command for %s on %s: %s", repr(worker_obj),
c.topic, c.payload)
global_topic_prefix = userdata['global_topic_prefix']
topic = c.topic[len(global_topic_prefix + '/'
):] if global_topic_prefix is not None else c.topic
self._queue_command(
self.Command(worker_obj.on_command, worker_obj.command_timeout,
[topic, c.payload]))
class Competitions:
def __init__(self, clist_user_name, clist_api_key, mount_point, bot, fallback):
self.clist_user_name = clist_user_name
self.clist_api_key = clist_api_key
self.bot = bot
self.ong = None
self.upc = None
self.mount_point = mount_point
self.utility = ContestUtility(mount_point)
self.jobstores = {
'default': SQLAlchemyJobStore(url='sqlite:///' + mount_point + 'coders1.db')
}
self.schedule = BackgroundScheduler(jobstores=self.jobstores)
self.schedule.start()
self.conv_handler = ConversationHandler(
entry_points=[CommandHandler('upcoming', self.upcoming)],
allow_reentry=True,
states={
SCHED: [CallbackQueryHandler(self.remind, pattern=r"^[0-9]*$")]
},
fallbacks=[fallback]
)
self.conv_handler1 = ConversationHandler(
entry_points=[CommandHandler('dontRemindMe', self.removeRemind)],
allow_reentry=True,
states={
REMNOTI: [CallbackQueryHandler(self.remnoti, pattern=r'^.*notiplz.*$')]
},
fallbacks=[fallback]
)
@staticmethod
def clist_requester(url, payload):
response = requests.get(url, headers={'Content-Type': 'application/json', 'User-agent': 'Mozilla/5.0'},
params=payload, verify=False)
return response.text
# COMMAND HANDLER FUNCTION TO SHOW LIST OF ONGOING COMPETITIONS
@timeouts.wrapper_for_class_methods
def ongoing(self, bot, update):
# PARSING JSON
date1 = update.message.date
payload = {'limit': '15', 'start__lt': str(date1), 'end__gt': str(date1),
'username': self.clist_user_name, 'api_key': self.clist_api_key,
'format': 'json', 'order_by': 'end'}
raw_data = self.clist_requester(url="https://clist.by/api/v1/contest/", payload=payload)
try:
json_data = json.loads(raw_data)
search_results = json_data['objects']
self.utility.ongoing_sender(update=update, contest_list=search_results)
self.ong = search_results
except:
self.utility.ongoing_sender(update, self.ong)
@timeouts.wrapper_for_class_methods
def upcoming(self, bot, update):
# PARSING JSON
date1 = update.message.date
payload = {'limit': '15', 'start__gt': str(date1), 'order_by': 'start',
'username': self.clist_user_name,
'api_key': self.clist_api_key, 'format': 'json'}
raw_data = self.clist_requester(url="https://clist.by/api/v1/contest/", payload=payload)
try:
json_data = json.loads(raw_data)
search_results = json_data['objects']
self.utility.upcoming_sender(update=update, contest_list=search_results)
self.upc = search_results
except:
self.utility.upcoming_sender(update, self.upc)
return SCHED
# FUNCTION TO SET REMINDER
def remind(self, bot, update):
query = update.callback_query
msg = query.data
if str(msg).isdigit():
msg = int(msg) - 1
start1 = ContestUtility.time_converter(self.upc[msg]['start'], '-0030')
dateT = str(self.upc[msg]['start']).replace("T", " ").split(" ")
start1 = start1.replace("T", " ").split(" ")
date = dateT[0].split("-")
date1 = start1[0].split("-")
time1 = start1[1].split(":")
cur_time = datetime.now()
if not cur_time >= datetime(int(date[0]), int(date[1]), int(date[2]), 0, 0):
self.schedule.add_job(self.remindmsgDay, 'cron', year=date[0], month=date[1], day=date[2], replace_existing=True,
id=str(query.message.chat_id) + str(self.upc[msg]['id']) + "0",
args=[str(query.message.chat_id),
str(self.upc[msg]['event']) + "\n" + str(self.upc[msg]['href'])])
if not cur_time >= datetime(int(date1[0]), int(date1[1]), int(date1[2]), int(time1[0]), int(time1[1])):
self.schedule.add_job(self.remindmsg, 'cron', year=date1[0], month=date1[1], day=date1[2], hour=time1[0],
minute=time1[1],
replace_existing=True,
id=str(query.message.chat_id) + str(self.upc[msg]['id']) + "1",
args=[str(query.message.chat_id),
str(self.upc[msg]['event'] + "\n" + str(self.upc[msg]['href']))])
bot.edit_message_text(chat_id=query.message.chat_id, message_id=query.message.message_id,
text="I will remind you about " + self.upc[msg][
'event'] + "\nYou can use command /dontremindme to cancel reminder")
if query.message.chat_id < 0:
bot.send_message(chat_id=query.message.chat_id,
text="I detected that this is a group. The reminder will be sent to the group. If you want to get reminder personally then use this command in private message")
else:
bot.edit_message_text(chat_id=query.message.chat_id, message_id=query.message.message_id,
text="Sorry contest has already started")
return ConversationHandler.END
# WHAT HAPPENSWHEN REMINDER IS DEPLOYED
def remindmsgDay(self, chatId, message):
self.bot.send_message(chat_id=chatId, text="You have a contest within 24 hours\n " + message)
def remindmsg(self, chatId, message):
self.bot.send_message(chat_id=chatId, text="Your contest starts in half an hour\n " + message)
@timeouts.wrapper_for_class_methods
def removeRemind(self, bot, update):
conn = sqlite3.connect(self.mount_point + 'coders1.db')
c = conn.cursor()
c.execute("SELECT id FROM apscheduler_jobs WHERE id LIKE " + "'" + str(
update.message.chat_id) + "%' AND id LIKE " + "'%1'")
if c.fetchone():
c.execute("SELECT id FROM apscheduler_jobs WHERE id LIKE " + "'" + str(
update.message.chat_id) + "%' AND id LIKE " + "'%1'")
a = c.fetchall()
keyboard = []
for i in range(0, len(a)):
s = str(a[i]).replace("('", "").replace("',)", "").replace(
'("', "").replace('",)', "")
print(s)
keyboard.append([InlineKeyboardButton(str(self.schedule.get_job(job_id=s).args[1].split("\n")[0]),
callback_data=s[:-1] + "notiplz")])
reply_markup = InlineKeyboardMarkup(keyboard)
update.message.reply_text("Here are your pending reminders\nSelect the reminder you want to remove",
reply_markup=reply_markup)
c.close()
return REMNOTI
else:
c.close()
update.message.reply_text("You have no pending reminders")
return ConversationHandler.END
def remnoti(self, bot, update):
query = update.callback_query
val = str(query.data).replace("notiplz", "")
try:
self.schedule.remove_job(val + "0")
except:
pass
try:
self.schedule.remove_job(val + "1")
except:
pass
bot.edit_message_text(text="Reminder removed", message_id=query.message.message_id,
chat_id=query.message.chat_id)
return ConversationHandler.END
class CronManager:
def __init__(self, use_mongo_db=True):
self.scheduler = BackgroundScheduler(timezone=shanghai_tz)
self.scheduler.configure()
if use_mongo_db:
self.job_store = MongoDBJobStore(database='apscheduler',
collection='cronTab',
client=db)
self.scheduler.add_jobstore(self.job_store)
self.is_replace_existing = True
else:
self.is_replace_existing = False
def add_cron(self, cron_instance):
if not isinstance(cron_instance, Cron):
raise TypeError('please add correct cron！')
if cron_instance.trigger_type == 'interval':
seconds = cron_instance.trigger_args.get('seconds')
if not isinstance(seconds,
int) and not common.can_convert_to_int(seconds):
raise TypeError('请输入合法的时间间隔！')
seconds = int(seconds)
if seconds <= 0:
raise TypeError('请输入大于0的时间间隔！')
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
seconds=seconds,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_id(),
max_instances=5,
jitter=0) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'date':
run_date = cron_instance.trigger_args.get('run_date')
# TODO 判断run_date类型
job = self.scheduler.add_job(
func=cron_instance.cron_mission,
trigger=cron_instance.trigger_type,
run_date=run_date,
replace_existing=self.is_replace_existing,
coalesce=True,
id=cron_instance.get_id()) # 玄学，新增job的时候不用加args，直接加对象调用的func
elif cron_instance.trigger_type == 'cron':
raise TypeError('暂时不支持 trigger_type 等于 \'cron\'')
return cron_instance.get_id()
def start(self, paused=False):
self.scheduler.start(paused=paused)
def pause_cron(self, cron_id=None, pause_all=False):
if pause_all:
self.scheduler.pause()
elif cron_id:
self.scheduler.pause_job(job_id=cron_id)
def resume_cron(self, cron_id=None, resume_all=False):
if resume_all:
self.scheduler.resume()
elif cron_id:
self.scheduler.resume_job(job_id=cron_id)
def del_cron(self, cron_id=None, del_all=False):
if del_all:
self.scheduler.remove_all_jobs()
elif cron_id:
self.scheduler.remove_job(job_id=cron_id)
def update_cron(self, cron_id, cron_info):
if not isinstance(cron_id, str):
raise TypeError('cron_id must be str')
if not isinstance(cron_info, dict):
raise TypeError('cron_info must be dict')
trigger_type = cron_info.get('triggerType')
interval = cron_info.get('interval')
run_date = cron_info.get('runDate')
test_case_suite_id_list = cron_info.get('testCaseSuiteIdList')
is_execute_forbiddened_case = cron_info.get('isExecuteForbiddenedCase')
test_case_id_list = cron_info.get('testCaseIdList')
test_domain = cron_info.get('testDomain')
alarm_mail_list = cron_info.get('alarmMailList')
is_ding_ding_notify = cron_info.get('isDingDingNotify')
ding_ding_access_token = cron_info.get('dingdingAccessToken')
ding_ding_notify_strategy = cron_info.get('dingdingNotifyStrategy')
is_enterprise_wechat_notify = cron_info.get('isEnterpriseWechatNotify')
enterprise_wechat_access_token = cron_info.get(
'enterpriseWechatAccessToken')
enterprise_wechat_notify_strategy = cron_info.get(
'enterpriseWechatNotifyStrategy')
cron_name = cron_info.get('name')
try:
if trigger_type == 'interval' and int(interval) > 0:
self.scheduler.modify_job(
job_id=cron_id, trigger=IntervalTrigger(seconds=interval))
elif trigger_type == 'date':
# TODO 判断run_date类型
self.scheduler.modify_job(
job_id=cron_id, trigger=DateTrigger(run_date=run_date))
else:
raise TypeError('更新定时任务触发器失败！')
if run_date:
cron = Cron(
test_case_suite_id_list=test_case_suite_id_list,
is_execute_forbiddened_case=is_execute_forbiddened_case,
test_domain=test_domain,
alarm_mail_list=alarm_mail_list,
is_ding_ding_notify=is_ding_ding_notify,
ding_ding_access_token=ding_ding_access_token,
ding_ding_notify_strategy=ding_ding_notify_strategy,
is_enterprise_wechat_notify=is_enterprise_wechat_notify,
enterprise_wechat_access_token=
enterprise_wechat_access_token,
enterprise_wechat_notify_strategy=
enterprise_wechat_notify_strategy,
trigger_type=trigger_type, # 更新定时器时，此参数并没有真正起到作用, 仅修改展示字段
test_case_id_list=test_case_id_list,
run_date=run_date,
cron_name=cron_name) # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
else:
cron = Cron(
test_case_suite_id_list=test_case_suite_id_list,
is_execute_forbiddened_case=is_execute_forbiddened_case,
test_domain=test_domain,
alarm_mail_list=alarm_mail_list,
is_ding_ding_notify=is_ding_ding_notify,
ding_ding_access_token=ding_ding_access_token,
ding_ding_notify_strategy=ding_ding_notify_strategy,
is_enterprise_wechat_notify=is_enterprise_wechat_notify,
enterprise_wechat_access_token=
enterprise_wechat_access_token,
enterprise_wechat_notify_strategy=
enterprise_wechat_notify_strategy,
trigger_type=trigger_type, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
test_case_id_list=test_case_id_list,
seconds=interval, # 更新定时器时，此参数并没有起到作用, 仅修改展示字段
cron_name=cron_name)
# 玄学，更改job的时候必须改args，不能改func
self.scheduler.modify_job(job_id=cron_id,
coalesce=True,
args=[cron])
except BaseException as e:
raise TypeError('更新定时任务失败: %s' % e)
def shutdown(self, force_shutdown=False):
if force_shutdown:
self.scheduler.shutdown(wait=False)
else:
self.scheduler.shutdown(wait=True)
def get_crons(self):
return self.scheduler.get_jobs()
class MainRunner(object):
workers = {}
dirpath = "."
defaultOutputPath = "output"
def __init__(self, dirpath="."):
"""
local path for load config
"""
logger.info("Initialing Main Runner for Fuzzy Testing")
if os.path.isdir(dirpath):
self.dirpath = dirpath
logger.info("loading runner config folder: " + dirpath)
else:
logger.info(dirpath + " is invalid, use default path instead")
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.load_dir(dirpath)
event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True)
event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove)
observer = Observer()
observer.schedule(event_handler, self.dirpath, recursive=True)
observer.start()
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fname in filenames:
if "json" in fname[-4:]:
self.load(os.path.join(dirpath, fname))
def load(self, fp):
"""
given a json file, load and create a task run regularly
"""
logger.info(fp + " was loaded!")
with open(fp) as in_data:
try:
data = json.load(in_data)
data["path"] = fp
except ValueError as e:
logger.warning(fp + " loaded failed: " + e.message)
return None
interval = 30
if "interval" in data:
interval = int(data["interval"])
if "output" in data:
if "defaultOutputPath" in data["output"]:
self.defaultOutputPath = data["output"]["defaultOutputPath"]
if "dirpath" in data["output"]:
data["output"]["outputPath"] = os.path.join(self.defaultOutputPath, data["output"]["dirpath"])
else:
data["output"] = {"outputPath": self.defaultOutputPath}
if "type" not in data:
logger.error(
"Missing type attribute in \
your configuration file [%s]"
% fp
)
return None
if fp in self.workers: # existing runner found
logger.info("Update exisitng runner [%s]" % fp)
runner = self.workers[fp]
runner.update(**data)
# //memo: Interval can't be modified
self.scheduler.modify_job(job_id=fp, func=runner.run, name=runner.name)
else: # Create new
logger.info("Create new runner [%s]" % fp)
module_path = data["type"][: data["type"].rfind(".")]
object_name = data["type"][data["type"].rfind(".") + 1 :]
try:
runner_module = getattr(importlib.import_module(module_path), object_name)
except Exception as e:
logger.exception(e)
return None
runner = runner_module(**data)
self.workers[fp] = runner
self.scheduler.add_job(runner.run, "interval", id=fp, name=runner.name, seconds=interval)
return runner
return None
def list(self):
"""
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
"""
for (fp, worker) in self.workers:
logger.info("path=" + fp + "," + str(worker) + ";")
def remove(self, fp):
"""
given file path, stop running instance if possible
"""
if fp in self.workers:
self.workers[fp].onstop()
self.scheduler.remove_job(job_id=fp)
del self.workers[fp]
return True
return False
def remove_advanced(self):
"""
TODO:
1. remove by start, end
2. by directory(?)
"""
pass
def unload_all(self):
"""
stop all running instances
"""
self.scheduler.shutdown()
def pause(self, fp):
"""
simply stop running instance but not remove config
TODO: should have timeout if stop failed
"""
self.scheduler.pause(job_id=fp)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
"""
For periodical minions, waking them according to timing
"""
pass
class MQ7:
"""Detector de gas MQ7"""
# resistencia del sensor
LOAD_RESISTANCE = 1
# pin salida digital
DOutput = Button(25)
# Ro value of the sensor
ro = 0.255
# Curva logaritmica con base 10
HIDROGENO_CURVA = [1.698970, 0.133539, -0.619674]
CO_CURVA = [1.698970, 0.220108, -0.790349]
# gas en ppm
hidrogeno_ppm = 0
co_ppm = 0
voltaje = 0
funcionamiento = True
# Contadores
voltaje_acumulado = 0
cantidad_muestras = 0
def __init__(self, convertor=ads1115.ADS1115, pin=PIN, address=ADDRESS):
'''Inicia con la lectura periodica de gases del sensor MQ7'''
# Permique que el resto del programa sigua funcionando aunque el ADC no funcione
try:
i2c = busio.I2C(board.SCL, board.SDA)
adc = convertor(i2c=i2c, address=address)
self.channel = AnalogIn(adc, pin)
self.sched_mq7 = BackgroundScheduler()
self.sched_mq7.add_job(self.leer_voltaje,
'interval',
seconds=5,
id="sched_mq7")
self.sched_mq7.start()
except (ValueError, OSError):
self.funcionamiento = False
logging.error('El ADC del MQ7 ha dejado de funcionar.')
def leer_voltaje(self):
'''leerá el voltaje cada cierto tiempo y lo irá acumulando, además
lleva la cuenta de cuantos ciclos de lectura lleva.'''
# Evita que se rompa el programa en caso de que el ADC deje de funcionar a mitad
# de una lectura de voltaje.
try:
self.voltaje_acumulado += self.channel.voltage
self.cantidad_muestras += 1
except (ValueError, OSError):
self.funcionamiento = False
logging.error('El ADC del MQ7 ha dejado de funcionar.')
self.sched_mq7.remove_job("sched_mq7")
def finalizar_ciclo(self):
'''Llama a las funciones necesarias para finalizar el ciclo de lectura
de datos y empezar uno nuevo'''
self.calcular_concentracion()
self.reiniciar_contadores()
def calcular_concentracion(self):
'''Hace los calculos necesarios para obtener las concentraciones
de los gases en ppm'''
calculos = Calculos()
self.voltaje = calculos.calcular_voltaje(self.voltaje_acumulado,
self.cantidad_muestras)
rs = calculos.calcular_resistencia(self.voltaje, self.LOAD_RESISTANCE)
ratio = rs / self.ro
self.hidrogeno_ppm = calculos.calcular_ppm(ratio, self.HIDROGENO_CURVA)
self.co_ppm = calculos.calcular_ppm(ratio, self.CO_CURVA)
def activar_alarma(self):
'''Pasa la variable alarma a True en el caso de que el pin digital
envien una señal.'''
global alarma_mq7
alarma_mq7 = 1
def reiniciar_contadores(self):
'''Reinicia los valores de los contadores para tomar nuevas lecturas.'''
global alarma_mq7
self.voltaje_acumulado = 0
self.cantidad_muestras = 0
alarma_mq7 = 0
def get_hidrogeno(self):
'''Devuelve un string de la variable hidrogeno_ppm con tres decimales'''
return self.hidrogeno_ppm
def get_co(self):
'''Devuelve un string de la variable co_ppm con tres decimales'''
return self.co_ppm
def get_alarma(self):
'''Devuelve un String de la variable alarma'''
global alarma_mq7
return str(alarma_mq7)
def get_voltaje(self):
'''Devuelve un String de la variable voltaje'''
return str(self.voltaje)
def get_funcionamiento(self):
'''Nos dice mediante un boolean si el sensor funciona. En el caso de True
es que el sensor funciona'''
return self.funcionamiento
# cuando el sensor detecte una concentración peligrosa lanzará una alarma
DOutput.when_pressed = activar_alarma
class MycroftRoutineSkill(MycroftSkill):
def __init__(self):
super(MycroftRoutineSkill, self).__init__(name="MycroftRoutineSkill")
self._in_progress_tasks = dict()
self._in_progress_tasks_lock = Lock()
def initialize(self):
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self._routines = defaultdict(dict)
self._routines.update(self._load_routine_data())
self._routine_to_sched_id_map = {}
self._register_routines()
path = dirname(abspath(__file__))
path_to_stop_words = join(path, 'vocab', self.lang, 'ThatsAll.voc')
self._stop_words = self._lines_from_path(path_to_stop_words)
path_to_cancel_words = join(path, 'vocab', self.lang, 'Cancel.voc')
self._cancel_words = self._lines_from_path(path_to_cancel_words)
path_to_days_of_week = join(path, 'vocab', self.lang, 'DaysOfWeek.voc')
self._days_of_week = self._lines_from_path(path_to_days_of_week)
self.add_event("mycroft.skill.handler.complete",
self._handle_completed_event)
def _handle_completed_event(self, message):
task_id = message.context.get("task_id")
with self._in_progress_tasks_lock:
if task_id not in self._in_progress_tasks:
return
LOGGER.info(task_id + " completed.")
self._in_progress_tasks[task_id].mark_finished()
def _await_completion_of_task(self, task_id):
LOGGER.info("Waiting for " + task_id)
start = time()
while start + TIMEOUT_IN_SECONDS > time():
with self._in_progress_tasks_lock:
try:
if self._in_progress_tasks[task_id].is_done():
del (self._in_progress_tasks[task_id])
return
except KeyError:
sleep(0.1)
LOGGER.warn("Timed out wating for {task}".format(task=task_id))
del (self._in_progress_tasks[task_id])
def send_message(self, message: str):
task_id = "{name}.{uuid}".format(name=self.name, uuid=uuid())
with self._in_progress_tasks_lock:
self._in_progress_tasks[task_id] = _Task(task_id)
self.bus.emit(
Message(msg_type="recognizer_loop:utterance",
data={"utterances": [message]},
context={"task_id": task_id}))
return task_id
def _lines_from_path(self, path):
with open(path, 'r') as file:
lines = [line.strip().lower() for line in file]
return lines
def _load_routine_data(self):
try:
with self.file_system.open(ROUTINES_FILENAME, 'r') as conf_file:
return json.loads(conf_file.read())
except FileNotFoundError:
log_message = "Routines file not found."
except PermissionError:
log_message = "Permission denied when reading routines file."
except json.decoder.JSONDecodeError:
log_message = "Error decoding json from routines file."
log_message += " Initializing empty dictionary."
return {}
def _register_routines(self):
for routine in self._routines:
self._register_routine(routine)
def _register_routine(self, name):
self.register_vocabulary(name, "RoutineName")
schedule = self._routines[name].get('schedule')
if schedule and self._routines.get('enabled', True):
self._schedule_routine(name, schedule)
def _schedule_routine(self, name, cronstring):
trigger = CronTrigger.from_crontab(cronstring)
job = self.scheduler.add_job(func=self._run_routine,
args=[name],
trigger=trigger,
name=name)
self._routine_to_sched_id_map[name] = job.id
def _write_routine_data(self):
with self.file_system.open(ROUTINES_FILENAME, 'w') as conf_file:
conf_file.write(json.dumps(self._routines, indent=4))
@intent_handler(
IntentBuilder("CreateRoutine").require("Create").require("Routine"))
def _create_routine(self, message):
name = self.get_response("name.it")
if not name:
return
name = name.lower()
if name in self._cancel_words:
return
tasks = self._get_task_list()
if not tasks:
return
self._routines[name]['tasks'] = tasks
self._write_routine_data()
self._register_routine(name)
self.speak_dialog('created', data={"name": name})
def _get_task_list(self):
first_task = self.get_response("first.task")
if not first_task:
return []
first_task = first_task.lower()
if first_task in self._cancel_words:
return []
tasks = [first_task]
while True:
task = self.get_response("next")
if not task:
return []
task = task.lower()
if task in self._cancel_words:
return []
if task in self._stop_words:
break
tasks.append(task)
return tasks
@intent_handler(
IntentBuilder("RunRoutine").optionally("Run").require("RoutineName"))
def _trigger_routine(self, message):
name = message.data["RoutineName"]
self._run_routine(name)
def _run_routine(self, name):
for task in self._routines[name]['tasks']:
task_id = self.send_message(task)
self._await_completion_of_task(task_id)
@intent_handler(
IntentBuilder("ListRoutine").require("List").require("Routines"))
def _list_routines(self, message):
if not self._routines:
self.speak_dialog('no.routines')
return
routines = ". ".join(self._routines.keys())
self.speak_dialog('list.routines')
self.speak(routines)
@intent_handler(
IntentBuilder("DeleteRoutine").require("Delete").require("RoutineName")
)
def _delete_routine(self, message):
name = message.data["RoutineName"]
del (self._routines[name])
self._write_routine_data()
self.speak_dialog('deleted', data={"name": name})
@intent_handler(
IntentBuilder("DescribeRoutine").require("Describe").require(
"RoutineName"))
def _describe_routine(self, message):
name = message.data["RoutineName"]
tasks = ". ".join(self._routines[name]['tasks'])
self.speak_dialog('describe', data={"name": name})
self.speak(tasks)
@intent_handler(
IntentBuilder("ScheduleRoutine").require("Schedule").require(
"RoutineName"))
def _add_routine_schedule(self, message):
name = message.data["RoutineName"]
days = self._get_days()
hour, minute = self._get_time()
cronstring = self._generate_cronstring(days, hour, minute)
self._routines[name]['schedule'] = cronstring
self._routines[name]['enabled'] = True
self._write_routine_data()
self._schedule_routine(name, cronstring)
self.speak_dialog("scheduled", data={'name': name})
@intent_handler(
IntentBuilder("DisableRoutine").require("Disable").require(
"RoutineName"))
def _disable_scheduled_routine(self, message):
name = message.data["RoutineName"]
self._routines[name]['enabled'] = False
self._write_routine_data()
self.scheduler.remove_job(self._routine_to_sched_id_map[name])
self.speak_dialog("disabled", data={"name": name})
@intent_handler(
IntentBuilder("EnableRoutine").require("Enable").require("RoutineName")
)
def _enable_scheduled_routine(self, message):
name = message.data["RoutineName"]
self._routines[name]['enabled'] = True
self._write_routine_data()
self._schedule_routine(name, self._routines[name]["schedule"])
self.speak_dialog("enabled", data={"name": name})
def _get_days(self):
days_to_run = []
days_from_user = self.get_response('which.days')
if not days_from_user:
return
days_from_user = days_from_user.lower()
for i in range(len(self._days_of_week)):
if self._days_of_week[i] in days_from_user:
days_to_run.append(str(i))
return ','.join(days_to_run)
def _get_time(self):
regex = '(?P<hour>[0-9]{1,2})[: ](?P<minute>[0-9]{1,2}) (?P<time_of_day>[ap].?m.?)'
time_from_user = self.get_response('what.time')
if not time_from_user:
return
time_from_user = time_from_user.lower()
matches = re.match(regex, time_from_user)
if not matches:
self.speak_dialog('could.not.parse.time')
return
matches = matches.groupdict()
hour = int(matches['hour'])
minute = int(matches['minute'])
pm = matches['time_of_day'] == 'pm'
hour = hour % 12
hour += 12 if pm else 0
return hour, minute
def _generate_cronstring(self, days, hour, minute):
return '{m} {h} * * {d}'.format(m=minute, h=hour, d=days)
class SchedulerService(rpyc.Service):
def __init__(self, **config):
self._scheduler = BackgroundScheduler()
self._scheduler.configure(**config)
self._scheduler.start()
self.logger = logging.getLogger("Heartbeat.core")
self.logger.info("Heartbeat Core Initalized")
def on_connect(self, conn):
# code that runs when a connection is created
# (to init the service, if needed)
self.logger.info("----------Begin New Client----------")
self.logger.info(conn)
self.logger.info("----------End New Client----------")
def on_disconnect(self, conn):
# code that runs after the connection has already closed
# (to finalize the service, if needed)
self.logger.info("----------Begin Goodbye Client----------")
self.logger.info(conn)
self.logger.info("----------End Goodbye Client----------")
def exposed_add_job(self, func, *args, **kwargs):
self.logger.info("----------Begin New Job----------")
self.logger.info("Function: %s", str(func))
self.logger.info("*args: %s", str(args))
self.logger.info("**kwargs: %s", str(dict(kwargs)))
self.logger.info("----------Eng New Job----------")
return self._scheduler.add_job(func, *args, **kwargs)
def exposed_modify_job(self, job_id, jobstore=None, **changes):
return self._scheduler.modify_job(job_id, jobstore, **changes)
def exposed_reschedule_job(self,
job_id,
jobstore=None,
trigger=None,
**trigger_args):
return self._scheduler.reschedule_job(job_id, jobstore, trigger,
**trigger_args)
def exposed_pause_job(self, job_id, jobstore=None):
return self._scheduler.pause_job(job_id, jobstore)
def exposed_resume_job(self, job_id, jobstore=None):
return self._scheduler.resume_job(job_id, jobstore)
def exposed_remove_job(self, job_id, jobstore=None):
self._scheduler.remove_job(job_id, jobstore)
def exposed_get_job(self, job_id, jobstore=None):
return self._scheduler.get_job(job_id, jobstore=jobstore)
def exposed_get_jobs(self, jobstore=None):
results = self._scheduler.get_jobs(jobstore)
return results
def exposed_get_tasks(self):
"""Return a list of schedule-able function"""
tasks = []
for module_file in os.listdir(
os.path.join(os.path.dirname(__file__), "task")):
if module_file == "__init__.py" or module_file[-3:] != ".py":
continue
module_name = "server.task.{}".format(module_file[:-3])
module = importlib.import_module(module_name)
if not hasattr(module, "__all__"):
continue
for function_name in module.__all__:
function = getattr(module, function_name)
if not callable(function):
continue
parameters = inspect.signature(function).parameters
parameters_str = ", ".join(
[str(val) for key, val in parameters.items()])
tasks.append("{}:{}({})".format(module_name, function_name,
parameters_str))
return tasks
class Scheduler():
jobstores = {'default': SQLAlchemyJobStore(url='sqlite:///jobs.sqlite')}
executors = {'default': ThreadPoolExecutor(1)}
job_defaults = {'coalesce': True, 'max_instances': 1}
periods = {
's': 'second',
'm': 'minute',
'h': 'hour',
'd': 'day',
'w': 'week',
'M': 'month'
}
def __init__(self):
self._apscheduler = BackgroundScheduler(jobstores=self.jobstores,
executors=self.executors,
job_defaults=self.job_defaults)
def parse_frequency(self, freq):
freq_regex = re.compile("(\d{1,3})((s|m|h|d|w|M))")
match = freq_regex.match(freq)
if match is None:
raise FrequencyException('time frequency not supported')
num = match.group(1)
period = match.group(2)
return int(num), period
def get_cron_kwarg(self, num, period):
return {self.periods[period]: '*/%d' % num}
def add_job(self, func, freq, job_id, args=None):
num, period = self.parse_frequency(freq)
cron_kwarg = self.get_cron_kwarg(num, period)
self._apscheduler.add_job(func,
'cron',
id=job_id,
args=args,
**cron_kwarg)
def remove_job(self, job_id):
self._apscheduler.remove_job(job_id)
def job_exists(self, job_id):
jobs = self._apscheduler.get_jobs()
return any([job.id == job_id for job in jobs])
def start(self):
try:
self._apscheduler.start()
except SchedulerAlreadyRunningError as e:
log.error(str(e))
def pause(self):
self.shutdown()
def shutdown(self):
try:
self._apscheduler.shutdown()
except SchedulerNotRunningError as e:
log.error(str(e))
class Scheduler(Flask):
days = {
"0": "sun",
"1": "mon",
"2": "tue",
"3": "wed",
"4": "thu",
"5": "fri",
"6": "sat",
"7": "sun",
"*": "*",
}
seconds = {"seconds": 1, "minutes": 60, "hours": 3600, "days": 86400}
def __init__(self):
super().__init__(__name__)
with open(Path.cwd().parent / "setup" / "scheduler.json", "r") as file:
self.settings = load(file)
dictConfig(self.settings["logging"])
self.configure_scheduler()
self.register_routes()
@staticmethod
def aps_date(date):
if not date:
return
date = datetime.strptime(date, "%d/%m/%Y %H:%M:%S")
return datetime.strftime(date, "%Y-%m-%d %H:%M:%S")
def configure_scheduler(self):
self.scheduler = BackgroundScheduler(self.settings["config"])
self.scheduler.start()
def register_routes(self):
@self.route("/delete_job/<job_id>", methods=["POST"])
def delete_job(job_id):
if self.scheduler.get_job(job_id):
self.scheduler.remove_job(job_id)
return jsonify(True)
@self.route("/next_runtime/<task_id>")
def next_runtime(task_id):
job = self.scheduler.get_job(task_id)
if job and job.next_run_time:
return jsonify(job.next_run_time.strftime("%Y-%m-%d %H:%M:%S"))
return jsonify("Not Scheduled")
@self.route("/schedule", methods=["POST"])
def schedule():
if request.json["mode"] in ("resume", "schedule"):
result = self.schedule_task(request.json["task"])
if not result:
return jsonify({"alert": "Cannot schedule in the past."})
else:
return jsonify({
"response": "Task resumed.",
"active": True
})
else:
try:
self.scheduler.pause_job(request.json["task"]["id"])
return jsonify({"response": "Task paused."})
except JobLookupError:
return jsonify(
{"alert": "There is no such job scheduled."})
@self.route("/time_left/<task_id>")
def time_left(task_id):
job = self.scheduler.get_job(task_id)
if job and job.next_run_time:
delta = job.next_run_time.replace(tzinfo=None) - datetime.now()
hours, remainder = divmod(delta.seconds, 3600)
minutes, seconds = divmod(remainder, 60)
days = f"{delta.days} days, " if delta.days else ""
return jsonify(f"{days}{hours}h:{minutes}m:{seconds}s")
return jsonify("Not Scheduled")
@staticmethod
def run_service(task_id):
post(
f"{environ.get('ENMS_ADDR')}/rest/run_task",
json=task_id,
auth=HTTPBasicAuth(environ.get("ENMS_USER"),
environ.get("ENMS_PASSWORD")),
verify=int(environ.get("VERIFY_CERTIFICATE", 1)),
)
def schedule_task(self, task):
if task["scheduling_mode"] == "cron":
crontab = task["crontab_expression"].split()
crontab[-1] = ",".join(self.days[day]
for day in crontab[-1].split(","))
trigger = {"trigger": CronTrigger.from_crontab(" ".join(crontab))}
elif task["frequency"]:
trigger = {
"trigger":
"interval",
"start_date":
self.aps_date(task["start_date"]),
"end_date":
self.aps_date(task["end_date"]),
"seconds":
int(task["frequency"]) * self.seconds[task["frequency_unit"]],
}
else:
trigger = {
"trigger": "date",
"run_date": self.aps_date(task["start_date"])
}
if not self.scheduler.get_job(task["id"]):
job = self.scheduler.add_job(
id=str(task["id"]),
replace_existing=True,
func=self.run_service,
args=[task["id"]],
**trigger,
)
else:
job = self.scheduler.reschedule_job(str(task["id"]), **trigger)
return job.next_run_time > datetime.now(job.next_run_time.tzinfo)
class MainRunner(object):
workers = {}
dirpath = '.'
defaultOutputPath = 'output'
class NoRunningFilter(logging.Filter):
def filter(self, record):
return not record.msg.startswith('Execution')
def __init__(self, dirpath='.'):
'''
local path for load config
'''
logger.info("Initialing Main Runner for Hasal agent")
if os.path.isdir(dirpath):
self.dirpath = dirpath
logger.info("loading runner config folder: " + dirpath)
else:
logger.info(dirpath + " is invalid, use default path instead")
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.load_dir(self.dirpath)
event_handler = JsonHandler(patterns=["*.json"],
ignore_directories=True)
event_handler.set_handler(oncreated=self.load,
onmodified=self.load,
ondeleted=self.remove)
observer = Observer()
observer.schedule(event_handler, self.dirpath, recursive=True)
observer.start()
my_filter = self.NoRunningFilter()
logging.getLogger("apscheduler.scheduler").addFilter(my_filter)
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fname in filenames:
if 'json' in fname[-4:]:
self.load(os.path.join(dirpath, fname))
def load(self, fp):
'''
given a json file, load and create a task run regularly
'''
logger.info(fp + " was loaded!")
with open("agent.log", 'w+') as f:
f.write(fp + " was loaded!")
data = {}
loaded = False
for _ in range(10):
try:
with open(fp) as in_data:
data = json.load(in_data)
# default will load JOB_NAME parameter in Jenkins created json file
data['name'] = data.get('JOB_NAME', "Jenkins Job")
data['path'] = fp
loaded = True
except ValueError as e:
logger.warning(fp + " loaded failed: " + e.message)
return None
except Exception as e:
logger.warning(
"File is not ready. Wait 1 second for another try.")
time.sleep(1)
if not loaded:
logger.warning(fp + " is not ready for 10 seconds.")
return None
# load interval value from Jenkins created json file (default : 30 )
interval = int(data.get('interval', 30))
# load outputpath and defaultoutputpath from Jenkins created json file
if 'output' in data:
if 'defaultOutputPath' in data['output']:
self.defaultOutputPath = data['output']['defaultOutputPath']
if 'dirpath' in data['output']:
data['output']['outputPath'] = os.path.join(
self.defaultOutputPath, data['output']['dirpath'])
else:
data['output'] = {'outputPath': self.defaultOutputPath}
if fp in self.workers: # existing runner found
logger.info("Update exisitng runner [%s]" % fp)
runner = self.workers[fp]
runner.update(**data)
# //memo: Interval can't be modified
self.scheduler.modify_job(job_id=fp,
func=runner.run,
name=runner.name)
else: # Create new
logger.info("Create new runner [%s]" % fp)
module_path = data.get('AGENT_MODULE_PATH', "hasalTask")
object_name = data.get('AGENT_OBJECT_NAME', "HasalTask")
try:
runner_module = getattr(importlib.import_module(module_path),
object_name)
except Exception as e:
logger.exception(e)
return None
runner = runner_module(**data)
self.workers[fp] = runner
self.scheduler.add_job(runner.run,
'interval',
id=fp,
name=runner.name,
seconds=interval)
return runner
def list(self):
'''
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
'''
for (fp, worker) in self.workers:
logger.info("path=" + fp + "," + str(worker) + ";")
def remove(self, fp):
'''
given file path, stop running instance if possible
'''
if fp in self.workers:
self.workers[fp].onstop()
self.scheduler.remove_job(job_id=fp)
del self.workers[fp]
return True
return False
def remove_advanced(self):
'''
TODO:
1. remove by start, end
2. by directory(?)
'''
pass
def unload_all(self):
'''
stop all running instances
'''
self.scheduler.shutdown()
def pause(self, fp):
'''
simply stop running instance but not remove config
TODO: should have timeout if stop failed
'''
self.scheduler.pause(job_id=fp)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
'''
For periodical minions, waking them according to timing
'''
pass
class SchedulerService(Service):
def __init__(self, env, db, collection, worker_num):
super(SchedulerService, self).__init__(env)
self._db_name = db
self._collection_name = collection
self._worker_num = worker_num
self._sched = None
global _cur_sched
_cur_sched = self
def on_active(self):
super(SchedulerService, self).on_active()
self._init_scheduler()
def on_inactive(self):
if self._sched is not None:
self._sched.shutdown()
def _init_scheduler(self):
jobstores = {
'default': MongoDBJobStore(self._db_name, self._collection_name,
host=self.mongodb_service.get_connection_info()),
}
executors = {
'default': ThreadPoolExecutor(self._worker_num),
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
self._sched = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
self._sched.start()
def add_job(self, handler, dt, args=None, kwargs=None, job_id=None, replace_existing=True):
'''
增加单一任务。执行即消亡
:param handler: 回调函数。需要考虑多线程问题
def handler(env, *args, **kwargs):pass
:param dt: datetime
:param job_id: string, 自定义id
:param kwargs: handler的参数
:return: job id
'''
args = args or []
args.insert(0, handler)
job = self._sched.add_job(dispatch_expire_job, 'date', run_date=dt, id=job_id, args=args, kwargs=kwargs,
replace_existing=replace_existing)
return job.id
def add_interval_job(self, handler,
weeks=0, days=0, hours=0, minutes=0, seconds=0, args=None, kwargs=None, job_id=None,
replace_existing=True):
'''
增加定期任务
:param handler: 回调函数。需要考虑多线程问题.
def handler(env, *args, **kwargs):pass
:param weeks:
:param days:
:param hours:
:param minutes:
:param seconds:
:param job_id: string, 自定义id
:param kwargs: handler的参数
:return: job id
'''
args = args or []
args.insert(0, handler)
job = self._sched.add_job(dispatch_expire_job, 'interval', weeks=weeks, days=days, hours=hours,
minutes=minutes, seconds=seconds, id=job_id, args=args, kwargs=kwargs,
replace_existing=replace_existing)
return job.id
def remove_job(self, job_id):
try:
self._sched.remove_job(job_id)
except JobLookupError:
pass
def pause_job(self, job_id):
self._sched.pause_job(job_id)
def resume_job(self, job_id):
self._sched.resume_job(job_id)
class Scheduler:
def __init__(self):
self.__running_tasks = []
self.aps_scheduler = BackgroundScheduler()
# task listener
def task_listener_add(event):
if event.job_id not in self.__running_tasks:
self.__running_tasks.append(event.job_id)
event_stream(type='task')
def task_listener_remove(event):
if event.job_id in self.__running_tasks:
self.__running_tasks.remove(event.job_id)
event_stream(type='task')
self.aps_scheduler.add_listener(task_listener_add, EVENT_JOB_SUBMITTED)
self.aps_scheduler.add_listener(task_listener_remove, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
# configure all tasks
self.__cache_cleanup_task()
self.__check_health_task()
self.update_configurable_tasks()
self.aps_scheduler.start()
def update_configurable_tasks(self):
self.__sonarr_update_task()
self.__radarr_update_task()
self.__sonarr_full_update_task()
self.__radarr_full_update_task()
self.__update_bazarr_task()
self.__search_wanted_subtitles_task()
self.__upgrade_subtitles_task()
self.__randomize_interval_task()
self.__automatic_backup()
if args.no_tasks:
self.__no_task()
def add_job(self, job, name=None, max_instances=1, coalesce=True, args=None, kwargs=None):
self.aps_scheduler.add_job(
job, DateTrigger(run_date=datetime.now()), name=name, id=name, max_instances=max_instances,
coalesce=coalesce, args=args, kwargs=kwargs)
def execute_job_now(self, taskid):
self.aps_scheduler.modify_job(taskid, next_run_time=datetime.now())
def get_running_tasks(self):
return self.__running_tasks
def get_task_list(self):
def get_time_from_interval(td_object):
seconds = int(td_object.total_seconds())
periods = [
('year', 60 * 60 * 24 * 365),
('month', 60 * 60 * 24 * 30),
('day', 60 * 60 * 24),
('hour', 60 * 60),
('minute', 60),
('second', 1)
]
strings = []
for period_name, period_seconds in periods:
if seconds > period_seconds:
period_value, seconds = divmod(seconds, period_seconds)
has_s = 's' if period_value > 1 else ''
strings.append("%s %s%s" % (period_value, period_name, has_s))
return ", ".join(strings)
def get_time_from_cron(cron):
year = str(cron[0])
if year == "2100":
return "Never"
day = str(cron[4])
hour = str(cron[5])
if day == "*":
text = "everyday"
else:
text = "every " + day_name[int(day)]
if hour != "*":
text += " at " + hour + ":00"
return text
task_list = []
for job in self.aps_scheduler.get_jobs():
next_run = 'Never'
if job.next_run_time:
next_run = pretty.date(job.next_run_time.replace(tzinfo=None))
if isinstance(job.trigger, CronTrigger):
if job.next_run_time and str(job.trigger.__getstate__()['fields'][0]) != "2100":
next_run = pretty.date(job.next_run_time.replace(tzinfo=None))
if job.id in self.__running_tasks:
running = True
else:
running = False
if isinstance(job.trigger, IntervalTrigger):
interval = "every " + get_time_from_interval(job.trigger.__getstate__()['interval'])
task_list.append({'name': job.name, 'interval': interval, 'next_run_in': next_run,
'next_run_time': next_run, 'job_id': job.id, 'job_running': running})
elif isinstance(job.trigger, CronTrigger):
task_list.append({'name': job.name, 'interval': get_time_from_cron(job.trigger.fields),
'next_run_in': next_run, 'next_run_time': next_run, 'job_id': job.id,
'job_running': running})
return task_list
def __sonarr_update_task(self):
if settings.general.getboolean('use_sonarr'):
self.aps_scheduler.add_job(
update_series, IntervalTrigger(minutes=int(settings.sonarr.series_sync)), max_instances=1,
coalesce=True, misfire_grace_time=15, id='update_series', name='Update Series list from Sonarr',
replace_existing=True)
self.aps_scheduler.add_job(
sync_episodes, IntervalTrigger(minutes=int(settings.sonarr.episodes_sync)), max_instances=1,
coalesce=True, misfire_grace_time=15, id='sync_episodes', name='Sync episodes with Sonarr',
replace_existing=True)
def __radarr_update_task(self):
if settings.general.getboolean('use_radarr'):
self.aps_scheduler.add_job(
update_movies, IntervalTrigger(minutes=int(settings.radarr.movies_sync)), max_instances=1,
coalesce=True, misfire_grace_time=15, id='update_movies', name='Update Movie list from Radarr',
replace_existing=True)
def __cache_cleanup_task(self):
self.aps_scheduler.add_job(cache_maintenance, IntervalTrigger(hours=24), max_instances=1, coalesce=True,
misfire_grace_time=15, id='cache_cleanup', name='Cache maintenance')
def __check_health_task(self):
self.aps_scheduler.add_job(check_health, IntervalTrigger(hours=6), max_instances=1, coalesce=True,
misfire_grace_time=15, id='check_health', name='Check health')
def __automatic_backup(self):
backup = settings.backup.frequency
if backup == "Daily":
self.aps_scheduler.add_job(
backup_to_zip, CronTrigger(hour=settings.backup.hour), max_instances=1, coalesce=True,
misfire_grace_time=15, id='backup', name='Backup database and configuration file',
replace_existing=True)
elif backup == "Weekly":
self.aps_scheduler.add_job(
backup_to_zip, CronTrigger(day_of_week=settings.backup.day, hour=settings.backup.hour),
max_instances=1, coalesce=True, misfire_grace_time=15, id='backup',
name='Backup database and configuration file', replace_existing=True)
elif backup == "Manually":
try:
self.aps_scheduler.remove_job(job_id='backup')
except JobLookupError:
pass
def __sonarr_full_update_task(self):
if settings.general.getboolean('use_sonarr'):
full_update = settings.sonarr.full_update
if full_update == "Daily":
self.aps_scheduler.add_job(
update_all_episodes, CronTrigger(hour=settings.sonarr.full_update_hour), max_instances=1,
coalesce=True, misfire_grace_time=15, id='update_all_episodes',
name='Update all Episode Subtitles from disk', replace_existing=True)
elif full_update == "Weekly":
self.aps_scheduler.add_job(
update_all_episodes,
CronTrigger(day_of_week=settings.sonarr.full_update_day, hour=settings.sonarr.full_update_hour),
max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_episodes',
name='Update all Episode Subtitles from disk', replace_existing=True)
elif full_update == "Manually":
self.aps_scheduler.add_job(
update_all_episodes, CronTrigger(year='2100'), max_instances=1, coalesce=True,
misfire_grace_time=15, id='update_all_episodes',
name='Update all Episode Subtitles from disk', replace_existing=True)
def __radarr_full_update_task(self):
if settings.general.getboolean('use_radarr'):
full_update = settings.radarr.full_update
if full_update == "Daily":
self.aps_scheduler.add_job(
update_all_movies, CronTrigger(hour=settings.radarr.full_update_hour), max_instances=1,
coalesce=True, misfire_grace_time=15,
id='update_all_movies', name='Update all Movie Subtitles from disk', replace_existing=True)
elif full_update == "Weekly":
self.aps_scheduler.add_job(
update_all_movies,
CronTrigger(day_of_week=settings.radarr.full_update_day, hour=settings.radarr.full_update_hour),
max_instances=1, coalesce=True, misfire_grace_time=15, id='update_all_movies',
name='Update all Movie Subtitles from disk', replace_existing=True)
elif full_update == "Manually":
self.aps_scheduler.add_job(
update_all_movies, CronTrigger(year='2100'), max_instances=1, coalesce=True, misfire_grace_time=15,
id='update_all_movies', name='Update all Movie Subtitles from disk', replace_existing=True)
def __update_bazarr_task(self):
if not args.no_update and os.environ["BAZARR_VERSION"] != '':
task_name = 'Update Bazarr'
if settings.general.getboolean('auto_update'):
self.aps_scheduler.add_job(
check_if_new_update, IntervalTrigger(hours=6), max_instances=1, coalesce=True,
misfire_grace_time=15, id='update_bazarr', name=task_name, replace_existing=True)
else:
self.aps_scheduler.add_job(
check_if_new_update, CronTrigger(year='2100'), hour=4, id='update_bazarr', name=task_name,
replace_existing=True)
self.aps_scheduler.add_job(
check_releases, IntervalTrigger(hours=3), max_instances=1, coalesce=True, misfire_grace_time=15,
id='update_release', name='Update Release Info', replace_existing=True)
else:
self.aps_scheduler.add_job(
check_releases, IntervalTrigger(hours=3), max_instances=1, coalesce=True, misfire_grace_time=15,
id='update_release', name='Update Release Info', replace_existing=True)
def __search_wanted_subtitles_task(self):
if settings.general.getboolean('use_sonarr'):
self.aps_scheduler.add_job(
wanted_search_missing_subtitles_series, IntervalTrigger(hours=int(settings.general.wanted_search_frequency)),
max_instances=1, coalesce=True, misfire_grace_time=15, id='wanted_search_missing_subtitles_series',
name='Search for wanted Series Subtitles', replace_existing=True)
if settings.general.getboolean('use_radarr'):
self.aps_scheduler.add_job(
wanted_search_missing_subtitles_movies, IntervalTrigger(hours=int(settings.general.wanted_search_frequency_movie)),
max_instances=1, coalesce=True, misfire_grace_time=15, id='wanted_search_missing_subtitles_movies',
name='Search for wanted Movies Subtitles', replace_existing=True)
def __upgrade_subtitles_task(self):
if settings.general.getboolean('upgrade_subs') and \
(settings.general.getboolean('use_sonarr') or settings.general.getboolean('use_radarr')):
self.aps_scheduler.add_job(
upgrade_subtitles, IntervalTrigger(hours=int(settings.general.upgrade_frequency)), max_instances=1,
coalesce=True, misfire_grace_time=15, id='upgrade_subtitles',
name='Upgrade previously downloaded Subtitles', replace_existing=True)
def __randomize_interval_task(self):
for job in self.aps_scheduler.get_jobs():
if isinstance(job.trigger, IntervalTrigger):
self.aps_scheduler.modify_job(job.id, next_run_time=datetime.now() + timedelta(seconds=randrange(job.trigger.interval.total_seconds()*0.75, job.trigger.interval.total_seconds())))
def __no_task(self):
for job in self.aps_scheduler.get_jobs():
self.aps_scheduler.modify_job(job.id, next_run_time=None)
class Scheduler(object):
def __init__(self):
self._scheduler = BackgroundScheduler()
self._scheduler.start()
def runCron(self, function, minute=None, hour=None, day=None, month=None, day_week=None, year=None, job_id=None, args=[], kwargs={}):
if job_id is None:
job_id=str(function)
logging.info('adding cron job: {}'.format(str(job_id)))
self._scheduler.add_job(function, 'cron', args=args, kwargs=kwargs, minute=minute, hour=hour, day=None, month=month, day_of_week=day_week, year=year, id=job_id, replace_existing=True)
#TODO: Build cron class
def runSimpleWeekCron(self, function, minute=None, hour=None, days_of_week=None, job_id=None, args=[], kwargs={}):
'''days_of_week takes a list of days'''
if job_id is None:
job_id=str(function)
if days_of_week is not None and days_of_week != '*':
run_days = []
for d in DAYS_OF_WEEK:
if d in days_of_week:
run_days.append(d)
days_of_week = ','.join(run_days)
logging.info('adding simple weekly cron job: {}'.format(str(job_id)))
self._scheduler.add_job(function, 'cron', args=args, kwargs=kwargs, minute=minute, hour=hour, day='*', month='*', day_of_week=days_of_week, year='*', id=job_id, replace_existing=True)
def runEveryS(self,delay, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runEveryS job: {}'.format(str(job_id)))
self._scheduler.add_job(function, 'interval', seconds=delay, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def runEveryM(self,delay, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runEveryM job: {}'.format(str(job_id)))
self._scheduler.add_job(function, 'interval', minutes=delay, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def runEveryH(self,delay, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runEveryH job: {}'.format(str(job_id)))
self._scheduler.add_job(function, 'interval', hours=delay, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def runInS(self,delay, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runInS job: {}'.format(str(job_id)))
run_time = datetime.now() + timedelta(seconds=delay)
self._scheduler.add_job(function, 'date', run_date=run_time, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def runInM(self,delay, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runInM job: {}'.format(str(job_id)))
run_time = datetime.now() + timedelta(minutes=delay)
self._scheduler.add_job(function, 'date', run_date=run_time, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def runInH(self,delay, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runInH job: {}'.format(str(job_id)))
run_time = datetime.now() + timedelta(hours=delay)
self._scheduler.add_job(function, 'date', run_date=run_time, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def runAt(self, date, function, args=[], kwargs={}, job_id=None, replace=True):
if job_id is None:
job_id=str(function)
logging.info('runAt job: {} date: {}'.format(str(job_id),str(date)))
self._scheduler.add_job(function, 'date', run_date=date, args=args, kwargs=kwargs, id=job_id, replace_existing=replace)
def cancel(self, job_id):
try:
logging.info('canceling job: {}'.format(str(job_id)))
self._scheduler.remove_job(job_id)
return True
except:
return False
def myfunc():
print("myfunc O(∩_∩)O")
import random
if random.random() > 0.6:
raise "hey raise"
def my_listener(event):
if event.exception:
log_job = {
"code": event.code,
"jobid": event.job_id,
"jobstore": event.jobstore
}
print(f'The job {event.job_id} crashed :( | {log_job}')
else:
print(f'The job {event.job_id} worked :)')
scheduler.add_listener(my_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
scheduler.start()
scheduler.add_job(myfunc, 'interval', seconds=1, id='test_job_id') # minutes=1
print(f"scheduler.get_jobs: {scheduler.get_jobs()}")
import time
time.sleep(6)
scheduler.remove_job('test_job_id')
scheduler.shutdown(wait=True)
class ScheduleCommand(Command):
schedule_sender = None
def __init__(self, logger, message_sender):
super().__init__(logger, message_sender)
# Message scheduler configuration
self.scheduler = BackgroundScheduler()
configuration = self.message_sender.get_configuration()
sdb_url = configuration.get('Message Scheduler', 'db_path')
self.scheduler.add_jobstore('sqlalchemy', url=sdb_url)
self.scheduler.start()
ScheduleCommand.schedule_sender = message_sender
def process(self, chat_id, user_id, username, arguments):
if len(arguments) < 1:
return self.help()
operation = arguments[0]
command_arguments = arguments[1:]
if operation == "add":
return self.add_message(username, chat_id, command_arguments)
elif operation == "time":
return self.get_local_time()
elif operation == "remove":
return self.remove_message(username, command_arguments)
else:
return self.help()
def get_local_time(self):
date_str = str(datetime.today().strftime('%Y-%m-%d %H:%M:%S'))
return "My time is {}".format(date_str)
def add_message(self, username, chat_id, arguments):
if len(arguments) <2:
return self.help()
date = None
message = None
if arguments[0] == 'relative':
if len(arguments)<3:
return self.help()
date = self.on_delta_parse(arguments[1])
message = arguments[2]
else:
date = self.on_date_parse(arguments[0])
message = arguments[1]
if message.startswith('"') and message.endswith('"'):
message = message[1:-1]
reference = self.user_reference(username)
if date is None:
return "{}Date format not recognized".format(reference)
current_date = datetime.today()
if date < current_date:
current_str = self.get_human_string_date(current_date)
return "{}Sorry, I can't travel to the past, my current date is: {}".format(reference, current_str)
message_id = str(random.randrange(0, 999999999))
self.scheduler.add_job(ScheduleCommand.send_programmed_message, \
'date', run_date=date, args=[username, \
chat_id, message], id = message_id)
date_str = self.get_human_string_date(date)
return "{}The message [{}] has been successfully scheduled on {}"\
.format(reference, message_id, date_str)
def send_programmed_message(username, chat_id, message):
reference = ScheduleCommand.user_reference(username)
text = "{}{}".format(reference, message)
ScheduleCommand.schedule_sender.send_message(chat_id, text)
send_programmed_message = staticmethod(send_programmed_message)
def remove_message(self, username, arguments):
if len(arguments) < 1:
return self.help()
founds = 0
ids = self.get_comma_arguments(arguments[0])
for message_id in ids:
job = self.scheduler.get_job(message_id)
if job == None:
continue
if job.args[0] != username:
continue
self.scheduler.remove_job(message_id)
founds += 1
reference = self.user_reference(username)
if founds == len(ids):
if len(ids) ==1:
return "{}The scheduled message was canceled.".format(reference)
else:
return "{}The scheduled messages were canceled.".format(reference)
elif founds > 0:
return "{}Some scheduled messages were canceled, but some others were not found or you aren't the owner.".format(reference)
else:
if len(ids) == 1:
return "{}The scheduled message was not found or you aren't the owner.".format(reference)
else:
return "{}The scheduled message were not found or you aren't the owner.".format(reference)
def get_human_string_date(self,datetime):
return str(datetime.strftime('%Y-%m-%d %H:%M:%S'))
def on_delta_parse(self, text_date):
regex = re.compile(r'^((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?$')
parts = regex.match(text_date)
if not parts:
return None
parts = parts.groupdict()
time_params = {}
valid_params = 0
for (name, param) in parts.items():
if param:
time_params[name] = int(param)
valid_params = valid_params + 1
if valid_params == 0:
return None
return datetime.now() + timedelta(**time_params)
def on_date_parse(self, text_date):
try:
return parser.parse(text_date, dayfirst = True, yearfirst = True)
except ValueError:
return None
def user_reference(username):
if username is None:
return ""
else:
return "@{}: ".format(username)
user_reference = staticmethod(user_reference)
def help(self):
self.logger.info("Printing help for schedule message command.")
return self.get_file_help(__file__, "schedule_message.man")
def name(self):
return "schedule_message"
def description(self):
return "Schedule a message to be sent in a specified date."
#
@app.before_request
def before_request():
db.connect()
logging.info('connection opened')
@app.after_request
def after_request(response):
db.close()
logging.info('connection closed')
return response
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s - %(levelname)s- %(message)s',
filename='error.log',
level=logging.INFO)
app.secret_key = 'super_secret_key'
app.debug = True
scheduler = BackgroundScheduler()
scheduler.add_job(Control().update, 'interval', hours=24, id='updates')
try:
scheduler.start()
app.run(host='0.0.0.0', port=8080)
except (KeyboardInterrupt, SystemExit):
scheduler.remove_job('updates')
scheduler.shutdown()
class SchedulerUtil(object):
# 类属性：记录实例是否存在
instance = None
# 初始化执行标志
init_flag = False
# 创建对象,首先会为对象分配空间
def __new__(cls, *args, **kwargs):
# 判断类属性是否为空对象
if cls.instance is None:
# 调用父类方法,为对象分配空间
logger.info("SchedulerUtil-->分配空间")
cls.instance = super().__new__(cls)
# 返回对象的引用
return cls.instance
# 分配空间后,对象属性初始化
def __init__(self):
# 判断是否执行过初始化动作
if SchedulerUtil.init_flag:
return
logger.info("SchedulerUtil-->对象属性初始化")
self.scheduler = BackgroundScheduler()
SchedulerUtil.init_flag = True
def __del__(self):
logger.info("del done!")
@classmethod
def flush(cls):
"""强制回收对象"""
cls.instance = None
SchedulerUtil.init_flag = False
def add_job(self, jobid, **kwargs):
logger.info("添加job----> %s" % jobid)
self.scheduler.add_job(eval("self.%s" % jobid),
'cron',
**kwargs,
id=jobid)
def remove_job(self, jobid):
logger.info("删除job----> %s" % jobid)
self.scheduler.remove_job(jobid)
def start_scheduler(self):
"""启动调度器
:return:
"""
logger.info("启动调度器")
self.scheduler.start()
logger.info("调度器状态：%s" % self.scheduler.state)
def shutdown_scheduler(self):
"""停止调度器
:return:
"""
logger.info("停止调度器")
self.scheduler.shutdown(wait=False)
def get_scheduler_status(self):
"""获取调度器状态
:return:
"""
logger.info("获取调度器状态：%s" % self.scheduler.state)
return self.scheduler.state
def get_scheduler_job(self, jobid):
"""获取调度中的任务
:return:
"""
return self.scheduler.get_job(jobid)
def get_scheduler_jobs(self):
"""获取调度中的任务列表
:return:
"""
return self.scheduler.get_jobs()
def exam_job(self):
"""考试信息任务
:return:
"""
logger.info("exam_job正在执行中...")
exam_server.ExamServer().run()
def employ_job(self):
"""招聘信息任务
:return:
"""
logger.info("employ_job正在执行中...")
zhaopin_server.ZhaopinServer().run()
def anjuke_job(self):
"""安居客数据任务
:return:
"""
logger.info("anjuke_job正在执行中...")
anjuke_server.AnJuKeServer().run()
def mafengwo_job(self):
"""马蜂窝数据任务（测试任务）
:return:
"""
logger.info("mafengwo_job正在执行中...")
mafengwo_server.MaFengWoServer("西安").run()
class BileanScheduler(object):
"""Billing scheduler based on apscheduler"""
job_types = (
NOTIFY, DAILY, FREEZE,
) = (
'notify', 'daily', 'freeze',
)
trigger_types = (DATE, CRON) = ('date', 'cron')
def __init__(self, **kwargs):
super(BileanScheduler, self).__init__()
self._scheduler = BackgroundScheduler()
self.notifier = notifier.Notifier()
self.engine_id = kwargs.get('engine_id', None)
self.context = kwargs.get('context', None)
if not self.context:
self.context = bilean_context.get_admin_context()
if cfg.CONF.bilean_task.store_ap_job:
self._scheduler.add_jobstore(cfg.CONF.bilean_task.backend,
url=cfg.CONF.bilean_task.connection)
def init_scheduler(self):
"""Init all jobs related to the engine from db."""
jobs = db_api.job_get_all(self.context, engine_id=self.engine_id)
if not jobs:
LOG.info(_LI("No job found from db"))
return True
for job in jobs:
if self.bilean_scheduler.is_exist(job.id):
continue
task_name = "_%s_task" % (job.job_type)
task = getattr(self, task_name)
self.bilean_task.add_job(task, job.id,
job_type=job.job_type,
params=job.parameters)
def add_job(self, task, job_id, trigger_type='date', **kwargs):
"""Add a job to scheduler by given data.
:param str|unicode user_id: used as job_id
:param datetime alarm_time: when to first run the job
"""
mg_time = cfg.CONF.bilean_task.misfire_grace_time
job_time_zone = cfg.CONF.bilean_task.time_zone
user_id = job_id.split('-')[1]
if trigger_type == 'date':
run_date = kwargs.get('run_date')
if run_date is None:
msg = "Param run_date cannot be None for trigger type 'date'."
raise exception.InvalidInput(reason=msg)
self._scheduler.add_job(task, 'date',
timezone=job_time_zone,
run_date=run_date,
args=[user_id],
id=job_id,
misfire_grace_time=mg_time)
return True
# Add a cron type job
hour = kwargs.get('hour', None)
minute = kwargs.get('minute', None)
if not hour or not minute:
hour, minute = self._generate_timer()
self._scheduler.add_job(task, 'cron',
timezone=job_time_zone,
hour=hour,
minute=minute,
args=[user_id],
id=job_id,
misfire_grace_time=mg_time)
return True
def modify_job(self, job_id, **changes):
"""Modifies the properties of a single job.
Modifications are passed to this method as extra keyword arguments.
:param str|unicode job_id: the identifier of the job
"""
self._scheduler.modify_job(job_id, **changes)
def remove_job(self, job_id):
"""Removes a job, preventing it from being run any more.
:param str|unicode job_id: the identifier of the job
"""
self._scheduler.remove_job(job_id)
def start(self):
LOG.info(_('Starting Billing scheduler'))
self._scheduler.start()
def stop(self):
LOG.info(_('Stopping Billing scheduler'))
self._scheduler.shutdown()
def is_exist(self, job_id):
"""Returns if the Job exists that matches the given ``job_id``.
:param str|unicode job_id: the identifier of the job
:return: True|False
"""
job = self._scheduler.get_job(job_id)
return job is not None
def _notify_task(self, user_id):
user = user_mod.User.load(self.context, user_id=user_id)
msg = {'user': user.id, 'notification': 'The balance is almost use up'}
self.notifier.info('billing.notify', msg)
if user.status != user.FREEZE and user.rate > 0:
user.do_bill(self.context)
try:
db_api.job_delete(
self.context, self._generate_job_id(user.id, 'notify'))
except exception.NotFound as e:
LOG.warn(_("Failed in deleting job: %s") % six.text_type(e))
self._add_freeze_job(user)
def _daily_task(self, user_id):
user = user_mod.User.load(self.context, user_id=user_id)
if user.status != user.FREEZE and user.rate > 0:
user.do_bill(self.context)
try:
db_api.job_delete(
self.context, self._generate_job_id(user.id, 'daily'))
except exception.NotFound as e:
LOG.warn(_("Failed in deleting job: %s") % six.text_type(e))
def _freeze_task(self, user_id):
user = user_mod.User.load(self.context, user_id=user_id)
if user.status != user.FREEZE and user.rate > 0:
user.do_bill(self.context)
try:
db_api.job_delete(
self.context, self._generate_job_id(user.id, 'freeze'))
except exception.NotFound as e:
LOG.warn(_("Failed in deleting job: %s") % six.text_type(e))
def _add_notify_job(self, user):
if not user.rate:
return False
total_seconds = user['balance'] / user['rate']
prior_notify_time = cfg.CONF.bilean_task.prior_notify_time * 3600
notify_seconds = total_seconds - prior_notify_time
notify_seconds = notify_seconds if notify_seconds > 0 else 0
run_date = timeutils.utcnow() + timedelta(seconds=notify_seconds)
job_params = {'run_date': run_date}
job_id = self._generate_job_id(user['id'], self.NOTIFY)
self.add_job(self._notify_task, job_id, params=job_params)
# Save job to database
job = {'id': job_id,
'job_type': self.NOTIFY,
'engine_id': self.engine_id,
'parameters': {'run_date': run_date}}
db_api.job_create(self.context, job)
def _add_freeze_job(self, user):
if not user.rate:
return False
total_seconds = user.balance / user.rate
run_date = timeutils.utcnow() + timedelta(seconds=total_seconds)
job_params = {'run_date': run_date}
job_id = self._generate_job_id(user.id, self.FREEZE)
self.add_job(self._freeze_task, job_id, params=job_params)
# Save job to database
job = {'id': job_id,
'job_type': self.FREEZE,
'engine_id': self.engine_id,
'parameters': {'run_date': run_date}}
db_api.job_create(self.context, job)
return True
def _add_daily_job(self, user):
job_id = self._generate_job_id(user.id, self.DAILY)
params = {'hour': random.randint(0, 23),
'minute': random.randint(0, 59)}
self.add_job(self._daily_task, job_id, trigger_type='cron',
params=params)
# Save job to database
job = {'id': job_id,
'job_type': self.DAILY,
'engine_id': self.engine_id,
'parameters': params}
db_api.job_create(self.context, job)
return True
def _delete_all_job(self, user):
for job_type in self.job_types:
job_id = self._generate_job_id(user.id, job_type)
if self.is_exist(job_id):
self.remove_job(job_id)
try:
db_api.job_delete(self.context, job_id)
except exception.NotFound as e:
LOG.warn(_("Failed in deleting job: %s") % six.text_type(e))
def update_user_job(self, user):
"""Update user's billing job"""
if user.status not in [user.ACTIVE, user.WARNING]:
self._delete_all_job(user.id)
return
for job_type in self.NOTIFY, self.FREEZE:
job_id = self._generate_job_id(user.id, job_type)
if self.is_exist(job_id):
self.remove_job(job_id)
try:
db_api.job_delete(self.context, job_id)
except exception.NotFound as e:
LOG.warn(_("Failed in deleting job: %s") % six.text_type(e))
daily_job_id = self._generate_job_id(user.id, self.DAILY)
if not self.is_exist(daily_job_id):
self._add_daily_job(user)
if user.status == user.ACTIVE:
self._add_notify_job(user)
else:
self._add_freeze_job(user)
def _generate_timer(self):
"""Generate a random timer include hour and minute."""
hour = random.randint(0, 23)
minute = random.randint(0, 59)
return hour, minute
def _generate_job_id(self, user_id, job_type):
"""Generate job id by given user_id and job type"""
return "%s-%s" % (job_type, user_id)
class ScheduleManager(IScheduleManager):
def __init__(self, config=None, event_listener=None):
self.__config = config
self.__event_listener = event_listener
jobstores, executors, job_defaults, timezone = self.__get_apscheduler_settings(
)
# initial apscheduler
self.__scheduler = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=timezone)
if self.__event_listener:
self.__scheduler.add_listener(self.__event_listener.event_start,
EVENT_SCHEDULER_STARTED)
self.__scheduler.add_listener(self.__event_listener.event_shutdown,
EVENT_SCHEDULER_SHUTDOWN)
self.__scheduler.add_listener(self.__event_listener.event_pause,
EVENT_SCHEDULER_PAUSED)
self.__scheduler.add_listener(self.__event_listener.event_resume,
EVENT_SCHEDULER_RESUMED)
self.__scheduler.add_listener(
self.__event_listener.event_job_submit, EVENT_JOB_SUBMITTED)
self.__scheduler.add_listener(
self.__event_listener.event_job_max_instances,
EVENT_JOB_MAX_INSTANCES)
self.__scheduler.add_listener(
self.__event_listener.event_job_execute, EVENT_JOB_EXECUTED)
self.__scheduler.add_listener(
self.__event_listener.event_job_error, EVENT_JOB_ERROR)
self.__scheduler.add_listener(self.__event_listener.event_job_miss,
EVENT_JOB_MISSED)
def __get_apscheduler_settings(self):
try:
jobstore_url = "oracle+cx_oracle://{username}:{password}${host}:{port}/{dbname}".format(
username=self.__config.db_user,
password=self.__config.db_pwd,
host=self.__config.db_host,
port=self.__config.db_port,
dbname=self.__config.db_name,
)
jobstores = {
"default":
SQLAlchemyJobStore(url=jobstore_url,
tablename=self.__config.tablename)
}
executors = {
"default": ThreadPoolExecutor(self.__config.max_workers),
"processpool": ProcessPoolExecutor(2)
}
job_defaults = {
"coalesce": True,
"max_instances": 10,
"misfire_grace_time": 30
}
timezone = self.__config.timezone
return jobstores, executors, job_defaults, timezone
except Exception as e:
raise e
def start(self, paused=False) -> bool:
try:
if self.__scheduler.state == STATE_RUNNING:
return True
self.__scheduler.start(paused=paused)
except Exception as e:
logging.error(f"scheduler start error...... {str(e)}")
raise e
def shutdown(self, wait=False) -> bool:
try:
self.__scheduler.shutdown(wait=wait)
if self.__scheduler.state == STATE_STOPPED:
return True
return False
except Exception as e:
logging.error(f"scheduler shutdown error...... {str(e)}")
raise e
def pause(self) -> bool:
try:
self.__scheduler.pause()
if self.__scheduler.state == STATE_PAUSED:
return True
return False
except Exception as e:
logging.error(f"scheduler pause error...... {str(e)}")
raise e
def resume(self) -> bool:
try:
self.__scheduler.resume()
if self.__scheduler.state == STATE_RUNNING:
return True
return False
except Exception as e:
logging.error(f"scheduler resume error...... {str(e)}")
raise e
def get_state(self) -> str:
try:
if self.__scheduler.state == STATE_RUNNING:
return "RUNNING"
if self.__scheduler.state == STATE_PAUSED:
return "PAUSED"
return "STOPPED"
except Exception as e:
logging.error(f"scheduler get_state error...... {str(e)}")
raise e
def get_job(self, id="") -> object:
try:
job = self.__scheduler.get_job(id)
return job
except Exception as e:
logging.error(f"scheduler get_job error...... {str(e)}")
raise e
def add_job(self, task_id="", trigger=None, task=None) -> bool:
try:
settings = {}
settings["id"] = task_id if task_id else str(uuid4())
settings["func"] = task.run
settings["coalesce"] = True
settings["replace_existing"] = True
settings = {**settings, **(trigger.get_settings())}
job = self.__scheduler.add_job(**settings)
return True if self.get_job(job.id) else False
except Exception as e:
logging.error(f"scheduler add_job error...... {str(e)}")
raise e
def remove_job(self, id: str) -> bool:
try:
self.__scheduler.remove_job(id)
return True if not self.get_job(id) else False
except JobLookupError as e:
return False
except Exception as e:
logging.error(f"scheduler remove_job error...... {str(e)}")
raise e
def pause_job(self, id: str) -> bool:
try:
job = self.__scheduler.pause_job(id)
return True if job else False
except Exception as e:
logging.error(f"scheduler pause_job error...... {str(e)}")
raise e
def resume_job(self, id: str) -> bool:
try:
job = self.__scheduler.resume_job(id)
return True if job else False
except Exception as e:
logging.error(f"scheduler resume_job error...... {str(e)}")
raise e
class Scheduler(object):
def __init__(self, tasks, settings):
self.settings = settings
self.logger = get_logger('scheduler', self.settings)
self.intervals = self.settings['scheduler']['intervals']
self.apscheduler = { 'apscheduler': self.settings['scheduler']['apscheduler'] }
if not isinstance(tasks, dict):
self.logger.error('tasks is not a dictionary')
return
if not isinstance(self.intervals, dict):
self.logger.error('intervals is not a dictionary')
return
self.tasks = self._flatten_dict(tasks, '')
self.logger.debug('Tasks found:')
self.logger.debug(LINE_SPLITTER)
for key in self.tasks:
self.logger.debug('%45s %30s' % (key, self.tasks[key].task_type))
self.logger.debug(LINE_SPLITTER)
#self.logger.debug('Checking tasks paths!')
# TODO: Check if paths are valid
def init(self):
""" Initializes the queue, and adds the tasks """
self.logger.info('Initilizing APScheduler...')
apsched_kwargs = self._flatten_dict(self.apscheduler, '')
apsched_kwargs['apscheduler.logger'] = get_logger('apscheduler', self.settings)
self.sched = BackgroundScheduler(apsched_kwargs)
for (id, task) in self.tasks.items():
task_type = task.task_type
self.logger.debug('Adding task "%s" [%s]' % (id, task_type))
if not task_type in self.intervals:
self.logger.info('Interval not defined for "%s" class. Assuming it is an once-time task' % task_type)
self.add_task(id, task)
continue
self.add_task(id, task, self.intervals[task_type])
self.logger.info('APScheduler initialized!')
def clear(self):
""" Removes all jobs from scheduler """
if not isinstance(self.sched, BaseScheduler):
self.logger.error('Scheduler is not initialized')
return
for job in self.sched.get_jobs():
job.remove()
def start(self):
""" Start the scheduler by starting the instance of APScheduler """
if not isinstance(self.sched, BaseScheduler):
self.logger.error('Scheduler is not initialized')
return
try:
self.sched.start()
except SchedulerAlreadyRunningError as e:
self.logger.warning(e)
def stop(self, wait=True):
""" Stop the scheduler. If wait=True, then it will be stopped after
all jobs that are currently executed will finish """
if not isinstance(self.sched, BaseScheduler):
self.logger.warning('Scheduler is not initialized')
return
try:
self.sched.shutdown(wait=wait)
except SchedulerNotRunningError as e:
self.logger.warning(e)
def add_task(self, id, func, interval=None):
""" Adds a new task into the queue. If interval is None then the task
will be executed once. """
if not isinstance(id, basestring):
self.logger.error('"id" argument is not an instance of basestring')
return
if not hasattr(func, '__call__'):
self.logger.error('"func" is not callable')
return
try:
if isinstance(interval, dict):
self.sched.add_job(func, trigger='interval', id=id, **interval)
elif interval is None: # Run once (ommit trigger)
self.sched.add_job(func, id=id)
else:
self.logger.error('"interval" is not an instance of [time|None]')
return
except ConflictingIdError as e:
self.logger.warning(e)
def remove_task(self, id):
""" Remove a job from the queue """
if not isinstance(id, basestring):
self.logger.error('"id" argument is not an instance of basestring')
return
try:
self.sched.remove_job(id)
except JobLookupError as e:
self.logger.warning(e)
def force_update(self, job_id=None):
""" Updates a job with id == job_id, or all jobs if no id is given """
if not isinstance(self.sched, BaseScheduler):
self.logger.warning('Scheduler is not initialized')
return
if not job_id:
self.logger.info("Forcing update of all jobs")
for job in self.sched.get_jobs():
self.__run_job(job)
else:
self.logger.info("Forcing update of job %s" % job_id)
job = self.sched.get_job(job_id)
if not job:
self.logger.warn("Job %s not found" % job_id)
else:
self.__run_job(job)
def __run_job(self, job):
if job.func:
# Add the job to the scheduler and run it just once
self.sched.add_job(job.func)
# If we explicity call job.func() then we block the thread and we get multiple
# missed executions from apscheduler
# job.func()
else:
self.logger.warn("Job %s has a None type callable func" % job.id)
def _flatten_dict(self, d, path):
new_dict = { }
for key in d:
if isinstance(d[key], dict):
new_path = '%s.%s' % (path, key) if path else key
x = self._flatten_dict(d[key],new_path).copy()
new_dict.update(x)
else:
new_key = '%s.%s' % (path, key)
new_dict[new_key] = d[key]
return new_dict
class ExScheduler(object):
def __init__(self):
self.__scheduler = BackgroundScheduler()
self.__jobstatuslist = {}
def start_scheduler(self, **options):
self.__scheduler.configure(**options)
self.__scheduler.start()
def shutdownscheduler(self):
self.__scheduler.shutdown(wait=False)
def getjob(self, jobid):
return self.__scheduler.get_job(jobid)
def scheduledjob(self, *args, **kw):
return self.__scheduler.scheduled_job(*args, **kw)
def addjob(self, *args, **kw):
job = self.__scheduler.add_job(*args, **kw)
self.__scheduler.wakeup()
return job
def removejob(self, job_id, jobstore=None):
self.__scheduler.remove_job(job_id=job_id, jobstore=jobstore)
def pausejob(self, job_id, jobstore=None):
self.__scheduler.pause_job(job_id=job_id, jobstore=jobstore)
self.setjobstatus(job_id, JobStatus.PAUSED)
def resumejob(self, job_id, jobstore=None):
self.__scheduler.resume_job(job_id=job_id, jobstore=jobstore)
self.setjobstatus(job_id, JobStatus.SCHEDULING)
def modifyjob(self, job_id, jobstore=None, **kw):
job = self.__scheduler.modify_job(job_id=job_id,
jobstore=jobstore,
**kw)
return job
def getjoblist(self):
joblist = self.__scheduler.get_jobs()
return map(self._getjobinfo, joblist)
def addlistener(self, callback, mask=EVENT_ALL):
self.__scheduler.add_listener(callback, mask)
def setjobstatus(self, jobid, jobstatus):
self.__jobstatuslist[jobid] = jobstatus
def jobstatusinitial(self, job_id, jobstore=None):
job = self.__scheduler.get_job(job_id=job_id, jobstore=jobstore)
if job_id not in self.__jobstatuslist:
status = (JobStatus.SCHEDULING
if job.next_run_time else JobStatus.PAUSED) if hasattr(
job, 'next_run_time') else JobStatus.PENDING
self.setjobstatus(job_id, status)
def _getjobinfo(self, job):
self.jobstatusinitial(job.id)
return {
"id":
str(job.id),
"name":
str(job.name),
"kwargs":
job.kwargs,
"trigger":
trigger_str_to_dict(str(job.trigger)),
"next_run_time":
datetime_repr(job.next_run_time)
if self.__jobstatuslist[str(job.id)] == JobStatus.SCHEDULING or
self.__jobstatuslist[str(job.id)] == JobStatus.RUNNING else "--",
"status":
self.__jobstatuslist[str(job.id)]
}
class Scheduler(threading.Thread):
def __init__(self, gc) -> None:
super().__init__()
self.name = "scheduler"
self.exit = threading.Event()
self.gc = gc
self._sched = BackgroundScheduler(daemon=True)
self._event_prefixes = tuple((f"{e.name.lower()}:" for e in EventType))
add_event_listener(EventType.SHUTDOWN, self.shutdown)
def run(self):
self._sched.start()
if ArgumentParser.args.scheduler_config:
self.read_config(ArgumentParser.args.scheduler_config)
self.exit.wait()
def shutdown(self, event: Event):
log.debug(f"Received request to shutdown scheduler {event.event_type}")
if self._sched.running:
self._sched.shutdown(wait=False)
self.exit.set()
def read_config(self, config_file: str) -> None:
log.debug(f"Reading scheduler configuration file {config_file}")
try:
with open(config_file, "r") as fp:
for line in fp:
line = line.strip()
if not line.startswith("#") and len(line) > 0:
self.add_job(line)
except Exception:
log.exception(
f"Failed to read scheduler configuration file {config_file}")
def scheduled_command(self, command):
log.debug(f"Running scheduled command {command}")
return cli_event_handler(command, graph=self.gc.graph)
def add_job(self, args: str) -> Job:
args = args.strip()
cron = re.split(r"\s+", args, 5)
if len(cron) != 6:
raise ValueError(f"Invalid job {args}")
minute, hour, day, month, day_of_week, command = cron
if str(command).startswith(self._event_prefixes):
event, cmd = command.split(":", 1)
log.debug(
(f"Scheduling to register command '{cmd}' for event {event} at"
f" minute={minute}, hour={hour}, day={day}, month={month},"
f" day_of_week={day_of_week}"))
job = self._sched.add_job(
register_cli_action,
"cron",
args=[command, True],
minute=minute,
hour=hour,
day=day,
month=month,
day_of_week=day_of_week,
)
else:
log.debug(
(f"Scheduling command '{command}' at"
f" minute={minute}, hour={hour}, day={day}, month={month},"
f" day_of_week={day_of_week}"))
job = self._sched.add_job(
self.scheduled_command,
"cron",
args=[command],
minute=minute,
hour=hour,
day=day,
month=month,
day_of_week=day_of_week,
)
return job
def remove_job(self, job_id: str) -> bool:
try:
self._sched.remove_job(job_id)
except JobLookupError:
log.error(f"Couldn't find job with id {job_id}")
else:
return True
return False
def get_jobs(self) -> Iterable:
for job in self._sched.get_jobs():
if isinstance(job.trigger, CronTrigger):
trigger_map = {}
for field in job.trigger.fields:
trigger_map[field.name] = str(field)
cron_line = (
f"{job.id}: {trigger_map.get('minute')} {trigger_map.get('hour')}"
f" {trigger_map.get('day')} {trigger_map.get('month')}"
f" {trigger_map.get('day_of_week')} {job.args[0]}")
yield cron_line
@staticmethod
def add_args(arg_parser: ArgumentParser) -> None:
arg_parser.add_argument(
"--scheduler-config",
help="Scheduler config in crontab format",
default=None,
dest="scheduler_config",
type=str,
)
class WorkersManager:
class Command:
def __init__(self, callback, timeout, args=(), options=dict()):
self._callback = callback
self._timeout = timeout
self._args = args
self._options = options
self._source = "{}.{}".format(
callback.__self__.__class__.__name__
if hasattr(callback, "__self__")
else callback.__module__,
callback.__name__,
)
def execute(self):
messages = []
try:
with timeout(
self._timeout,
exception=WorkerTimeoutError(
"Execution of command {} timed out after {} seconds".format(
self._source, self._timeout
)
),
):
if inspect.isgeneratorfunction(self._callback):
for message in self._callback(*self._args):
messages += message
else:
messages = self._callback(*self._args)
except WorkerTimeoutError as e:
if messages:
logger.log_exception(
_LOGGER, "%s, sending only partial update", e, suppress=True
)
else:
raise e
_LOGGER.debug("Execution result of command %s: %s", self._source, messages)
return messages
def __init__(self, config):
self._mqtt_callbacks = []
self._config_commands = []
self._update_commands = []
self._scheduler = BackgroundScheduler(timezone=utc)
self._daemons = []
self._config = config
self._command_timeout = config.get("command_timeout", DEFAULT_COMMAND_TIMEOUT)
def register_workers(self, global_topic_prefix):
for (worker_name, worker_config) in self._config["workers"].items():
module_obj = importlib.import_module("workers.%s" % worker_name)
klass = getattr(module_obj, "%sWorker" % worker_name.title())
if module_obj.REQUIREMENTS is not None:
self._pip_install_helper(module_obj.REQUIREMENTS)
command_timeout = worker_config.get(
"command_timeout", self._command_timeout
)
worker_obj = klass(
command_timeout, global_topic_prefix, **worker_config["args"]
)
if "sensor_config" in self._config and hasattr(worker_obj, "config"):
_LOGGER.debug(
"Added %s config with a %d seconds timeout", repr(worker_obj), 2
)
command = self.Command(worker_obj.config, 2, [])
self._config_commands.append(command)
if hasattr(worker_obj, "status_update"):
_LOGGER.debug(
"Added %s worker with %d seconds interval and a %d seconds timeout",
repr(worker_obj),
worker_config["update_interval"],
worker_obj.command_timeout,
)
command = self.Command(
worker_obj.status_update, worker_obj.command_timeout, []
)
self._update_commands.append(command)
if "update_interval" in worker_config:
job_id = "{}_interval_job".format(worker_name)
interval_job = self._scheduler.add_job(
partial(self._queue_command, command),
"interval",
seconds=worker_config["update_interval"],
id=job_id,
)
self._mqtt_callbacks.append(
(
worker_obj.format_topic("update_interval"),
partial(self._update_interval_wrapper, command, job_id),
)
)
elif hasattr(worker_obj, "run"):
_LOGGER.debug("Registered %s as daemon", repr(worker_obj))
self._daemons.append(worker_obj)
else:
raise "%s cannot be initialized, it has to define run or status_update method" % worker_name
if "topic_subscription" in worker_config:
self._mqtt_callbacks.append(
(
worker_config["topic_subscription"],
partial(self._on_command_wrapper, worker_obj),
)
)
if "topic_subscription" in self._config:
for (callback_name, options) in self._config["topic_subscription"].items():
self._mqtt_callbacks.append(
(
options["topic"],
lambda client, _, c: self._queue_if_matching_payload(
self.Command(
getattr(self, callback_name), self._command_timeout
),
c.payload,
options["payload"],
),
)
)
return self
def start(self, mqtt):
mqtt.callbacks_subscription(self._mqtt_callbacks)
if "sensor_config" in self._config:
self._publish_config(mqtt)
self._scheduler.start()
self.update_all()
for daemon in self._daemons:
threading.Thread(target=daemon.run, args=[mqtt], daemon=True).start()
def _queue_if_matching_payload(self, command, payload, expected_payload):
if payload.decode("utf-8") == expected_payload:
self._queue_command(command)
def update_all(self):
_LOGGER.debug("Updating all workers")
for command in self._update_commands:
self._queue_command(command)
@staticmethod
def _queue_command(command):
_WORKERS_QUEUE.put(command)
@staticmethod
def _pip_install_helper(package_names):
for package in package_names:
pip_main(["install", "-q", package])
logger.reset()
def _update_interval_wrapper(self, command, job_id, client, userdata, c):
_LOGGER.info("Recieved updated interval for %s with: %s", c.topic, c.payload)
try:
new_interval = int(c.payload)
self._scheduler.remove_job(job_id)
self._scheduler.add_job(
partial(self._queue_command, command),
"interval",
seconds=new_interval,
id=job_id,
)
except ValueError:
logger.log_exception(
_LOGGER, "Ignoring invalid new interval: %s", c.payload
)
def _on_command_wrapper(self, worker_obj, client, userdata, c):
_LOGGER.debug(
"Received command for %s on %s: %s", repr(worker_obj), c.topic, c.payload
)
global_topic_prefix = userdata["global_topic_prefix"]
topic = (
c.topic[len(global_topic_prefix + "/") :]
if global_topic_prefix is not None
else c.topic
)
self._queue_command(
self.Command(
worker_obj.on_command, worker_obj.command_timeout, [topic, c.payload]
)
)
def _publish_config(self, mqtt):
for command in self._config_commands:
messages = command.execute()
for msg in messages:
msg.topic = "{}/{}".format(
self._config["sensor_config"].get("topic", "homeassistant"),
msg.topic,
)
msg.retain = self._config["sensor_config"].get("retain", True)
mqtt.publish(messages)
class PeriodicManager(LoggingMixin):
def __init__(self, mailbox: Mailbox):
super().__init__()
self.mailbox = mailbox
self.sc = BackgroundScheduler()
def start(self):
self.sc.start()
def shutdown(self):
self.sc.shutdown()
def _generate_job_id(self, run_id, task_id):
return '{}:{}'.format(run_id, task_id)
def add_task(self, run_id, task_id, periodic_config):
if 'cron' in periodic_config:
self.sc.add_job(id=self._generate_job_id(run_id, task_id),
func=trigger_periodic_task, args=(self.mailbox, run_id, task_id),
trigger=CronTrigger.from_crontab(periodic_config['cron']))
elif 'interval' in periodic_config:
interval_config: dict = periodic_config['interval']
if 'seconds' in interval_config:
seconds = interval_config['seconds']
else:
seconds = 0
if 'minutes' in interval_config:
minutes = interval_config['minutes']
else:
minutes = 0
if 'hours' in interval_config:
hours = interval_config['hours']
else:
hours = 0
if 'days' in interval_config:
days = interval_config['days']
else:
days = 0
if 'weeks' in interval_config:
weeks = interval_config['weeks']
else:
weeks = 0
if seconds < 10 and 0 >= minutes and 0 >= hours and 0 >= days and 0 >= weeks:
self.log.error('Interval mast greater than 20 seconds')
return
self.sc.add_job(id=self._generate_job_id(run_id, task_id),
func=trigger_periodic_task, args=(self.mailbox, run_id, task_id),
trigger=IntervalTrigger(seconds=seconds,
minutes=minutes,
hours=hours,
days=days,
weeks=weeks))
else:
self.log.error('Periodic support type cron or interval. current periodic config {}'.format(periodic_config))
def remove_task(self, run_id, task_id):
self.sc.remove_job(job_id=self._generate_job_id(run_id, task_id))
class Scheduler:
TRIGGERS = {
"trig_5minutes": {
"id": "trig_5minutes",
"name": "Every five minutes",
"options": [],
"schema": {},
"trigger_args": lambda args: dict(minute="*/5"),
"from_trigger": lambda trig: []
},
"trig_hourly": {
"id": "trig_hourly",
"name": "Each hour",
"options": [],
"schema": {},
"trigger_args": lambda args: dict(hour="*"),
"from_trigger": lambda trig: []
},
"trig_daily": {
"id": "trig_daily",
"name": "Each day",
"options": [],
"schema": {},
"trigger_args": lambda args: dict(day="*"),
"from_trigger": lambda trig: []
},
"trig_weekday": {
"id":
"trig_weekday",
"name":
"Each weekday",
"options": [{
"id": i,
"name": el,
"active": True
} for i, el in enumerate("Mon Tue Wed Thu Fri Sat Sun".split())],
"schema": {
"id": {
"type": "integer",
"coerce": int,
"min": 0,
"max": 6
},
"name": {
"type": "string",
"required": False
},
"active": {
"type": "boolean",
"coerce": utility.coerce_bool,
"required": True
}
},
"trigger_args":
lambda args: dict(day_of_week=",".join(str(a) for a in args)),
"from_trigger":
lambda trig: [int(d) for d in str(trig.fields[4]).split(",")]
},
"trig_monthly": {
"id":
"trig_monthly",
"name":
"Each month",
"options": [{
"id": i + 1,
"name": el,
"active": True
} for i, el in enumerate(("Jan Feb Mar Apr May Jun "
"Jul Aug Sep Oct Nov Dec").split())],
"schema": {
"id": {
"type": "integer",
"coerce": int,
"min": 0,
"max": 12
},
"name": {
"type": "string",
"required": False
},
"active": {
"type": "boolean",
"coerce": utility.coerce_bool,
"required": True
}
},
"trigger_args":
lambda args: dict(month=",".join(str(a) for a in args)),
"from_trigger":
lambda trig: [int(d) for d in str(trig.fields[1]).split(",")]
},
}
"""Predefined triggers and their argument checks."""
def __init__(self,
elastic,
crawler_dir="crawlers",
crawler_args={},
**cron_defaults):
"""Initializes the scheduler by binding it to it's elasticsearch db.
Args:
elastic (elasticsearch.Elasticsearh): The es-client to save the
crawling jobs in.
crawler_dir (str): the directory, where the crawlers will be found.
Defaults to "crawlers".
job_defaults (dict): a dictionary of keyword arguments for
the schedulers job_defaults.
**cron_defaults (dict): a dictionary of keyword arguments for
the schedulers job_defaults.
Returns:
Scheduler: a fresh Scheduler instance.
"""
jobstores = {
"default": {
"type": "memory"
},
"elastic": InjectorJobStore(kwargs=crawler_args, client=elastic)
}
executors = {
"default": ThreadPoolExecutor(10),
"processpool": ProcessPoolExecutor(10)
}
job_defaults = {
"misfire_grace_time": 5 * 60, # 5min
"coalesce": True,
}
self.cron_defaults = utility.DefaultDict(
{
# standard is every day at 00:00:00
"hour": 0,
"minute": 0,
"second": 0
},
**cron_defaults)
self.scheduler = BackgroundScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
self.crawlers = _detect_crawlers()
# set up the validator schema.
self.job_validator = cerberus.Validator(SCHEMATA["job"]({
"trigger_ids":
list(self.TRIGGERS)
}),
allow_unknown=True)
self.scheduler.start()
def upsert_job(self, job_dict, **runtime_args):
"""Adds or updates a job using the provided user_input.
If an id field is present in the dict, the job is updated, otherwise
a new one is created.
Args:
job_dict (dict): user input for a job, as defined in `SCHEMATA`.
**runtime_args (dict): additional runtime arguments for the
crawler.
Returns:
apscheduler.job.Job: a new job Object.
"""
if not self.job_validator.validate(job_dict):
raise (AssertionError(str(self.job_validator.errors)))
doc = utility.SDA(job_dict)
job = self.crawlers.get(doc["crawler.id"], None)
# default to the SearchPlugin, and give the search name as argument.
if job is None:
inst = {
"args": ("SearchPlugin", runtime_args),
"kwargs": dict(search_id=doc["crawler.id"])
}
else:
inst = {"args": (doc["crawler.id"], runtime_args), "kwargs": {}}
trigger = self._make_trigger(doc["schedule"])
if doc["id"]:
self.scheduler.modify_job(doc["id"],
jobstore="elastic",
func=_run_plugin,
name=doc["name.name"],
**inst)
new_job = self.scheduler.reschedule_job(doc["id"],
jobstore="elastic",
trigger=trigger)
else:
# use the crawler id as name, when the job is created.
new_job = self.scheduler.add_job(_run_plugin,
jobstore="elastic",
trigger=trigger,
name=doc["crawler.id"],
**inst)
return new_job
def get_triggers(self):
"""Returns a list of triggers, that are predefined in the system.
Returns:
list: a list of tuples, holding id and name for each trigger.
"""
return [{
"id": v["id"],
"name": v["name"],
"options": v["options"]
} for v in self.TRIGGERS.values()]
def sync_jobs(self, joblist):
"""Synchronize the current jobs with a given list of jobs.
This means, that all jobs not included in the list will be removed,
existing ones will be updated and new ones will be added to the
scheduler.
Args:
joblist (list): a list of jobs in the format of the schema.
Returns:
bool: whether this operation was successful or not.
"""
logger.debug("Syncing job lists ...")
current_jobs = self.get_jobs()
jobs_to_keep = {j["id"] for j in joblist if j.get("id")}
# remove old jobs
for job in current_jobs:
if job["id"] not in jobs_to_keep:
self.scheduler.remove_job(job["id"], jobstore="elastic")
# update and add jobs
for job in joblist:
self.upsert_job(job)
return True
def _make_trigger(self, trigger_doc):
"""Creates a trigger from a given dictionary of user input."""
# we can assume, that an id for the trigger is given in the input.
cur_trigger = self.TRIGGERS[trigger_doc["id"]]
option_validator = cerberus.Validator(cur_trigger["schema"])
args = [
o["id"] for o in trigger_doc["options"]
if option_validator(o) and o["active"]
]
trigger_args = cur_trigger["trigger_args"](args)
return CronTrigger(**trigger_args)
def _serialize_trigger(self, trigger):
"""Serializes a trigger into a json array, as defined in TRIGGERS."""
# since we only have a defined set of triggers, the following is
# possible.
mapping = [(v["trigger_args"]([]).keys(), k)
for k, v in self.TRIGGERS.items()]
trigger_doc = None
result = {}
for keys, name in mapping:
# all keys for the mapping need to be defined.
def_keys = [f.name for f in trigger.fields if not f.is_default]
if all([(key in def_keys) for key in keys]):
trigger_doc = self.TRIGGERS[name]
break
if not trigger_doc:
return result
result["name"] = trigger_doc["name"]
result["id"] = trigger_doc["id"]
args = set(trigger_doc["from_trigger"](trigger))
# copy the list of options (otherwise this leads to nasty side effects)
options = [dict(**item) for item in trigger_doc["options"]]
for option in options:
option["active"] = option["id"] in args
result["options"] = options
return result
def get_jobs(self):
"""Returns a list of jobs that are scheduled in the system.
Returns:
list: a list of job-dicts, holding the id and the runtimes.
"""
jobs = self.scheduler.get_jobs()
joblist = []
for job in jobs:
joblist.append({
"id": job.id,
"name": {
"name": job.name
},
"crawler": {
"id": job.args[0]
},
"schedule": self._serialize_trigger(job.trigger),
"next_run": {
"name": job.next_run_time
}
})
logger.debug(f"Retrieved {len(joblist)} jobs from the jobstore.")
return joblist
def run_job(self, job_id):
"""Runs the job with the specified id immediately.
Args:
job_id: the id of the job that should be run.
Returns:
bool: whether running the job succeeded or not.
"""
logger.debug(f"Running job '{job_id}' directly.")
cur_job = self.scheduler.get_job(job_id, jobstore="elastic")
if cur_job is None:
return False
cur_job.func(*cur_job.args, **cur_job.kwargs)
return True
class Boss(object):
workers = {}
dirpath = '.'
output = None
def __init__(self, dirpath='.', output='output'):
'''
local path for load config
'''
logger.info("Initialing BOSS")
if os.path.isdir(dirpath):
self.dirpath = dirpath
logger.info("loading job config folder: " + dirpath)
else:
logger.info(dirpath + " is invalid, use default path instead")
self.output = output
logger.info("Setup output folder: " + output)
if not os.path.isdir(output):
logger.info("target directory " + output +
" doesn't exist, creating..")
os.makedirs(output)
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.load_dir(dirpath)
event_handler = JsonHandler(patterns=["*.json"],
ignore_directories=True)
event_handler.set_handler(oncreated=self.load,
onmodified=self.load,
ondeleted=self.remove)
observer = Observer()
observer.schedule(event_handler, self.dirpath, recursive=True)
observer.start()
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fname in filenames:
if 'json' in fname[-4:]:
self.load(os.path.join(dirpath, fname))
def load(self, fp):
'''
given a json file, load and create a task run regularly
'''
logger.info(fp + " was loaded!")
with open(fp) as in_data:
try:
data = json.load(in_data)
data['path'] = fp
except ValueError as e:
logger.warning(fp + " loaded failed: " + e.message)
return None
interval = 30
if 'interval' in data:
interval = int(data['interval'])
if self.output:
# TODO: test case for no 'output' key in data
if 'output' not in data:
data['output'] = {}
output = data['output']
if 'dirpath' in output:
output['dirpath'] = os.path.join(self.output,
output['dirpath'])
else:
output['dirpath'] = self.output
if 'type' not in data:
logger.error("Missing type attribute in \
your configruation file [%s]" % fp)
return None
if fp in self.workers: # existing minion found
logger.info("Update exisitng minion [%s]" % fp)
minion = self.workers[fp]
minion.update(**data)
# //memo: Interval can't be modified
self.scheduler.modify_job(job_id=fp,
func=minion.collect,
name=minion.name + '_' +
minion.serial)
else: # Create new
logger.info("Create new minion [%s]" % fp)
module_path = data['type'][:data['type'].rfind(".")]
object_name = data['type'][data['type'].rfind(".") + 1:]
try:
minion_module = getattr(
importlib.import_module(module_path), object_name)
except Exception as e:
logger.exception(e)
return None
minion = minion_module(**data)
self.workers[fp] = minion
self.scheduler.add_job(minion.collect,
'interval',
id=fp,
name=minion.name + '_' + minion.serial,
seconds=interval)
return minion
return None
def list(self):
'''
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
'''
for (fp, worker) in self.workers:
logger.info("path=" + fp + "," + str(worker) + ";")
def remove(self, fp):
'''
given file path, stop running instance if possible
'''
if fp in self.workers:
self.workers[fp].onstop()
self.scheduler.remove_job(job_id=fp)
del self.workers[fp]
return True
return False
def remove_advanced(self):
'''
TODO:
1. remove by start, end
2. by directory(?)
'''
pass
def unload_all(self):
'''
stop all running instances
'''
self.scheduler.shutdown()
def pause(self, fp):
'''
simply stop running instance but not remove config
TODO: should have timeout if stop failed
'''
self.scheduler.pause(job_id=fp)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
'''
For periodical minions, waking them according to timing
'''
pass
class Scheduler(object):
def __init__(self):
self.cron_job_args = [
'year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute',
'second', 'start_date', 'end_date'
]
self.scheduler = BackgroundScheduler(
timezone=getattr(config, 'TIME_ZONE', 'Asia/Shanghai'))
self.jobs = {}
self.already_init = False
def init(self):
if not self.already_init:
for job in Job.query.filter_by(enabled=True).all():
self.add_job(job)
self.scheduler.start()
self.already_init = True
def __parse_args(self, trigger, trigger_args):
if trigger == 'cron':
args = {
k: v
for k, v in zip(self.cron_job_args, trigger_args.split(';'))
if v
}
# 周需要单独处理，0 对应周一，与页面上的说明不一致
day_of_week = int(
args['day_of_week']) if args.get('day_of_week') else None
if day_of_week == 0:
args['day_of_week'] = 6
elif day_of_week is not None:
args['day_of_week'] = day_of_week - 1
return args
elif trigger == 'interval':
return {'seconds': int(trigger_args)}
elif trigger == 'date':
return {'run_date': trigger_args}
else:
raise ValueError('未知的调度策略: %r' % trigger)
def add_job(self, job):
job_id = str(job.id)
args = self.__parse_args(job.trigger, job.trigger_args)
instance = self.scheduler.add_job(agent,
job.trigger,
id=job_id,
args=(job.id, job.command_user,
job.command, job.targets),
**args)
self.jobs[job_id] = instance
def valid_job_trigger(self, trigger, trigger_args):
try:
args = self.__parse_args(trigger, trigger_args)
job = self.scheduler.add_job(agent,
trigger,
args=(None, None, None, None),
next_run_time=None,
**args)
job.remove()
return True
except ValueError:
return False
def remove_job(self, job_id):
job_id = str(job_id)
if self.scheduler.get_job(job_id):
self.scheduler.remove_job(job_id)
def update_job(self, job):
job_id = str(job.id)
if self.scheduler.get_job(job_id):
args = self.__parse_args(job.trigger, job.trigger_args)
self.scheduler.reschedule_job(job_id, trigger=job.trigger, **args)
elif job.enabled:
self.add_job(job)
class SchedulerEdge(object):
#instacia do objeto e inicia o escalonador
def __init__(self):
#self.verifica_sensores()
def run_thread():
while(True):
time.sleep(1)
self.scheduler = BackgroundScheduler() # atribui um agendador background
self.scheduler.start() # inicia o agendador
self.th = threading.Thread(target= run_thread) # thread executa outtro fluxo para o agendador rodar
self.th.start()
#def verifica_DB():
# print("Aqui")
#self.verifica_sensores()
def add_job(self, a): # cria uma nova tarefa no escalonador
# analisar o modo (interval,date,cron) para executar de forma correta
jsonObject = json.loads(a)
if(jsonObject['modo']=='cron'):
self.scheduler.add_job(self.tick, jsonObject['modo'], second = jsonObject['info']['second'], minute = jsonObject['info']['minute'],
hour = jsonObject['info']['hour'], day = jsonObject['info']['day'], month = jsonObject['info']['month'], year = jsonObject['info']['year'],id = a, args = [a])
# self.scheduler.add_job(self.tick, jsonObject['modo'], second = 0, minute = jsonObject['info']['minute'],
# hour = jsonObject['info']['hour'], day = jsonObject['info']['day'], month = jsonObject['info']['month'], year = jsonObject['info']['year'],id = a, args = [a])
elif(jsonObject['modo']=='interval'):
self.scheduler.add_job(self.tick, jsonObject['modo'], second = jsonObject['info']['second'], minute = jsonObject['info']['minute'],
hour = jsonObject['info']['hour'], day = jsonObject['info']['day'], month = jsonObject['info']['month'], year = jsonObject['info']['year'],id = a, args = [a])
elif(jsonObject['modo']=='date'):
variabledate = datetime.date(int(jsonObject['info']['year']),int(jsonObject['info']['month'])
,int(jsonObject['info']['day']))
self.scheduler.add_job(self.tick, jsonObject['modo'],run_time = datetime.date(variabledate),id = a, args = [a])
def remove_job(self, a):
self.scheduler.remove_job(a)
def tick(self,response):
object_events = Event_Treatment()
object_events.event(1,response)
#print("Pi")
#print(response)
def verifica_sensores(self): # Verifica os sensores cadastrados no DB, colocando-os em uma tabela para
print("Entrou") # comparar quando ocorrer modificaçõe no DB. Dessa forma, é possivel
buffer = BytesIO() # adicionar ou remover novos sensores em tempo de execução.
c = pycurl.Curl()
c.setopt(c.URL, 'http://localhost:8000/sensors/?format=json') # Local onde se encontra todos os sensores
c.setopt(c.WRITEDATA, buffer) # cadastrados no DB
c.perform()
c.close()
body = buffer.getvalue()
# Body is a byte string.
# We have to know the encoding in order to print it to a text file
# such as standard output.
jsonObject = json.loads(body.decode('iso-8859-1'))
i=0
j=0
lista_sensores = []
for row in jsonObject: # Pega todos os links referente a cada sensor cadastrado
print(row['url'])
# Coloca em uma tabela como a finalidade que foi descrita
##self.tab_temp[i] = row['url']
lista_sensores.insert(i,row['url'])
#lista_sensores.append(row['url'])
i=i+1
#print(lista_sensores)
#teste = lista_sensores[0]
#print(teste)
tab_permanente = []
if len(tab_permanente): # Se tiver algo na tabela, ocorre atualização dos dispositivos
for j in range(i): # no T_EVENTO
print("tabela permanente existe") # Criar uma função para comparar as tabelas || talvez uma fila
else:
for w in lista_sensores:
tab_permanente.insert(j,lista_sensores[j])
#Chamar um metodo para retornar um json.
print("Json sensor", j)
#-------------------------------------------------------------------------------
buffer = BytesIO() # adicionar ou remover novos sensores em tempo de execução.
c = pycurl.Curl()
c.setopt(c.URL, lista_sensores[j]) # Local onde se encontra todos os sensores
c.setopt(c.WRITEDATA, buffer) # cadastrados no DB
c.perform()
c.close()
body = buffer.getvalue()
jsonObject = json.loads(body.decode('iso-8859-1'))
print(jsonObject)
#-------------------------------------------------------------------------------
self.add_job(body.decode('iso-8859-1'))
j=j+1
class Boss(object):
workers = {}
dirpath = '.'
output = None
def __init__(self, dirpath='.', output='output'):
'''
local path for load config
'''
logger.info("Initialing BOSS")
if os.path.isdir(dirpath):
self.dirpath = dirpath
logger.info("loading job config folder: " + dirpath)
else:
logger.info(dirpath + " is invalid, use default path instead")
self.output = output
logger.info("Setup output folder: " + output)
if not os.path.isdir(output):
logger.info("target directory "
+ output
+ " doesn't exist, creating..")
os.makedirs(output)
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.load_dir(dirpath)
event_handler = JsonHandler(patterns=["*.json"],
ignore_directories=True)
event_handler.set_handler(oncreated=self.load,
onmodified=self.load,
ondeleted=self.remove)
observer = Observer()
observer.schedule(event_handler, self.dirpath, recursive=True)
observer.start()
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fname in filenames:
if 'json' in fname[-4:]:
self.load(os.path.join(dirpath, fname))
def load(self, fp):
'''
given a json file, load and create a task run regularly
'''
logger.info(fp + " was loaded!")
with open(fp) as in_data:
try:
data = json.load(in_data)
data['path'] = fp
except ValueError as e:
logger.warning(fp + " loaded failed: " + e.message)
return None
interval = 30
if 'interval' in data:
interval = int(data['interval'])
if self.output:
# TODO: test case for no 'output' key in data
if 'output' not in data:
data['output'] = {}
output = data['output']
if 'dirpath' in output:
output['dirpath'] = os.path.join(self.output, output['dirpath'])
else:
output['dirpath'] = self.output
if 'type' not in data:
logger.error("Missing type attribute in \
your configruation file [%s]" % fp)
return None
if fp in self.workers: # existing minion found
logger.info("Update exisitng minion [%s]" % fp)
minion = self.workers[fp]
minion.update(**data)
# //memo: Interval can't be modified
self.scheduler.modify_job(job_id=fp,
func=minion.collect,
name=minion.name+'_'+minion.serial
)
else: # Create new
logger.info("Create new minion [%s]" % fp)
module_path = data['type'][:data['type'].rfind(".")]
object_name = data['type'][data['type'].rfind(".") + 1:]
try:
minion_module = getattr(importlib.import_module(
module_path), object_name)
except Exception as e:
logger.exception(e)
return None
minion = minion_module(**data)
self.workers[fp] = minion
self.scheduler.add_job(minion.collect, 'interval',
id=fp,
name=minion.name + '_' + minion.serial,
seconds=interval
)
return minion
return None
def list(self):
'''
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
'''
for (fp, worker) in self.workers:
logger.info("path=" + fp + "," + str(worker) + ";")
def remove(self, fp):
'''
given file path, stop running instance if possible
'''
if fp in self.workers:
self.workers[fp].onstop()
self.scheduler.remove_job(job_id=fp)
del self.workers[fp]
return True
return False
def remove_advanced(self):
'''
TODO:
1. remove by start, end
2. by directory(?)
'''
pass
def unload_all(self):
'''
stop all running instances
'''
self.scheduler.shutdown()
def pause(self, fp):
'''
simply stop running instance but not remove config
TODO: should have timeout if stop failed
'''
self.scheduler.pause(job_id=fp)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
'''
For periodical minions, waking them according to timing
'''
pass
class Scheduler():
""" This class handles event adding, removing and listing """
def __init__(self):
self.is_running = False
self.scheduler = BackgroundScheduler()
self.supported = ['mail', 'slack', 'whatsapp']
self.counter = 0
def process_event(self, event):
""" Used adding jobs to scheduler. """
event_description = json.loads(event.description)
message = 'This is default message.'
if 'message' in event_description:
message = event_description['message']
receiver = None
if 'receiver' in event_description:
receiver = event_description['receiver']
subject = None
if 'subject' in event_description:
subject = event_description['subject']
if not event_description['how'] in self.supported:
raise NameError(
'Unknown option. Only mail, slack and whatsapp are supported now.'
)
id = str(self.counter)
if event_description['how'] == 'mail':
self.scheduler.add_job(Mail.send,
next_run_time=event.begin.datetime,
kwargs=dict(message=message,
receiver=receiver,
subject=subject),
id=id,
name=event.name)
if event_description['how'] == 'slack':
self.scheduler.add_job(Slack.send,
next_run_time=event.begin.datetime,
kwargs=dict(message=message,
channel=receiver),
id=id,
name=event.name)
if event_description['how'] == 'whatsapp':
self.scheduler.add_job(WhatsApp.send,
next_run_time=event.begin.datetime,
kwargs=dict(message=message,
receiver=receiver),
id=id,
name=event.name)
self.counter += 1
return id
def get_events(self):
"""
Method returns list of scheduled events.
"""
events = []
for event in self.scheduler.get_jobs():
function_name = str(event.func)
fc = 'Mail'
if 'WhatsApp' in function_name:
fc = 'WhatsApp'
if 'Slack' in function_name:
fc = 'Slack'
events.append(
str('ID: ' + event.id + ', Name: ' + event.__str__() + ' ' +
fc + ' ' + str(event.kwargs)))
# print(events)
return events
def run(self):
"""
Starts scheduler.
"""
if not self.is_running:
self.scheduler.start()
self.is_running = True
def stop(self):
"""
Stops scheduler.
"""
if self.is_running:
self.scheduler.shutdown()
self.is_running = False
def read_calendar(self, file):
"""
Method will read specified calendar from file in ics format.
"""
with open(file) as f:
ics = f.read()
return Calendar(ics)
def load_calendar(self, calendar_file):
"""
Method will add events from calendar to scheduler.
"""
c = self.read_calendar(calendar_file)
for event in c.events:
self.process_event(event)
@staticmethod
def create_description(how='mail',
receiver=None,
message=None,
subject=None):
"""
Method will create description to match for adding job.
"""
d = '{'
d += '"how": "'
d += how
d += '"'
if receiver != None:
d += ', "receiver": "'
d += receiver
d += '"'
if message != None:
d += ', "message": "'
d += message
d += '"'
if subject != None:
d += ', "subject": "'
d += subject
d += '"'
d += '}'
# print(d)
return d
@staticmethod
def create_event(name,
when,
how='mail',
receiver=None,
message=None,
subject=None):
"""
Method will create event from specified parameters which can be later added to scheduler.
"""
e = Event()
e.name = name
e.begin = when
e.description = Scheduler.create_description(how, receiver, message,
subject)
return e
def remove_event(self, event_id):
"""
Remove event from scheduler specified by his id.
"""
try:
self.scheduler.remove_job(str(event_id))
return True
except:
return False
class Boss(object):
default_path = None
workers = []
def __init__(self):
'''
local path for load config
'''
self.scheduler = BackgroundScheduler()
self.scheduler.start()
if self.default_path:
self.load(self.default_path)
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fn in filenames:
self.load(os.path.join(dirpath, fn))
def load(self, fp):
'''
given a file
TBI: directory
'''
with open(fp) as in_data:
data = json.load(in_data)
minion = ShellMinion(**data)
self.workers.append(minion)
self.scheduler.add_job(minion.collect, 'interval',
name=minion.name+'_'+minion.serial, seconds=2
)
def list(self):
'''
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
'''
for worker in self.workers:
print(str(worker))
def remove(self, sn):
'''
given an SN, stop running instance if possible
TODO: remove it from the list
'''
self.scheduler.remove_job(sn)
def remove_advanced(self):
'''
TBD
TODO:
1. remove by start, end
2. by directory(?)
'''
pass
def unload_all(self):
'''
stop all running instances
'''
self.scheduler.shutdown()
def stop(self, sn):
'''
simply stop running instance but not remove config
TODO: should have timeout if stop failed
'''
self.scheduler.stop(sn)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
'''
For periodical minions, waking them according to timing
'''
pass
class BotBase:
def __init__(self, chatter: ChatterBase):
self.session = Session()
self.subscribers = {}
# I never expect to have any significant amount of users, so I'll just load all in memory on start
self.load_subscribers()
self.chatter = chatter
self.chatter.set_new_message_event(self.new_message_event)
self.scheduler = BackgroundScheduler()
self.schedule_jobs()
def load_subscribers(self):
for row in self.session.query(Subscriber):
s: Subscriber = row
self.subscribers[s.user_id] = {
'repeat': s.repeat,
'sleep_from': s.sleep_from,
'sleep_to': s.sleep_to
}
# bot_base has essential commands: subscribe/unsubscribe/sleep
def user_subscribe(self, user_id, period):
session = self.session
subscriber = session.query(Subscriber).filter_by(
user_id=user_id).first()
if subscriber:
subscriber.repeat = period
self.scheduler.remove_job(user_id)
else:
subscriber = Subscriber(user_id=user_id, repeat=period)
session.add(subscriber)
session.commit()
self.scheduler.add_job(self.send_to_periodic_subscribers,
'interval',
minutes=period,
args=[user_id],
id=user_id)
self.subscribers[user_id] = {
'repeat': period,
'sleep_from': 0,
'sleep_to': 0
}
def user_unsubscribe(self, user_id):
session = self.session
subscriber = session.query(Subscriber).filter(
Subscriber.user_id == user_id).first()
if subscriber:
self.scheduler.remove_job(user_id)
session.query(Subscriber).filter(
Subscriber.user_id == user_id).delete()
session.commit()
if user_id in self.subscribers.keys():
del self.subscribers[user_id]
def user_sleep(self, user_id, sleep_from, sleep_to):
session = self.session
subscriber = session.query(Subscriber).filter(
Subscriber.user_id == user_id).first()
subscriber.sleep_from = sleep_from
subscriber.sleep_to = sleep_to
session.add(subscriber)
session.commit()
self.subscribers[user_id]['sleep_from'] = sleep_from
self.subscribers[user_id]['sleep_to'] = sleep_to
print('sleep configured for {}'.format(user_id))
def user_get_subscription(self, user_id):
session = self.session
subscriber = session.query(Subscriber).filter(
Subscriber.user_id == user_id).first()
return subscriber.repeat if subscriber else 0
def schedule_jobs(self):
print('scheduling jobs...')
for key, value in self.subscribers.items():
self.scheduler.add_job(self.send_to_periodic_subscribers,
'interval',
minutes=value['repeat'],
args=[key],
id=key)
print('scheduling {} every {}'.format(key, value['repeat']))
self.scheduler.start()
def is_sleep_time(self, user_id):
subscriber_settings: dict = self.subscribers.get(user_id)
sleep_from = subscriber_settings.get('sleep_from', 0)
sleep_to = subscriber_settings.get('sleep_to', 0)
if not sleep_from and not sleep_to:
return False
sleep_from = time(sleep_from, 0, 0)
sleep_to = time(sleep_to, 0, 0)
cur_time = datetime.now().time()
def time_in_range(start, end, x):
if start <= end:
return start <= x <= end
else:
return start <= x or x <= end
if time_in_range(sleep_from, sleep_to, cur_time):
return True
else:
return False
def get_next_run_time(self, user_id):
t = self.scheduler.get_job(user_id).next_run_time
t = t.strftime('%H:%M:%S')
return t
def get_sleep_hours(self, user_id):
subscriber = self.session.query(Subscriber).filter(
Subscriber.user_id == user_id).first()
sleep_from = subscriber.sleep_from
sleep_to = subscriber.sleep_to
return sleep_from, sleep_to
# respond or distribute message to all subscribers
def say(self, message, chat_id, bold=False, mono=False, colour=''):
self.chatter.send_message(message, chat_id, bold, colour, mono)
# classes can implement it
def new_message_event(self, message_text, message_source, chat_id):
pass
def start_listening(self): # start bot's event loop
while True:
self.chatter.cycle()
# self.chatter.start_listening()
def send_to_periodic_subscribers(self, chat_id):
if self.is_sleep_time(chat_id):
raise SleepTimeException
class MainRunner(object):
workers = {}
dirpath = '.'
defaultOutputPath = 'output'
class NoRunningFilter(logging.Filter):
def filter(self, record):
return not record.msg.startswith('Execution')
def __init__(self, dirpath='.'):
'''
local path for load config
'''
logger.info("Initialing Main Runner for Hasal agent")
if os.path.isdir(dirpath):
self.dirpath = dirpath
logger.info("loading runner config folder: " + dirpath)
else:
logger.info(dirpath + " is invalid, use default path instead")
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.load_dir(self.dirpath)
event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True)
event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove)
observer = Observer()
observer.schedule(event_handler, self.dirpath, recursive=True)
observer.start()
my_filter = self.NoRunningFilter()
logging.getLogger("apscheduler.scheduler").addFilter(my_filter)
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fname in filenames:
if 'json' in fname[-4:]:
self.load(os.path.join(dirpath, fname))
def load(self, fp):
'''
given a json file, load and create a task run regularly
'''
logger.info(fp + " was loaded!")
with open("agent.log", 'w+') as f:
f.write(fp + " was loaded!")
data = {}
loaded = False
for _ in range(10):
try:
with open(fp) as in_data:
data = json.load(in_data)
# default will load JOB_NAME parameter in Jenkins created json file
data['name'] = data.get('JOB_NAME', "Jenkins Job")
data['path'] = fp
loaded = True
except ValueError as e:
logger.warning(fp + " loaded failed: " + e.message)
return None
except Exception as e:
logger.warning("File is not ready. Wait 1 second for another try.")
time.sleep(1)
if not loaded:
logger.warning(fp + " is not ready for 10 seconds.")
return None
# load interval value from Jenkins created json file (default : 30 )
interval = int(data.get('interval', 30))
# load outputpath and defaultoutputpath from Jenkins created json file
if 'output' in data:
if 'defaultOutputPath' in data['output']:
self.defaultOutputPath = data['output']['defaultOutputPath']
if 'dirpath' in data['output']:
data['output']['outputPath'] = os.path.join(self.defaultOutputPath, data['output']['dirpath'])
else:
data['output'] = {'outputPath': self.defaultOutputPath}
if fp in self.workers: # existing runner found
logger.info("Update exisitng runner [%s]" % fp)
runner = self.workers[fp]
runner.update(**data)
# //memo: Interval can't be modified
self.scheduler.modify_job(job_id=fp,
func=runner.run,
name=runner.name
)
else: # Create new
logger.info("Create new runner [%s]" % fp)
module_path = data.get('AGENT_MODULE_PATH', "hasalTask")
object_name = data.get('AGENT_OBJECT_NAME', "HasalTask")
try:
runner_module = getattr(importlib.import_module(
module_path), object_name)
except Exception as e:
logger.exception(e)
return None
runner = runner_module(**data)
self.workers[fp] = runner
self.scheduler.add_job(runner.run, 'interval',
id=fp,
name=runner.name,
seconds=interval
)
return runner
def list(self):
'''
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
'''
for (fp, worker) in self.workers:
logger.info("path=" + fp + "," + str(worker) + ";")
def remove(self, fp):
'''
given file path, stop running instance if possible
'''
if fp in self.workers:
self.workers[fp].onstop()
self.scheduler.remove_job(job_id=fp)
del self.workers[fp]
return True
return False
def remove_advanced(self):
'''
TODO:
1. remove by start, end
2. by directory(?)
'''
pass
def unload_all(self):
'''
stop all running instances
'''
self.scheduler.shutdown()
def pause(self, fp):
'''
simply stop running instance but not remove config
TODO: should have timeout if stop failed
'''
self.scheduler.pause(job_id=fp)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
'''
For periodical minions, waking them according to timing
'''
pass
class Jobs_Scheduler():
__lock_sending_to_msg_q = threading.Lock()
def __init__(self):
self.__scheduler = None
self.__init_scheduler()
def __init_scheduler(self):
try:
self.__scheduler = BackgroundScheduler()
self.__load_scheduled_jobs()
except Exception as e:
Logger.log_debug('ERROR @ Job_Scheduler -> __init_scheduler')
Logger.log_error(str(e))
finally:
Logger.log_info('Done with initializing job scheduler service')
def start_scheduler(self):
self.__scheduler.start()
def stop_scheduler(self):
self.__scheduler.stop()
def __load_scheduled_jobs(self):
try:
file_path = cfg.SCHEDULED_JOBS_FILES_PATH
for file in glob.glob(file_path + "*.shed"):
dict_msg = CF.load_dictionary_object(file)
jobid = str(dict_msg["cronjobid"])
m = str(dict_msg["cronminute"])
h = dict_msg["cronhour"]
dow = dict_msg["crondateofweek"]
self.schedule_a_job(jobid, m, h, dow, None)
except Exception as e:
Logger.log_debug("ERROR @ load_scheduled_jobs")
Logger.log_error(str(e))
def schedule_a_job(self, cronjobid, m="0",h='*',dow='*', message_dict=None):
#-----------------------------------------
# jobid format :- <device_com_pk><hub_pk>
#-----------------------------------------
ret_val = False
#Jobs_Scheduler.__job_lock.acquire()
try:
if m>=0:
#dict_obj={"jobid":jobid,"m":m,"h":h,"dow":dow}
ret_val = True
if not message_dict == None:
file_name = '%s%s%s' % (cfg.SCHEDULED_JOBS_FILES_PATH, cronjobid , ".shed")
ret_val = CF.save_dictionary_object(file_name, message_dict)
if ret_val:
#print 'adding a schedule'
Logger.log_info('Adding scheduler job ' + str(cronjobid))
self.__scheduler.add_job(Jobs_Scheduler.scheduler_callback_function
,minute=m,hour=h,day_of_week=dow, id=cronjobid,trigger="cron",kwargs={"jobid":cronjobid})
return ret_val
except Exception as e:
Logger.log_debug("ERROR @ schedule_a_job")
Logger.log_error(str(e))
return False
finally:
#Jobs_Scheduler.__job_lock.release()
pass
def remove_a_scheduled_job(self, cronjobid):
#Jobs_Scheduler.__job_lock.acquire()
try:
Logger.log_debug("Removing scheduled job " + cronjobid)
self.__scheduler.remove_job(cronjobid)
file_name = cfg.SCHEDULED_JOBS_FILES_PATH + cronjobid + ".shed"
os.remove(file_name)
return (True,"")
except Exception as e:
Logger.log_debug("ERROR @ remove_a_scheduled_job")
Logger.log_error(str(e))
return (False,str(e).replace('u',''))
finally:
#Jobs_Scheduler.__job_lock.release()
pass
def print_jobs(self):
try:
self.__scheduler.print_jobs()
except Exception as e:
Logger.log_debug("ERROR @ print_jobs")
Logger.log_error(str(e))
@staticmethod
def scheduler_callback_function(jobid):
Jobs_Scheduler.__lock_sending_to_msg_q.acquire()
try:
jobid = str(jobid)
Logger.log_info("Scheduler triggered for job "+ jobid)
file_path = cfg.SCHEDULED_JOBS_FILES_PATH
file_name = file_path + jobid + ".shed"
dict_msg = CF.load_dictionary_object(file_name)
if not dict_msg == None:
# Send the message to MessageProc q
result_msg = Jobs_Scheduler.send_job_command(dict_msg)
except Exception as e:
Logger.log_debug("ERROR @ scheduler_callback_function")
Logger.log_error(str(e))
finally:
Logger.log_debug("End of scheduler_callback_function")
Jobs_Scheduler.__lock_sending_to_msg_q.release()
@staticmethod
def send_job_command(dict_command):
try:
cronjobid = dict_command['cronjobid']
device_code = dict_command['devicecode']
com_id = dict_command['comid']
hubcondeviceid = dict_command['hubcondeviceid']
comcode = dict_command['comcode']
dict_command.update({'u_o_s': 'S'})
zc = ZC()
mp_sock = zc.connect_zmq(cfg.msg_proc_ip, cfg.msg_proc_port, 15000)
mp_sock.send(json.dumps(dict_command))
ret_msg = None
ret_msg = zc.zmq_recv(mp_sock, 15000)
mp_sock.close()
del mp_sock
if not ret_msg == None:
ret_msg = json.loads(ret_msg)
# Send to notification Q
hub_sno = CF.get_hub_serial_no()
ret_msg.update({'hub_sno': hub_sno})
ret_msg.update({'notification_type': 'CRON_EVENT'})
ret_msg.update({'msg_type' :'NOTIFICATION'})
ret_msg.update({'notification_dt': datetime.datetime.now().strftime("%Y%m%d %H%M%S")})
ret_msg.update({'cronjobid': cronjobid})
ret_msg.update({'devicecode': device_code})
ret_msg.update({'hubcondeviceid': hubcondeviceid})
ret_msg.update({'comcode': comcode})
Logger.log_debug('Sending ack to notification queue')
CF.send_to_notifiction_queue(ret_msg)
else:
Logger.log_error('No response back from the message processing queue for the scheduled event.')
except Exception as e:
Logger.log_debug("ERROR @ Jobs_Scheduler -> send_job_command")
Logger.log_error(str(e))
"""
@staticmethod
def send_to_notifiction_queue(dict_msg):
try:
zc = ZC()
en_sock = zc.connect_zmq(cfg.EVENTS_PROCESSOR_IP, cfg.EVENTS_PROCESSOR_PORT)
en_sock.send(json.dumps(dict_msg))
ret_msg = None
ret_msg = zc.zmq_recv(en_sock)
en_sock.close()
del en_sock
if ret_msg == None:
return False
else:
return True
except Exception as e:
Logger.log_debug("ERROR @ Jobs_Scheduler -> send_to_notifiction_queue()")
Logger.log_error(str(e))
return False
"""
class Trigger:
def __init__(self, app):
self.scheduler = None
self.app = app
def setup(self):
self.scheduler = BackgroundScheduler({
'apscheduler.jobstores.default': {
'type': 'sqlalchemy',
'url': self.app.config["TRIGGER_DATABASE_URL"] #os.environ.get('TRIGGER_DATABASE_URL')
},
'apscheduler.executors.processpool': {
'type': 'processpool',
'max_workers': '30'
},
'apscheduler.job_defaults.coalesce': 'false',
'apscheduler.job_defaults.max_instances': '20',
'apscheduler.timezone': 'UTC',
})
def start(self):
self.scheduler.start()
def is_running(self):
return self.scheduler.running()
def shutdown(self):
self.scheduler.shutdown()
def load_job_list(self):
with self.app.app_context():
projects = AutoProject.query.all()
# key_list = ("minute", "hour", "day", "month", "day_of_week")
for p in projects:
if p.enable and self.scheduler.get_job(p.id) is None:
cron = p.cron.replace("\n", "").strip().split(" ")
#print(cron)
if len(cron) < 5:
continue
j = self.scheduler.add_job(func=run_job, trigger='cron', name=p.name, replace_existing=True,
minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4],
id="%s" % p.id, args=(p.id,))
else:
self.update_job(p.id)
def update_job(self, id):
with self.app.app_context():
p = AutoProject.query.filter_by(id=id).first()
if p.enable:
cron = p.cron.replace("\n", "").strip().split(" ")
if len(cron) < 5:
return False
print(self.scheduler.get_job(id))
if self.scheduler.get_job(id) is None:
self.scheduler.add_job(func=run_job, trigger='cron', name=p.name,
minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4],
id="%s" % id, args=(id,))
else:
self.remove_job(id)
self.scheduler.add_job(func=run_job, trigger='cron', name=p.name,
minute=cron[0], hour=cron[1], day=cron[2], month=cron[3], day_of_week=cron[4],
id="%s" % id, args=(id,))
return True
def remove_job(self, id):
if self.scheduler.get_job(id) is not None:
self.scheduler.remove_job(id)
def pause_job(self, id):
pass
def resume_job(self, id):
pass
def get_jobs(self):
to_zone = tz.gettz("CST")
#jobs = self.scheduler.get_jobs()
urls = {
"pass": "******",
"fail": "fail.png",
"running": "run.gif",
"none": "project.png"
}
projects = AutoProject.query.order_by(AutoProject.id.desc()).all()
data = {"total": len(projects), "rows": []}
for p in projects:
next_run_time = "调度未启动"
job = self.scheduler.get_job(p.id)
if job is not None:
next_run_time = job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S")
# 获取该job下最后一次运行状态
task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first()
if task is not None:
output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no)
if os.path.exists(output_dir + "/report.html"):
tree = ET.parse(output_dir + "/output.xml")
root = tree.getroot()
# passed = root.find("./statistics/suite/stat").attrib["pass"]
fail = root.find("./statistics/suite/stat").attrib["fail"]
if int(fail) != 0:
status = 'fail'
else:
status = 'pass'
else:
status = "none"
data["rows"].append({"id": "%s" % p.id,
"name": p.name,
"enable": p.enable,
"status": status,
"url": url_for('static', filename='images/%s' % urls[status]),
"cron": p.cron,
"next_run_time": next_run_time
})
"""
for job in jobs:
status = "running"
task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first()
if task is None:
continue
output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no)
if os.path.exists(output_dir + "/report.html"):
tree = ET.parse(output_dir + "/output.xml")
root = tree.getroot()
#passed = root.find("./statistics/suite/stat").attrib["pass"]
fail = root.find("./statistics/suite/stat").attrib["fail"]
if int(fail) != 0:
status = 'fail'
else:
status = 'pass'
data["rows"].append({"id": "%s" % job.id,
"name": job.name,
"status": status,
"url": url_for('static', filename='images/%s' % urls[status]),
"cron": AutoProject.query.filter_by(id=job.id).first().cron,
"next_run_time": job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S")
})
"""
return data
def print_jobs(self):
pass
class MainRunner(object):
class FilterAllLog(logging.Filter):
# default we will filter logger from apscheduler.executors.default, apscheduler.scheduler,
# you can config filter logger in config.json
def filter(self, record):
return ""
def __init__(self, input_cmd_config_fp, input_job_config_fp, input_config_fp):
# init value
cmd_config_fp = os.path.abspath(input_cmd_config_fp)
job_config_fp = os.path.abspath(input_job_config_fp)
config_fp = os.path.abspath(input_config_fp)
# load configuration json files
self.cmd_config = CommonUtil.load_json_file(cmd_config_fp)
self.job_config = CommonUtil.load_json_file(job_config_fp)
self.config = CommonUtil.load_json_file(config_fp)
# init schedulers
self.scheduler = BackgroundScheduler()
self.scheduler.add_jobstore('sqlalchemy', url=self.config['job_store_url'])
self.scheduler.start()
# init variables
mananger = Manager()
self.sync_queue = mananger.Queue()
self.async_queue = mananger.Queue()
self.current_job_list = []
# Slack Sending Queue
# TODO: prevent the Slack bot is disable, the sending queue will use too much memory.
self.slack_sending_queue = mananger.Queue(50)
# init logger
self.set_logging(self.config['log_level'], self.config['log_filter'])
def set_logging(self, log_level, log_filter_list):
default_log_format = '%(asctime)s %(levelname)s [%(name)s.%(funcName)s] %(message)s'
default_datefmt = '%Y-%m-%d %H:%M'
if log_level.lower() == "debug":
logging.basicConfig(level=logging.DEBUG, format=default_log_format, datefmt=default_datefmt)
else:
logging.basicConfig(level=logging.INFO, format=default_log_format, datefmt=default_datefmt)
my_filter = self.FilterAllLog()
for target_logger in log_filter_list:
logging.getLogger(target_logger).addFilter(my_filter)
def scheduler_del_job(self, **kwargs):
input_cmd_str = kwargs.get("input_cmd_str", "")
cmd_str_list = input_cmd_str.split(" ")
if len(cmd_str_list) == 2:
job_id = cmd_str_list[1]
current_job_list = self.scheduler.get_jobs()
current_job_id_list = [j.id for j in current_job_list]
if job_id in current_job_id_list:
self.scheduler.remove_job(job_id)
else:
logging.error("Cannot find the specify job id [%s]" % job_id)
else:
logging.error("Incorrect cmd format! [%s]" % input_cmd_str)
def scheduler_list_job(self, **kwargs):
self.scheduler.print_jobs()
def scheduler_shutdown(self, **kwargs):
self.scheduler.shutdown()
sys.exit(0)
def list_all_commands(self, **kwargs):
print "Current supported commands as below:"
print "-" * 80
for cmd_str in self.cmd_config['cmd-settings']:
print '{:30s} {:50s} '.format(cmd_str, self.cmd_config['cmd-settings'][cmd_str]['desc'])
print "-" * 80
def scheduler_job_handler(self, input_cmd_obj, input_cmd_str):
cmd_match_pattern = input_cmd_obj.keys()[0]
func_point = getattr(self, input_cmd_obj[cmd_match_pattern]['func-name'])
func_point(cmd_configs=input_cmd_obj[cmd_match_pattern]['configs'], input_cmd_str=input_cmd_str)
def cmd_queue_composer(self, input_cmd_str):
for cmd_pattern in self.cmd_config['cmd-settings']:
re_compile_obj = re.compile(cmd_pattern)
re_match_obj = re_compile_obj.search(input_cmd_str)
if re_match_obj:
current_command_obj = self.cmd_config['cmd-settings'][cmd_pattern]
logging.debug("job matched [%s]" % cmd_pattern)
target_queue_type = current_command_obj.get('queue-type', None)
if target_queue_type == "async":
self.async_queue.put({"cmd_obj": current_command_obj, "cmd_pattern": cmd_pattern, "input_cmd_str": input_cmd_str})
elif target_queue_type == "sync":
self.sync_queue.put({"cmd_obj": current_command_obj, "cmd_pattern": cmd_pattern, "input_cmd_str": input_cmd_str})
else:
self.scheduler_job_handler({cmd_pattern: current_command_obj}, input_cmd_str)
break
def load_default_jobs(self, input_scheduler, input_job_config):
current_jobs = input_scheduler.get_jobs()
current_jobs_name = [job.name for job in current_jobs]
for job_name in input_job_config:
if input_job_config[job_name]['default-loaded']:
if job_name not in current_jobs_name:
func_point = getattr(importlib.import_module(input_job_config[job_name]['module-path']), job_name)
self.scheduler.add_job(func_point, input_job_config[job_name]['trigger-type'],
id=job_name,
seconds=input_job_config[job_name]['interval'],
max_instances=input_job_config[job_name]['max-instances'],
kwargs={
'async_queue': self.async_queue,
'sync_queue': self.sync_queue,
'slack_sending_queue': self.slack_sending_queue,
'configs': input_job_config[job_name]['configs'],
'cmd_config': self.cmd_config}
)
def job_exception_listener(self, event):
if event.exception:
logging.error("Job [%s] crashed [%s]" % (event.job_id, event.exception))
logging.error(event.traceback)
def add_event_listener(self):
self.scheduler.add_listener(self.job_exception_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
def run(self):
# load default job into scheduler if the job is not exist
self.load_default_jobs(self.scheduler, self.job_config)
# add event listener into scheduler
self.add_event_listener()
# enter the loop to receive the interactive command
while True:
user_input = raw_input()
self.cmd_queue_composer(user_input)
time.sleep(3)
class APSModule(object):
"""处理post python模块请求,单例模式运行
EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED |EVENT_JOB_EXECUTED |
EVENT_JOB_ERROR | EVENT_JOB_MISSED |EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES
"""
_instance_lock = threading.Lock()
def __init__(self):
self.ModuleJobsScheduler = BackgroundScheduler()
self.ModuleJobsScheduler.add_listener(self.deal_result)
self.ModuleJobsScheduler.start()
def __new__(cls, *args, **kwargs):
if not hasattr(APSModule, "_instance"):
with APSModule._instance_lock:
if not hasattr(APSModule, "_instance"):
APSModule._instance = object.__new__(cls)
return APSModule._instance
def putin_post_python_module_queue(self, post_module_intent=None):
try:
# 存储uuid
tmp_self_uuid = str(uuid.uuid1())
# 清空历史记录
post_module_intent._clean_log()
logger.warning("模块放入列表:{} job_id: {} uuid: {}".format(
post_module_intent.NAME, None, tmp_self_uuid))
post_module_intent.module_self_uuid = tmp_self_uuid
self.ModuleJobsScheduler.add_job(
func=post_module_intent._thread_run,
max_instances=1,
id=tmp_self_uuid)
# 放入缓存队列,用于后续删除任务,存储结果等
req = {
'broker': post_module_intent.MODULE_BROKER,
'uuid': tmp_self_uuid,
'module': post_module_intent,
'time': int(time.time()),
'job_id': None,
}
Xcache.create_module_task(req)
Notice.send_info("模块: {} {} 开始执行".format(
post_module_intent.NAME, post_module_intent._target_str))
return True
except Exception as E:
logger.error(E)
return False
def deal_result(self, event=None):
flag = False
if event.code == EVENT_JOB_ADDED:
# print("EVENT_JOB_ADDED")
pass
elif event.code == EVENT_JOB_REMOVED:
# print("EVENT_JOB_REMOVED")
pass
elif event.code == EVENT_JOB_MODIFIED:
# print("EVENT_JOB_MODIFIED")
pass
elif event.code == EVENT_JOB_EXECUTED: # 执行完成
flag = self.store_executed_result(event.job_id)
elif event.code == EVENT_JOB_ERROR:
# print("EVENT_JOB_ERROR")
flag = self.store_error_result(event.job_id, event.exception)
elif event.code == EVENT_JOB_MISSED:
# print("EVENT_JOB_MISSED")
pass
elif event.code == EVENT_JOB_SUBMITTED:
# print("EVENT_JOB_SUBMITTED")
pass
elif event.code == EVENT_JOB_MAX_INSTANCES:
# print("EVENT_JOB_MAX_INSTANCES")
pass
else:
pass
return flag
@staticmethod
def store_executed_result(task_uuid=None):
req = Xcache.get_module_task_by_uuid(task_uuid=task_uuid)
if req is None:
logger.warning("缓存中无对应实例,可能已经模块已经中途退出")
return False
module_common_instance = req.get("module")
# 存储运行结果
try:
module_common_instance._store_result_in_history()
Notice.send_success("模块: {} {} 执行完成".format(
module_common_instance.NAME,
module_common_instance._target_str))
logger.warning("多模块实例执行完成:{}".format(module_common_instance.NAME))
Xcache.del_module_task_by_uuid(task_uuid=task_uuid) # 清理缓存信息
return True
except Exception as E:
Xcache.del_module_task_by_uuid(task_uuid=task_uuid) # 清理缓存信息
logger.error("多模块实例执行异常:{} 异常信息: {}".format(
module_common_instance.NAME, E))
Notice.send_exception("模块: {} 执行异常,异常信息: {}".format(
module_common_instance.NAME, E))
logger.error(E)
return False
@staticmethod
def store_error_result(task_uuid=None, exception=None):
req = Xcache.get_module_task_by_uuid(task_uuid=task_uuid)
Xcache.del_module_task_by_uuid(task_uuid=task_uuid) # 清理缓存信息
module_common_instance = req.get("module")
# 存储运行结果
try:
module_common_instance.log_except(exception)
module_common_instance._store_result_in_history()
logger.error("多模块实例执行异常:{} 异常信息: {}".format(
module_common_instance.NAME, exception))
Notice.send_exception("模块: {} 执行异常,异常信息: {}".format(
module_common_instance.NAME, exception))
return True
except Exception as E:
logger.error("多模块实例执行异常:{} 异常信息: {}".format(
module_common_instance.NAME, E))
Notice.send_exception("模块: {} 执行异常,异常信息: {}".format(
module_common_instance.NAME, E))
logger.error(E)
return False
def delete_job_by_uuid(self, task_uuid=None):
req = Xcache.get_module_task_by_uuid(task_uuid=task_uuid)
Xcache.del_module_task_by_uuid(task_uuid=task_uuid) # 清理缓存信息
# 删除后台任务
try:
self.ModuleJobsScheduler.remove_job(task_uuid)
except Exception as E:
logger.error(E)
try:
module_common_instance = req.get("module")
except Exception as E:
logger.error(E)
return False
# 存储已经生成的结果
try:
module_common_instance.log_info("用户手动删除任务")
module_common_instance._store_result_in_history()
except Exception as E:
logger.error("删除多模块实例异常:{} 异常信息: {}".format(
module_common_instance.NAME, E))
Notice.send_exception("模块: {} 执行异常,异常信息: {}".format(
module_common_instance.NAME, E))
logger.error(E)
return False
# 发送通知
Notice.send_info("模块: {} {} 手动删除".format(
module_common_instance.NAME, module_common_instance._target_str))
logger.warning("多模块实例手动删除:{}".format(module_common_instance.NAME))
return True
class MainRunner(object):
workers = {}
dirpath = '.'
defaultOutputPath = 'output'
def __init__(self, dirpath='.'):
'''
local path for load config
'''
logger.info("Initialing Main Runner for Hasal agent")
if os.path.isdir(dirpath):
self.dirpath = dirpath
logger.info("loading runner config folder: " + dirpath)
else:
logger.info(dirpath + " is invalid, use default path instead")
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.load_dir(self.dirpath)
event_handler = JsonHandler(patterns=["*.json"], ignore_directories=True)
event_handler.set_handler(oncreated=self.load, onmodified=self.load, ondeleted=self.remove)
observer = Observer()
observer.schedule(event_handler, self.dirpath, recursive=True)
observer.start()
def load_dir(self, folder):
(dirpath, dirnames, filenames) = os.walk(folder).next()
for fname in filenames:
if 'json' in fname[-4:]:
self.load(os.path.join(dirpath, fname))
def load(self, fp):
'''
given a json file, load and create a task run regularly
'''
logger.info(fp + " was loaded!")
with open(fp) as in_data:
try:
data = json.load(in_data)
data['name'] = "Jenkins Job"
data['path'] = fp
except ValueError as e:
logger.warning(fp + " loaded failed: " + e.message)
return None
interval = 30
if 'interval' in data:
interval = int(data['interval'])
if 'output' in data:
if 'defaultOutputPath' in data['output']:
self.defaultOutputPath = data['output']['defaultOutputPath']
if 'dirpath' in data['output']:
data['output']['outputPath'] = os.path.join(self.defaultOutputPath, data['output']['dirpath'])
else:
data['output'] = {'outputPath': self.defaultOutputPath}
if fp in self.workers: # existing runner found
logger.info("Update exisitng runner [%s]" % fp)
runner = self.workers[fp]
runner.update(**data)
# //memo: Interval can't be modified
self.scheduler.modify_job(job_id=fp,
func=runner.run,
name=runner.name
)
else: # Create new
logger.info("Create new runner [%s]" % fp)
module_path = "hasalTask"
object_name = "HasalTask"
try:
runner_module = getattr(importlib.import_module(
module_path), object_name)
except Exception as e:
logger.exception(e)
return None
runner = runner_module(**data)
self.workers[fp] = runner
self.scheduler.add_job(runner.run, 'interval',
id=fp,
name=runner.name,
seconds=interval
)
return runner
def list(self):
'''
to list all configs loaded
format: [squence number] [minion name] [config_path] [status]
'''
for (fp, worker) in self.workers:
logger.info("path=" + fp + "," + str(worker) + ";")
def remove(self, fp):
'''
given file path, stop running instance if possible
'''
if fp in self.workers:
self.workers[fp].onstop()
self.scheduler.remove_job(job_id=fp)
del self.workers[fp]
return True
return False
def remove_advanced(self):
'''
TODO:
1. remove by start, end
2. by directory(?)
'''
pass
def unload_all(self):
'''
stop all running instances
'''
self.scheduler.shutdown()
def pause(self, fp):
'''
simply stop running instance but not remove config
TODO: should have timeout if stop failed
'''
self.scheduler.pause(job_id=fp)
def resume(self, sn):
# not sure we can do this
pass
def __del__(self):
self.unload_all()
def get_config(self):
conf = {}
return conf
def _wake(self):
'''
For periodical minions, waking them according to timing
'''
pass
class MetricsTaskManager(manager.Manager):
"""manage periodical tasks"""
RPC_API_VERSION = '1.0'
def __init__(self, service_name=None, *args, **kwargs):
super(MetricsTaskManager, self).__init__(*args, **kwargs)
scheduler = schedule_manager.SchedulerManager()
scheduler.start()
partitioner = ConsistentHashing()
partitioner.start()
partitioner.join_group()
self.watch_job_id = None
self.cleanup_job_id = None
self.group = None
self.watcher = None
self.scheduler = None
self.rpcapi = rpcapi.SubprocessAPI()
self.executor_map = {}
self.enable_sub_process = CONF.telemetry.enable_dynamic_subprocess
if self.enable_sub_process:
self.scheduler = BackgroundScheduler()
self.scheduler.start()
self.schedule_boot_jobs(self.host)
def assign_job(self, context, task_id, executor):
if not self.enable_sub_process:
instance = JobHandler.get_instance(context, task_id)
instance.schedule_job(task_id)
else:
if not self.watch_job_id:
self.init_watchers(executor)
local_executor = self.get_local_executor(context, task_id, None,
executor)
self.rpcapi.assign_job_local(context, task_id, local_executor)
def remove_job(self, context, task_id, executor):
if not self.enable_sub_process:
instance = JobHandler.get_instance(context, task_id)
instance.remove_job(task_id)
else:
job = db.task_get(context, task_id)
storage_id = job['storage_id']
for name in self.executor_map.keys():
if storage_id in self.executor_map[name]["storages"]:
local_executor = "{0}:{1}".format(executor, name)
self.rpcapi.remove_job_local(context, task_id,
local_executor)
tasks, failed_tasks = self.get_all_tasks(storage_id)
if len(failed_tasks) == 0 and len(tasks) == 0:
self.stop_executor(name, local_executor, storage_id)
def assign_failed_job(self, context, failed_task_id, executor):
if not self.enable_sub_process:
instance = FailedJobHandler.get_instance(context, failed_task_id)
instance.schedule_failed_job(failed_task_id)
else:
if not self.watch_job_id:
self.init_watchers(executor)
local_executor = self.get_local_executor(context, None,
failed_task_id, executor)
self.rpcapi.assign_failed_job_local(context, failed_task_id,
local_executor)
def remove_failed_job(self, context, failed_task_id, executor):
if not self.enable_sub_process:
instance = FailedJobHandler.get_instance(context, failed_task_id)
instance.remove_failed_job(failed_task_id)
else:
job = db.failed_task_get(context, failed_task_id)
storage_id = job['storage_id']
for name in self.executor_map.keys():
if storage_id in self.executor_map[name]["storages"]:
local_executor = "{0}:{1}".format(executor, name)
self.rpcapi.remove_failed_job_local(
context, failed_task_id, local_executor)
tasks, failed_tasks = self.get_all_tasks(storage_id)
if len(failed_tasks) == 0 and len(tasks) == 0:
self.stop_executor(name, local_executor, storage_id)
def schedule_boot_jobs(self, executor):
"""Schedule periodic collection if any task is currently assigned to
this executor """
try:
filters = {'executor': executor, 'deleted': False}
context = ctxt.get_admin_context()
tasks = db.task_get_all(context, filters=filters)
failed_tasks = db.failed_task_get_all(context, filters=filters)
LOG.info("Scheduling boot time jobs for this executor: total "
"jobs to be handled :%s" % len(tasks))
for task in tasks:
self.assign_job(context, task['id'], executor)
LOG.debug('Periodic collection job assigned for id: '
'%s ' % task['id'])
for failed_task in failed_tasks:
self.assign_failed_job(context, failed_task['id'], executor)
LOG.debug('Failed job assigned for id: '
'%s ' % failed_task['id'])
except Exception as e:
LOG.error(
"Failed to schedule boot jobs for this executor "
"reason: %s.", six.text_type(e))
else:
LOG.debug("Boot job scheduling completed.")
def init_watchers(self, group):
watcher = GroupMembership(agent_id=group)
watcher.start()
watcher.create_group(group)
LOG.info('Created child process membership group {0}.'
'Initial members of group: {1}'.format(
group, watcher.get_members(group)))
watcher.register_watcher_func(group, self.on_process_join,
self.on_process_leave)
self.group = group
self.watcher = watcher
self.watch_job_id = uuidutils.generate_uuid()
self.scheduler.add_job(
watcher.watch_group_change,
'interval',
seconds=CONF.telemetry.group_change_detect_interval,
next_run_time=datetime.datetime.now(),
id=self.watch_job_id)
LOG.info(
'Created watch for group membership change for group {0}.'.format(
group))
self.cleanup_job_id = uuidutils.generate_uuid()
self.scheduler.add_job(self.process_cleanup,
'interval',
seconds=CONF.telemetry.process_cleanup_interval,
next_run_time=datetime.datetime.now(),
id=self.cleanup_job_id)
LOG.info(
'Created process cleanup background job for group {0}.'.format(
group))
def on_process_join(self, event):
LOG.info('Member %s joined the group %s' %
(event.member_id, event.group_id))
host = event.group_id.decode('utf-8')
if self.watcher:
LOG.info('Processes in current node {0}'.format(
self.watcher.get_members(host)))
def on_process_leave(self, event):
LOG.info('Member %s left the group %s' %
(event.member_id, event.group_id))
executor_topic = event.member_id.decode('utf-8')
name = executor_topic.split(':')[1]
if name in self.executor_map.keys():
host = event.group_id.decode('utf-8')
LOG.info(
"Re-create process {0} in {1} that is handling tasks".format(
executor_topic, host))
launcher = self.create_process(executor_topic, host)
self.executor_map[name]["launcher"] = launcher
context = ctxt.get_admin_context()
for storage_id in self.executor_map[name]["storages"]:
tasks, failed_tasks = self.get_all_tasks(storage_id)
for task in tasks:
LOG.info("Re-scheduling task {0} of storage {1}".format(
task['id'], storage_id))
self.rpcapi.assign_job_local(context, task['id'],
executor_topic)
for f_task in failed_tasks:
LOG.info("Re-scheduling failed failed task {0},"
" of storage {1}".format(f_task['id'],
storage_id))
self.rpcapi.assign_failed_job_local(
context, f_task['id'], executor_topic)
def process_cleanup(self):
LOG.info('Periodic process cleanup called')
executor_names = self.executor_map.keys()
# Collect all names to delete
names_to_delete = []
for name in executor_names:
if len(self.executor_map[name]["storages"]) == 0:
delay = self.executor_map[name]["cleanup_delay"]
if delay < 0:
LOG.info(
"Cleanup delay for local executor {0} expired".format(
name))
names_to_delete.append(name)
else:
LOG.info(
"Delay cleanup for local executor {0} for {1}".format(
name, delay))
delay = delay - CONF.telemetry.process_cleanup_interval
self.executor_map[name]["cleanup_delay"] = delay
# Delete names
for name in names_to_delete:
self.executor_map[name]["launcher"].stop()
self.executor_map.pop(name)
def create_process(self, topic=None, host=None):
metrics_task_server = service. \
MetricsService.create(binary='delfin-task',
topic=topic,
host=host,
manager='delfin.'
'task_manager.'
'subprocess_manager.'
'SubprocessManager',
coordination=False)
launcher = oslo_ser.ProcessLauncher(CONF)
launcher.launch_service(metrics_task_server, workers=1)
return launcher
def get_local_executor(self, context, task_id, failed_task_id, executor):
executor_names = self.executor_map.keys()
storage_id = None
if task_id:
job = db.task_get(context, task_id)
storage_id = job['storage_id']
elif failed_task_id:
job = db.failed_task_get(context, failed_task_id)
storage_id = job['storage_id']
else:
raise exception.InvalidInput("Missing task id")
# Storage already exists
for name in executor_names:
executor_topic = "{0}:{1}".format(executor, name)
if storage_id in self.executor_map[name]["storages"]:
return executor_topic
# Return existing executor_topic
for name in executor_names:
no_of_storages = len(self.executor_map[name]["storages"])
if no_of_storages and (no_of_storages <
CONF.telemetry.max_storages_in_child):
executor_topic = "{0}:{1}".format(executor, name)
LOG.info(
"Selecting existing local executor {0} for {1}".format(
executor_topic, storage_id))
self.executor_map[name]["storages"].append(storage_id)
return executor_topic
# Return executor_topic after creating one
for index in range(CONF.telemetry.max_childs_in_node):
name = "executor_{0}".format(index + 1)
if name not in executor_names:
executor_topic = "{0}:{1}".format(executor, name)
LOG.info("Create a new local executor {0} for {1}".format(
executor_topic, storage_id))
launcher = self.create_process(topic=executor_topic,
host=executor)
self.executor_map[name] = {
"storages": [storage_id],
"launcher": launcher,
"cleanup_delay": 0
}
return executor_topic
msg = "Reached maximum number of ({0}) local executors". \
format(CONF.telemetry.max_childs_in_node)
LOG.error(msg)
raise RuntimeError(msg)
def get_all_tasks(self, storage_id):
filters = {'storage_id': storage_id, 'deleted': False}
context = ctxt.get_admin_context()
tasks = db.task_get_all(context, filters=filters)
failed_tasks = db.failed_task_get_all(context, filters=filters)
return tasks, failed_tasks
def stop_executor(self, name, local_executor, storage_id):
LOG.info("Stop and remove local executor {0}".format(local_executor))
if storage_id in self.executor_map[name]["storages"]:
self.executor_map[name]["storages"].remove(storage_id)
self.executor_map[name]["cleanup_delay"] = \
CONF.telemetry.task_cleanup_delay
def stop(self):
"""Cleanup periodic jobs"""
if self.watch_job_id:
self.scheduler.remove_job(self.watch_job_id)
if self.cleanup_job_id:
self.scheduler.remove_job(self.cleanup_job_id)
if self.group and self.watcher:
self.watcher.delete_group(self.group)
if self.watcher:
self.watcher.stop()
if self.scheduler:
self.scheduler.shutdown()
self.watch_job_id = None
self.cleanup_job_id = None
self.group = None
self.watcher = None
class Scheduler(object):
def __init__(self):
self._scheduler = BackgroundScheduler()
self._scheduler.start()
def runCron(self,
function,
minute=None,
hour=None,
day=None,
month=None,
day_week=None,
year=None,
job_id=None,
args=[],
kwargs={}):
if job_id is None:
job_id = str(function)
logging.info('adding cron job: {}'.format(str(job_id)))
self._scheduler.add_job(function,
'cron',
args=args,
kwargs=kwargs,
minute=minute,
hour=hour,
day=None,
month=month,
day_of_week=day_week,
year=year,
id=job_id,
replace_existing=True)
#TODO: Build cron class
def runSimpleWeekCron(self,
function,
minute=None,
hour=None,
days_of_week=None,
job_id=None,
args=[],
kwargs={}):
'''days_of_week takes a list of days'''
if job_id is None:
job_id = str(function)
if days_of_week is not None and days_of_week != '*':
run_days = []
for d in DAYS_OF_WEEK:
if d in days_of_week:
run_days.append(d)
days_of_week = ','.join(run_days)
logging.info('adding simple weekly cron job: {}'.format(str(job_id)))
self._scheduler.add_job(function,
'cron',
args=args,
kwargs=kwargs,
minute=minute,
hour=hour,
day='*',
month='*',
day_of_week=days_of_week,
year='*',
id=job_id,
replace_existing=True)
def runEveryS(self,
delay,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runEveryS job: {}'.format(str(job_id)))
self._scheduler.add_job(function,
'interval',
seconds=delay,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def runEveryM(self,
delay,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runEveryM job: {}'.format(str(job_id)))
self._scheduler.add_job(function,
'interval',
minutes=delay,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def runEveryH(self,
delay,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runEveryH job: {}'.format(str(job_id)))
self._scheduler.add_job(function,
'interval',
hours=delay,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def runInS(self,
delay,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runInS job: {}'.format(str(job_id)))
run_time = datetime.now() + timedelta(seconds=delay)
self._scheduler.add_job(function,
'date',
run_date=run_time,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def runInM(self,
delay,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runInM job: {}'.format(str(job_id)))
run_time = datetime.now() + timedelta(minutes=delay)
self._scheduler.add_job(function,
'date',
run_date=run_time,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def runInH(self,
delay,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runInH job: {}'.format(str(job_id)))
run_time = datetime.now() + timedelta(hours=delay)
self._scheduler.add_job(function,
'date',
run_date=run_time,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def runAt(self,
date,
function,
args=[],
kwargs={},
job_id=None,
replace=True):
if job_id is None:
job_id = str(function)
logging.info('runAt job: {} date: {}'.format(str(job_id), str(date)))
self._scheduler.add_job(function,
'date',
run_date=date,
args=args,
kwargs=kwargs,
id=job_id,
replace_existing=replace)
def cancel(self, job_id):
try:
logging.info('canceling job: {}'.format(str(job_id)))
self._scheduler.remove_job(job_id)
return True
except:
return False
class Trigger:
def __init__(self, app):
self.scheduler = None
self.app = app
def setup(self):
self.scheduler = BackgroundScheduler({
'apscheduler.jobstores.default': {
'type': 'sqlalchemy',
'url': self.app.config[
"TRIGGER_DATABASE_URL"] #os.environ.get('TRIGGER_DATABASE_URL')
},
'apscheduler.executors.processpool': {
'type': 'threadpool',
'max_workers': '30'
},
'apscheduler.job_defaults.coalesce':
'false',
'apscheduler.job_defaults.max_instances':
'4',
'apscheduler.timezone':
'UTC',
})
def start(self):
self.scheduler.start()
def is_running(self):
return self.scheduler.running()
def shutdown(self):
self.scheduler.shutdown()
def load_job_list(self):
with self.app.app_context():
projects = AutoProject.query.all()
# key_list = ("minute", "hour", "day", "month", "day_of_week")
for p in projects:
if p.enable and self.scheduler.get_job(p.id) is None:
cron = p.cron.replace("\n", "").strip().split(" ")
#print(cron)
if len(cron) < 5:
continue
j = self.scheduler.add_job(func=run_job,
trigger='cron',
name=p.name,
replace_existing=True,
minute=cron[0],
hour=cron[1],
day=cron[2],
month=cron[3],
day_of_week=cron[4],
id="%s" % p.id,
args=(p.id, ))
else:
self.update_job(p.id)
def add_job(self, func, name, id, cron):
if self.scheduler.get_job(id) is None:
self.scheduler.add_job(func=func,
trigger='cron',
name=name,
minute=cron[0],
hour=cron[1],
day=cron[2],
month=cron[3],
day_of_week=cron[4],
id="%s" % id)
def update_job(self, id):
with self.app.app_context():
p = AutoProject.query.filter_by(id=id).first()
if p.enable:
cron = p.cron.replace("\n", "").strip().split(" ")
if len(cron) < 5:
return False
print(self.scheduler.get_job(id))
if self.scheduler.get_job(id) is None:
self.scheduler.add_job(func=run_job,
trigger='cron',
name=p.name,
minute=cron[0],
hour=cron[1],
day=cron[2],
month=cron[3],
day_of_week=cron[4],
id="%s" % id,
args=(id, ))
else:
self.remove_job(id)
self.scheduler.add_job(func=run_job,
trigger='cron',
name=p.name,
minute=cron[0],
hour=cron[1],
day=cron[2],
month=cron[3],
day_of_week=cron[4],
id="%s" % id,
args=(id, ))
return True
def remove_job(self, id):
if self.scheduler.get_job(id) is not None:
self.scheduler.remove_job(id)
def pause_job(self, id):
pass
def resume_job(self, id):
pass
def get_jobs(self):
to_zone = tz.gettz("CST")
#jobs = self.scheduler.get_jobs()
urls = {
"pass": "******",
"fail": "fail.png",
"running": "run.gif",
"none": "project.png"
}
projects = AutoProject.query.order_by(AutoProject.id.desc()).all()
data = {"total": len(projects), "rows": []}
for p in projects:
next_run_time = "调度未启动"
status = "pass"
job = self.scheduler.get_job(p.id)
if job is not None:
next_run_time = job.next_run_time.astimezone(to_zone).strftime(
"%Y-%m-%d %H:%M:%S")
# 获取该job下最后一次运行状态
task = AutoTask.query.filter_by(project_id=job.id).order_by(
AutoTask.build_no.desc()).first()
if task is not None:
output_dir = os.getcwd() + "/logs/%s/%s" % (
task.project_id, task.build_no)
if os.path.exists(output_dir + "/report.html"):
tree = ET.parse(output_dir + "/output.xml")
root = tree.getroot()
# passed = root.find("./statistics/suite/stat").attrib["pass"]
fail = root.find(
"./statistics/suite/stat").attrib["fail"]
if int(fail) != 0:
status = 'fail'
else:
status = 'pass'
else:
status = "none"
data["rows"].append({
"id":
"%s" % p.id,
"name":
p.name,
"enable":
p.enable,
"status":
status,
"url":
url_for('static', filename='images/%s' % urls[status]),
"cron":
p.cron,
"next_run_time":
next_run_time
})
"""
for job in jobs:
status = "running"
task = AutoTask.query.filter_by(project_id=job.id).order_by(AutoTask.build_no.desc()).first()
if task is None:
continue
output_dir = os.getcwd() + "/logs/%s/%s" % (task.project_id, task.build_no)
if os.path.exists(output_dir + "/report.html"):
tree = ET.parse(output_dir + "/output.xml")
root = tree.getroot()
#passed = root.find("./statistics/suite/stat").attrib["pass"]
fail = root.find("./statistics/suite/stat").attrib["fail"]
if int(fail) != 0:
status = 'fail'
else:
status = 'pass'
data["rows"].append({"id": "%s" % job.id,
"name": job.name,
"status": status,
"url": url_for('static', filename='images/%s' % urls[status]),
"cron": AutoProject.query.filter_by(id=job.id).first().cron,
"next_run_time": job.next_run_time.astimezone(to_zone).strftime("%Y-%m-%d %H:%M:%S")
})
"""
return data
def print_jobs(self):
pass
class PeriodicManager(LoggingMixin):
"""
Support cron and interval config
cron: second minute hour day month day_of_week option(year)
interval: weeks,days,hours,minutes,seconds
"""
def __init__(self, mailbox: Mailbox):
super().__init__()
self.mailbox = mailbox
self.sc = BackgroundScheduler()
def start(self):
self.sc.start()
def shutdown(self):
self.sc.shutdown()
def _generate_job_id(self, run_id, task_id):
return '{}:{}'.format(run_id, task_id)
def add_task(self, run_id, task_id, periodic_config: Dict):
if 'cron' in periodic_config:
def build_cron_trigger(expr) -> CronTrigger:
cron_items = expr.split()
if len(cron_items) == 7:
return CronTrigger(second=cron_items[0],
minute=cron_items[1],
hour=cron_items[2],
day=cron_items[3],
month=cron_items[4],
day_of_week=cron_items[5],
year=cron_items[6])
elif len(cron_items) == 6:
return CronTrigger(second=cron_items[0],
minute=cron_items[1],
hour=cron_items[2],
day=cron_items[3],
month=cron_items[4],
day_of_week=cron_items[5])
else:
raise ValueError(
'The cron expression {} is incorrect format, follow the pattern: '
'second minute hour day month day_of_week optional(year).'
.format(expr))
self.sc.add_job(id=self._generate_job_id(run_id, task_id),
func=trigger_periodic_task,
args=(self.mailbox, run_id, task_id),
trigger=build_cron_trigger(
periodic_config['cron']))
elif 'interval' in periodic_config:
interval_expr: str = periodic_config['interval']
interval_items = interval_expr.split(',')
if len(interval_items) != 5:
raise ValueError(
'The interval expression {} is incorrect format, follow the pattern: '
'weeks,days,hours,minutes,seconds.'.format(interval_expr))
temp_list = []
is_zero = True
for item in interval_items:
if item is None or '' == item.strip():
v = 0
else:
v = int(item.strip())
if v < 0:
raise Exception(
'The item of interval expression must be greater than or equal to 0.'
)
if v > 0:
is_zero = False
temp_list.append(v)
if is_zero:
raise Exception('The interval config must be greater than 0.')
self.sc.add_job(id=self._generate_job_id(run_id, task_id),
func=trigger_periodic_task,
args=(self.mailbox, run_id, task_id),
trigger=IntervalTrigger(seconds=temp_list[4],
minutes=temp_list[3],
hours=temp_list[2],
days=temp_list[1],
weeks=temp_list[0]))
else:
self.log.error(
'Periodic support type cron or interval. current periodic config {}'
.format(periodic_config))
def remove_task(self, run_id, task_id):
self.sc.remove_job(job_id=self._generate_job_id(run_id, task_id))
def main():
TIMEZONE = os.getenv('TIMEZONE', 'Asia/Shanghai')
client = docker.DockerClient(base_url='unix://var/run/docker.sock',
timeout=100)
events = client.events(decode=True)
scheduler = BackgroundScheduler(
executors={'default': ThreadPoolExecutor(40)}, timezone=TIMEZONE)
logging.basicConfig(stream=sys.stdout)
logging.getLogger('apscheduler').setLevel(logging.INFO)
try:
scheduler.start()
except:
pass
# 监控services的启动、更新、关闭，实时进行定时任务的创建和修改
while True:
# 反复查询正在运行中的容器的cron.d目录，调整job列表
try:
containers = client.containers.list()
except:
continue
scheduled_jobs = scheduler.get_jobs()
hashed_scheduled_jobs = {}
for job in scheduled_jobs:
# 若存储器中的任务所属容器当前不存在，则在存储请中删除此任务
job_container = job.args[0]
job_command = job.args[1]
job_hash = job.args[2]
if job_container not in containers:
scheduler.remove_job(job_id=job.id)
else:
hashed_scheduled_jobs[job_hash] = job
for container in containers:
try:
cmd = "sh -c '[ -d /etc/cron.d ] && find /etc/cron.d ! -name \".*\" -type f -exec cat \{\} \;'"
exit_code, output = container.exec_run(cmd=cmd,
stderr=False,
tty=True)
tab = output.decode().replace('\t', ' ')
except:
continue
if tab == '':
continue
cron_jobs = CronTab(tab=tab, user=False)
job_hashes = []
for job in cron_jobs:
if not job.is_enabled():
continue
job_hash = hashsum(container.name + ': ' + str(job))
if job_hash not in hashed_scheduled_jobs:
slices = str(job.slices)
if slices.startswith('@'):
slices = SPECIALS[slices.lstrip('@')]
scheduler.add_job(lambda c, cmd, _: c.exec_run(
cmd=cmd, stderr=False, tty=True),
CronTrigger.from_crontab(slices),
args=[container, job.command, job_hash],
name=job.command)
else:
del hashed_scheduled_jobs[job_hash]
for job_hash, job in hashed_scheduled_jobs.items():
# 未命中的
scheduler.remove_job(job_id=job.id)
time.sleep(10)
