class PeriodGather(object):
def __init__(self, db):
self.__db = db
self.__scheduler = BlockingScheduler()
self.__scheduler.add_job(self.gather, 'cron', day_of_week='mon-fri', hour=16, minute=30)
def start(self):
self.__scheduler.start()
def gather(self):
_logger.info('period gather stock basic and history data, begin.....')
try:
StockBasicCollector(self.__db).collect()
stock_list = self.__get_stock_list()
for stock in stock_list:
HistDataCollector(stock, self.__db).collect()
except Exception as e:
_logger.exception(e)
_logger.info('period gather stock basic and history data, end.....')
def __get_stock_list(self):
collection = Collection(Constants.BASIC_COLLECTION, self.__db)
stock_infos = collection.find()
stock_list = []
for stock_info in stock_infos:
stock_list.append(stock_info['code'])
return stock_list
def stop(self):
if self.__scheduler:
self.__scheduler.shutdown()
class PeriodicRetrievalManager(RetrievalManager):
"""
Manages the periodic retrieval of updates.
"""
def __init__(self, retrieval_period: TimeDeltaInSecondsT, update_mapper: UpdateMapper,
logger: Logger=PythonLoggingLogger()):
"""
Constructor.
:param retrieval_period: the period that dictates the frequency at which data is retrieved
:param update_mapper: the object through which updates can be retrieved from the source
:param logger: log recorder
"""
super().__init__(update_mapper, logger)
self._retrieval_period = retrieval_period
self._running = False
self._state_lock = Lock()
self._updates_since = None # type: datetime
self._scheduler = BlockingScheduler()
self._scheduler.add_job(self._do_periodic_retrieval, "interval", seconds=self._retrieval_period, coalesce=True,
max_instances=1, next_run_time=datetime.now())
def run(self, updates_since: datetime=datetime.min):
self._updates_since = localise_to_utc(updates_since)
with self._state_lock:
if self._running:
raise RuntimeError("Already running")
self._running = True
self._scheduler.start()
def start(self, updates_since: datetime=datetime.min):
"""
Starts the periodic retriever in a new thread. Cannot start if already running.
:param updates_since: the time from which to get updates from (defaults to getting all updates).
"""
Thread(target=self.run, args=(updates_since, )).start()
def stop(self):
"""
Stops the periodic retriever.
"""
with self._state_lock:
if self._running:
self._scheduler.shutdown(wait=False)
self._running = False
logging.debug("Stopped periodic retrieval manger")
def _do_periodic_retrieval(self):
assert self._updates_since is not None
updates = self._do_retrieval(self._updates_since)
if len(updates) > 0:
# Next time, get all updates since the most recent that was received last time
self._updates_since = updates.get_most_recent()[0].timestamp
else:
# Get all updates since same time in future (not going to move since time forward to simplify things - there
# is no risk of getting duplicates as no updates in range queried previously). Therefore not changing
# `self._updates_since`.
pass
def cronjob():
scheduler = BlockingScheduler()
print "*******"
scheduler.add_job(checkupdate,'cron', second='0', hour='2',minute='0')
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
def startCrawlerTask():
from apscheduler.schedulers.blocking import BlockingScheduler
scheduler = BlockingScheduler()
scheduler.add_job(crawlerTask, 'cron', second='0',minute='15', hour='8')
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
def startschedule():
scheduler = BlockingScheduler()
scheduler.add_job(test,'cron', second='*/3', hour='*')
scheduler.add_job(dotraveljobs,'cron', second='*/3', hour='*')
scheduler.add_job(doindexjobs,'cron', second='*/3', hour='*')
scheduler.add_job(docontentjobs,'cron', second='*/3', hour='*')
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
class Worker(threading.Thread):
def __init__(self, hours=0, minutes=0, seconds=0):
super(Worker, self).__init__()
if not (hours or minutes or seconds):
raise ValueError('Hours, minutes and seconds cannot be all equal to 0')
self.scheduler = BlockingScheduler()
self.job = self.scheduler.add_job(run, 'interval', hours=hours, minutes=minutes, seconds=seconds)
def run(self):
self.scheduler.start()
def stop(self, *args):
self.scheduler.shutdown()
class AnalysisTask(object):
def __init__(self,parent,cellList,interval_min=1):
# cellList is a list of CellItem
super(AnalysisTask, self).__init__()
self.parent = parent
self.cellList = cellList
self.stage = parent.stage
self.interval_min = interval_min
self.taskOn = True
self.scheduler = BlockingScheduler()
def start(self):
self.t = threading.Thread(target=self.analyseJob,args=(None,))
self.t.start()
def stop(self):
self.taskOn = False
if self.scheduler.running:
self.scheduler.shutdown()
print("Scheduler stopped")
# self.t.join()
def analyseJob(self,s):
self.scheduler.add_job(self.analyseList, 'interval', minutes=self.interval_min)
self.analyseList()
self.scheduler.start()
def analyseList(self):
analysis_start_time = time.time()
print("Starting cell analysis")
for cellItem in self.cellList:
self.analyseCell(cellItem)
if not self.taskOn:
print("AnalysisTask stopped")
return
now = time.time()
next_analysis_time = analysis_start_time + 60*self.interval_min
waiting_time = next_analysis_time - now
next_analysis_time_str = time.strftime("%d %b %Y %H:%M:%S",time.localtime(next_analysis_time))
print("Next analysis in {} min (on {})".format(waiting_time/60,next_analysis_time_str))
def analyseCell(self,cellItem):
print("Going to cell {}".format(cellItem.cellName))
self.stage.moveAbsolute(cellItem.position,wait=True)
assert self.stage.position == cellItem.position
self.parent.saveData(cellItem.cellName)
def build_cube_from_names_or_file(buildType, cube_name=None, names_file=None, start_time=None, end_time=None, timed_build=None, crontab_options=None):
cube_name_list_from_file = []
if names_file is not None:
cube_name_list_from_file = CSVReader.get_cube_names_from_csv(names_file)
if cube_name is not None:
cube_name_list = cube_name.split(',')
cube_name_list += cube_name_list_from_file
print "building cubes for", cube_name_list
if timed_build is not None and crontab_options is not None:
scheduler = BlockingScheduler()
if timed_build is 'i' and crontab_options is not None:
scheduler.add_job(ClientJob.build, 'interval', hours=int(crontab_options),
args=[cube_name_list, buildType, start_time, end_time])
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
elif timed_build is 't' and crontab_options is not None:
time_list = crontab_options.split(',')
if time_list.__len__() == 6:
scheduler.add_job(ClientJob.build, 'date',
run_date=datetime.datetime(int(time_list[0]), int(time_list[1]),
int(time_list[2]), int(time_list[3]),
int(time_list[4]), int(time_list[5])),
args=[cube_name_list, buildType, start_time, end_time])
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
else:
print 'Bad command line!'
else:
print 'Bad command line!'
else:
status = ClientJob.build(cube_name_list, buildType, start_time, end_time)
if(status != True):
sys.exit(1)
def transferfile(results):
src_file_path = results['file_abs_path']
dest_file_path = "~/example-copy"
save_json(results, src_file_path)
auth = get_access_token()
src = "mattbest#NICHO-LENO5" # source endpoint
dst = "mattbest#Oba-Nicho5-Dell" # destination endpoint
# authenticate using access token
api = api_client.TransferAPIClient(
username=auth.username,
goauth=auth.token
)
# activate endpoints
status, message, data = api.endpoint_autoactivate(src)
status, message, data = api.endpoint_autoactivate(dst)
# get submission id
code, reason, result = api.transfer_submission_id()
submission_id = result["value"]
# designate endpoints(1) and items(2) for transfer(3)
def transfer(id, source, dest):
t = Transfer(id, source, dest)
t.add_item(src_file_path, dest_file_path)
status, reason, result = api.transfer(t)
os._exit(1)
### SCHEDULE TRANSFER JOB ###
scheduler = BlockingScheduler()
now = datetime.now()
scheduler.add_job(lambda:
transfer(submission_id, src, dst),
'date',
run_date=datetime(now.year, now.month, now.day, now.hour, now.minute+1, 0)
)
# returns 0 in the child, pid of the child in the parent
if os.fork():
sys.exit()
scheduler.start()
scheduler.shutdown()
class StockJob:
def __init__(self):
self.scheduler = BlockingScheduler()
self.pool = None
if (platform.system() == 'Windows'):
self.stockDeal = Trader_gxzq(no=TDX_USER,pwd=TDX_PWD,dimpwd=TDX_DIMPWD)
self.isopen = False
self.localData = LocalData.getInstance()
self.stockData = StockData.getInstance()
self.tdxData = TdxData.getInstance()
self.mailUtil = MailUtil.getInstance()
def add_interval_job(self,fun,days=1,start='2020-05-07 20:56:00'):
# 在 2020-05-07 20:00:00，每隔1天执行一次
self.scheduler.add_job(fun, 'interval', days=days, start_date=start)
'''
cron: 在特定时间周期性地触发：
year: 4位数字
month: 月 (1-12)
day: 天 (1-31)
week: 标准周 (1-53)
day_of_week: 周中某天 (0-6 or mon,tue,wed,thu,fri,sat,sun)
hour: 小时 (0-23)
minute:分钟 (0-59)
second: 秒 (0-59)
start_date: 最早执行时间
end_date: 最晚执行时间
timezone: 执行时间区间
'''
def add_cron_job(self,fun,month='*',day='*',day_of_week='*',hour='*',minute='*'):
self.scheduler.add_job(func=fun, trigger='cron', month=month, day=day,day_of_week=day_of_week, hour=hour, minute=minute)
def add_date_job(self,fun,date):
self.scheduler.add_job(fun, 'date', run_date=date)
def start(self):
self.scheduler.start()
def stop(self):
self.scheduler.shutdown()
# 回测
def test(self,default=True):
config.putByKey('TDX_CATEGORY','6')
if default:# 默认情况只有交易日可回测
now = time.strftime("%Y%m%d", time.localtime())
if not DEBUG and self.stockData.isOpen(now) <= 0:
return
# 判断是否交易日，才回测否则不做操作
codes = self.localData.codes(type=1)
codes = codes[codes['zs'].eq(0)] #只回测个股
for index, code in codes.iterrows():
logging.debug('%s开始更新，当前索引%s,剩余%s'%(code['code'],index,len(codes)-index-1))
try:
g57 = StockTest2(code['code'], 0, 'Test2')
g57.run()
except Exception as e:
print(e)
# 回测完成后设置股票池
self.pool = self.localData.result_report()
# 下载数据
def download(self):
config.putByKey('TDX_CATEGORY', '6')
codes = self.localData.codes()
codes = codes[codes['zs'].eq(0)]
for index,code in codes.iterrows():
try:
self.localData.data(code['code'],code['zs'])
time.sleep(0.2) # 每分钟60次限制
except Exception as e:
print(e)
# 更新股票代码、板块、关注股东初始化
def update_codes(self):
self.stockData.updateStocks()
# self.tdxData.updateBk()
self.localData.init_gdwarn()
# self.localData.update_stock_pool()
# 更新板块
def update_bks(self):
self.tdxData.updateBk()
# 更新股东
def update_gds(self):
self.tdxData.updateGDs()
# 更新分红
def update_fhs(self):
self.tdxData.updateFhYears()
# 打开交易软件
def open_tdx(self):
now = time.strftime("%Y%m%d", time.localtime())
if not DEBUG and self.stockData.isOpen(now) <= 0:
return
logging.debug('软件开始登陆。。。。')
self.stockDeal.login()
time.sleep(10)
self.stockDeal.click_tree_item('买入')
# 登录成功后，休眠5秒获取持仓信息
time.sleep(3)
results, orders = self.stockDeal.orders()
self.les_money = float(results['可用'])
# 可买入的股票，保存在数据库表中(月线为卖出时删除该自选，在关闭时合并股票到表中，如果表中没有股票从当月可操作股票中随机获取)
# 获取可买入数量，可买入的股票池，每只买入金额，可卖出的股票池
nums,buy_data,level_money,seal_data = self.localData.get_buy_stocks(self.les_money)
self.level_money = level_money
self.seal_data = seal_data
self.buy_pools = {} # 讲买卖股票池改为{code:'',num:''}形式
if len(buy_data)>0:
for i in range(nums):
self.buy_pools[buy_data[i]]=0
# 根据持仓查询对应的月回测记录，讲交易状态放入缓存
self.seal_pools = {} # 可卖出的股票
try:
for i in range(len(orders)):
self.seal_pools[orders[i][0]]=orders[i][4]
self.isopen = True
logging.debug('软件开始登陆成功。\n可用资金%s,买入股票池%s,持仓%s'%(self.les_money,str(self.buy_pools),str(self.seal_pools)))
except Exception as e:
logging.debug('获取持仓失败%s'%e)
# 关闭交易软件
def close_tdx(self):
now = time.strftime("%Y%m%d", time.localtime())
if not DEBUG and self.stockData.isOpen(now) <= 0:
return
logging.debug('软件即将关闭。。。。')
try:
# 当日成交,如果有成交记录发送邮件通知
dayorders = self.stockDeal.day_orders()
# 关闭交易软件
self.stockDeal.destory()
# 根据当日成交更新表成交
self.localData.update_stock_pool(dayorders,handle=1)
self.mailUtil.sendEmail([MAIL_USER], '当日成交', self.localData.get_order_msg(dayorders))
logging.debug('软件已关闭.当日成交明细：\n%s'%("\n".join('%s' % id for id in dayorders)))
except Exception as e:
pass
self.isopen = False
# 个股预警交易
def update_datas(self):
now = time.strftime("%Y%m%d", time.localtime())
if not DEBUG and self.stockData.isOpen(now) <= 0:
return
if not DEBUG and not self.localData.is_deal_time():
return
if not self.isopen:
self.close_tdx()
self.open_tdx()
# 设置初始数据
config.putByKey('TDX_CATEGORY','9')
pools = {}
pools.update(self.buy_pools)
pools.update(self.seal_pools)
pools_data = []
for code,num in pools.items():
# 更新数据
df = self.localData.data(code,0)
current_price = df.tail(1)['close'].values[0]
# 回测
g57 = StockTest3(code, 0, 'Test3',data=df)
order,result,msgs = g57.run()
# 解析回测结果:日期','类型','价格','手数','成交额','扣除费
# base = self.localData.get_base(code, 0)
arr = order.split('\t')
if arr[0] != now: # 当天买组交易
continue
# 手数需要重新计算
if arr[1]=='1' :# 买入
self.stockDeal.click_toolbar('买入')
logging.debug('买入：%s'%order)
if len(self.buy_pools)>0:# 当月线为买入状态方可交易
num = int(self.level_money[code] / current_price / 100) * 100
# 根据编号
# self.stockDeal.buy(code=code,type=base['type'],price=current_price,num=num)
self.buy_pools.pop(code,'404') # 买入后移出股票池
pools_data.append({'code': code, 'state': 1, 'num': num})
logging.info('买入成功：代码：%s，价格：%s，数量：%s' % (code,current_price,num))
else:
logging.error('买入失败，买入股票池%s,待买入%s,价格%s'%(self.buy_pools,code,current_price))
self.mailUtil.sendEmail([MAIL_USER], '交易失败，请及时关注', '买入失败%s'%order)
elif arr[1]=='0':# 卖出
self.stockDeal.click_toolbar('卖出')
logging.debug('卖出：%s' % order)
num = self.seal_pools.get(code,0)
if self.seal_pools.get(code,0) ==0:
err_msg = '卖出失败，代码：%s，当前价格：%s，数量：%s' % (code,current_price,num)
logging.error(err_msg)
self.mailUtil.sendEmail([MAIL_USER], '交易失败，请及时关注', err_msg)
continue
# self.stockDeal.seal(code=code,type=base['type'],price=current_price,num=num)
pools_data.append({'code': code, 'state': 0, 'num': 0})
self.seal_pools.pop(code,'404')
logging.info('卖出成功：代码：%s，价格：%s，数量：%s' % (code, current_price, num))
# 如果月线为买入状态，需加入到self.pools第一个位置
if self.seal_data.get(code,0)>0:
self.buy_pools[code] = self.stockDeal.get_money(current_price,num)
else:
logging.error('卖出池%s未找到%s'%(str(self.seal_data,code)))
@classmethod
def getInstance(cls):
if not hasattr(StockJob, "_instance"):
StockJob._instance = StockJob()
return StockJob._instance
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Process BF2 manually or \
through scheduler.')
parser.add_argument("-s", "--sched", action="store_true", dest="sched",
default=False, help="use scheduler to run")
parser.add_argument("-m", "--man", action="store_true", dest="man",
default=False, help="run manually")
parser.add_argument("-r", "--repo", dest="repo",
default="/Users/Christina/Projects/BF2",
help="repo working directory")
args = parser.parse_args()
if os.path.isfile('.git/config.lock'):
os.remove('.git/config.lock')
repo = Repo(args.repo)
config = repo.config_writer()
config.set_value("user", "email", "*****@*****.**")
config.set_value("user", "name", "cmh2166")
index = repo.index
if not args.sched and not args.man:
parser.print_help()
exit()
elif args.man:
main()
elif args.sched:
sched.start()
sched.shutdown(wait=True)
class MyScheduler:
def __init__(self):
self._scheduler = BlockingScheduler()
self._hue = HueWrapper()
self._enviro = EnviroWrapper()
self._kasa = KasaWrapper()
self._bright = 0
self._init()
self._jobs_list = [
self._manage_heater,
self._manage_lights,
]
self._jobs_cycle = itertools.cycle(self._jobs_list)
self.heater_on_for = 0
self.heater_off_for = 0
self.switch_on = False
def _init(self):
logging.basicConfig(level=logging.INFO)
logging.getLogger("apscheduler.scheduler").addFilter(MuteFilter())
self._hue.connect()
self._scheduler.add_job(func=self._get_sunset_sunrise, trigger=at_midnight)
self._scheduler.add_job(func=self._get_sunset_sunrise)
self._scheduler.add_job(self._manage_next, trigger=every_thirty_seconds)
self._scheduler.add_job(self._manage_all)
def start(self):
self._scheduler.print_jobs()
self._scheduler.start()
def stop(self):
self._scheduler.shutdown()
# if self._hue:
# self._hue.off()
def _manage_all(self):
for job in self._jobs_list:
job()
time.sleep(5)
def _manage_next(self):
job = next(self._jobs_cycle)
job()
def _manage_lights(self):
if self._hue.is_on:
self._hue.do_whatever()
def _manage_heater(self):
now = datetime.datetime.now()
weekday = now.weekday()
hour = now.hour
mins = now.minute
month = now.month
monday = 0
friday = 4
on_holiday = False
in_work_hours = not on_holiday \
and monday <= weekday <= friday \
and 8 <= hour <= 16 \
and (hour != 16 or mins <= 30)
is_spring = 3 <= month <= 5
is_summer = 6 <= month <= 8
is_autumn = 9 <= month <= 11
is_winter = month == 12 or month <= 2
is_morning = 0 <= hour <= 12
# is_early_morning = 0 <= hour <= 8
logging.info('weekday: {} hour: {} in_work_hours: {}'.format(weekday, hour, in_work_hours))
temperature = self._enviro.get_temperature()
logging.info('temperature: {}'.format(temperature))
if is_spring:
target_temperature = 18.0
elif is_summer:
target_temperature = 10.0
elif is_autumn:
target_temperature = 18.0
elif is_winter:
target_temperature = 18.0
else:
target_temperature = 18.0
if is_winter:
# if is_early_morning:
# target_temperature += 1
if is_morning:
target_temperature += 1
is_on = self._kasa.is_on()
logging.info('is_on: {0}'.format(is_on))
if is_on:
self.heater_on_for += 1
self.heater_off_for = 0
else:
self.heater_on_for = 0
self.heater_off_for += 1
logging.info('heater_on_for: {0}'.format(self.heater_on_for))
logging.info('heater_off_for: {0}'.format(self.heater_off_for))
cooler_thx = temperature > target_temperature or self.heater_on_for > 5
warmer_plz = in_work_hours and temperature < target_temperature - 2
if cooler_thx:
logging.info('cooler_thx')
self.switch_on = False
elif warmer_plz:
logging.info('warmer_plz')
if self.heater_off_for > 1:
logging.info('Duty cycle on')
self.switch_on = True
logging.info('switch_on: {0}'.format(self.switch_on))
if self.switch_on:
logging.info('Switching heater on')
self._kasa.switch_on()
elif cooler_thx:
logging.info('Switching heater off')
self._kasa.switch_off()
def _get_sunset_sunrise(self):
a = Astral()
leeds = a['Leeds']
today = datetime.date.today()
self._today_sun_data = leeds.sun(date=today, local=True)
self.timezone = leeds.timezone
logging.info(pprint.pformat(self._today_sun_data))
self.dawn = self._today_sun_data['dawn']
self.sunrise = self._today_sun_data['sunrise']
self.sunset = self._today_sun_data['sunset']
self.dusk = self._today_sun_data['dusk']
at_dawn = _get_cron_trigger_for_datetime(self.dawn)
at_sunrise = _get_cron_trigger_for_datetime(self.sunrise)
at_sunset = _get_cron_trigger_for_datetime(self.sunset)
at_dusk = _get_cron_trigger_for_datetime(self.dusk)
during_sunrise = IntervalTrigger(seconds=5, start_date=self.dawn, end_date=self.sunrise)
during_sunset = IntervalTrigger(seconds=5, start_date=self.sunset, end_date=self.dusk)
self._scheduler.add_job(func=self._at_dawn, trigger=at_dawn)
self._scheduler.add_job(func=self._during_sunrise, trigger=during_sunrise)
self._scheduler.add_job(func=self._at_sunrise, trigger=at_sunrise)
self._scheduler.add_job(func=self._at_sunset, trigger=at_sunset)
self._scheduler.add_job(func=self._during_sunset, trigger=during_sunset)
self._scheduler.add_job(func=self._at_dusk, trigger=at_dusk)
now = datetime.datetime.now(tz)
if now <= self.dawn:
day_factor = 0.0
elif self.dawn < now <= self.sunrise:
day_factor = colour_helper.get_day_factor(self.dawn, now, self.sunrise, True)
elif self.sunrise < now <= self.sunset:
day_factor = 1.0
elif self.sunset < now <= self.dusk:
day_factor = colour_helper.get_day_factor(self.sunset, now, self.dusk, False)
elif now < self.dusk:
day_factor = 0.0
else:
day_factor = 0.25
self._set_day_factor(day_factor)
def _at_dawn(self):
day_factor = 0.0
self._set_day_factor(day_factor)
logging.info('dawn')
def _at_sunrise(self):
day_factor = 1.0
self._set_day_factor(day_factor)
logging.info('sunrise')
def _at_sunset(self):
day_factor = 1.0
self._set_day_factor(day_factor)
logging.info('sunset')
def _at_dusk(self):
day_factor = 0.0
self._set_day_factor(day_factor)
logging.info('dusk')
def _during_sunrise(self):
day_factor = colour_helper.get_day_factor(self.dawn, datetime.datetime.now(tz), self.sunrise, True)
self._set_day_factor(day_factor)
def _during_sunset(self):
day_factor = colour_helper.get_day_factor(self.sunset, datetime.datetime.now(tz), self.dusk, False)
self._set_day_factor(day_factor)
def _set_day_factor(self, day_factor):
logging.info('day factor: {}'.format(day_factor))
colour_helper.set_day_factor(day_factor)
class Controller:
def __init__(self, configs):
self.main_loop_lock = False
self.scheduler = None
self.configs = configs
# self.installer = None
self.agent_holder = None
self.need_download = False
self.need_install = False
self.need_start = False
self.need_version_recheck = False
self.download_path = None
self.checksum = None
self.ssl_context = ssl.create_default_context()
self.recycle_queue = Queue.Queue()
# self.block_count = 0
def start(self):
LOG.debug('controller started')
recycle_thread = ProcessRecycle(self.recycle_queue)
recycle_thread.start()
self.scheduler = None
if os.path.isfile('/etc/timezone'):
with open('/etc/timezone', 'a+') as f:
tz = f.read().strip()
self.scheduler = BlockingScheduler(timezone=tz)
if not self.scheduler:
self.scheduler = BlockingScheduler()
self.scheduler.add_job(self.update_check,
trigger='interval',
seconds=self.configs['update_check_period'],
next_run_time=datetime.datetime.now())
self.scheduler.add_job(self.main_job,
trigger='cron',
minute='*/%d' %
self.configs['main_loop_period_minute'],
second='0')
self.scheduler.start()
LOG.debug('controller stopped')
def stop(self):
self.stop_agent()
if self.scheduler:
self.scheduler.shutdown()
def main_job(self):
if self.main_loop_lock:
# if self.block_count >= 3:
# self.terminate_installer()
# self.block_count = 0
return
self.main_loop_lock = True
# self.block_count = 0
self.health_check()
if self.need_version_recheck:
self.version_recheck()
try:
if self.need_download:
LOG.info('********************* download')
if self.download_agent():
self.need_download = False
self.need_install = True
if self.need_install:
LOG.info('********************* install')
old_folder = self.get_latest_install_folder()
update = self.need_install_check()
# reinstall current agent package
if not update:
self.stop_agent()
self.uninstall_agent(old_folder)
package_path = self.get_latest_install_package()
if package_path:
agent_dir = self.install_agent(package_path)
# new agent installed, remove old agent folder
if update:
self.stop_agent()
self.uninstall_agent(old_folder)
self.start_agent(agent_dir)
self.need_install = False
self.need_start = False
elif self.need_start:
LOG.info('********************* restart')
self.need_start = False
self.stop_agent()
agent_dir = self.get_latest_install_folder()
if not agent_dir:
package_path = self.get_latest_install_package()
if package_path:
LOG.debug('********************* restart install')
agent_dir = self.install_agent(package_path)
else:
return
LOG.debug('********************* restart start')
self.start_agent(agent_dir)
except Exception as e:
pass
finally:
self.main_loop_lock = False
def install_agent(self, package_path):
checksum = utils.get_checksum(package_path, self.get_checksum_dir())
if not utils.check_file(package_path, checksum):
LOG.debug("check downloaded package failed, removing %s",
package_path)
os.remove(package_path)
utils.remove_checksum(package_path, self.get_checksum_dir())
return None
dirname = utils.get_file_name_without_extension(package_path)
agent_dir = '%s/%s' % (self.get_install_dir(), dirname)
r = os.system('mkdir -p %s && tar xzf %s -C %s --strip-components 1' %
(agent_dir, package_path, agent_dir)) != 0
if r != 0:
LOG.debug('decompress failed, %d', r)
os.remove(package_path)
os.rmdir(agent_dir)
return None
else:
LOG.debug('decompress success')
installer = Installer(agent_dir, self.recycle_queue)
installer.install()
return agent_dir
def uninstall_agent(self, agent_dir=None):
# if self.installer is None:
# if agent_dir and os.path.isdir(agent_dir):
# self.installer = Installer(agent_dir, self.recycle_queue)
# else:
# return
# self.installer.uninstall()
# self.installer = None
if agent_dir and os.path.isdir(agent_dir):
installer = Installer(agent_dir, self.recycle_queue)
installer.uninstall()
# def terminate_installer(self):
# if self.installer:
# self.installer.terminate()
# self.installer = None
def start_agent(self, agent_dir):
if agent_dir:
self.agent_holder = AgentHolder(agent_dir, self.recycle_queue)
agent_process_name = self.agent_holder.start()
self.save_process_name_to_disk(agent_process_name)
# self.agent_abnormal = False
def stop_agent(self):
if self.agent_holder is None:
agent_process_name = self.get_process_name_from_disk()
if agent_process_name:
os.popen('killall -g %s' % agent_process_name)
self.save_process_name_to_disk()
else:
self.agent_holder.stop()
self.agent_holder = None
self.save_process_name_to_disk()
def health_check(self):
if self.agent_holder:
LOG.info('.............. last check before: %d seconds' %
(int(time.time()) - self.agent_holder.last_check_time))
if int(time.time()
) - self.agent_holder.last_check_time > self.configs[
'health_check_threshold']:
LOG.info('.............. check health, failed')
self.need_install = True
else:
self.need_start = False
else:
LOG.info('.............. agent_holder is None')
self.need_start = True
def update_check(self):
if self.check_new_version():
self.need_download = True
else:
self.need_download = False
if self.need_install_check():
LOG.info('a later version downloaded, need install it')
self.need_install = True
def need_install_check(self):
download_version = int(
utils.get_version_from_path(self.get_latest_install_package()))
install_version = int(
utils.get_version_from_path(self.get_latest_install_folder()))
if download_version > install_version:
return True
else:
return False
def version_recheck(self):
self.need_version_recheck = False
if self.check_new_version():
self.need_download = True
else:
self.need_download = False
def check_new_version(self):
try:
url = '%s%s' % (self.configs['update_server_url'],
self.configs['update_info_path'])
req = urllib2.Request(url)
response = urllib2.urlopen(req,
timeout=3,
context=self.ssl_context)
response_json = json.loads(response.read())
new_version = response_json['version']
new_path = response_json['path']
checksum = response_json['checksum']
# print new_version
# print new_path
version = int(
utils.get_version_from_path(self.get_latest_install_package()))
if new_version > version:
self.download_path = new_path
self.checksum = checksum
LOG.info('check new version: new version valid')
return True
else:
LOG.info('check new version: it is the latest version')
return False
except Exception as e:
self.need_version_recheck = True
LOG.exception('check new version failed, err: %s', e)
return False
def download_agent(self):
if self.download_path and self.checksum:
url = '%s/%s' % (self.configs['update_server_url'],
self.download_path)
f = urllib2.urlopen(url, timeout=300, context=self.ssl_context)
package_name = utils.get_file_name(url)
package_path = '%s/%s' % (self.get_download_dir(), package_name)
with open(package_path, 'wb') as target:
target.write(f.read())
if not utils.check_file(package_path, self.checksum):
os.remove(package_path)
LOG.debug('checksum failed, removing %s', package_name)
return False
else:
utils.save_checksum(self.checksum, package_path,
self.get_checksum_dir())
LOG.debug('download %s success', package_name)
self.download_path = None
self.checksum = None
return True
def get_install_dir(self):
path = os.path.join(self.configs['home_dir'], 'install_folders')
if not os.path.isdir(path):
os.mkdir(path)
return path
def get_download_dir(self):
path = os.path.join(self.configs['home_dir'], 'download')
if not os.path.isdir(path):
os.mkdir(path)
return path
def get_checksum_dir(self):
path = os.path.join(self.configs['home_dir'], 'checksum')
if not os.path.isdir(path):
os.mkdir(path)
return path
def get_latest_install_folder(self):
latest_version = 0
latest_folder_path = None
if not os.path.isdir(self.get_install_dir()):
return None
for folder in os.listdir(self.get_install_dir()):
version = utils.get_version(folder)
if version.isdigit() and int(version) > latest_version:
latest_version = int(version)
latest_folder_path = folder
if latest_folder_path:
return '%s/%s' % (self.get_install_dir(), latest_folder_path)
else:
return None
def get_latest_install_package(self):
latest_version = 0
latest_package = None
if not os.path.isdir(self.get_download_dir()):
return None
for package in os.listdir(self.get_download_dir()):
version = utils.get_version(package)
if version.isdigit() and int(version) > latest_version:
latest_version = int(version)
latest_package = package
if latest_package:
return '%s/%s' % (self.get_download_dir(), latest_package)
else:
return None
# for debug
def get_tmp_dir(self):
return os.path.join(self.configs['home_dir'], 'tmp')
def save_process_name_to_disk(self, process_name=''):
with open('%s/process_name' % self.configs['home_dir'], 'w+') as f:
f.truncate()
f.write(process_name)
def get_process_name_from_disk(self):
with open('%s/process_name' % self.configs['home_dir'], 'a+') as f:
return f.read()
class scheduler():
"""This Class provide the main scheduler, it consumes the config
and schedules the drivers to run for their respective metrics"""
def __init__(self, loggingLevel=logging.ERROR):
self.logger = logging.getLogger('sensipy')
self.logger.setLevel(loggingLevel)
fh = logging.FileHandler('error.log')
fh.setLevel(logging.ERROR)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
self.logger.addHandler(fh)
self.logger.addHandler(ch)
self.sender = Sender()
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
job_defaults = {
'coalesce': False,
'max_instances': 3
}
self.queue = Queue(connection=Redis())
self.scheduler = BlockingScheduler(executors=executors, job_defaults=job_defaults)
self.configFile = 'config.json'
def start(self):
"""Runs all the initialization methods, then starts the
scheduler in standalone (blocking) mode"""
self.loadConf()
self.loadDrivers()
self.loadFeeds()
self.runScheduler()
self.scheduler.print_jobs()
self.scheduler.start()
self.printConf("test")
print("scheduler started")
def stop(self):
print("test")
self.scheduler.shutdown()
def loadDrivers(self):
"""Iterates through the Driver instances stored in the config,
and loads corresponding instances into the driver dict"""
self.sources = {}
for source in self.config['sources']:
sourceConf = self.config['sources'][source]
baseClass = sourceConf['baseClass']
self.logger.debug("Loading: " + source +
" instance of: " + baseClass)
sourceArgs = sourceConf['source-config']
self.sources[source] = {}
try:
print(baseClass)
tempModule = import_module('sources.' + baseClass)
"""tempModule = __import__('sources.' + baseClass,
globals(), locals(), [baseClass], -1)
"""
self.sources[source]['source'] = getattr(tempModule, str(
baseClass))(sourceArgs)
except Exception as e:
self.logger.error("exception: " + str(e))
return None
def loadConf(self):
"""Retrieves config from file specified in __init__"""
with open(self.configFile) as f:
self.config = json.load(f)
def printConf(self, args):
print(args)
print(self.config)
def loadFeeds(self):
"""Sets up each metric in it's corresponding driver instance nice name
"""
metrics = self.config['metrics']
for metric in metrics:
metricConf = self.config['metrics'][metric]
metricConf['name'] = metric
source = metricConf['source']['driver']
if 'metrics' not in self.sources[source['name']]:
self.sources[source['name']]['metrics'] = []
self.sources[source['name']]['metrics'].append(metricConf)
def runScheduler(self):
"""Sets up base scheduler interval for each configured
driver instance"""
for source in self.sources:
intervals = [
int(self.sources[source]['metrics'][x]['interval']) for x
in range(0, len(self.sources[source]['metrics']))]
sourceInterval = self.gcd(intervals)
self.sources[source]['sourceInterval'] = sourceInterval
self.logger.debug(self.sources[source]['metrics'])
self.scheduler.add_job(
self.getDriverData, 'interval', args=[
self.sources[source]['metrics']],
seconds=sourceInterval)
def getDriverData(self, metricSet):
"""Gets data from a single driver instance, on the intervals in
each metrics config, data is put on the queue with all information
needed to send to service"""
driverNiceName = metricSet[0]['source']['driver']['name']
if 'driverCounter' not in self.drivers[driverNiceName]:
self.drivers[driverNiceName]['driverCounter'] = self.drivers[
driverNiceName]['driverInterval']
else:
self.drivers[driverNiceName]['driverCounter'] += self.drivers[
driverNiceName]['driverInterval']
for metric in metricSet:
count = self.drivers[driverNiceName]['driverCounter']
metricInterval = int(metric['interval'])
if count % metricInterval == 0:
metricId = metric['id']
value = self.drivers[driverNiceName]['driver'].getData(metric)
dt = datetime.utcnow()
self.queue.enqueue(
self.sender.send_metric, metricId, value, dt)
def gcd(self, nums):
"""Recursively computes Greatest Common Divisor for a list of
numbers, used to compute the base scheduler interval for a
given set of metric intervals"""
if len(nums) == 1:
return nums[0]
if len(nums) == 0:
return None
if len(nums) >= 2:
a = nums[-1:][0]
b = nums[-2:-1][0]
while b:
a, b = b, a % b
nums = nums[:-2]
nums.append(a)
return self.gcd(nums)
def showConf(self):
"""Debug method to ensure config is loading correctly, and to
pretty print a config to clean up one with miffed formatting"""
return json.dumps(
self.config, sort_keys=True, indent=4, separators=(',', ': '))
class DakokuManager(object):
def __init__(self, config_path, schedule_path):
self.config_path = config_path
self.schedule_path = schedule_path
cfg = self._load_config()
try:
human_mode_min = cfg["human_mode"]
except:
human_mode_min = 0
try:
self.log_dir = cfg["log_dir"]
file_handler = logging.FileHandler(os.path.join(self.log_dir, "dakoku.log"), 'a+')
file_handler.level = LEVEL
log.addHandler(file_handler)
log.info("saving log to %s", self.log_dir)
except:
self.log_dir = None
sched = self._load_schedule()
start_date = dt.datetime.strptime(sched["valid"]["start"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
end_date = dt.datetime.strptime(sched["valid"]["end"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
log.info("dakoku is valid for %s - %s", start_date, end_date)
holidays = []
if "api_key" in cfg:
try:
holidays = self._get_holidays(cfg["api_key"], start_date, end_date)
log.info("holidays are skipped: %s", holidays)
except Exception as e:
log.error("failed to import holidays: %s", e)
else:
log.warn("No api_key specified. No holiday is registered!")
self.worker = DakokuWorker(cfg["host"], cfg["user"], cfg["pass"], holidays, self.log_dir)
self.register(sched["working"], start_date, end_date, holidays, human_mode_min)
def _load_config(self):
log.debug("loading config from %s", self.config_path)
with open(self.config_path, 'r') as f:
cfg = json.load(f)
return cfg
def _load_schedule(self):
log.debug("loading schedule from %s", self.schedule_path)
with open(self.schedule_path, 'r') as f:
cfg = json.load(f)
return cfg
def _get_holidays(self, api_key, start_date, end_date):
srv = Google(serviceName='calendar',
version='v3',
developerKey=api_key)
calendar_id = '*****@*****.**'
cal = srv.events().list(calendarId=calendar_id,
maxResults=30,
orderBy="startTime",
singleEvents=True,
timeMin=start_date.strftime('%Y-%m-%dT%H:%M:%SZ'),
timeMax=end_date.strftime('%Y-%m-%dT%H:%M:%SZ')).execute()
return [dt.datetime.strptime(e["start"]["date"], '%Y-%m-%d').replace(tzinfo=pytz.timezone("Asia/Tokyo")) for e in cal["items"]]
def register(self, working, start_date, end_date, holidays, human_mode_min=0):
self.scheduler = Scheduler(timezone=pytz.timezone('Asia/Tokyo'), logger=log)
today = dt.date.today()
for w in working:
# schedule shukkin
h, m = map(int, w["from"].split(':'))
fromtime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
d = dt.datetime.combine(today, fromtime) - dt.timedelta(minutes=human_mode_min)
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=d.hour, minute=d.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_start),
trigger)
# schedule taikin
h, m = map(int, w["till"].split(':'))
tilltime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=tilltime.hour, minute=tilltime.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_end),
trigger)
self.scheduler.print_jobs()
def start(self):
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
]
for s in servers:
url = s + today + '.txt'
logger.info(url)
try:
html = requests.get(url).content.splitlines()
except:
logger.error('http get failed.')
continue
logger.info('total: ' + str(len(html)))
hashes = set()
for h in html:
h = h.strip().lower()
if not h:
continue
hashes.add(h)
if hashes:
save(hashes)
logger.info('over')
atexit.register(lambda: sched.shutdown(wait=False))
if __name__ == '__main__':
sched.start()
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 3 17:03:38 2021
@author: S. Hiram Rubio
Script principal
"""
#Run de code / Correr código
if __name__ == "__main__":
#Libs / librerias
from apscheduler.schedulers.blocking import BlockingScheduler
from BTVis import generate_graph
# / Generar gráfica
generate_graph(FLAG=True, DELETE=False)
#Scheduler / planificador
#Creation and job initialization / Creacion e inicio de trabajo
scheduler = BlockingScheduler()
job = scheduler.add_job(lambda: generate_graph(DELETE=False),
'interval',
hours=2.22)
scheduler.start()
scheduler.print_jobs()
if (False): scheduler.shutdown()
class JobScheduler(object):
def __init__(self, every=30, unit='second'):
self.mongo = mongopool.get()
self.cursor = self.mongo.get_database('apscheduler').get_collection(
'jobs')
self.every = every
self.unit = unit
self.scheduler = BlockingScheduler(logger=logger)
self.scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=pytz.timezone('Asia/Saigon'))
self._set_trigger(every, unit)
def _set_trigger(self, every, unit):
now = datetime.now().astimezone(pytz.timezone('Asia/Saigon'))
if unit == 'second':
self.trigger = CronTrigger(second='*/{}'.format(every),
start_date=now)
elif unit == 'minute':
self.trigger = CronTrigger(minute='*/{}'.format(every),
start_date=now)
elif unit == 'hour':
self.trigger = CronTrigger(hour='*/{}'.format(every),
start_date=now)
elif unit == 'day':
self.trigger = CronTrigger(day='*/{}'.format(every),
start_date=now)
else:
raise Exception(message='Unknown time unit')
def add_jobstore(self, jobstore, alias):
self.scheduler.add_jobstore(jobstore, alias)
def add_executor(self, executor, alias):
self.scheduler.add_executor(executor, alias)
def add_job(self,
job_fn,
id='id1',
name='job1',
jobstore='default',
executor='default',
args=None,
kwargs=None):
now = datetime.now().astimezone(pytz.timezone('Asia/Saigon'))
history = list(self.cursor.find({'_id': id}))
if history:
#TODO: process missing jobs
self.cursor.delete_one({'_id': id})
next_run_time = self.trigger.get_next_fire_time(None, now)
if kwargs:
kwargs['run_time'] = next_run_time
else:
kwargs = {'run_time': next_run_time}
self.scheduler.add_job(job_fn,
trigger=self.trigger,
next_run_time=next_run_time,
id=id,
name=name,
jobstore=jobstore,
executor=executor,
args=args,
kwargs=kwargs)
def remove_job(self, id, jobstore='default'):
self.scheduler.remove_job(job_id=id, jobstore=jobstore)
def callback(self, callback_fn, mark=EVENT_ALL):
self.scheduler.add_listener(callback_fn)
def start(self):
mongopool.put(self.mongo)
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
self.scheduler.scheduled_job
if options.password == '' or options.username == '':
print u'请输入用户名密码'
exit(1)
'''
# longin
jd.login_by_QR()
good_data = jd.good_detail(options.good, options.area)
#print(good_data)
# time task
sched = BlockingScheduler()
if len(good_data['link']) >10 and jd.pre_add_cart(good_data):
# 快捷抢购，预先添加购物车
print("预添加成功！快捷抢购中...")
sched.add_job(lite_Q,'date', run_date=options.time)
else:
# 无法预添加购物车则进行常规抢购
print("预添加失败！常规抢购中...")
sched.add_job(reg_Q,'date', run_date=options.time, args=[good_data])
#sched.print_jobs()
print('Press Ctrl+{0} to exit'.format('Pause(Break)' if os.name == 'nt' else 'C'))
try:
sched.start()
except (KeyboardInterrupt, SystemExit):
sched.shutdown(wait=False)
print('Exit The Job!')
else:
sched.shutdown(wait=True)
'default': ThreadPoolExecutor(10),
'processpool': ProcessPoolExecutor(3),
}
def my_job():
cur_day = datetime.now().strftime('%Y-%m-%d')
cur_time = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
report_path = '../Fanli/Report/' + cur_day + '/'
path = '../Fanli/'
result = os.system('nosetests ' + path + ' --with-html --html-report=' +
report_path + 'fanli_report_' + cur_time + '.html ')
if result != 0:
files = os.listdir(report_path)
file = report_path + files[-1]
mail.send_mail(file, 'getResouces接口Fail')
def create_json():
CreateFile().create_file('getResource')
s = BlockingScheduler(executors=executors)
s.add_job(my_job, 'interval', seconds=5)
s.add_job(create_json, 'interval', seconds=5)
try:
s.start()
except (SystemExit, KeyboardInterrupt):
s.shutdown()
class ApsPlanner(BasePlanner):
"""
Planner implementing scheduling using the |APS|_. Scheduling sets the :any:`APS Job <apscheduler.job.Job>` as links' job.
.. |APS| replace:: Advanced Python Scheduler
.. _APS: https://apscheduler.readthedocs.io/en/stable/index.html
.. _configuring-scheduler: https://apscheduler.readthedocs.io/en/stable/userguide.html#configuring-the-scheduler
"""
def __init__(self,
links: Union[Link, List[Link]] = None,
threads: int = 30,
executors_override: dict = None,
job_defaults_override: dict = None,
ignore_exceptions: bool = False,
catch_exceptions: bool = None):
"""
:type links: :any:`Link` or list[:any:`Link`]
:param links: Links that should be added and scheduled.
|default| :code:`None`
:type threads: int
:param threads: Number of threads available for job execution. Each link will be run on a separate thread job.
|default| :code:`30`
:type executors_override: dict
:param executors_override: Overrides for executors option of `APS configuration <configuring-scheduler_>`__
|default| :code:`None`
:type job_defaults_override: dict
:param job_defaults_override: Overrides for job_defaults option of `APS configuration <configuring-scheduler_>`__
|default| :code:`None`
:type ignore_exceptions: bool
:param ignore_exceptions: Whether exceptions should be ignored or halt the planner.
|default| :code:`False`
"""
self._threads = threads
self._ignore_exceptions = ignore_exceptions
if catch_exceptions is not None: # pragma: no cover
self._ignore_exceptions = catch_exceptions
warnings.warn(
'\'catch_exceptions\' was renamed to \'ignore_exceptions\' in version 0.2.0 and will be permanently changed in version 1.0.0',
DeprecationWarning)
if executors_override is None:
executors_override = {}
if job_defaults_override is None:
job_defaults_override = {}
executors = {
'default': ThreadPoolExecutor(threads),
**executors_override
}
job_defaults = {
'coalesce': False,
'max_instances': threads,
**job_defaults_override
}
self._scheduler = BlockingScheduler(executors=executors,
job_defaults=job_defaults,
timezone='UTC')
# self._scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults, timezone=utc)
self._scheduler.add_listener(self._on_exception, EVENT_JOB_ERROR)
super().__init__(links)
def _on_exception(self, event):
if event.code is EVENT_JOB_ERROR:
try:
# It would be amazing if we could print the entire Link, but APS serialises Link.transfer to a string and that's all we have from Job's perspective.
extra_info = f'\n\nRaised when executing {self._scheduler.get_job(event.job_id)}'
exception_message = str(event.exception) + f'{extra_info}'
traceback = event.exception.__traceback__
try:
raise type(event.exception)(
exception_message).with_traceback(traceback)
except TypeError as type_exception:
# Some custom exceptions won't let you use the common constructor and will throw an error on initialisation. We catch these and just throw a generic RuntimeError.
raise Exception(exception_message).with_traceback(
traceback) from None
except Exception as e:
_LOGGER.exception(e)
if not self._ignore_exceptions and self.running:
self.shutdown(wait=False)
def _schedule(self, link: Link):
"""
Schedule a link. Sets :any:`APS Job <apscheduler.job.Job>` as this link's job.
:type link: :any:`Link`
:param link: Link to be scheduled
"""
job = self._scheduler.add_job(
link.transfer,
trigger=IntervalTrigger(seconds=link.interval.total_seconds()))
link.set_job(job)
def _unschedule(self, link: Link):
"""
Unschedule a link.
:type link: :any:`Link`
:param link: Link to be unscheduled
"""
if link.job is not None:
link.job.remove()
link.set_job(None)
def start(self):
"""
Start this planner. Calls :any:`APS Scheduler.start() <apscheduler.schedulers.base.BaseScheduler.start>`
See :ref:`Start and Shutdown <start_shutdown>` to learn more about starting and shutdown.
"""
super().start()
def _start_planner(self):
self._scheduler.start()
def pause(self):
"""
Pause this planner. Calls :any:`APScheduler.pause() <apscheduler.schedulers.base.BaseScheduler.pause>`
"""
_LOGGER.info('Pausing %s' % str(self))
self._scheduler.pause()
def resume(self):
"""
Resume this planner. Calls :any:`APScheduler.resume() <apscheduler.schedulers.base.BaseScheduler.resume>`
"""
_LOGGER.info('Resuming %s' % str(self))
self._scheduler.resume()
def shutdown(self, wait: bool = True):
"""
Shutdown this planner. Calls :any:`APScheduler.shutdown() <apscheduler.schedulers.base.BaseScheduler.shutdown>`
See :ref:`Start and Shutdown <start_shutdown>` to learn more about starting and shutdown.
:type wait: bool
:param wait: Whether to wait until all currently executing jobs have finished.
|default| :code:`True`
"""
super().shutdown(wait)
def _shutdown_planner(self, wait: bool = True):
"""
Shutdown this planner. Calls :any:`APScheduler.shutdown() <apscheduler.schedulers.base.BaseScheduler.shutdown>`
:type wait: bool
:param wait: Whether to wait until all currently executing jobs have finished.
|default| :code:`True`
"""
self._scheduler.shutdown(wait=wait)
def purge(self):
"""
Unschedule and clear all links. It can be used while planner is running. APS automatically removes jobs, so we only clear the links.
"""
for link in self.links:
try:
link.job.remove()
except JobLookupError:
pass # APS already removed jobs if shutdown was called before purge, otherwise let's do it ourselves
link.set_job(None)
self._links = []
@property
def running(self):
"""
Whether this planner is currently running. Changed by calls to :any:`start` and :any:`shutdown`.
:return: State of this planner
:rtype: bool
"""
return self._scheduler.state == STATE_RUNNING
def __repr__(self):
return 'ApsPlanner(threads:%s)' % (self._threads)
class SendScheduler(threading.Thread):
def __init__(self):
self._running = True
self.scheduler = BlockingScheduler()
def get_transfer_latency(self, send_files_list, band_list):
sum_latency = len(band_list) - 1
last_latency = len(send_files_list[-1]) * BUFFER_SIZE / band_list[-1]
return sum_latency + last_latency
def send_job(self, sock, send_file, current_band, times):
sock.send(send_file)
print("[Client][{}kB/s] sending file packet {}...".format(
current_band / 1000, times),
end="")
sent_size = len(send_file)
print(len(send_file))
# print(send_file)
size_pack = sock.recv(4)
recv_size, = struct.unpack("!I", size_pack)
while sent_size != recv_size:
print("times: {}|sent_size:{}|recv_size:{}".format(
times, sent_size, recv_size))
sock.send(send_file)
print("[Client][{}kB/s] sending file packet {}...".format(
current_band / 1000, times),
end="")
sent_size = len(send_file)
print(len(send_file))
# print(send_file)
size_pack = sock.recv(4)
recv_size, = struct.unpack("!I", size_pack)
def run(self, sock, send_files_list, band_list):
if len(send_files_list) != len(band_list):
print(
"[Send Scheduler] Send ERROR: The send files fail to match the bandwidth list."
)
return
else:
band_num = len(band_list)
i = 0
j = 0
for i in range(band_num):
current_band = band_list[i]
packet_num = math.ceil(current_band / BUFFER_SIZE)
if len(send_files_list[i]) != packet_num and i != band_num - 1:
print("[Send Scheduler]: Send ERROR: Wrong packet numbers.")
print("i: {}|len: {}|packet_num:{}|band_num: {}".format(
i, len(send_files_list[i]), packet_num, band_num))
return
latency_delta = 1 / len(send_files_list[i])
print("Cur_band: {}|packt_num:{}|latency_delta:{}".format(
current_band, packet_num, latency_delta))
send_files = send_files_list[i]
count = 0
for send_file in send_files:
self.scheduler.add_job(
func=self.send_job,
args=(
sock,
send_file,
current_band,
j,
),
next_run_time=datetime.datetime.now() +
datetime.timedelta(seconds=latency_delta * count))
j += 1
count += 1
try:
self.scheduler.start()
except (KeyboardInterrupt, SystemExit):
self.scheduler.shutdown()
def terminate(self):
self.scheduler.shutdown()
print("[Client] Send scheduler terminated.")
self._running = False
if args.action == 'add':
add_char()
elif args.action == 'list':
list_char()
elif args.action == 'remove':
rm_char()
elif args.action == 'start':
km = KeyManager(key_store_path)
if len(km.keys) == 0:
print('no keys, add one with the add command')
exit(3)
for k in KeyManager(key_store_path).keys:
iteration[tuple(k)] = 0 # init 24 hour counter for every key
notify_store = StorageDictList(notify_store_path)
scheduler = BlockingScheduler(timezone=utc)
scheduler.add_job(do_stuff, 'interval', hours=1)
try:
print('starting scheduler...')
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown(wait=False)
print('scheduler stopped')
elif args.action == 'test':
test_mail()
else:
print('unknown action {act}'.format(act=args.action))
def corn_trigger():
global SCHEDULER
SCHEDULER.add_job(func=alarm,
args=['cron'],
trigger='cron',
second='*/5',
id='corn_job')
# # 循环执行
# def interval_trigger():
# global SCHEDULER
# SCHEDULER.add_job(func=alarm, args=['interval'], trigger='interval', seconds=5, id='interval_job')
#
#
# # 一次执行
# def date_trigger():
# global SCHEDULER
# SCHEDULER.add_job(func=alarm, args=['date'], trigger='date', run_date=datetime.now(), id='date_job')
SCHEDULER = BlockingScheduler()
if __name__ == '__main__':
corn_trigger()
# interval_trigger()
# date_trigger()
try:
SCHEDULER.start()
except (KeyboardInterrupt, SystemExit):
SCHEDULER.shutdown()
class Heating(object):
def __init__(self):
with open('config.json') as json_data:
self.config = json.load(json_data)
json_data.close()
logger.debug('Configuration: ' + str(self.config))
self.processing_lock = threading.Lock()
self.calendar_lock = threading.Lock()
self.relay_lock = threading.Lock()
self.heating_trigger = None
self.preheat_trigger = None
self.event_trigger = None
#Sensible defaults
self.events = None
self.desired_temp = self.config['heating_settings']['minimum_temperature']
self.current_temp = None
self.proportional_time = 0
self.time_on = None
self.time_off = None
self.event_sync_id = None
self.relays = None
self.relays_heating = None
self.relays_preheat = None
self.http_server = None
self.temp_sensors = {}
self.sched = None
self.outside_temp = None
self.outside_apparent_temp = None
def start(self):
logger.info('Starting')
self.credentials = self.get_credentials()
self.darksky_details = self.get_darksky_details()
logger.debug('Setting up scheduler and error handler')
self.sched = BlockingScheduler()
self.sched.add_listener(self.scheduler_listener, EVENT_JOB_ERROR)
logger.debug('Searching for temperature sensors')
try:
self.find_temp_sensors()
except NoTagsFoundException as e:
pass
logger.debug('Searching for relay')
#self.relay = BTRelay.find_relay()
#self.relay = USBRelay.find_relay()
self.relays = USBMultipleRelays.find_relays()
for relay in self.relays._relays:
if relay.port_numbers == tuple(self.config['relays']['heating']):
self.relays_heating = relay
logger.info('Found heating relay at ' + str(relay.port_numbers))
elif relay.port_numbers == tuple(self.config['relays']['preheat']):
self.relays_preheat = relay
logger.info('Found preheat relay at ' + str(relay.port_numbers))
if self.relays_heating is None:
raise Exception('No heating relay found')
logger.debug('Creating scheduler jobs')
#Get new events every X minutes
self.sched.add_job(self.get_next_event, trigger = 'cron', \
next_run_time = pytz.utc.localize(datetime.datetime.utcnow()), hour = '*/' + str(self.config['calendar_settings']['update_calendar_interval_hours']), minute = 0)
self.sched.add_job(self.update_outside_temperature, trigger = 'cron', \
next_run_time = pytz.utc.localize(datetime.datetime.utcnow()), hour = '*', minute = '*/15')
#Scan for new devices every minute
self.sched.add_job(self.find_temp_sensors, trigger = 'cron', \
next_run_time = pytz.utc.localize(datetime.datetime.utcnow()), hour = '*', minute = '*')
HttpHandler.heating = self
logger.debug('Starting HTTP server')
self.http_server = ThreadedHTTPServer(('localhost', 8080), HttpHandler)
http_server_thread = threading.Thread(target=self.http_server.serve_forever)
http_server_thread.setDaemon(True) # don't hang on exit
http_server_thread.start()
logger.debug('Starting scheduler')
try:
self.sched.start()
except Exception as e:
logger.error('Error in scheduler: ' + str(e))
self.http_server.shutdown()
self.sched.shutdown(wait = False)
def scheduler_listener(self, event):
if event.exception is not None or event.code == EVENT_JOB_MAX_INSTANCES:
logger.error('Error in scheduled event: ' + str(event))
logger.debug(type(event.exception))
if not isinstance(event.exception, NoTemperatureException) and not isinstance(event.exception, NoTagsFoundException):
logger.error('Killing all the things')
raise Exception(str(event))
#self.http_server.shutdown()
#self.sched.shutdown(wait = False)
#exit(1)
def find_temp_sensors(self):
self.temp_sensors = TempSensor.find_temp_sensors(self.temp_sensors)
for sensor in list(self.temp_sensors.values()):
if sensor.temp_job_id is None:
logger.info('Setting scheduler job for ' + sensor.mac)
#Get a new temperature every minute
sensor.temp_job_id = self.sched.add_job(self.get_temperature, trigger = 'interval', \
start_date = datetime.datetime.now(), seconds = self.config['heating_settings']['update_temperature_interval_seconds'], \
name = sensor.mac + ' temperature job', args = (sensor,))
def heating_on(self, proportion):
self.time_on = pytz.utc.localize(datetime.datetime.utcnow())
self.time_off = None
self.proportional_time = proportion
logger.debug('Getting relay lock')
self.relay_lock.acquire()
logger.debug('Got relay lock')
self.relays_heating.on()
logger.debug('Releasing relay lock')
self.relay_lock.release()
self.set_heating_trigger(proportion, True)
def heating_off(self, proportion):
self.time_off = pytz.utc.localize(datetime.datetime.utcnow())
self.time_on = None
logger.debug('Getting relay lock')
self.relay_lock.acquire()
logger.debug('Got relay lock')
self.relays_heating.off()
logger.debug('Releasing relay lock')
self.relay_lock.release()
self.set_heating_trigger(proportion, False)
def preheat_on(self, time_off):
logger.debug('Getting relay lock')
self.relay_lock.acquire()
logger.debug('Got relay lock')
self.relays_preheat.on()
logger.debug('Releasing relay lock')
self.relay_lock.release()
self.set_preheat_trigger(time_off)
def preheat_off(self):
logger.debug('Getting relay lock')
self.relay_lock.acquire()
logger.debug('Got relay lock')
self.relays_preheat.off()
logger.debug('Releasing relay lock')
self.relay_lock.release()
def check_relay_states(self):
logger.debug('Checking states ' + str(self.relays.all_status()))
iter = 0
for s in self.relays.all_status():
iter += 1
if s == 0:
self.relays.one_off(iter)
else:
self.relays.one_on(iter)
def set_heating_trigger(self, proportion, on):
self.proportional_time = proportion
if self.heating_trigger is not None:
try:
self.heating_trigger.remove()
except JobLookupError as e:
pass
self.heating_trigger = None
if on:
if proportion < self.config['heating_settings']['proportional_heating_interval_minutes']:
run_date = self.time_on + datetime.timedelta(0,self.proportional_time * 60)
logger.info('New proportional time: ' + str(proportion) + '/' + str(self.config['heating_settings']['proportional_heating_interval_minutes']) +\
' mins - will turn off at ' + str(run_date.astimezone(get_localzone())))
self.heating_trigger = self.sched.add_job(\
self.process, trigger='date', run_date=run_date, name='Proportional off at ' + str(run_date.astimezone(get_localzone())))
else:
if proportion > 0:
if self.time_off is None:
self.time_off = pytz.utc.localize(datetime.datetime.utcnow())
run_date = self.time_off + datetime.timedelta(0,(self.config['heating_settings']['proportional_heating_interval_minutes'] - self.proportional_time) * 60)
logger.info('New proportional time: ' + str(proportion) + '/' + str(self.config['heating_settings']['proportional_heating_interval_minutes']) +\
' mins - will turn on at ' + str(run_date.astimezone(get_localzone())))
self.heating_trigger = self.sched.add_job(\
self.process, trigger='date', run_date=run_date, name='Proportional on at ' + str(run_date.astimezone(get_localzone())))
def set_preheat_trigger(self, time_off):
if self.preheat_trigger is not None:
try:
self.preheat_trigger.remove()
except JobLookupError as e:
pass
self.preheat_trigger = None
logger.info('Preheat off at ' + str(time_off.astimezone(get_localzone())))
self.preheat_trigger = self.sched.add_job(\
self.process, trigger='date', run_date=time_off, name='Preheat off at ' + str(time_off.astimezone(get_localzone())))
def get_temperature(self, sensor):
try:
sensor.get_ambient_temp()
except NoTemperatureException as e:
logger.warn('Removing sensor ' + sensor.mac + ' from sensors list due to disconnection')
try:
sensor.temp_job_id.remove()
except JobLookupError as e:
pass
del self.temp_sensors[sensor.mac]
self.update_current_temp()
def update_current_temp(self):
temps = []
for mac, sensor in self.temp_sensors.items():
if sensor.amb_temp is not None:
temps.append(sensor.amb_temp)
if not temps:
raise NoTemperatureException()
#self.current_temp = sum(temps) / float(len(temps))
self.current_temp = min(temps)
logger.info('Overall temperature is now ' + str(self.current_temp) + ' from ' + str(temps))
self.process()
def get_next_event(self):
self.calendar_lock.acquire()
http = self.credentials.authorize(httplib2.Http(timeout=self.config['calendar_settings']['calendar_timeout_seconds']))
service = discovery.build('calendar', 'v3', http=http)
now = datetime.datetime.utcnow().isoformat() + 'Z'
logger.debug('Getting the next event')
try:
eventsResult = service.events().list(
calendarId=self.config['calendar_settings']['calendar_id'], timeMin=now, maxResults=3, singleEvents=True, orderBy='startTime').execute()
events = eventsResult.get('items', [])
self.event_sync_id = str(uuid.uuid4())
logger.debug('Sending request: ' + str({'id':self.event_sync_id, \
'type':'web_hook', \
'address':'https://www.steev.me.uk/heating/events', \
'expiration':(int(time.time())+(self.config['calendar_settings']['update_calendar_interval_hours']*60*60))*1000 \
}))
hook_response = service.events().watch(calendarId=self.config['calendar_settings']['calendar_id'], \
body={'id':self.event_sync_id, \
'type':'web_hook', \
'address':'https://www.steev.me.uk/heating/events', \
'expiration':(int(time.time())+(self.config['calendar_settings']['update_calendar_interval_hours']*60*60))*1000 \
})\
.execute()
if hook_response is not None:
logger.debug('Got response' + str(hook_response) + ' from web_hook call')
except HttpError as e:
logger.error('HttpError, resp = ' + str(e.resp) + '; content = ' + str(e.content))
logger.exception(e)
self.calendar_lock.release()
return
except Exception as e:
logger.exception(e)
self.calendar_lock.release()
return
parsed_events = []
if events:
counter = 0
for event in events:
counter += 1
start = event['start'].get('dateTime', event['start'].get('date'))
start_date = parser.parse(start)
end = event['end'].get('dateTime', event['end'].get('date'))
end_date = parser.parse(end)
try:
desired_temp = float(event['summary'])
except ValueError:
if event['summary'].lower() == 'on':
desired_temp = 'On'
if event['summary'].lower() == 'preheat':
desired_temp = 'Preheat'
logger.info('Event ' + str(counter) + ' is ' + str(start_date.astimezone(get_localzone())) + \
' to ' + str(end_date.astimezone(get_localzone())) + ': ' + str(desired_temp))
parsed_events.append({'start_date': start_date, 'end_date': end_date, 'desired_temp': desired_temp})
if counter == 1:
#Set a schedule to get the one after this
if self.event_trigger is not None:
try:
self.event_trigger.remove()
except JobLookupError as e:
pass
self.event_trigger = None
self.event_trigger = self.sched.add_job(self.get_next_event, \
trigger='date', run_date=end_date, name='Event end at ' + str(end_date.astimezone(get_localzone())))
#Tell the processing that this is a new event so it resets the proportion to start again
if self.events is None or start_date != self.events[0]['start_date'] or end_date != self.events[0]['end_date'] or desired_temp != self.events[0]['desired_temp']:
logger.info('New event starting, resetting time off.')
self.time_off = None
self.events = parsed_events
else:
self.events = None
self.calendar_lock.release()
self.process()
def update_outside_temperature(self):
try:
logger.info('Getting new outside temperature')
with urllib.request.urlopen('https://api.darksky.net/forecast/' + self.darksky_details['api_key'] + '/' + self.darksky_details['latlong'] + '?exclude=[minutely,hourly,daily]&units=si') as darksky_url:
data = json.loads(darksky_url.read().decode())
logger.debug(str(data))
if data['currently']:
if data['currently']['apparentTemperature']:
self.outside_apparent_temp = data['currently']['apparentTemperature']
logger.info('Got outside apparent temperature: ' + str(self.outside_apparent_temp))
if data['currently']['temperature']:
self.outside_temp = data['currently']['temperature']
logger.info('Got outside temperature: ' + str(self.outside_temp))
except Exception as e:
pass
def process(self):
logger.debug('Processing')
#Main calculations. Figure out whether the heating needs to be on or not.
if self.current_temp is None:
return
self.processing_lock.acquire()
current_time = pytz.utc.localize(datetime.datetime.utcnow())
current_temp = self.current_temp
time_due_on = None
have_temp_event = False
forced_on = False
have_preheat = False
if current_temp < self.config['heating_settings']['minimum_temperature']:
#If we're below the minimum allowed temperature, turn on at full blast.
logger.info('Temperature is below minimum, turning on')
self.desired_temp = str(self.config['heating_settings']['minimum_temperature'])
self.heating_on(self.config['heating_settings']['proportional_heating_interval_minutes'])
elif self.events is not None:
#Find preheat events
index = -1
while index < 3:
index += 1
if index >= len(self.events):
break
if self.events[index]['desired_temp'] == 'Preheat':
if self.events[index]['start_date'] < current_time and not self.events[index]['end_date'] < current_time:
have_preheat = True
if not(self.relays_preheat._status):
logger.info('Preheat on')
self.preheat_on(self.events[index]['end_date'])
break
if (not have_preheat) and self.relays_preheat._status:
self.preheat_off()
#Find normal events
index = -1
next_time = None
while index < 3:
index += 1
if index >= len(self.events):
break
if self.events[index]['desired_temp'] == 'Preheat':
continue
elif self.events[index]['desired_temp'] == 'On':
if self.events[index]['start_date'] < current_time and not self.events[index]['end_date'] < current_time:
forced_on = True
if not(self.relays_heating._status):
logger.info('Heating forced on')
self.heating_on(self.config['heating_settings']['proportional_heating_interval_minutes'])
else:
have_temp_event = True
break
if have_temp_event:
next_time = self.events[index]['start_date']
next_time_end = self.events[index]['end_date']
next_temp = self.events[index]['desired_temp']
logger.debug('Processing data: ' + str(next_time.astimezone(get_localzone())) + \
' to ' + str(next_time_end.astimezone(get_localzone())) + ', ' + str(next_temp))
self.desired_temp = str(next_temp)
if next_time_end < current_time:
#If the last event ended in the past, off.
logger.warn('Event end time is in the past.')
self.heating_off(0)
elif not forced_on:
temp_diff = next_temp - current_temp
new_proportional_time = None
if next_time < current_time:
time_due_on = next_time
logger.info('Currently in an event starting at ' + str(next_time.astimezone(get_localzone())) + \
' ending at ' + str(next_time_end.astimezone(get_localzone())) + ' temp diff is ' + str(temp_diff))
#Check all events for warm-up temperature
for event in self.events:
if event['desired_temp'] == 'On' or event['desired_temp'] == 'Preheat':
continue
event_next_time = event['start_date']
if event_next_time > current_time:
event_desired_temp = event['desired_temp']
event_temp_diff = event_desired_temp - current_temp
logger.debug('Future event starting at ' + str(event_next_time.astimezone(get_localzone())) + \
' temp difference is ' + str(event_temp_diff))
if event_temp_diff > 0:
#Start X minutes earlier for each degree the heating is below the desired temp, plus Y minutes.
event_time_due_on = event_next_time - datetime.timedelta(0,(event_temp_diff * self.config['heating_settings']['minutes_per_degree'] * 60) + (self.config['heating_settings']['effect_delay_minutes'] * 60))
logger.debug('Future event needs warm up, due on at ' + str(event_time_due_on.astimezone(get_localzone())))
if time_due_on is None or event_time_due_on < time_due_on or event_time_due_on < current_time:
time_due_on = event_time_due_on
next_temp = event_desired_temp
temp_diff = event_temp_diff
logger.debug('Future event starting at ' + str(event_next_time.astimezone(get_localzone())) + \
' warm-up, now due on at ' + str(time_due_on.astimezone(get_localzone())))
#Full blast until 0.3 degrees difference
if event_temp_diff > 0.3:
new_proportional_time = 30
elif time_due_on is None or event_next_time < time_due_on:
time_due_on = event_next_time
elif time_due_on is None or event_next_time < time_due_on:
time_due_on = event_next_time
if time_due_on < next_time:
logger.info('Before an event starting at ' + str(next_time.astimezone(get_localzone())) +\
' temp diff is ' + str(temp_diff) + ' now due on at ' + str(time_due_on.astimezone(get_localzone())))
if time_due_on <= current_time:
if temp_diff < 0:
logger.info('Current temperature ' + str(current_temp) + ' is higher than the desired temperature ' + str(next_temp))
self.heating_off(0)
else:
if new_proportional_time is None:
#Calculate the proportional amount of time the heating needs to be on to reach the desired temperature
new_proportional_time = temp_diff * self.config['heating_settings']['proportional_heating_interval_minutes'] / 2
if new_proportional_time < self.config['heating_settings']['minimum_active_period_minutes']: #Minimum time boiler can be on to be worthwhile
new_proportional_time = self.config['heating_settings']['minimum_active_period_minutes']
elif new_proportional_time > self.config['heating_settings']['proportional_heating_interval_minutes']:
new_proportional_time = self.config['heating_settings']['proportional_heating_interval_minutes']
#Are we currently on or off?
if not(self.relays_heating._status) or self.time_on is None: #Off
if self.time_off is None:
time_due_on = next_time
new_time_due_on = next_time
elif new_proportional_time <= self.proportional_time:
#Need to be on for less time - turn on in a bit
time_due_on = self.time_off + datetime.timedelta(0,(self.config['heating_settings']['proportional_heating_interval_minutes'] * 60) - (self.proportional_time * 60))
new_time_due_on = self.time_off + datetime.timedelta(0,(self.config['heating_settings']['proportional_heating_interval_minutes'] * 60) - (new_proportional_time * 60))
else:
#Need to be on for more time - turn on now
time_due_on = self.time_off + datetime.timedelta(0,(self.config['heating_settings']['proportional_heating_interval_minutes'] * 60) - (self.proportional_time * 60))
new_time_due_on = current_time
if new_time_due_on <= current_time:
logger.info('Heating is off, due on at ' + str(new_time_due_on.astimezone(get_localzone())) +'; Turning on')
self.heating_on(new_proportional_time)
else:
if new_proportional_time != self.proportional_time:
logger.info('Changing time next due on.')
self.set_heating_trigger(new_proportional_time, self.relays_heating._status)
if time_due_on != new_time_due_on:
logger.info('Heating was off, due on at ' + str(time_due_on.astimezone(get_localzone())) +\
'. Now due on at ' + str(new_time_due_on.astimezone(get_localzone())))
else: #On
time_due_off = self.time_on + datetime.timedelta(0,self.proportional_time * 60)
if new_proportional_time < self.config['heating_settings']['proportional_heating_interval_minutes']:
#Must have a time_on at this point
new_time_due_off = self.time_on + datetime.timedelta(0,new_proportional_time * 60)
else:
new_time_due_off = next_time_end
if new_time_due_off < current_time:
logger.info('Heating was on, due off at ' + str(time_due_off.astimezone(get_localzone())) +'; Turning off')
self.heating_off(new_proportional_time)
else:
if new_proportional_time != self.proportional_time:
logger.info('Changing time next due off.')
self.set_heating_trigger(new_proportional_time, self.relays_heating._status)
if new_time_due_off != time_due_off:
logger.info('Heating was on, due off at ' + str(time_due_off.astimezone(get_localzone())) +\
'. Now due off at ' + str(new_time_due_off.astimezone(get_localzone())))
else:
self.desired_temp = str(self.config['heating_settings']['minimum_temperature'])
#If we don't have an event yet, warn and ensure relay is off
logger.info('No events available')
if self.relays_heating._status:
logger.debug('Heating off')
self.heating_off(0)
if have_preheat:
logger.info('Preheat but no normal event available.')
else:
if self.relays_preheat._status:
logger.debug('Preheat off')
self.preheat_off()
self.check_relay_states()
self.processing_lock.release()
def get_credentials(self):
'''Gets valid user credentials from storage.
If nothing has been stored, or if the stored credentials are invalid,
the OAuth2 flow is completed to obtain the new credentials.
Returns:
Credentials, the obtained credential.
'''
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
logger.debug('Getting credentials from ' + credential_dir)
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir, 'calendar-heating.json')
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
parser = argparse.ArgumentParser(parents=[tools.argparser])
flags = parser.parse_args()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets('client_secret.json', 'https://www.googleapis.com/auth/calendar.readonly')
flow.user_agent = 'Heating'
credentials = tools.run_flow(flow, store, flags)
logger.info('Storing credentials to ' + credential_path)
return credentials
def get_darksky_details(self):
with open('darksky_details.json') as json_data:
details = json.load(json_data)
json_data.close()
logger.debug('DarkSky details: ' + str(details))
return details
def my_listener(event):
if event.exception:
print('任务出错了。' + event)
# sms.send_wrong_sms()
sched.shutdown()
# time.sleep(5)
# sched.start()
else:
pass
# 正常发送短信
def semd_sms():
sms.send_normal_sms()
# sched.add_job(func=my_email, trigger='interval', seconds=60)
sched.add_job(func=my_bd, trigger='interval', seconds=150)
# sched.add_job(func=semd_sms, trigger='cron', hour=15, minute=00)
sched.add_listener(my_listener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
sched._logger = logging
sched.start()
except Exception as e:
# sms.send_wrong_sms()
nowtime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
newsql = "insert into tab_send_email (address_to,mail_subject,mail_text) values('*****@*****.**','定时任务出现问题'+'" + \
nowtime + "','" + str(e) + "')"
tools.warning(str(e))
ms.ExecNonQuery(newsql)
sched.shutdown()
time.sleep(5)
sched.start()
class FreezerScheduler(object):
def __init__(self, apiclient, interval, job_path):
# config_manager
self.client = apiclient
self.freezerc_executable = spawn.find_executable('freezerc')
if self.freezerc_executable is None:
# Needed in the case of a non-activated virtualenv
self.freezerc_executable = spawn.find_executable(
'freezerc', path=':'.join(sys.path))
self.job_path = job_path
self._client = None
self.lock = threading.Lock()
self.execution_lock = threading.Lock()
job_defaults = {
'coalesce': True,
'max_instances': 1
}
self.scheduler = BlockingScheduler(job_defaults=job_defaults)
if self.client:
self.scheduler.add_job(self.poll, 'interval',
seconds=interval, id='api_poll')
self.add_job = self.scheduler.add_job
self.remove_job = self.scheduler.remove_job
self.jobs = {}
def get_jobs(self):
if self.client:
job_doc_list = utils.get_active_jobs_from_api(self.client)
try:
utils.save_jobs_to_disk(job_doc_list, self.job_path)
except Exception as e:
logging.error('Unable to save jobs to {0}. '
'{1}'.format(self.job_path, e))
return job_doc_list
else:
return utils.get_jobs_from_disk(self.job_path)
def start_session(self, session_id, job_id, session_tag):
if self.client:
return self.client.sessions.start_session(session_id,
job_id,
session_tag)
else:
raise Exception("Unable to start session: api not in use.")
def end_session(self, session_id, job_id, session_tag, result):
if self.client:
return self.client.sessions.end_session(session_id,
job_id,
session_tag,
result)
else:
raise Exception("Unable to end session: api not in use.")
def upload_metadata(self, metadata_doc):
if self.client:
self.client.backups.create(metadata_doc)
def start(self):
utils.do_register(self.client)
self.poll()
self.scheduler.start()
def update_job(self, job_id, job_doc):
if self.client:
try:
return self.client.jobs.update(job_id, job_doc)
except Exception as e:
logging.error("[*] Job update error: {0}".format(e))
def update_job_status(self, job_id, status):
doc = {'job_schedule': {'status': status}}
self.update_job(job_id, doc)
def is_scheduled(self, job_id):
return self.scheduler.get_job(job_id) is not None
def create_job(self, job_doc):
job = Job.create(self, self.freezerc_executable, job_doc)
if job:
self.jobs[job.id] = job
logging.info("Created job {0}".format(job.id))
return job
def poll(self):
try:
work_job_doc_list = self.get_jobs()
except Exception as e:
logging.error("[*] Unable to get jobs: {0}".format(e))
return
work_job_id_list = []
# create job if necessary, then let it process its events
for job_doc in work_job_doc_list:
job_id = job_doc['job_id']
work_job_id_list.append(job_id)
job = self.jobs.get(job_id, None) or self.create_job(job_doc)
if job:
job.process_event(job_doc)
# request removal of any job that has been removed in the api
for job_id, job in self.jobs.iteritems():
if job_id not in work_job_id_list:
job.remove()
remove_list = [job_id for job_id, job in self.jobs.items()
if job.can_be_removed()]
for k in remove_list:
self.jobs.pop(k)
def stop(self):
try:
self.scheduler.shutdown(wait=False)
except:
pass
def reload(self):
logging.warning("reload not supported")
name = name.replace(' ', '%20')
insertPrice(price, name, currency)
return {'name': name, 'price': price, 'currency': currency}
elif req.status_code == 400:
print('400: Resource not found')
return {}
# SCHEDULER ############################################################
sched = BlockingScheduler()
# @sched.scheduled_job('interval', minutes=1)
# def interval_job():
# scrapBookDepository()
@sched.scheduled_job('cron', day_of_week='mon-sun', hour=15)
def cron_job():
scrapBookDepository()
atexit.register(lambda: sched.shutdown())
sched.start()
status = cachet.PERFORMANCE_ISSUES
elif first == 'FAILED' and second == 'SUCCESS':
status = cachet.PARTIAL_OUTAGE
else:
status = cachet.MAJOR_OUTAGE
monitors.append(CachetComponent(id='synthetics-{}'.format(monitor_id),
name=monitor_name[monitor_id],
status=status))
return monitors
def update():
fetch_snitches()
fetch_newrelic()
fetch_synthetics()
if DMS_PING_URL:
requests.get(DMS_PING_URL)
if __name__ == '__main__':
from apscheduler.schedulers.blocking import BlockingScheduler
scheduler = BlockingScheduler()
scheduler.add_job(update, 'interval', minutes=2, max_instances=1, coalesce=True)
try:
scheduler.start()
except (KeyboardInterrupt, SystemExit):
scheduler.shutdown()
# else:
# forecast.forecast(res.keys()[0][0])
if __name__ == '__main__':
jobstores = {'default': MemoryJobStore()}
executors = {'default': ThreadPoolExecutor(20)}
job_defaults = {'coalesce': False, 'max_instances': 1}
logger.info("Starting preprocessor")
scheduler = BlockingScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
for query_key in queries_config:
kwArgs = {
"query": queries_config[query_key],
"rawDataDB": rawDataDB,
"preprocessOutputDB": preprocessOutputDB
}
logger.debug(kwArgs)
logger.info("Adding job to preprocess %s", query_key)
scheduler.add_job(loadAndPreprocess,
'interval',
name=query_key,
kwargs=kwArgs,
seconds=int(default_config['query_mean_interval']))
try:
scheduler.start()
finally:
scheduler.shutdown()
class ScrapeScheduler(Executor):
def __init__(self):
Executor.register(ScrapeScheduler)
self._logger = logging.getLogger(type(self).__name__)
self._scheduler = None
self._executor = None
self._job_list = list()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._executor.shutdown()
def register_cache(self, cache_name):
self._cache.register_catelog(cache_name)
def register_job(self, job: BaseJob):
'''
Default to run every minute
:param job:
:return:
'''
self._job_list.append(job)
def _create_executors(self):
max_instances = len(
[job for job in self._job_list if job.schedule_type != 'one-off'])
max_instances = LogicUtil.if_else_default(max_instances, 1,
lambda x: x > 0)
job_defaults = {'max_instances': max_instances}
self._scheduler = BlockingScheduler(job_defaults=job_defaults)
self._executor = TaskExecutor()
def _add_hooks(self):
_self = self
def shutdown_hook(event):
e = event.exception
if e:
_self._logger.error(
f'{Formatter.get_timestamp()} - Scheduler crashed!, {type(e)} - {e}'
)
if isinstance(e, KeyboardInterrupt):
if None is not _self._scheduler:
_self._scheduler.remove_all_jobs()
_self._scheduler.shutdown()
if None is not _self._executor:
_self._executor.shutdown()
self._scheduler.add_listener(shutdown_hook, EVENT_JOB_ERROR)
def _add_registred_jobs(self):
for job in self._job_list:
if job.schedule_type == "sec":
self._scheduler.add_job(job.run,
'interval',
id=type(job).__name__,
seconds=job.sec)
elif job.schedule_type == "cron":
self._scheduler.add_job(job.run,
CronTrigger.from_crontab(job.cron),
id=type(job).__name__)
elif job.schedule_type == "one-off":
self._scheduler.add_job(job.run, id=type(job).__name__)
def start(self):
# Lazy init
self._create_executors()
self._add_hooks()
self._add_registred_jobs()
self._scheduler.start()
def shutdown(self):
self._scheduler.shutdown()
class DakokuManager(object):
def __init__(self, config_path, schedule_path):
self.config_path = config_path
self.schedule_path = schedule_path
cfg = self._load_config()
try:
human_mode_min = cfg["human_mode"]
except:
human_mode_min = 0
try:
self.log_dir = cfg["log_dir"]
file_handler = logging.FileHandler(os.path.join(self.log_dir, "dakoku.log"), 'a+')
file_handler.level = LEVEL
log.addHandler(file_handler)
log.info("saving log to %s", self.log_dir)
except:
self.log_dir = None
self.api_key = cfg["api_key"]
sched = self._load_schedule()
start_date = dt.datetime.strptime(sched["valid"]["start"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
end_date = dt.datetime.strptime(sched["valid"]["end"], '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo'))
log.info("dakoku is valid for %s - %s", start_date, end_date)
holidays = self._get_holidays(start_date, end_date)
self.worker = DakokuWorker(cfg["host"], cfg["user"], cfg["pass"], holidays, self.log_dir)
self.register(sched["working"], start_date, end_date, holidays, human_mode_min)
def _load_config(self):
log.debug("loading config from %s", self.config_path)
with open(self.config_path, 'r') as f:
cfg = json.load(f)
return cfg
def _load_schedule(self):
log.debug("loading schedule from %s", self.schedule_path)
with open(self.schedule_path, 'r') as f:
cfg = json.load(f)
return cfg
def _get_holidays(self, start_date, end_date):
holidays = japanese_holiday.getholidays(
str(self.api_key), # this function does not allow unicode
japanese_holiday.HOLIDAY_TYPE_OFFICIAL_JA,
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'),
)
ret = []
for hd in holidays:
date_str = hd['start']['date']
ret.append(dt.datetime.strptime(date_str, '%Y-%m-%d').replace(tzinfo=pytz.timezone('Asia/Tokyo')))
return ret
def register(self, working, start_date, end_date, holidays, human_mode_min=0):
self.scheduler = Scheduler(timezone=pytz.timezone('Asia/Tokyo'), logger=log)
today = dt.date.today()
for w in working:
# schedule shukkin
h, m = map(int, w["from"].split(':'))
fromtime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
d = dt.datetime.combine(today, fromtime) - dt.timedelta(minutes=human_mode_min)
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=d.hour, minute=d.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_start),
trigger)
# schedule taikin
h, m = map(int, w["till"].split(':'))
tilltime = dt.time(h,m,tzinfo=pytz.timezone('Asia/Tokyo'))
trigger = CronTrigger(day_of_week=w["dayOfWeek"],
hour=tilltime.hour, minute=tilltime.minute,
start_date=start_date,
end_date=end_date,
timezone=pytz.timezone('Asia/Tokyo'))
self.scheduler.add_job(dispatch_after(human_mode_min * 60,
self.worker.work_end),
trigger)
self.scheduler.print_jobs()
def start(self):
self.scheduler.start()
def shutdown(self):
self.scheduler.shutdown()
def plantaskdsfsdfsafdsafsd(self,
func,
targger="cron",
args=None,
year=None,
month=None,
week="*",
day_of_week='*',
day=None,
hour=None,
minute=None,
second='0'):
"""添加定时器任务、计划任务
func 函数名
args 函数参数 例：(1,)
year 年
month 月
day 日
hour 几点
minute 几分
second 几秒
"""
BlockingSchedulers = BlockingScheduler()
if targger == 'cron':
BlockingSchedulers.add_job(func,
targger,
args=args,
year=year,
month=month,
week=week,
day_of_week=day_of_week,
day=day,
hour=hour,
minute=minute,
second=second)
elif targger == 'interval':
if day:
BlockingSchedulers.add_job(func,
targger,
args=args,
days=int(day))
elif hour:
BlockingSchedulers.add_job(func,
targger,
args=args,
hours=int(hour))
elif minute:
BlockingSchedulers.add_job(func,
targger,
args=args,
minutes=int(minute))
elif second:
BlockingSchedulers.add_job(func,
targger,
args=args,
seconds=int(second))
try:
BlockingSchedulers.start()
except:
BlockingSchedulers.shutdown()
class JobManage():
def __init__(self):
jobstores = {'default': MemoryJobStore()}
executors = {
'default': ThreadPoolExecutor(50)
# 'processpool': ProcessPoolExecutor(3)
}
job_defaults = {'coalesce': False, 'max_instances': 50}
self.sched = BlockingScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
self.addError()
self.addJobExecuted()
def addJob(self, func, jobId=None, cron=None, args=[], kwargs={}):
'''
只支持cron的形式
* * * * * command
分 时 日 月 周 命令
第1列表示分钟1～59 每分钟用*或者 */1表示
第2列表示小时1～23（0表示0点）
第3列表示日期1～31
第4列表示月份1～12
第5列标识号星期0～6（0表示星期天）
第6列要运行的命令
'''
if cron is None:
raise Exception("cron cannot be Null")
(minute, hour, day, month, week) = cron.split(" ")
self.sched.add_job(func,
trigger='cron',
id=jobId,
hour=hour,
minute=minute,
day=day,
month=month,
week=week,
args=args,
kwargs=kwargs)
def removeJob(self, jobId):
self.sched.remove_job(jobId)
def start(self):
self.sched.start()
def shutdown(self):
self.sched.shutdown()
def printJobs(self):
self.sched.print_jobs()
def getJobs(self):
return self.sched.get_jobs()
def addError(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_ERROR)
def addJobExecuted(self, func=None):
if func is None:
func = self.listener
self.sched.add_listener(func, EVENT_JOB_EXECUTED)
def listener(self, event):
if event.exception:
log.error("任务【%s】 任务出错 : %s" % (event.job_id, event.traceback))
else:
log.debug("任务【%s】已经跑完，结束时间 : %s " % (event.job_id, getNow()))
# jobMange = JobManage()
