def run(self): self.logger.info("{} 数据采集同步线程启动".format(self.market)) self.db = SqliteMemDriver() hikyuu_init(self.hku_config_file, ignore_preload=True) stk_list = self.get_stock_list() hku_warn_if(not stk_list, "从数据库中获取的股票列表为空!", logger=self.logger) self.mutex.tryLock() end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day( ) start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day( ) start = Datetime.now() if self._phase1_start_time >= self._phase1_end_time: delta = TimeDelta(0) elif start >= self._phase1_end_time: delta = (self._phase1_start_time + TimeDelta(1) - start) elif start < self._phase1_start_time: delta = (self._phase1_start_time - start) else: delta = self._interval * ceil( (start - self._phase1_start_time) / self._interval) - ( start - self._phase1_start_time) hku_info('{} 下次采集时间:{}', self.market, start + delta, logger=self.logger) delta = int(delta.total_milliseconds()) while self.working and not self.cond.wait(self.mutex, int(delta)): last_time = Datetime.today() + end_delta start = Datetime.now() if start >= last_time: next_time = Datetime.today() + TimeDelta(1) + start_delta hku_info('{} 明日采集时间:{}', self.market, next_time, logger=self.logger) delta = next_time - start delta = int(delta.total_milliseconds()) continue hku_trace("start:{}".format(start)) self.collect(stk_list) end = Datetime.now() x = end - start if x + TimeDelta(seconds=1) < self._interval: delta = self._interval - x - TimeDelta(seconds=1) delta = int(delta.total_milliseconds()) self.logger.info("{} {:<.2f} 秒后重新采集".format( self.market, delta / 1000)) #self.sleep(delta) self.db.close() self.logger.info("{} 数据采集同步线程终止!".format(self.market))
def next_delta(start_time, interval, phase1_delta, phase2_delta, ignore_weekend): interval_delta = TimeDelta(seconds=interval) phase1_start_delta, phase1_end_delta = phase1_delta phase2_start_delta, phase2_end_delta = phase2_delta today = Datetime.today() phase1_start, phase1_end = today + phase1_start_delta, today + phase1_end_delta phase2_start, phase2_end = today + phase2_start_delta, today + phase2_end_delta current_time = Datetime.now() maybe_time = current_time + interval_delta if phase1_start_delta == phase1_end_delta and phase2_start_delta == phase2_end_delta: # 两个时间周期各自的起止时间相等,则认为未做限制 delta = interval_delta - (current_time - start_time) elif maybe_time > phase2_end: # 如果预计的下一次时间大于周期2的结束日期, 则取下一日的周期1起始时间计算 next_time = today + TimeDelta(1) + phase1_start_delta if ignore_weekend and next_time.day_of_week() in (0, 6): next_time = today.next_week() + phase1_start_delta delta = next_time - start_time elif maybe_time in (phase1_start, phase1_end, phase2_start, phase2_end): # 如果下一次时间刚好等于时间周期的起止点,则直接返回预计的时间间隔 delta = interval_delta elif start_time < phase1_start and phase1_start < maybe_time < phase1_end: # 如果本次的时间小于周期1的起始时间且预计下一次的时间在phase1内,则取phase1起始时间计算 delta = phase1_start - start_time elif phase1_end < maybe_time < phase2_start: delta = phase2_start - start_time else: delta = interval_delta - (current_time - start_time) return delta
def run(use_proxy, source, seconds, phase1, phase2, ignore_weekend): phase1_delta = parse_phase(phase1) hku_error_if(phase1_delta is None or len(phase1_delta) != 2, "无效参数 phase1: {}".format(phase1), callback=lambda: exit(1)) hku_error_if(phase1_delta[0] > phase1_delta[1], "无效参数 phase1: {}, 结束时间应大于等于起始时间".format(phase1), callback=lambda: exit(1)) phase2_delta = parse_phase(phase2) hku_error_if(phase2_delta is None or len(phase2_delta) != 2, "无效参数 phase2: {}".format(phase2), callback=lambda: exit(1)) hku_error_if(phase2_delta[0] > phase2_delta[1], "无效参数 phase2: {}, 结束时间应大于等于起始时间".format(phase2), callback=lambda: exit(1)) hku_error_if( phase1_delta[1] > phase2_delta[0], "无效参数 phase1: {}, phase2: {}, phase2 起始时间应大于等于 phase1 结束时间".format( phase1, phase2), callback=lambda: exit(1)) hku_logger.info("采集时间段1:{}".format(phase1)) hku_logger.info("采集时间段2:{}".format(phase2)) config_file = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini' if not os.path.exists(config_file): print("未找到配置文件,请先运行 HikyuuTDX 进行配置与数据导入") exit(1) hikyuu_init(config_file, ignore_preload=True) print("采集程序运行中,可使用 Ctrl-C 终止!") sm = StockManager.instance() stk_list = [ stk.market_code.lower() for stk in sm if stk.valid and stk.type in (constant.STOCKTYPE_A, constant.STOCKTYPE_INDEX, constant.STOCKTYPE_GEM) ] spot_topic = ':spot:' def batch_func(records): spot = bytearray(spot_topic.encode('utf-8')) buf = create_fb_spot(records) spot.extend(buf) pub_sock.send(bytes(spot)) address = "ipc:///tmp/hikyuu_real_pub.ipc" pub_sock = pynng.Pub0() pub_sock.listen(address) today = Datetime.today() phase1_time = [today + x for x in phase1_delta] phase2_time = [today + x for x in phase2_delta] start_time = Datetime.now() if not (phase1_time[0] <= start_time <= phase1_time[1] or phase2_time[0] <= start_time <= phase2_time[1]): delta = next_delta(start_time, seconds, phase1_delta, phase2_delta, ignore_weekend) next_time = start_time + delta hku_info("启动采集时间:{}".format(next_time)) time.sleep(delta.total_seconds()) while True: try: start_time = Datetime.now() pub_sock.send("{}{}".format(spot_topic, '[start spot]').encode('utf-8')) records = get_spot_parallel(stk_list, source, use_proxy, batch_func) hku_info("{}:{}:{} 采集数量: {}".format(start_time.hour, start_time.minute, start_time.second, len(records))) pub_sock.send('{}{}'.format(spot_topic, '[end spot]').encode('utf-8')) delta = next_delta(start_time, seconds, phase1_delta, phase2_delta, ignore_weekend) time.sleep(delta.total_seconds()) except Exception as e: hku_error(e) time.sleep(10)