def next_delta(start_time, interval, phase1_delta, phase2_delta, ignore_weekend): interval_delta = TimeDelta(seconds=interval) phase1_start_delta, phase1_end_delta = phase1_delta phase2_start_delta, phase2_end_delta = phase2_delta today = Datetime.today() phase1_start, phase1_end = today + phase1_start_delta, today + phase1_end_delta phase2_start, phase2_end = today + phase2_start_delta, today + phase2_end_delta current_time = Datetime.now() maybe_time = current_time + interval_delta if phase1_start_delta == phase1_end_delta and phase2_start_delta == phase2_end_delta: # 两个时间周期各自的起止时间相等,则认为未做限制 delta = interval_delta - (current_time - start_time) elif maybe_time > phase2_end: # 如果预计的下一次时间大于周期2的结束日期, 则取下一日的周期1起始时间计算 next_time = today + TimeDelta(1) + phase1_start_delta if ignore_weekend and next_time.day_of_week() in (0, 6): next_time = today.next_week() + phase1_start_delta delta = next_time - start_time elif maybe_time in (phase1_start, phase1_end, phase2_start, phase2_end): # 如果下一次时间刚好等于时间周期的起止点,则直接返回预计的时间间隔 delta = interval_delta elif start_time < phase1_start and phase1_start < maybe_time < phase1_end: # 如果本次的时间小于周期1的起始时间且预计下一次的时间在phase1内,则取phase1起始时间计算 delta = phase1_start - start_time elif phase1_end < maybe_time < phase2_start: delta = phase2_start - start_time else: delta = interval_delta - (current_time - start_time) return delta
def __init__(self, config, hku_config_file, market='SH'): super(self.__class__, self).__init__() self.working = True self._config = config self.hku_config_file = hku_config_file self.market = market.lower() self.marketid = None self._interval = TimeDelta(seconds=config.getint('collect', 'interval', fallback=60 * 60)) self._phase1_start_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat( (config.get('collect', 'phase1_start', fallback='09:05')) ) ) ) self._phase1_end_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat( (config.get('collect', 'phase1_end', fallback='09:05')) ) ) ) self._use_zhima_proxy = config.getboolean('collect', 'use_zhima_proxy', fallback=False) self.cond = QWaitCondition() self.mutex = QMutex()
def insert_bars(self, market, bars, ktype): """批量插入一批Bar, 如果 bar['market'] 和 指定的 market 不一致,将被忽略 :param str market: 市场简称 :param bar: 可通过键值方式取值,必须包含以下key:market, code, datetime, open, high low, close, amount, volumn, bid1,bid1_amount .. bid5, ask1, ask1_amount... """ kline = [ (bar['code'], Datetime(bar['datetime']).start_of_day().number if ktype == 'day' else Datetime(bar['datetime']).number, bar['open'], bar['high'], bar['low'], bar['close'], bar['amount'], bar['volumn'], bar['bid1'], bar['bid1_amount'], bar['bid2'], bar['bid2_amount'], bar['bid3'], bar['bid3_amount'], bar['bid4'], bar['bid4_amount'], bar['bid5'], bar['bid5_amount'], bar['ask1'], bar['ask1_amount'], bar['ask2'], bar['ask2_amount'], bar['ask3'], bar['ask3_amount'], bar['ask4'], bar['ask4_amount'], bar['ask5'], bar['ask5_amount']) for bar in bars if market.lower() == bar['market'].lower() ] sql = 'INSERT OR REPLACE INTO {}_{}(code, datetime, open, high, low, close, amount, \ volumn, bid1, bid1_amount, bid2, bid2_amount, bid3, bid3_amount, bid4, bid4_amount, \ bid5, bid5_amount, ask1, ask1_amount, ask2, ask2_amount, ask3, ask3_amount, \ ask4, ask4_amount, ask5, ask5_amount) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,\ ?,?,?,?,?,?,?,?,?,?)'.format(market, ktype) self.cursor.executemany(sql, kline)
def run(self): self.logger.info("{} 数据采集同步线程启动".format(self.market)) self.db = SqliteMemDriver() hikyuu_init(self.hku_config_file, ignore_preload=True) stk_list = self.get_stock_list() hku_warn_if(not stk_list, "从数据库中获取的股票列表为空!", logger=self.logger) self.mutex.tryLock() end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day( ) start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day( ) start = Datetime.now() if self._phase1_start_time >= self._phase1_end_time: delta = TimeDelta(0) elif start >= self._phase1_end_time: delta = (self._phase1_start_time + TimeDelta(1) - start) elif start < self._phase1_start_time: delta = (self._phase1_start_time - start) else: delta = self._interval * ceil( (start - self._phase1_start_time) / self._interval) - ( start - self._phase1_start_time) hku_info('{} 下次采集时间:{}', self.market, start + delta, logger=self.logger) delta = int(delta.total_milliseconds()) while self.working and not self.cond.wait(self.mutex, int(delta)): last_time = Datetime.today() + end_delta start = Datetime.now() if start >= last_time: next_time = Datetime.today() + TimeDelta(1) + start_delta hku_info('{} 明日采集时间:{}', self.market, next_time, logger=self.logger) delta = next_time - start delta = int(delta.total_milliseconds()) continue hku_trace("start:{}".format(start)) self.collect(stk_list) end = Datetime.now() x = end - start if x + TimeDelta(seconds=1) < self._interval: delta = self._interval - x - TimeDelta(seconds=1) delta = int(delta.total_milliseconds()) self.logger.info("{} {:<.2f} 秒后重新采集".format( self.market, delta / 1000)) #self.sleep(delta) self.db.close() self.logger.info("{} 数据采集同步线程终止!".format(self.market))
def next_time_delta(self): now = Datetime.now() next_time = Datetime(now.year, now.month, now.day, self.hour, self.minute) if next_time < now: current_date = Datetime(now.year, now.month, now.day) if current_date.day_of_week() == 5: next_time = next_time + TimeDelta(3) else: next_time = next_time + TimeDelta(1) return (next_time, next_time - now)
def loadKData(self, market, code, ktype, start_ix, end_ix, out_buffer): """ 【重载接口】(必须)按指定的位置[start_ix, end_ix)读取K线数据至out_buffer :param str market: 市场标识 :param str code: 证券代码 :param KQuery.KType ktype: K线类型 :param int start_ix: 起始位置 :param int end_ix: 结束位置 :param KRecordListPtr out_buffer: 传入的数据缓存,读取数据后使用 out_buffer.append(krecord) 加入数据 """ if start_ix >= end_ix or start_ix <0 or end_ix <0: return data = self._get_bars(market, code, ktype) if len(data) < start_ix: return total = end_ix if end_ix < len(data) else len(data) for i in range(start_ix, total): record = KRecord() record.datetime = Datetime(data[i].get('datetime')) record.openPrice = data[i].get('open') record.highPrice = data[i].get('high') record.lowPrice = data[i].get('low') record.closePrice = data[i].get('close') record.transAmount = data[i].get('amount') record.transCount = data[i].get('vol') out_buffer.append(record)
def getKRecordList(self, market, code, query): """ 【重载接口】(必须)按指定的位置[start_ix, end_ix)读取K线数据至out_buffer :param str market: 市场标识 :param str code: 证券代码 :param Query query: 查询条件 :rtype: KRecordList """ if query.query_type == Query.DATE: print("未实现按日期查询") return KRecordList() start_ix = query.start end_ix = query.end if start_ix >= end_ix or start_ix < 0 or end_ix < 0: return KRecordList() data = self._get_bars(market, code, query.ktype) if len(data) < start_ix: return KRecordList() total = end_ix if end_ix < len(data) else len(data) result = KRecordList() for i in range(start_ix, total): record = KRecord() record.datetime = Datetime(data[i].get('datetime')) record.open = data[i].get('open') record.high = data[i].get('high') record.low = data[i].get('low') record.close = data[i].get('close') record.amount = data[i].get('amount') record.volume = data[i].get('vol') result.append(record) return result
def getKRecord(self, market, code, pos, ktype): """ 【重载接口】(必须)获取指定位置的K线记录 :param str market: 市场标识 :param str code: 证券代码 :param int pos: 指定位置(大于等于0) :param Query.KType ktype: K线类型 """ record = KRecord() if pos < 0: return record data = self._get_bars(market, code, ktype) if data is None: return record if pos < len(data): record.datetime = Datetime(data.index[pos]) record.open = data['open'][pos] record.high = data['high'][pos] record.low = data['low'][pos] record.close = data['close'][pos] record.amount = data['money'][pos] record.volume = data['volume'][pos] return record
def loadKData(self, market, code, ktype, start_ix, end_ix, out_buffer): """ 【重载接口】(必须)按指定的位置[start_ix, end_ix)读取K线数据至out_buffer :param str market: 市场标识 :param str code: 证券代码 :param Query.KType ktype: K线类型 :param int start_ix: 起始位置 :param int end_ix: 结束位置 :param KRecordListPtr out_buffer: 传入的数据缓存,读取数据后使用 out_buffer.append(krecord) 加入数据 """ if start_ix >= end_ix or start_ix < 0 or end_ix < 0: return data = self._get_bars(market, code, ktype) if len(data) < start_ix: return total = end_ix if end_ix < len(data) else len(data) for i in range(start_ix, total): record = KRecord() record.datetime = Datetime(data.index[i]) record.open = data['open'][i] record.high = data['high'][i] record.low = data['low'][i] record.close = data['close'][i] record.amount = data['money'][i] record.volume = data['volume'][i] out_buffer.append(record)
def getKRecord(self, market, code, pos, ktype): """ 【重载接口】(必须)获取指定位置的K线记录 :param str market: 市场标识 :param str code: 证券代码 :param int pos: 指定位置(大于等于0) :param KQuery.KType ktype: K线类型 """ record = KRecord() if pos < 0: return record data = self._get_bars(market, code, ktype) if data is None: return record if pos < len(data): record.datetime = Datetime(data[pos].get('datetime')) record.openPrice = data[pos].get('open') record.highPrice = data[pos].get('high') record.lowPrice = data[pos].get('low') record.closePrice = data[pos].get('close') record.transAmount = data[pos].get('amount') record.transCount = data[pos].get('vol') return record
def PositionList_to_np(pos_list): """转化为numpy结构数组""" t_type = np.dtype({ 'names': [ '证券代码', '证券名称', '买入日期', '已持仓天数', '持仓数量', '投入金额', '当前市值', '盈亏金额', '盈亏比例' ], 'formats': ['U10', 'U20', 'datetime64[D]', 'i', 'i', 'd', 'd', 'd', 'd'] }) sm = StockManager.instance() query = Query(-1) data = [] for pos in pos_list: invest = pos.buyMoney - pos.sellMoney + pos.totalCost k = pos.stock.getKData(query) cur_val = k[0].closePrice * pos.number bonus = cur_val - invest date_list = sm.getTradingCalendar( QueryByDate(Datetime(pos.takeDatetime.date()))) data.append( (pos.stock.market_code, pos.stock.name, pos.takeDatetime, len(date_list), pos.number, invest, cur_val, bonus, 100 * bonus / invest)) return np.array(data, dtype=t_type)
def __init__(self, config, market='SH'): super(self.__class__, self).__init__() self.working = True self._config = config self.market = market.lower() self.marketid = None assert config.getboolean("mysql", "enable", fallback=False) self._db_config = { 'user': config['mysql']['usr'], 'password': config['mysql']['pwd'], 'host': config['mysql']['host'], 'port': config['mysql']['port'] } self._connect = None self.quotations = [ 'stock', ] # sina 不支持基金,qq支持,此处屏蔽基金 #if config['quotation']['stock']: # self.quotations.append('stock') #if config['quotation']['fund']: # self.quotations.append('fund') #self.logger.info(self.quotations) self._interval = TimeDelta( seconds=config.getint('collect', 'interval', fallback=60 * 60)) self._phase1_start_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat((config.get('collect', 'phase1_start', fallback='09:05'))))) self._phase1_end_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat((config.get('collect', 'phase1_end', fallback='09:05'))))) self._use_zhima_proxy = config.getboolean('collect', 'use_zhima_proxy', fallback=False) self.cond = QWaitCondition() self.mutex = QMutex()
def _sell(self, datetime, market, code, price, num): """实现 OrderBrokerBase 的 _sell 接口""" if self._real: import tushare as ts df = ts.get_realtime_quotes(code) new_price = float(df.ix[0]['bid']) self._broker.sell(code, new_price, num) return Datetime.now() else: self._broker.sell(code, price, num) return datetime
def _sell(self, datetime, market, code, price, num): """实现 OrderBrokerBase 的 _sell 接口""" if self._real: import tushare as ts df = ts.get_realtime_quotes(code) new_price = float(df.ix[0]['bid']) self._broker.sell(code, new_price, num) return Datetime.now() else: self._broker.sell(code, price, num) return datetime
def process_one_record(self, record): if not self.record_is_valid(record): return (0, 0) connect = self.get_connect() current_date = Datetime(record['datetime'].date()).number table = get_table(connect, self.market, record['code'], 'day') sql = 'replace into {} (date, open, high, low, close, amount, count) \ values ({}, {}, {}, {}, {}, {}, {})'.format( table, current_date, record['open'], record['high'], record['low'], record['close'], record['amount'], record['volumn']) cur = connect.cursor() cur.execute(sql) cur.close() return
def _buy(self, datetime, market, code, price, num): """实现 OrderBrokerBase 的 _buy 接口""" if self._real: import tushare as ts df = ts.get_realtime_quotes(code) new_price = float(df.ix[0]['ask']) if (abs(new_price - price) <= self._slip): self._broker.buy(code, new_price, num) else: print("out of slip, not buy!!!!!!!!!!") return Datetime.now() else: self._broker.buy(code, price, num) return datetime
def _buy(self, datetime, market, code, price, num): """实现 OrderBrokerBase 的 _buy 接口""" if self._real: import tushare as ts df = ts.get_realtime_quotes(code) new_price = float(df.ix[0]['ask']) if (abs(new_price - price) <= self._slip): self._broker.buy(code, new_price, num) else: print("out of slip, not buy!!!!!!!!!!") return Datetime.now() else: self._broker.buy(code, price, num) return datetime
def indicator_getitem(data, i): """ :param i: int | Datetime | slice | str 类型 """ if isinstance(i, int): length = len(data) index = length + i if i < 0 else i if index < 0 or index >= length: raise IndexError("index out of range: %d" % i) return data.get(index) elif isinstance(i, slice): return [data.get(x) for x in range(*i.indices(len(data)))] elif isinstance(i, Datetime): return data.get_by_date(i) elif isinstance(i, str): return data.get_by_date(Datetime(i)) else: raise IndexError("Error index type")
def run(use_proxy, source, seconds, phase1, phase2, ignore_weekend): phase1_delta = parse_phase(phase1) hku_error_if(phase1_delta is None or len(phase1_delta) != 2, "无效参数 phase1: {}".format(phase1), callback=lambda: exit(1)) hku_error_if(phase1_delta[0] > phase1_delta[1], "无效参数 phase1: {}, 结束时间应大于等于起始时间".format(phase1), callback=lambda: exit(1)) phase2_delta = parse_phase(phase2) hku_error_if(phase2_delta is None or len(phase2_delta) != 2, "无效参数 phase2: {}".format(phase2), callback=lambda: exit(1)) hku_error_if(phase2_delta[0] > phase2_delta[1], "无效参数 phase2: {}, 结束时间应大于等于起始时间".format(phase2), callback=lambda: exit(1)) hku_error_if( phase1_delta[1] > phase2_delta[0], "无效参数 phase1: {}, phase2: {}, phase2 起始时间应大于等于 phase1 结束时间".format( phase1, phase2), callback=lambda: exit(1)) hku_logger.info("采集时间段1:{}".format(phase1)) hku_logger.info("采集时间段2:{}".format(phase2)) config_file = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini' if not os.path.exists(config_file): print("未找到配置文件,请先运行 HikyuuTDX 进行配置与数据导入") exit(1) hikyuu_init(config_file, ignore_preload=True) print("采集程序运行中,可使用 Ctrl-C 终止!") sm = StockManager.instance() stk_list = [ stk.market_code.lower() for stk in sm if stk.valid and stk.type in (constant.STOCKTYPE_A, constant.STOCKTYPE_INDEX, constant.STOCKTYPE_GEM) ] spot_topic = ':spot:' def batch_func(records): spot = bytearray(spot_topic.encode('utf-8')) buf = create_fb_spot(records) spot.extend(buf) pub_sock.send(bytes(spot)) address = "ipc:///tmp/hikyuu_real_pub.ipc" pub_sock = pynng.Pub0() pub_sock.listen(address) today = Datetime.today() phase1_time = [today + x for x in phase1_delta] phase2_time = [today + x for x in phase2_delta] start_time = Datetime.now() if not (phase1_time[0] <= start_time <= phase1_time[1] or phase2_time[0] <= start_time <= phase2_time[1]): delta = next_delta(start_time, seconds, phase1_delta, phase2_delta, ignore_weekend) next_time = start_time + delta hku_info("启动采集时间:{}".format(next_time)) time.sleep(delta.total_seconds()) while True: try: start_time = Datetime.now() pub_sock.send("{}{}".format(spot_topic, '[start spot]').encode('utf-8')) records = get_spot_parallel(stk_list, source, use_proxy, batch_func) hku_info("{}:{}:{} 采集数量: {}".format(start_time.hour, start_time.minute, start_time.second, len(records))) pub_sock.send('{}{}'.format(spot_topic, '[end spot]').encode('utf-8')) delta = next_delta(start_time, seconds, phase1_delta, phase2_delta, ignore_weekend) time.sleep(delta.total_seconds()) except Exception as e: hku_error(e) time.sleep(10)
class CollectToMemThread(QThread): def __init__(self, config, hku_config_file, market='SH'): super(self.__class__, self).__init__() self.working = True self._config = config self.hku_config_file = hku_config_file self.market = market.lower() self.marketid = None self._interval = TimeDelta(seconds=config.getint('collect', 'interval', fallback=60 * 60)) self._phase1_start_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat( (config.get('collect', 'phase1_start', fallback='09:05')) ) ) ) self._phase1_end_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat( (config.get('collect', 'phase1_end', fallback='09:05')) ) ) ) self._use_zhima_proxy = config.getboolean('collect', 'use_zhima_proxy', fallback=False) self.cond = QWaitCondition() self.mutex = QMutex() def stop(self): if self.working: self.working = False self.cond.wakeAll() self.wait() def __del__(self): self.stop() @hku_catch() def run(self): self.logger.info("{} 数据采集同步线程启动".format(self.market)) self.db = SqliteMemDriver() hikyuu_init(self.hku_config_file, ignore_preload=True) stk_list = self.get_stock_list() hku_warn_if(not stk_list, "从数据库中获取的股票列表为空!", self.logger) self.mutex.tryLock() end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day() start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day() start = Datetime.now() if self._phase1_start_time >= self._phase1_end_time: delta = TimeDelta(0) elif start >= self._phase1_end_time: delta = (self._phase1_start_time + TimeDelta(1) - start) elif start < self._phase1_start_time: delta = (self._phase1_start_time - start) else: delta = self._interval * ceil((start - self._phase1_start_time) / self._interval ) - (start - self._phase1_start_time) hku_info('{} 下次采集时间:{}'.format(self.market, start + delta), self.logger) delta = int(delta.total_milliseconds()) while self.working and not self.cond.wait(self.mutex, int(delta)): last_time = Datetime.today() + end_delta start = Datetime.now() if start >= last_time: next_time = Datetime.today() + TimeDelta(1) + start_delta hku_info('{} 明日采集时间:{}'.format(self.market, next_time), self.logger) delta = next_time - start delta = int(delta.total_milliseconds()) continue hku_trace("start:{}".format(start)) self.collect(stk_list) end = Datetime.now() x = end - start if x + TimeDelta(seconds=1) < self._interval: delta = self._interval - x - TimeDelta(seconds=1) delta = int(delta.total_milliseconds()) self.logger.info("{} {:<.2f} 秒后重新采集".format(self.market, delta / 1000)) #self.sleep(delta) self.db.close() self.logger.info("{} 数据采集同步线程终止!".format(self.market)) @hku_catch(retry=2, ret=[]) def get_stock_list(self): sm = StockManager.instance() return [ stk.market_code.lower() for stk in sm if stk.type in (constant.STOCKTYPE_A, constant.STOCKTYPE_INDEX, constant.STOCKTYPE_GEM) and stk.valid and stk.market.lower() == self.market.lower() ] def record_is_valid(self, record): return record['amount'] > 0.0 and record['volumn'] > 0.0 \ and record['high'] >= record['open'] >= record['low'] > 0.0 \ and record['high'] >= record['close'] >= record['low'] @hku_catch() def collect(self, stk_list): record_list = get_spot_parallel(stk_list, source='sina', use_proxy=self._use_zhima_proxy) hku_info("{} 网络获取数量:{}".format(self.market, len(record_list))) record_list = [r for r in record_list if self.record_is_valid(r)] self.db.insert_bars(self.market, record_list, 'day') self.db.insert_bars(self.market, record_list, 'min')
class CollectToMySQLThread(QThread): def __init__(self, config, market='SH'): super(self.__class__, self).__init__() self.working = True self._config = config self.market = market.lower() self.marketid = None assert config.getboolean("mysql", "enable", fallback=False) self._db_config = { 'user': config['mysql']['usr'], 'password': config['mysql']['pwd'], 'host': config['mysql']['host'], 'port': config['mysql']['port'] } self._connect = None self.quotations = [ 'stock', ] # sina 不支持基金,qq支持,此处屏蔽基金 #if config['quotation']['stock']: # self.quotations.append('stock') #if config['quotation']['fund']: # self.quotations.append('fund') #self.logger.info(self.quotations) self._interval = TimeDelta( seconds=config.getint('collect', 'interval', fallback=60 * 60)) self._phase1_start_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat((config.get('collect', 'phase1_start', fallback='09:05'))))) self._phase1_end_time = Datetime( datetime.datetime.combine( datetime.date.today(), datetime.time.fromisoformat((config.get('collect', 'phase1_end', fallback='09:05'))))) self._use_zhima_proxy = config.getboolean('collect', 'use_zhima_proxy', fallback=False) self.cond = QWaitCondition() self.mutex = QMutex() def stop(self): if self.working: self.working = False self.cond.wakeAll() self.wait() if self._connect is not None: hku_trace('关闭数据库连接', logger=self.logger) self._connect.close() def __del__(self): self.stop() @hku_catch() def run(self): self.logger.info("{} 数据采集同步线程启动".format(self.market)) stk_list = self.get_stock_list() hku_warn_if(not stk_list, "从数据库中获取的股票列表为空!", logger=self.logger) self.mutex.tryLock() end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day( ) start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day( ) start = Datetime.now() if self._phase1_start_time >= self._phase1_end_time: delta = TimeDelta(0) elif start >= self._phase1_end_time: delta = (self._phase1_start_time + TimeDelta(1) - start) elif start < self._phase1_start_time: delta = (self._phase1_start_time - start) else: delta = self._interval * ceil( (start - self._phase1_start_time) / self._interval) - ( start - self._phase1_start_time) hku_info('{} 下次采集时间:{}', self.market, start + delta, logger=self.logger) delta = int(delta.total_milliseconds()) while self.working and not self.cond.wait(self.mutex, int(delta)): last_time = Datetime.today() + end_delta start = Datetime.now() if start >= last_time: next_time = Datetime.today() + TimeDelta(1) + start_delta hku_info('{} 明日采集时间:{}', self.market, next_time, logger=self.logger) delta = next_time - start delta = int(delta.total_milliseconds()) continue hku_trace("start:{}".format(start)) self.collect(stk_list) end = Datetime.now() x = end - start if x + TimeDelta(seconds=1) < self._interval: delta = self._interval - x - TimeDelta(seconds=1) delta = int(delta.total_milliseconds()) self.logger.info("{} {:<.2f} 秒后重新采集".format( self.market, delta / 1000)) #self.sleep(delta) self.logger.info("{} 数据采集同步线程终止!".format(self.market)) def record_is_valid(self, record): return record['amount'] > 0.0 and record['volumn'] > 0.0 \ and record['high'] >= record['open'] >= record['low'] > 0.0 \ and record['high'] >= record['close'] >= record['low'] @hku_catch(ret=False) def process_one_record(self, record): if not self.record_is_valid(record): return (0, 0) connect = self.get_connect() current_date = Datetime(record['datetime'].date()).number table = get_table(connect, self.market, record['code'], 'day') sql = 'replace into {} (date, open, high, low, close, amount, count) \ values ({}, {}, {}, {}, {}, {}, {})'.format( table, current_date, record['open'], record['high'], record['low'], record['close'], record['amount'], record['volumn']) cur = connect.cursor() cur.execute(sql) cur.close() return @hku_catch() def collect(self, stk_list): record_list = get_spot_parallel(stk_list, source='sina', use_proxy=self._use_zhima_proxy) hku_info("{} 网络获取数量:{}".format(self.market, len(record_list))) connect = self.get_connect() if self.marketid == None: self.marketid = get_marketid(connect, self.market) update_count = 0 for record in record_list: if not self.working: break if self.process_one_record(record): update_count += 1 connect.commit() hku_info("{} 数据库更新记录数:{}".format(self.market, update_count)) def get_connect(self): if self._connect: return self._connect try: self._connect = mysql.connector.connect(**self._db_config) except mysql.connector.Error as err: if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: self.logger.error("MYSQL密码或用户名错误!") elif err.errno == errorcode.ER_BAD_DB_ERROR: self.logger.error("MySQL数据库不存在!") else: self.logger.error("连接数据库失败,{}".format(err.msg)) except: self.logger.error("未知原因导致无法连接数据库!") return self._connect @hku_catch(retry=2, ret=[]) def get_stock_list(self): connect = self.get_connect() stk_list = get_stock_list(connect, self.market, self.quotations) return [ "{}{}".format(self.market.lower(), item[2]) for item in stk_list if item[3] == 1 ]
def _getIndexRangeByDate(self, market, code, query): """ 【重载接口】(必须)按日期获取指定的K线数据 :param str market: 市场标识 :param str code: 证券代码 :param KQuery query: 日期查询条件(QueryByDate) """ print("getIndexRangeByDate") if query.queryType != Query.DATE: return (0, 0) start_datetime = query.startDatetime end_datetime = query.endDatetime if start_datetime >= end_datetime or start_datetime > Datetime.max(): return (0, 0) data = self._get_bars(market, code, query.kType) total = len(data) if total == 0: return (0, 0) mid, low = 0, 0 high = total-1 while low <= high: tmp_datetime = Datetime(data[high].get('datetime')) if start_datetime > tmp_datetime: mid = high + 1 break tmp_datetime = Datetime(data[low].get('datetime')) if tmp_datetime >= start_datetime: mid = low break mid = (low + high) // 2 tmp_datetime = Datetime(data[mid].get('datetime')) if start_datetime > tmp_datetime: low = mid + 1 else: high = mid - 1 if mid >= total: return (0, 0) start_pos = mid low = mid high = total - 1 while low <= high: tmp_datetime = Datetime(data[high].get('datetime')) if end_datetime > tmp_datetime: mid = high + 1 break tmp_datetime = Datetime(data[low].get('datetime')) if tmp_datetime >= end_datetime: mid = low break mid = (low + high) // 2 tmp_datetime = Datetime(data[mid].get('datetime')) if end_datetime > tmp_datetime: low = mid + 1 else: high = mid - 1 end_pos = total if mid >= total else mid if start_pos >= end_pos: return (0,0) return (start_pos, end_pos)