def history_sysframe(exchange, url, force): for c in Collection.query({'exchange': exchange}): try: if not force and history_exists(c): continue # 拿到数据文件 theurl = ('{}/hqApplet/data/day/00{}.day.zip').format( url, c.symbol) log.info('fetching exchange {} url {}'.format(exchange, theurl)) r = session.get( theurl, timeout=(5, 10), verify=False) if r.status_code != 200: log.warning('{}_{}下载失败, 错误码: {}' ''.format(exchange, c.symbol, r.status_code)) continue content = gzip.decompress(r.content) # 解析之 num_rows = struct.unpack('>i', content[0:4])[0] kline_days = [] for i in range(num_rows): raw_row = content[4 + 40 * i: 4 + 40 * i + 40] row = struct.unpack('>i5f2ifi', raw_row) t = row[0] date = datetime(year=int(str(t)[0:2]) + 1997, month=int(str(t)[2:4]), day=int(str(t)[4:6]), minute=int(str(t)[6:8] or 0), second=int(str(t)[8:10] or 0)) # row[6]总是0, 不知道是啥 q = { 'exchange': exchange, 'symbol': c.symbol, 'quote_type': '1d', 'quote_at': date, 'open_': row[1], 'high': row[2], 'low': row[3], 'close': row[4], 'mean': row[5], 'volume': row[7], 'amount': row[8], 'quantity': row[9], } if kline_days: q['lclose'] = kline_days[-1]['close'] save_quotes(q, c, first_quote=False) else: save_quotes(q, c, first_quote=True) kline_days.append(q) except: log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
def history_sysframe(exchange, url, force): for c in Collection.query({'exchange': exchange}): try: if not force and history_exists(c): continue # 拿到数据文件 theurl = ('{}/hqApplet/data/day/00{}.day.zip').format( url, c.symbol) log.info('fetching exchange {} url {}'.format(exchange, theurl)) r = session.get(theurl, timeout=(5, 10), verify=False) if r.status_code != 200: log.warning('{}_{}下载失败, 错误码: {}' ''.format(exchange, c.symbol, r.status_code)) continue content = gzip.decompress(r.content) # 解析之 num_rows = struct.unpack('>i', content[0:4])[0] kline_days = [] for i in range(num_rows): raw_row = content[4 + 40 * i:4 + 40 * i + 40] row = struct.unpack('>i5f2ifi', raw_row) t = row[0] date = datetime(year=int(str(t)[0:2]) + 1997, month=int(str(t)[2:4]), day=int(str(t)[4:6]), minute=int(str(t)[6:8] or 0), second=int(str(t)[8:10] or 0)) # row[6]总是0, 不知道是啥 q = { 'exchange': exchange, 'symbol': c.symbol, 'quote_type': '1d', 'quote_at': date, 'open_': row[1], 'high': row[2], 'low': row[3], 'close': row[4], 'mean': row[5], 'volume': row[7], 'amount': row[8], 'quantity': row[9], } if kline_days: q['lclose'] = kline_days[-1]['close'] save_quotes(q, c, first_quote=False) else: save_quotes(q, c, first_quote=True) kline_days.append(q) except: log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
def history(site, force=False): conf = get_conf(site) exchange = conf['abbr'] url = conf['quote']['history']['url'] type_ = conf['quote']['history']['type'] if not url: log.warning('{}尚未配置'.format(exchange)) return if type_ == 'sysframe': history_sysframe(exchange, url, force) elif type_ == 'winner': history_winner(exchange, url, force) else: log.warning('{}无法识别'.format(type_)) return
def realtime(site): conf = get_conf(site) exchange = conf['abbr'] url = conf['quote']['realtime']['url'] type_ = conf['quote']['realtime']['type'] today = datetime.utcnow().replace( minute=0, second=0, microsecond=0) + timedelta(hours=8) if not url: log.warning('{}尚未配置实时行情url'.format(exchange)) return if today.hour < 9 or today.hour > 22: log.warning('不在9点到22点之间, 不做解析') return today = today.replace(hour=0) text = session.get(url, timeout=(3, 7)).text quotes = parse_quotes(type_, text) saved = 0 for q in quotes: Collection.update_one({'exchange': exchange, 'symbol': q['symbol'].strip()}, {'$set': {'name': q['name']}}, upsert=True) q['exchange'] = exchange q['quote_type'] = '1d' q['quote_at'] = today if q['open_'] in ['—', '-', None, '']: continue else: # 找到上一个交易日的数据, 如果和lclose不符则舍弃 # 需要保证数据每天更新/不足时需要把日线补足才能正常显示 lq = Quote.query_one({'exchange': exchange, 'symbol': q['symbol'].strip(), 'quote_type': '1d', 'quote_at': {'$lt': today}}, sort=[('quote_at', -1)]) if not lq or abs(lq.close - q['lclose']) < 0.01: Quote(q).upsert() saved += 1 log.info('{} 导入 {}/{} 条实时交易记录'.format(exchange, saved, len(quotes)))