def history_sysframe(exchange, url, force):
for c in Collection.query({'exchange': exchange}):
try:
if not force and history_exists(c):
continue
# 拿到数据文件
theurl = ('{}/hqApplet/data/day/00{}.day.zip').format(
url, c.symbol)
log.info('fetching exchange {} url {}'.format(exchange, theurl))
r = session.get(
theurl, timeout=(5, 10), verify=False)
if r.status_code != 200:
log.warning('{}_{}下载失败, 错误码: {}'
''.format(exchange, c.symbol, r.status_code))
continue
content = gzip.decompress(r.content)
# 解析之
num_rows = struct.unpack('>i', content[0:4])[0]
kline_days = []
for i in range(num_rows):
raw_row = content[4 + 40 * i: 4 + 40 * i + 40]
row = struct.unpack('>i5f2ifi', raw_row)
t = row[0]
date = datetime(year=int(str(t)[0:2]) + 1997,
month=int(str(t)[2:4]),
day=int(str(t)[4:6]),
minute=int(str(t)[6:8] or 0),
second=int(str(t)[8:10] or 0))
# row[6]总是0, 不知道是啥
q = {
'exchange': exchange,
'symbol': c.symbol,
'quote_type': '1d',
'quote_at': date,
'open_': row[1],
'high': row[2],
'low': row[3],
'close': row[4],
'mean': row[5],
'volume': row[7],
'amount': row[8],
'quantity': row[9],
}
if kline_days:
q['lclose'] = kline_days[-1]['close']
save_quotes(q, c, first_quote=False)
else:
save_quotes(q, c, first_quote=True)
kline_days.append(q)
except:
log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
def history_sysframe(exchange, url, force):
for c in Collection.query({'exchange': exchange}):
try:
if not force and history_exists(c):
continue
# 拿到数据文件
theurl = ('{}/hqApplet/data/day/00{}.day.zip').format(
url, c.symbol)
log.info('fetching exchange {} url {}'.format(exchange, theurl))
r = session.get(theurl, timeout=(5, 10), verify=False)
if r.status_code != 200:
log.warning('{}_{}下载失败, 错误码: {}'
''.format(exchange, c.symbol, r.status_code))
continue
content = gzip.decompress(r.content)
# 解析之
num_rows = struct.unpack('>i', content[0:4])[0]
kline_days = []
for i in range(num_rows):
raw_row = content[4 + 40 * i:4 + 40 * i + 40]
row = struct.unpack('>i5f2ifi', raw_row)
t = row[0]
date = datetime(year=int(str(t)[0:2]) + 1997,
month=int(str(t)[2:4]),
day=int(str(t)[4:6]),
minute=int(str(t)[6:8] or 0),
second=int(str(t)[8:10] or 0))
# row[6]总是0, 不知道是啥
q = {
'exchange': exchange,
'symbol': c.symbol,
'quote_type': '1d',
'quote_at': date,
'open_': row[1],
'high': row[2],
'low': row[3],
'close': row[4],
'mean': row[5],
'volume': row[7],
'amount': row[8],
'quantity': row[9],
}
if kline_days:
q['lclose'] = kline_days[-1]['close']
save_quotes(q, c, first_quote=False)
else:
save_quotes(q, c, first_quote=True)
kline_days.append(q)
except:
log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
def history(site, force=False):
conf = get_conf(site)
exchange = conf['abbr']
url = conf['quote']['history']['url']
type_ = conf['quote']['history']['type']
if not url:
log.warning('{}尚未配置'.format(exchange))
return
if type_ == 'sysframe':
history_sysframe(exchange, url, force)
elif type_ == 'winner':
history_winner(exchange, url, force)
else:
log.warning('{}无法识别'.format(type_))
return
def history(site, force=False):
conf = get_conf(site)
exchange = conf['abbr']
url = conf['quote']['history']['url']
type_ = conf['quote']['history']['type']
if not url:
log.warning('{}尚未配置'.format(exchange))
return
if type_ == 'sysframe':
history_sysframe(exchange, url, force)
elif type_ == 'winner':
history_winner(exchange, url, force)
else:
log.warning('{}无法识别'.format(type_))
return
def realtime(site):
conf = get_conf(site)
exchange = conf['abbr']
url = conf['quote']['realtime']['url']
type_ = conf['quote']['realtime']['type']
today = datetime.utcnow().replace(
minute=0, second=0, microsecond=0) + timedelta(hours=8)
if not url:
log.warning('{}尚未配置实时行情url'.format(exchange))
return
if today.hour < 9 or today.hour > 22:
log.warning('不在9点到22点之间, 不做解析')
return
today = today.replace(hour=0)
text = session.get(url, timeout=(3, 7)).text
quotes = parse_quotes(type_, text)
saved = 0
for q in quotes:
Collection.update_one({'exchange': exchange,
'symbol': q['symbol'].strip()},
{'$set': {'name': q['name']}},
upsert=True)
q['exchange'] = exchange
q['quote_type'] = '1d'
q['quote_at'] = today
if q['open_'] in ['—', '-', None, '']:
continue
else:
# 找到上一个交易日的数据, 如果和lclose不符则舍弃
# 需要保证数据每天更新/不足时需要把日线补足才能正常显示
lq = Quote.query_one({'exchange': exchange,
'symbol': q['symbol'].strip(),
'quote_type': '1d',
'quote_at': {'$lt': today}},
sort=[('quote_at', -1)])
if not lq or abs(lq.close - q['lclose']) < 0.01:
Quote(q).upsert()
saved += 1
log.info('{} 导入 {}/{} 条实时交易记录'.format(exchange, saved, len(quotes)))