def history_all(): for site in SITES: retries = 3 while retries > 0: retries -= 1 try: history(site, force=True) except: log.exception('站点{}历史行情获取失败, retries={}'.format(site, retries)) else: break
def realtime_all(): for site in SITES: retries = 3 while retries > 0: retries -= 1 try: realtime(site) except: log.exception('站点{}实时行情获取失败, retries={}'.format(site, retries)) else: break
def history_sysframe(exchange, url, force): for c in Collection.query({'exchange': exchange}): try: if not force and history_exists(c): continue # 拿到数据文件 theurl = ('{}/hqApplet/data/day/00{}.day.zip').format( url, c.symbol) log.info('fetching exchange {} url {}'.format(exchange, theurl)) r = session.get( theurl, timeout=(5, 10), verify=False) if r.status_code != 200: log.warning('{}_{}下载失败, 错误码: {}' ''.format(exchange, c.symbol, r.status_code)) continue content = gzip.decompress(r.content) # 解析之 num_rows = struct.unpack('>i', content[0:4])[0] kline_days = [] for i in range(num_rows): raw_row = content[4 + 40 * i: 4 + 40 * i + 40] row = struct.unpack('>i5f2ifi', raw_row) t = row[0] date = datetime(year=int(str(t)[0:2]) + 1997, month=int(str(t)[2:4]), day=int(str(t)[4:6]), minute=int(str(t)[6:8] or 0), second=int(str(t)[8:10] or 0)) # row[6]总是0, 不知道是啥 q = { 'exchange': exchange, 'symbol': c.symbol, 'quote_type': '1d', 'quote_at': date, 'open_': row[1], 'high': row[2], 'low': row[3], 'close': row[4], 'mean': row[5], 'volume': row[7], 'amount': row[8], 'quantity': row[9], } if kline_days: q['lclose'] = kline_days[-1]['close'] save_quotes(q, c, first_quote=False) else: save_quotes(q, c, first_quote=True) kline_days.append(q) except: log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
def history_sysframe(exchange, url, force): for c in Collection.query({'exchange': exchange}): try: if not force and history_exists(c): continue # 拿到数据文件 theurl = ('{}/hqApplet/data/day/00{}.day.zip').format( url, c.symbol) log.info('fetching exchange {} url {}'.format(exchange, theurl)) r = session.get(theurl, timeout=(5, 10), verify=False) if r.status_code != 200: log.warning('{}_{}下载失败, 错误码: {}' ''.format(exchange, c.symbol, r.status_code)) continue content = gzip.decompress(r.content) # 解析之 num_rows = struct.unpack('>i', content[0:4])[0] kline_days = [] for i in range(num_rows): raw_row = content[4 + 40 * i:4 + 40 * i + 40] row = struct.unpack('>i5f2ifi', raw_row) t = row[0] date = datetime(year=int(str(t)[0:2]) + 1997, month=int(str(t)[2:4]), day=int(str(t)[4:6]), minute=int(str(t)[6:8] or 0), second=int(str(t)[8:10] or 0)) # row[6]总是0, 不知道是啥 q = { 'exchange': exchange, 'symbol': c.symbol, 'quote_type': '1d', 'quote_at': date, 'open_': row[1], 'high': row[2], 'low': row[3], 'close': row[4], 'mean': row[5], 'volume': row[7], 'amount': row[8], 'quantity': row[9], } if kline_days: q['lclose'] = kline_days[-1]['close'] save_quotes(q, c, first_quote=False) else: save_quotes(q, c, first_quote=True) kline_days.append(q) except: log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
def history_winner(exchange, url, force): assert url.startswith('tcp://') host, port = url[6:].split(':') port = int(port) s = socket.socket() s.connect((host, port)) def get_day_data(symbol): bsymbol = symbol.encode('utf-8') bizdata = b''.join([ b'\xfe\x8f\x00\x00', b'\x00' * 12, b'\x01\x00', b',\x00', b'\x02\x04', b'\x00', b'\x00', b'\x00\x00\x00\x00', b'\x013' + bsymbol, b'\x03\x00', b'\x00\x00', b'\x00\x00\x00\x00\x00\x00\x00\x00', b'Z\x00\x10\x00\x013' + bsymbol, b'\x00\x00\x00\x00', b'\x00', b'\x00', ]) bdata = b''.join([ b'\x95\x00\x00\x95', b'11=524\x00', b'13=8\x00', b'5=36862\x00', b'4=109\x00', b'1=66\x008=', bizdata, b'\x00', b'1=40\x0069=markid=60ba7308cab942ee961536a74ec7c5f9\x00\x00', ]) s.sendall(bdata) batch = 8192 result = [] while True: result.append(s.recv(batch)) if len(result[-1]) != batch: break return b''.join(result) def parse_day_data(data): size = struct.unpack('>i', b'\x00' + data[1:4])[0] assert size + 4 == len(data) m = re.compile(b'1=(\d+)\x008=').search(data) if m: bizsize = int(m.group(1)) start = m.span()[1] try: bbiz = zlib.decompress(data[start:start + bizsize]) except: bbiz = data[start:start + bizsize] assert bbiz[:4] == b'\xfe\x8f\x00\x00' num_packs = struct.unpack('<H', bbiz[16:18])[0] sizes = [ struct.unpack('<H', bbiz[18 + 2 * i:20 + 2 * i])[0] for i in range(num_packs) ] for i in range(num_packs): start = 18 + 2 * num_packs + 0 if i == 0 else sizes[i - 1] if bbiz[start:start + 2] == b'\x02\x04': # K线数据 symbol = bbiz[start + 10:start + 16].decode('utf-8') count = struct.unpack('<I', bbiz[start + 16:start + 20])[0] kline_days = [] for i in range(count): begin = start + 20 + i * 32 row = struct.unpack('<Iiiiiiii', bbiz[begin:begin + 32]) q = { 'exchange': exchange, 'symbol': symbol, 'quote_type': '1d', 'quote_at': datetime.strptime(str(row[0]), "%Y%m%d"), 'open_': row[1] / 100, 'high': row[2] / 100, 'low': row[3] / 100, 'close': row[4] / 100, 'amount': row[5] / 1., 'volume': row[6], } if kline_days: q['lclose'] = kline_days[-1]['close'] save_quotes(q, c, first_quote=False) else: save_quotes(q, c, first_quote=True) kline_days.append(q) for c in Collection.query({'exchange': exchange}): try: if not force and history_exists(c): continue if '$' not in c.symbol: log.info('feching {}_{} on {}'.format(c.exchange, c.symbol, url)) data = get_day_data(c.symbol) parse_day_data(data) except: log.exception('{}_{}获取失败'.format(exchange, c.symbol))
def history_winner(exchange, url, force): assert url.startswith('tcp://') host, port = url[6:].split(':') port = int(port) s = socket.socket() s.connect((host, port)) def get_day_data(symbol): bsymbol = symbol.encode('utf-8') bizdata = b''.join([ b'\xfe\x8f\x00\x00', b'\x00' * 12, b'\x01\x00', b',\x00', b'\x02\x04', b'\x00', b'\x00', b'\x00\x00\x00\x00', b'\x013' + bsymbol, b'\x03\x00', b'\x00\x00', b'\x00\x00\x00\x00\x00\x00\x00\x00', b'Z\x00\x10\x00\x013' + bsymbol, b'\x00\x00\x00\x00', b'\x00', b'\x00', ]) bdata = b''.join([ b'\x95\x00\x00\x95', b'11=524\x00', b'13=8\x00', b'5=36862\x00', b'4=109\x00', b'1=66\x008=', bizdata, b'\x00', b'1=40\x0069=markid=60ba7308cab942ee961536a74ec7c5f9\x00\x00', ]) s.sendall(bdata) batch = 8192 result = [] while True: result.append(s.recv(batch)) if len(result[-1]) != batch: break return b''.join(result) def parse_day_data(data): size = struct.unpack('>i', b'\x00' + data[1:4])[0] assert size + 4 == len(data) m = re.compile(b'1=(\d+)\x008=').search(data) if m: bizsize = int(m.group(1)) start = m.span()[1] try: bbiz = zlib.decompress(data[start: start + bizsize]) except: bbiz = data[start: start + bizsize] assert bbiz[:4] == b'\xfe\x8f\x00\x00' num_packs = struct.unpack('<H', bbiz[16:18])[0] sizes = [struct.unpack('<H', bbiz[18 + 2 * i: 20 + 2 * i])[0] for i in range(num_packs)] for i in range(num_packs): start = 18 + 2 * num_packs + 0 if i == 0 else sizes[i - 1] if bbiz[start:start + 2] == b'\x02\x04': # K线数据 symbol = bbiz[start + 10: start + 16].decode('utf-8') count = struct.unpack( '<I', bbiz[start + 16: start + 20])[0] kline_days = [] for i in range(count): begin = start + 20 + i * 32 row = struct.unpack('<Iiiiiiii', bbiz[begin: begin + 32]) q = { 'exchange': exchange, 'symbol': symbol, 'quote_type': '1d', 'quote_at': datetime.strptime(str(row[0]), "%Y%m%d"), 'open_': row[1] / 100, 'high': row[2] / 100, 'low': row[3] / 100, 'close': row[4] / 100, 'amount': row[5] / 1., 'volume': row[6], } if kline_days: q['lclose'] = kline_days[-1]['close'] save_quotes(q, c, first_quote=False) else: save_quotes(q, c, first_quote=True) kline_days.append(q) for c in Collection.query({'exchange': exchange}): try: if not force and history_exists(c): continue if '$' not in c.symbol: log.info('feching {}_{} on {}'.format(c.exchange, c.symbol, url)) data = get_day_data(c.symbol) parse_day_data(data) except: log.exception('{}_{}获取失败'.format(exchange, c.symbol))