Esempio n. 1
0
def history_sysframe(exchange, url, force):
    for c in Collection.query({'exchange': exchange}):
        try:
            if not force and history_exists(c):
                continue

            # 拿到数据文件
            theurl = ('{}/hqApplet/data/day/00{}.day.zip').format(
                url, c.symbol)
            log.info('fetching exchange {} url {}'.format(exchange, theurl))
            r = session.get(
                theurl, timeout=(5, 10), verify=False)
            if r.status_code != 200:
                log.warning('{}_{}下载失败, 错误码: {}'
                            ''.format(exchange, c.symbol, r.status_code))
                continue
            content = gzip.decompress(r.content)

            # 解析之
            num_rows = struct.unpack('>i', content[0:4])[0]
            kline_days = []
            for i in range(num_rows):
                raw_row = content[4 + 40 * i: 4 + 40 * i + 40]
                row = struct.unpack('>i5f2ifi', raw_row)
                t = row[0]
                date = datetime(year=int(str(t)[0:2]) + 1997,
                                month=int(str(t)[2:4]),
                                day=int(str(t)[4:6]),
                                minute=int(str(t)[6:8] or 0),
                                second=int(str(t)[8:10] or 0))
                # row[6]总是0, 不知道是啥
                q = {
                    'exchange': exchange,
                    'symbol': c.symbol,
                    'quote_type': '1d',
                    'quote_at': date,
                    'open_': row[1],
                    'high': row[2],
                    'low': row[3],
                    'close': row[4],
                    'mean': row[5],
                    'volume': row[7],
                    'amount': row[8],
                    'quantity': row[9],
                }
                if kline_days:
                    q['lclose'] = kline_days[-1]['close']
                    save_quotes(q, c, first_quote=False)
                else:
                    save_quotes(q, c, first_quote=True)
                kline_days.append(q)
        except:
            log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
Esempio n. 2
0
def history_sysframe(exchange, url, force):
    for c in Collection.query({'exchange': exchange}):
        try:
            if not force and history_exists(c):
                continue

            # 拿到数据文件
            theurl = ('{}/hqApplet/data/day/00{}.day.zip').format(
                url, c.symbol)
            log.info('fetching exchange {} url {}'.format(exchange, theurl))
            r = session.get(theurl, timeout=(5, 10), verify=False)
            if r.status_code != 200:
                log.warning('{}_{}下载失败, 错误码: {}'
                            ''.format(exchange, c.symbol, r.status_code))
                continue
            content = gzip.decompress(r.content)

            # 解析之
            num_rows = struct.unpack('>i', content[0:4])[0]
            kline_days = []
            for i in range(num_rows):
                raw_row = content[4 + 40 * i:4 + 40 * i + 40]
                row = struct.unpack('>i5f2ifi', raw_row)
                t = row[0]
                date = datetime(year=int(str(t)[0:2]) + 1997,
                                month=int(str(t)[2:4]),
                                day=int(str(t)[4:6]),
                                minute=int(str(t)[6:8] or 0),
                                second=int(str(t)[8:10] or 0))
                # row[6]总是0, 不知道是啥
                q = {
                    'exchange': exchange,
                    'symbol': c.symbol,
                    'quote_type': '1d',
                    'quote_at': date,
                    'open_': row[1],
                    'high': row[2],
                    'low': row[3],
                    'close': row[4],
                    'mean': row[5],
                    'volume': row[7],
                    'amount': row[8],
                    'quantity': row[9],
                }
                if kline_days:
                    q['lclose'] = kline_days[-1]['close']
                    save_quotes(q, c, first_quote=False)
                else:
                    save_quotes(q, c, first_quote=True)
                kline_days.append(q)
        except:
            log.exception('{}:{} 抓取失败'.format(c.exchange, c.symbol))
Esempio n. 3
0
def history(site, force=False):
    conf = get_conf(site)
    exchange = conf['abbr']
    url = conf['quote']['history']['url']
    type_ = conf['quote']['history']['type']

    if not url:
        log.warning('{}尚未配置'.format(exchange))
        return

    if type_ == 'sysframe':
        history_sysframe(exchange, url, force)
    elif type_ == 'winner':
        history_winner(exchange, url, force)
    else:
        log.warning('{}无法识别'.format(type_))
        return
Esempio n. 4
0
def history(site, force=False):
    conf = get_conf(site)
    exchange = conf['abbr']
    url = conf['quote']['history']['url']
    type_ = conf['quote']['history']['type']

    if not url:
        log.warning('{}尚未配置'.format(exchange))
        return

    if type_ == 'sysframe':
        history_sysframe(exchange, url, force)
    elif type_ == 'winner':
        history_winner(exchange, url, force)
    else:
        log.warning('{}无法识别'.format(type_))
        return
Esempio n. 5
0
def realtime(site):
    conf = get_conf(site)
    exchange = conf['abbr']
    url = conf['quote']['realtime']['url']
    type_ = conf['quote']['realtime']['type']
    today = datetime.utcnow().replace(
        minute=0, second=0, microsecond=0) + timedelta(hours=8)

    if not url:
        log.warning('{}尚未配置实时行情url'.format(exchange))
        return
    if today.hour < 9 or today.hour > 22:
        log.warning('不在9点到22点之间, 不做解析')
        return

    today = today.replace(hour=0)
    text = session.get(url, timeout=(3, 7)).text
    quotes = parse_quotes(type_, text)
    saved = 0
    for q in quotes:
        Collection.update_one({'exchange': exchange,
                               'symbol': q['symbol'].strip()},
                              {'$set': {'name': q['name']}},
                              upsert=True)
        q['exchange'] = exchange
        q['quote_type'] = '1d'
        q['quote_at'] = today
        if q['open_'] in ['—', '-', None, '']:
            continue
        else:
            # 找到上一个交易日的数据, 如果和lclose不符则舍弃
            # 需要保证数据每天更新/不足时需要把日线补足才能正常显示
            lq = Quote.query_one({'exchange': exchange,
                                  'symbol': q['symbol'].strip(),
                                  'quote_type': '1d',
                                  'quote_at': {'$lt': today}},
                                 sort=[('quote_at', -1)])
            if not lq or abs(lq.close - q['lclose']) < 0.01:
                Quote(q).upsert()
                saved += 1

    log.info('{} 导入 {}/{} 条实时交易记录'.format(exchange, saved, len(quotes)))