Exemplo n.º 1
0
Arquivo: parser.py Projeto: maocis/ybk
def parse(site):
    rabbrs = {v: k for k, v in ABBRS.items()}
    abbr = rabbrs[site]
    parser = importlib.__import__('ybk.parsers.{}'.format(site),
                                  fromlist=['Parser']).Parser()
    log.info('解析交易所 {}'.format(abbr))
    num_parsed = 0
    num_failed = 0
    for a in Announcement.query({'exchange': abbr,
                                 'parsed': {'$ne': True}}):
        log.info('parsing {}'.format(a.url))
        try:
            for c in parser.parse(a.type_, a.html):
                c['from_url'] = a.url
                Collection(c).upsert()
            a.update({'$set': {'parsed': True}})
            num_parsed += 1
        except Exception as e:
            num_failed += 1
            if not isinstance(e, NotImplementedError):
                log.exception('解析错误')
            continue

    log.info('解析完毕, {}个成功, {}个失败'.format(num_parsed, num_failed))
Exemplo n.º 2
0
Arquivo: cli.py Projeto: maocis/ybk
def do_list(parser, args):
    for abbr, site in ABBRS.items():
        print('{abbr}: {site}'.format(**locals()))