示例#1
0
def getBrands(n_worker=1):
    brand_ids = jsm.Brand.IDS.keys()
    error_queue = Queue(len(brand_ids))
    brand_id_queue = Queue(len(brand_ids))
    [brand_id_queue.put(i) for i in brand_ids]
    print "total brands = %d" % len(brand_ids)
    pool = [BrandDownloadProcess(brand_id_queue, error_queue) for _ in xrange(n_worker)]
    for p in pool:
        p.start()
    for p in pool:
        p.join()
    
    print "collecting results"
    error_list = utils.dump_queue(error_queue)
    
    print "OK"
    return error_list
示例#2
0
def getFinances(n_worker=1):
    con = pymongo.MongoClient(host=mongo_info["host"], port=mongo_info["port"])
    col = con[mongo_info["db"]][mongo_info["collection"]]
    tick_ids = sorted(col.distinct("tick_id"))
    error_queue = Queue(len(tick_ids))
    tick_id_queue = Queue(len(tick_ids))
    [tick_id_queue.put(i) for i in tick_ids]
    print "total brands = %d" % len(tick_ids)
    pool = [FinanceDownloadProcess(tick_id_queue, error_queue) for _ in xrange(n_worker)]
    for p in pool:
        p.start()
    for p in pool:
        p.join()
    
    print "collecting results"
    error_list = utils.dump_queue(error_queue)
    
    print "OK"
    return error_list