def getBrands(n_worker=1):
brand_ids = jsm.Brand.IDS.keys()
error_queue = Queue(len(brand_ids))
brand_id_queue = Queue(len(brand_ids))
[brand_id_queue.put(i) for i in brand_ids]
print "total brands = %d" % len(brand_ids)
pool = [BrandDownloadProcess(brand_id_queue, error_queue) for _ in xrange(n_worker)]
for p in pool:
p.start()
for p in pool:
p.join()
print "collecting results"
error_list = utils.dump_queue(error_queue)
print "OK"
return error_list
def getFinances(n_worker=1):
con = pymongo.MongoClient(host=mongo_info["host"], port=mongo_info["port"])
col = con[mongo_info["db"]][mongo_info["collection"]]
tick_ids = sorted(col.distinct("tick_id"))
error_queue = Queue(len(tick_ids))
tick_id_queue = Queue(len(tick_ids))
[tick_id_queue.put(i) for i in tick_ids]
print "total brands = %d" % len(tick_ids)
pool = [FinanceDownloadProcess(tick_id_queue, error_queue) for _ in xrange(n_worker)]
for p in pool:
p.start()
for p in pool:
p.join()
print "collecting results"
error_list = utils.dump_queue(error_queue)
print "OK"
return error_list