def stock_filter(items): ret = [] for i in split_list_iter(items, 60): keys = async_http.quote(",".join(i)) url = stock_base.format(pids=keys, time=int(time.time() * 1000)) ret.append({"url": url}) return ret
def stock_filter(items): ret = [] for i in split_list_iter(items, 30): ret.append({ "url": api.format(mid= mid_table[spider.CONFIG["site_id"]]) + ','.join(i) }) return ret
def price_filter(items): ret = [] for group in split_list_iter(items, 60): url = price_url % (",".join("J_"+ i for i in group), random.randint(1000000, 10000000), int(time.time() * 1000)) ret.append({ "url": url}) return ret
def price_filter(items): ret = [] for group in split_list_iter(items, 60): url = price_url % (",".join( "J_" + i for i in group), random.randint( 1000000, 10000000), int(time.time() * 1000)) ret.append({"url": url}) return ret
def stock_filter(items): ret = [] for i in split_list_iter(items, 30): ret.append({ "url": api.format(mid=mid_table[spider.CONFIG["site_id"]]) + ','.join(i) }) return ret
def stock_filter(items): ret = [] for i in split_list_iter(items, 60): keys = async_http.quote(",".join(i)) url = stock_base.format(pids=keys, time=int(time.time() * 1000)) ret.append({ "url": url, }) return ret
def stock_task_filter(items): ret = [] for i in split_list_iter(items, 40): prices = dict(i) base = stburl + "&".join(["productIds=" + i for i in prices.keys()]) ret.append({ "url": base, "price": prices, }) return ret
def off_filter(items): gids = [item[0] for item in items] tasks = [] for i in split_list_iter(gids, 60): offcheck_hdr = async_http.json_header.copy() offcheck_hdr["Referer"] = "http://category.dangdang.com" tasks.append({ "url": priceurl, "payload": { "pids": ",".join(i), "time": str(int(time.time())), "type": "get_price", }, "header": offcheck_hdr }) return tasks
def off_filter(items): qids = [] for item in items: url = item[1] if "360buy" in url or "book" in url: qid = GET_QID.findall(url)[0] else: qid = item[0] qids.append(qid) ret = [] for group in split_list_iter(qids, 60): url = price_url % (",".join("J_"+ i for i in group), random.randint(1000000, 10000000), int(time.time() * 1000)) ret.append({ "url": url}) return ret