def stock_filter(items):
ret = []
for i in split_list_iter(items, 60):
keys = async_http.quote(",".join(i))
url = stock_base.format(pids=keys, time=int(time.time() * 1000))
ret.append({"url": url})
return ret
def stock_filter(items):
ret = []
for i in split_list_iter(items, 30):
ret.append({
"url": api.format(mid= mid_table[spider.CONFIG["site_id"]]) + ','.join(i)
})
return ret
def price_filter(items):
ret = []
for group in split_list_iter(items, 60):
url = price_url % (",".join("J_"+ i for i in group),
random.randint(1000000, 10000000),
int(time.time() * 1000))
ret.append({ "url": url})
return ret
def price_filter(items):
ret = []
for group in split_list_iter(items, 60):
url = price_url % (",".join(
"J_" + i for i in group), random.randint(
1000000, 10000000), int(time.time() * 1000))
ret.append({"url": url})
return ret
def stock_filter(items):
ret = []
for i in split_list_iter(items, 30):
ret.append({
"url":
api.format(mid=mid_table[spider.CONFIG["site_id"]]) + ','.join(i)
})
return ret
def stock_filter(items):
ret = []
for i in split_list_iter(items, 60):
keys = async_http.quote(",".join(i))
url = stock_base.format(pids=keys, time=int(time.time() * 1000))
ret.append({
"url": url,
})
return ret
def stock_task_filter(items):
ret = []
for i in split_list_iter(items, 40):
prices = dict(i)
base = stburl + "&".join(["productIds=" + i for i in prices.keys()])
ret.append({
"url": base,
"price": prices,
})
return ret
def off_filter(items):
gids = [item[0] for item in items]
tasks = []
for i in split_list_iter(gids, 60):
offcheck_hdr = async_http.json_header.copy()
offcheck_hdr["Referer"] = "http://category.dangdang.com"
tasks.append({
"url": priceurl,
"payload": {
"pids": ",".join(i),
"time": str(int(time.time())),
"type": "get_price",
},
"header": offcheck_hdr
})
return tasks
def off_filter(items):
qids = []
for item in items:
url = item[1]
if "360buy" in url or "book" in url:
qid = GET_QID.findall(url)[0]
else:
qid = item[0]
qids.append(qid)
ret = []
for group in split_list_iter(qids, 60):
url = price_url % (",".join("J_"+ i for i in group),
random.randint(1000000, 10000000),
int(time.time() * 1000))
ret.append({ "url": url})
return ret
def off_filter(items):
gids = [item[0] for item in items]
tasks = []
for i in split_list_iter(gids, 60):
offcheck_hdr = async_http.json_header.copy()
offcheck_hdr["Referer"] = "http://category.dangdang.com"
tasks.append({
"url": priceurl,
"payload": {
"pids": ",".join(i),
"time": str(int(time.time())),
"type": "get_price",
},
"header": offcheck_hdr
})
return tasks
