def list_parser(task, rule):
item = jsonp_json(task["text"])
skus = []
groups = []
dp_pairs = []
if "products" not in item:
log_with_time("found nothing: %s" % task["url"])
return
now = int(time.time())
dps_log = {}
shop = {}
comment = {}
promos = []
for p in item["products"]:
try:
s = p["skus"]
price = str(s["price"])
url = str(s["sUrl"])
title = s["name"]
except KeyError:
log_with_time("rule error: %s" % task["text"])
continue
dp_pairs.append((url, title))
if s["stock"] > 0:
stock = 1
else:
stock = 0
promos.append((url, s["skuNo"]))
skus.append((url, price, stock))
if p.get("shopId"):
shop[get_crc(url)] = "%s,%s" % (p["shopId"], p["sName"])
if p.get("evaluateCount"):
comment[get_crc(url)] = int(p["evaluateCount"])
result = format_price(skus)
for r in result:
dps_log[r[1]] = now
return {
"spider": result,
"dp": dp_pairs,
"dps_log": dps_log,
"shop": shop,
"comment": comment,
"promo": promos,
}
def list_parser(task, rule):
item = jsonp_json(task["text"])
skus = []
groups = []
dp_pairs = []
if "products" not in item:
log_with_time("found nothing: %s" % task["url"])
return
now = int(time.time())
dps_log = {}
shop = {}
comment = {}
promos = []
for p in item["products"]:
try:
s = p["skus"]
price = str(s["price"])
url = str(s["sUrl"])
title = s["name"]
except KeyError:
log_with_time("rule error: %s" % task["text"])
continue
dp_pairs.append((url, title))
if s["stock"] > 0:
stock = 1
else:
stock = 0
promos.append((url, s["skuNo"]))
skus.append((url, price, stock))
if p.get("shopId"):
shop[get_crc(url)] = "%s,%s" % (p["shopId"], p["sName"])
if p.get("evaluateCount"):
comment[get_crc(url)] = int(p["evaluateCount"])
result = format_price(skus)
for r in result:
dps_log[r[1]] = now
return {
"spider": result,
"dp": dp_pairs,
"dps_log": dps_log,
"shop": shop,
"comment": comment,
"promo": promos,
}
def promo_parser(task, rule):
j = jsonp_json(task["text"])
promos = []
log_with_time("url: %s" % task["old"])
crc = get_crc(task["old"])
#site_enable
#1 客户端
#2 手机网页
#3 手机端
if not j["proms"]:
log_with_time("no promo: %s" % task["old"])
return format_promo([(crc, promos)])
for item in j["proms"]:
tp = item["type"]
if tp == "ZENGPIN":
b = []
for title in item["titleList"]:
b.append(title["title"])
if b:
promos.append({
"keywords": "赠品",
"type": 3,
"desc": u"赠: %s" % "".join(b)
})
elif tp == "LYMANZENG":
b = []
for title in item["titleList"]:
b.append(title["title"])
if b:
promos.append({
"keywords": "满赠",
"type": 3,
"desc": u"%s: %s" % (item["desc"], "".join(b))
})
elif tp == "ZHIJIANG":
#ignore
pass
elif tp == "ZENGQUANRED" or tp == "ZENGQUANBLUE":
b = []
for title in item["titleList"]:
b.append(u"%s张%s元%s" % (title["couponNum"], title["couponPrice"], title["couponName"]))
if b:
promos.append({
"keywords": "赠券",
"type": 3,
"desc": u"赠: " + "".join(b)
})
elif tp == "MANJIAN_old":
promos.append({
"keywords": "满减",
"type": 1,
"desc": item["desc"],
})
elif tp == "MANJIAN":
promos.append({
"keywords": "满减",
"type": 1,
"desc": item["desc"],
})
elif tp == "LYMANJIAN":
i = {
"keywords": "满减",
"type": 1,
"desc": item["desc"],
}
if item.get("more") or item.get("more") == "true":
i["desc"] += item["more_msg"]
promos.append(i)
elif tp == "LYMANFAN":
promos.append({
"keywords": "返券",
"type": 2,
"desc": item["desc"],
})
elif tp == "MANFANRED":
desc = item["desc"].replace("<", "<")
desc = desc.replace(">", ">")
promos.append({
"keywords": "返券",
"type": 2,
"desc": desc,
})
elif tp == "ZHEKOU" or tp == "LYMANZHE":
promos.append({
"keywords": "多买优惠",
"type": 11,
"desc": item["desc"]
})
#VIPPRICE -> 会员价忽略
else:
log_with_time("other promo: %s" % str(item))
if not promos:
log_with_time("no promo: %s" % task["old"])
return format_promo([(crc, promos)])
def promo_parser(task, rule):
j = jsonp_json(task["text"])
promos = []
log_with_time("url: %s" % task["old"])
crc = get_crc(task["old"])
#site_enable
#1 客户端
#2 手机网页
#3 手机端
if not j["proms"]:
log_with_time("no promo: %s" % task["old"])
return format_promo([(crc, promos)])
for item in j["proms"]:
tp = item["type"]
if tp == "ZENGPIN":
b = []
for title in item["titleList"]:
b.append(title["title"])
if b:
promos.append({
"keywords": "赠品",
"type": 3,
"desc": u"赠: %s" % "".join(b)
})
elif tp == "LYMANZENG":
b = []
for title in item["titleList"]:
b.append(title["title"])
if b:
promos.append({
"keywords": "满赠",
"type": 3,
"desc": u"%s: %s" % (item["desc"], "".join(b))
})
elif tp == "ZHIJIANG":
#ignore
pass
elif tp == "ZENGQUANRED" or tp == "ZENGQUANBLUE":
b = []
for title in item["titleList"]:
b.append(u"%s张%s元%s" %
(title["couponNum"], title["couponPrice"],
title["couponName"]))
if b:
promos.append({
"keywords": "赠券",
"type": 3,
"desc": u"赠: " + "".join(b)
})
elif tp == "MANJIAN_old":
promos.append({
"keywords": "满减",
"type": 1,
"desc": item["desc"],
})
elif tp == "MANJIAN":
promos.append({
"keywords": "满减",
"type": 1,
"desc": item["desc"],
})
elif tp == "LYMANJIAN":
i = {
"keywords": "满减",
"type": 1,
"desc": item["desc"],
}
if item.get("more") or item.get("more") == "true":
i["desc"] += item["more_msg"]
promos.append(i)
elif tp == "LYMANFAN":
promos.append({
"keywords": "返券",
"type": 2,
"desc": item["desc"],
})
elif tp == "MANFANRED":
desc = item["desc"].replace("<", "<")
desc = desc.replace(">", ">")
promos.append({
"keywords": "返券",
"type": 2,
"desc": desc,
})
elif tp == "ZHEKOU" or tp == "LYMANZHE":
promos.append({
"keywords": "多买优惠",
"type": 11,
"desc": item["desc"]
})
#VIPPRICE -> 会员价忽略
else:
log_with_time("other promo: %s" % str(item))
if not promos:
log_with_time("no promo: %s" % task["old"])
return format_promo([(crc, promos)])