Beispiel #1
0
def list_parser(task, rule): 
    item = jsonp_json(task["text"]) 
    skus = [] 
    groups = [] 
    dp_pairs = [] 
    if "products" not in item: 
        log_with_time("found nothing: %s" % task["url"])
        return
    now = int(time.time())
    dps_log = {} 
    shop = {}
    comment = {}
    promos = []
    for p in item["products"]: 
        try: 
            s = p["skus"] 
            price = str(s["price"])
            url = str(s["sUrl"])
            title = s["name"]
        except KeyError:
            log_with_time("rule error: %s" % task["text"])
            continue 
        dp_pairs.append((url, title))
        if s["stock"] > 0:
            stock = 1
        else:
            stock = 0 
        promos.append((url, s["skuNo"]))
        skus.append((url, price, stock)) 
        if p.get("shopId"):
            shop[get_crc(url)] =  "%s,%s" % (p["shopId"], p["sName"])
        if p.get("evaluateCount"): 
            comment[get_crc(url)] = int(p["evaluateCount"])
    result = format_price(skus) 
    for r in result:
        dps_log[r[1]] = now 
    return {
            "spider": result,
            "dp": dp_pairs,
            "dps_log": dps_log,
            "shop": shop,
            "comment": comment,
            "promo": promos,
            }
Beispiel #2
0
def list_parser(task, rule):
    item = jsonp_json(task["text"])
    skus = []
    groups = []
    dp_pairs = []
    if "products" not in item:
        log_with_time("found nothing: %s" % task["url"])
        return
    now = int(time.time())
    dps_log = {}
    shop = {}
    comment = {}
    promos = []
    for p in item["products"]:
        try:
            s = p["skus"]
            price = str(s["price"])
            url = str(s["sUrl"])
            title = s["name"]
        except KeyError:
            log_with_time("rule error: %s" % task["text"])
            continue
        dp_pairs.append((url, title))
        if s["stock"] > 0:
            stock = 1
        else:
            stock = 0
        promos.append((url, s["skuNo"]))
        skus.append((url, price, stock))
        if p.get("shopId"):
            shop[get_crc(url)] = "%s,%s" % (p["shopId"], p["sName"])
        if p.get("evaluateCount"):
            comment[get_crc(url)] = int(p["evaluateCount"])
    result = format_price(skus)
    for r in result:
        dps_log[r[1]] = now
    return {
        "spider": result,
        "dp": dp_pairs,
        "dps_log": dps_log,
        "shop": shop,
        "comment": comment,
        "promo": promos,
    }
Beispiel #3
0
def promo_parser(task, rule): 
    j = jsonp_json(task["text"]) 
    promos = [] 
    log_with_time("url: %s" % task["old"])
    crc = get_crc(task["old"])
    #site_enable 
    #1 客户端
    #2 手机网页
    #3 手机端 
    if not j["proms"]: 
        log_with_time("no promo: %s" % task["old"]) 
        return format_promo([(crc, promos)])
    for item in j["proms"]:
        tp = item["type"] 
        if tp == "ZENGPIN": 
            b = []
            for title in item["titleList"]:
                b.append(title["title"])
            if b:
                promos.append({
                    "keywords": "赠品",
                    "type": 3,
                    "desc": u"赠: %s" % "".join(b)
                    })
        elif tp == "LYMANZENG": 
            b = []
            for title in item["titleList"]:
                b.append(title["title"])
            if b:
                promos.append({
                    "keywords": "满赠",
                    "type": 3,
                    "desc": u"%s: %s" % (item["desc"], "".join(b))
                    }) 
        elif tp == "ZHIJIANG": 
            #ignore
            pass 
        elif tp == "ZENGQUANRED" or tp == "ZENGQUANBLUE": 
            b = []
            for title in item["titleList"]:
                b.append(u"%s张%s元%s" % (title["couponNum"], title["couponPrice"], title["couponName"])) 
            if b:
                promos.append({
                    "keywords": "赠券",
                    "type": 3,
                    "desc": u"赠: " + "".join(b)
                    }) 
        elif tp == "MANJIAN_old": 
            promos.append({
                "keywords": "满减",
                "type": 1,
                "desc": item["desc"],
                })
        elif tp == "MANJIAN": 
            promos.append({
                "keywords": "满减",
                "type": 1,
                "desc": item["desc"],
                }) 
        elif tp == "LYMANJIAN": 
            i = {
                "keywords": "满减",
                "type": 1,
                "desc": item["desc"],
                } 
            if item.get("more") or item.get("more") == "true":
                i["desc"] += item["more_msg"]
            promos.append(i) 
        elif tp == "LYMANFAN":
            promos.append({
                    "keywords": "返券",
                    "type": 2, 
                    "desc": item["desc"],
                    }) 
        elif tp == "MANFANRED": 
            desc = item["desc"].replace("&lt;", "<")
            desc = desc.replace("&gt;", ">")
            promos.append({
                    "keywords": "返券",
                    "type": 2, 
                    "desc": desc,
                    }) 
        elif tp == "ZHEKOU" or tp == "LYMANZHE": 
            promos.append({
                "keywords": "多买优惠",
                "type": 11,
                "desc": item["desc"]
                })
        #VIPPRICE -> 会员价忽略
        else:
            log_with_time("other promo: %s" % str(item))
    if not promos:
        log_with_time("no promo: %s" % task["old"]) 
    return format_promo([(crc, promos)])
Beispiel #4
0
def promo_parser(task, rule):
    j = jsonp_json(task["text"])
    promos = []
    log_with_time("url: %s" % task["old"])
    crc = get_crc(task["old"])
    #site_enable
    #1 客户端
    #2 手机网页
    #3 手机端
    if not j["proms"]:
        log_with_time("no promo: %s" % task["old"])
        return format_promo([(crc, promos)])
    for item in j["proms"]:
        tp = item["type"]
        if tp == "ZENGPIN":
            b = []
            for title in item["titleList"]:
                b.append(title["title"])
            if b:
                promos.append({
                    "keywords": "赠品",
                    "type": 3,
                    "desc": u"赠: %s" % "".join(b)
                })
        elif tp == "LYMANZENG":
            b = []
            for title in item["titleList"]:
                b.append(title["title"])
            if b:
                promos.append({
                    "keywords": "满赠",
                    "type": 3,
                    "desc": u"%s: %s" % (item["desc"], "".join(b))
                })
        elif tp == "ZHIJIANG":
            #ignore
            pass
        elif tp == "ZENGQUANRED" or tp == "ZENGQUANBLUE":
            b = []
            for title in item["titleList"]:
                b.append(u"%s张%s元%s" %
                         (title["couponNum"], title["couponPrice"],
                          title["couponName"]))
            if b:
                promos.append({
                    "keywords": "赠券",
                    "type": 3,
                    "desc": u"赠: " + "".join(b)
                })
        elif tp == "MANJIAN_old":
            promos.append({
                "keywords": "满减",
                "type": 1,
                "desc": item["desc"],
            })
        elif tp == "MANJIAN":
            promos.append({
                "keywords": "满减",
                "type": 1,
                "desc": item["desc"],
            })
        elif tp == "LYMANJIAN":
            i = {
                "keywords": "满减",
                "type": 1,
                "desc": item["desc"],
            }
            if item.get("more") or item.get("more") == "true":
                i["desc"] += item["more_msg"]
            promos.append(i)
        elif tp == "LYMANFAN":
            promos.append({
                "keywords": "返券",
                "type": 2,
                "desc": item["desc"],
            })
        elif tp == "MANFANRED":
            desc = item["desc"].replace("&lt;", "<")
            desc = desc.replace("&gt;", ">")
            promos.append({
                "keywords": "返券",
                "type": 2,
                "desc": desc,
            })
        elif tp == "ZHEKOU" or tp == "LYMANZHE":
            promos.append({
                "keywords": "多买优惠",
                "type": 11,
                "desc": item["desc"]
            })
        #VIPPRICE -> 会员价忽略
        else:
            log_with_time("other promo: %s" % str(item))
    if not promos:
        log_with_time("no promo: %s" % task["old"])
    return format_promo([(crc, promos)])