def product_info_parse(self, response):
try:
rs = json.loads(response.text)
except:
logger.warning('返回结果JSON解析失败')
logger.warning(rs)
if rs['code'] != '0':
logger.warning('product_info_parse()_retcode返回异常参数:{}'.format(
rs['code']))
return
if rs['result'] == None:
logger.warning('product_info_parse()_result equals None')
return
searchResultVOList = []
for i in rs['result']['searchCatResultVOList']:
for j in i['searchResultVOList']:
searchResultVOList.append(j)
for i in searchResultVOList:
pipline = SKUItem()
promotionslist = []
for item in i['tags']:
promotionslist.append('<' + item.get('iconText') + ' ' +
item.get('words') + ' ' +
item.get('name') + '>')
if 'skuId' not in i.keys(): continue
pipline['catId'] = response.meta['catId']
pipline['catName'] = response.meta['catName']
pipline['type'] = response.meta['type']
pipline['stockCount'] = i.get('stockCount', None)
pipline['skuId'] = i['skuId'] if 'skuName' in i.keys() else None
pipline['skuName'] = i['skuName'] if 'skuName' in i.keys(
) else None
pipline['realTimePrice'] = i[
'realTimePrice'] if 'realTimePrice' in i.keys() else None
pipline['basicPrice'] = i['basicPrice'] if 'basicPrice' in i.keys(
) else None
pipline['promotions'] = helper.list2str(promotionslist)
pipline['mkPrice'] = i['mkPrice'] if 'mkPrice' in i.keys(
) else None
pipline['monthSales'] = i['monthSales'] if 'monthSales' in i.keys(
) else None
pipline['highOpinion'] = i[
'highOpinion'] if 'highOpinion' in i.keys() else None
pipline['imgUrl'] = i['imgUrl'] if 'imgUrl' in i.keys() else None
pipline['coupons'] = response.meta['coupons']
pipline['storeId'] = i['storeId'] if 'storeId' in i.keys(
) else None
pipline['storeName'] = response.meta['storeName']
pipline['storeAddress'] = response.meta['storeName']
yield pipline
def store_info_parse(self, response):
rs = json.loads(response.text)
if rs['code'] != '0':
logger.warning('retcode返回异常参数:{}'.format(rs['retcode']))
backupFailRequest(response.url)
return
productCount = 0
cateList = []
rs = rs['result']
for i in rs['cateList'] if 'cateList' in rs.keys() else []:
cateList.append({i['title']:i['catId']})
productCount += i['productCount']
storeInfo = rs['storeInfo']
pipline = MyJDstoreItem()
pipline['id'] = storeInfo['storeId'] if 'storeId' in storeInfo.keys() else None
pipline['name'] = storeInfo['storeName'] if 'storeName' in storeInfo.keys() else None
pipline['address'] = storeInfo['storeAddress'] if 'storeAddress' in storeInfo.keys() else None
pipline['productCount'] = productCount
pipline['monthSaleCount'] = storeInfo['monthSaleNum'] if 'monthSaleNum' in storeInfo.keys() else None
# pipline['lon'] = storeInfo['geoPoint']['lon']
# pipline['lat'] = storeInfo['geoPoint']['lat']
# pipline['state'] = storeInfo['state']
# pipline['city'] = storeInfo['city']
# pipline['district'] = storeInfo['district']
# pipline['deliverProvider'] = storeInfo['deliverProvider']
# pipline['commentCount'] = storeInfo['commentCount']
coupons = []
for i in rs['couponList']['coupons'] if 'couponList' in rs.keys() and 'coupons' in rs['couponList'].keys() else []:
coupons.append(i['couponTypeDesc']+':'+i['amount']+i['amountUnit']+i['limitRule']+':'+i['couponDetail'])
pipline['coupons'] = helper.list2str(coupons)
promotions = []
for i in storeInfo['tags'] if 'tags' in storeInfo.keys() else []:
promotions.append(i['name']+':'+i['words'])
pipline['promotions'] = helper.list2str(promotions)
pipline['telephones'] = storeInfo['storeTel'] if 'storeTel' in storeInfo.keys() else None
return pipline
def product_info_parse(self, response):
try:
rs = json.loads(response.text)
except:
logger.warning('返回结果JSON解析失败')
# logger.warning(rs)
if rs['code'] != '0':
logger.warning('product_info_parse()_retcode返回异常参数:{}'.format(
rs['code']))
return
if rs['result'] == None:
logger.warning('product_info_parse()_result equals None')
logger.warning(response.url)
searchResultVOList = []
for i in rs['result']['searchCatResultVOList']:
for j in i['searchResultVOList']:
searchResultVOList.append(j)
# for i in searchResultVOList:
# print(i)
for i in searchResultVOList:
print(i)
# pipline = SKUItem()
promotionslist = []
for item in i['tags']:
promotionslist.append('<' + item.get('iconText') + ' ' +
item.get('words') + ' ' +
item.get('name') + '>')
k = i.get('realTimePrice', '无')
h = i.get('basicPrice', '无')
print(k, h, helper.list2str(promotionslist))
def store_info_parse(self, response):
rs = json.loads(response.text)
if rs['code'] != '0':
logger.warning('store_info_parse()_retcode返回异常参数:{}'.format(
rs['code']))
return
rs = rs['result']
storeInfo = rs['storeInfo']
# productCount = 0
# cateList = []
# for i in rs['cateList'] if 'cateList' in rs.keys() else []:
# cateList.append({i['title']:i['catId']})
# productCount += i['productCount']
#
# pipline = JDstoreItem()
# pipline['id'] = storeInfo['storeId'] if 'storeId' in storeInfo.keys() else None
# pipline['name'] = storeInfo['storeName'] if 'storeName' in storeInfo.keys() else None
# pipline['address'] = storeInfo['storeAddress'] if 'storeAddress' in storeInfo.keys() else None
# pipline['productCount'] = productCount
# pipline['monthSaleCount'] = storeInfo['monthSaleNum'] if 'monthSaleNum' in storeInfo.keys() else None
# coupons = []
# for i in rs['couponList']['coupons'] if 'couponList' in rs.keys() and 'coupons' in rs['couponList'].keys() else []:
# coupons.append(i['couponTypeDesc']+':'+i['amount']+i['amountUnit']+i['limitRule']+':'+i['couponDetail'])
# pipline['coupons'] = helper.list2str(coupons)
# promotions = []
# for i in storeInfo['tags'] if 'tags' in storeInfo.keys() else []:
# promotions.append(i['name']+':'+i['words'])
# pipline['promotions'] = helper.list2str(promotions)
# pipline['telephones'] = storeInfo['storeTel'] if 'storeTel' in storeInfo.keys() else None
#
# yield pipline
coupons = []
for i in rs['storeCouponInfo'][
'couponModelList'] if 'storeCouponInfo' in rs.keys(
) and 'couponModelList' in rs['storeCouponInfo'].keys(
) else []:
coupons.append('<' + i['price'] + '元优惠券' + ' ' + i['couponLimit'] +
' ' + i['couponTitle'] + '>')
for item in rs['cateList']:
if 'childCategoryList' in item.keys() and len(
item['childCategoryList']) > 0:
for i in item['childCategoryList']:
type = 2 if len(i['catId']) > 0 else 4
catId = i['catId'] if len(
i['catId']) > 0 else i['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=i['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'storeAddress':
storeInfo['storeAddress'],
'coupons': helper.list2str(coupons),
'catId': i['catId'],
'catName': i['title'],
'type': type
})
else:
type = 2 if len(item['catId']) > 0 else 4
catId = item['catId'] if len(
item['catId']) > 0 else item['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=item['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'storeAddress': storeInfo['storeAddress'],
'coupons': helper.list2str(coupons),
'catId': item['catId'],
'catName': item['title'],
'type': type
})
def store_info_parse(self, response):
rs = json.loads(response.text)
if rs['code'] != '0':
logger.warning('store_info_parse()_retcode返回异常参数:{}'.format(
rs['code']))
return
rs = rs['result']
storeInfo = rs['storeInfo']
coupons = []
for i in rs['storeCouponInfo'][
'couponModelList'] if 'storeCouponInfo' in rs.keys(
) and 'couponModelList' in rs['storeCouponInfo'].keys(
) else []:
coupons.append('<' + i['price'] + '元优惠券' + ' ' + i['couponLimit'] +
' ' + i['couponTitle'] + '>')
for item in rs['cateList']:
if 'childCategoryList' in item.keys() and len(
item['childCategoryList']) > 0:
for i in item['childCategoryList']:
type = 2 if len(i['catId']) > 0 else 4
catId = i['catId'] if len(
i['catId']) > 0 else i['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=i['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'coupons': helper.list2str(coupons),
'catId': i['catId'],
'catName': i['title'],
'type': type
})
else:
type = 2 if len(item['catId']) > 0 else 4
catId = item['catId'] if len(
item['catId']) > 0 else item['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=item['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'coupons': helper.list2str(coupons),
'catId': item['catId'],
'catName': item['title'],
'type': type
})
def parse(self, response):
rs = json.loads(response.text)
if rs['retcode'] in errorCode:
logger.warning('retcode返回异常参数:{}'.format(rs['retcode']))
backupFailRequest(response.url)
return
for item in rs['data']:
pipline = JDstoreItem()
pipline['id'] = item['id']
pipline['lon'] = item['geoPoint']['lon']
pipline['lat'] = item['geoPoint']['lat']
pipline['name'] = item['title']
pipline['address'] = item['address']
pipline['productCount'] = item['productCount']
pipline['monthSaleCount'] = item['monthSaleCount']
pipline['state'] = item['state']
pipline['city'] = item['city']
pipline['district'] = item['district']
pipline['deliverProvider'] = item['deliverProvider']
pipline['commentCount'] = item['commentCount']
coupons = []
for i in item['coupons'] if item['coupons'] != None else []:
for j in i.keys():
coupons.append(i[j])
pipline['coupons'] = helper.list2str(coupons)
pipline['promotions'] = helper.list2str(
item['promotions']) if item['promotions'] != None else []
pipline['telephones'] = helper.list2str(
item['telephones']) if item['telephones'] != None else []
# self.pipeline.process_item(item=pipline,type='store')
yield pipline
# nextreq
geoPoint = rs['data'][random.randint(0, len(rs['data']) - 1)]
newbranchpars = {}
newbranchpars['lon'] = geoPoint['geoPoint']['lon']
newbranchpars['lat'] = geoPoint['geoPoint']['lat']
newbranchpars['pageToken'] = 1
newbranchpars['cityid'] = response.meta['cityid']
newbranchpars['apikey'] = self.apikey
masterbranchpars = {}
masterbranchpars['lon'] = response.meta['lon']
masterbranchpars['lat'] = response.meta['lat']
masterbranchpars['pageToken'] = rs['pageToken']
masterbranchpars['cityid'] = response.meta['cityid']
masterbranchpars['apikey'] = self.apikey
for i in [newbranchpars, masterbranchpars]:
# 'http://api01.idataapi.cn:8000/shopping/pdj?lon=113.762164&lat=23.043879&cityid=1601&pageToken=2&apikey=Rg44X2fsMoOFneg2QHNMSJIBaT28OXPdYgOmodOqWD2q7dRw2K6WbNsMfSrLWaRM'
yield Request(
url='http://api01.idataapi.cn:8000/shopping/pdj?{}'.format(
getUrlWithPars(i)),
method='GET',
callback=self.parse,
meta={
'cityid': i['cityid'],
'pageToken': i['pageToken'],
'lon': i['lon'],
'lat': i['lat']
},
headers=self.headers,
dont_filter=True)