def start_requests(self):
for id in self.storeId:
yield Request(url='https://daojia.jd.com/client?{}'.format(
helper.getUrlWithPars(
self.deal_with_storeDetailV220_param(id))),
method='GET',
callback=self.store_info_parse,
headers=self.headers,
dont_filter=True)
def parse(self, response):
rs = json.loads(response.text)
if rs['code'] != '0':
logger.warning('parse()_retcode返回异常参数:{}'.format(rs['code']))
return
storeId = []
for item in rs['result']['data']['data'] if 'result' in rs.keys(
) else []:
item = item['floorCellData']
storeId.append(item['storeId'])
for i in item['brosStore'] if 'brosStore' in item.keys() else []:
storeId.append(i['params']['storeId'])
for id in storeId:
yield Request(url='https://daojia.jd.com/client?{}'.format(
helper.getUrlWithPars(
self.deal_with_storeDetailV220_param(id))),
method='GET',
callback=self.store_info_parse,
headers=self.headers,
dont_filter=True)
if 'result' in rs.keys():
par = self.deal_with_recommendStoreList_param(
lng=response.meta['lon'],
lat=response.meta['lat'],
city_id=response.meta['cityid'],
curpage=response.meta['pageToken'] + 1)
yield Request(url='https://daojia.jd.com/client?{}'.format(
helper.getUrlWithPars(par)),
method='GET',
callback=self.parse,
headers=self.headers,
meta={
'cityid': response.meta['cityid'],
'pageToken': response.meta['pageToken'] + 1,
'lon': response.meta['lon'],
'lat': response.meta['lat']
},
dont_filter=True)
def start_requests(self):
for key in self.city_lon_lat.keys():
for j in self.city_lon_lat[key]:
par = self.deal_with_recommendStoreList_param(lng=j[0],
lat=j[1],
city_id=key,
curpage=1)
yield Request(url='https://daojia.jd.com/client?{}'.format(
helper.getUrlWithPars(par)),
callback=self.parse,
headers=self.headers,
meta={
'cityid': key,
'pageToken': 1,
'lon': j[0],
'lat': j[1]
},
dont_filter=True)
def store_info_parse(self, response):
rs = json.loads(response.text)
if rs['code'] != '0':
logger.warning('store_info_parse()_retcode返回异常参数:{}'.format(
rs['code']))
return
rs = rs['result']
storeInfo = rs['storeInfo']
# productCount = 0
# cateList = []
# for i in rs['cateList'] if 'cateList' in rs.keys() else []:
# cateList.append({i['title']:i['catId']})
# productCount += i['productCount']
#
# pipline = JDstoreItem()
# pipline['id'] = storeInfo['storeId'] if 'storeId' in storeInfo.keys() else None
# pipline['name'] = storeInfo['storeName'] if 'storeName' in storeInfo.keys() else None
# pipline['address'] = storeInfo['storeAddress'] if 'storeAddress' in storeInfo.keys() else None
# pipline['productCount'] = productCount
# pipline['monthSaleCount'] = storeInfo['monthSaleNum'] if 'monthSaleNum' in storeInfo.keys() else None
# coupons = []
# for i in rs['couponList']['coupons'] if 'couponList' in rs.keys() and 'coupons' in rs['couponList'].keys() else []:
# coupons.append(i['couponTypeDesc']+':'+i['amount']+i['amountUnit']+i['limitRule']+':'+i['couponDetail'])
# pipline['coupons'] = helper.list2str(coupons)
# promotions = []
# for i in storeInfo['tags'] if 'tags' in storeInfo.keys() else []:
# promotions.append(i['name']+':'+i['words'])
# pipline['promotions'] = helper.list2str(promotions)
# pipline['telephones'] = storeInfo['storeTel'] if 'storeTel' in storeInfo.keys() else None
#
# yield pipline
coupons = []
for i in rs['storeCouponInfo'][
'couponModelList'] if 'storeCouponInfo' in rs.keys(
) and 'couponModelList' in rs['storeCouponInfo'].keys(
) else []:
coupons.append('<' + i['price'] + '元优惠券' + ' ' + i['couponLimit'] +
' ' + i['couponTitle'] + '>')
for item in rs['cateList']:
if 'childCategoryList' in item.keys() and len(
item['childCategoryList']) > 0:
for i in item['childCategoryList']:
type = 2 if len(i['catId']) > 0 else 4
catId = i['catId'] if len(
i['catId']) > 0 else i['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=i['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'storeAddress':
storeInfo['storeAddress'],
'coupons': helper.list2str(coupons),
'catId': i['catId'],
'catName': i['title'],
'type': type
})
else:
type = 2 if len(item['catId']) > 0 else 4
catId = item['catId'] if len(
item['catId']) > 0 else item['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=item['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'storeAddress': storeInfo['storeAddress'],
'coupons': helper.list2str(coupons),
'catId': item['catId'],
'catName': item['title'],
'type': type
})
def store_info_parse(self, response):
rs = json.loads(response.text)
if rs['code'] != '0':
logger.warning('store_info_parse()_retcode返回异常参数:{}'.format(
rs['code']))
return
rs = rs['result']
storeInfo = rs['storeInfo']
coupons = []
for i in rs['storeCouponInfo'][
'couponModelList'] if 'storeCouponInfo' in rs.keys(
) and 'couponModelList' in rs['storeCouponInfo'].keys(
) else []:
coupons.append('<' + i['price'] + '元优惠券' + ' ' + i['couponLimit'] +
' ' + i['couponTitle'] + '>')
for item in rs['cateList']:
if 'childCategoryList' in item.keys() and len(
item['childCategoryList']) > 0:
for i in item['childCategoryList']:
type = 2 if len(i['catId']) > 0 else 4
catId = i['catId'] if len(
i['catId']) > 0 else i['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=i['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'coupons': helper.list2str(coupons),
'catId': i['catId'],
'catName': i['title'],
'type': type
})
else:
type = 2 if len(item['catId']) > 0 else 4
catId = item['catId'] if len(
item['catId']) > 0 else item['promotLabel']
searchByCategoryPost = self.deal_with_searchByCategoryPost(
storeId=storeInfo['storeId'],
catId=catId,
catName=item['title'],
type=type)
yield Request(url='https://daojia.jd.com/client?{}'.format(
getUrlWithPars(searchByCategoryPost)),
method='POST',
callback=self.product_info_parse,
body=json.dumps(searchByCategoryPost),
headers=self.headers,
meta={
'storeName': storeInfo['storeName'],
'coupons': helper.list2str(coupons),
'catId': item['catId'],
'catName': item['title'],
'type': type
})
def start_requests(self):
for i in range(0,5):
yield Request(url=self.url.format(helper.getUrlWithPars(self.querystring)), method="GET", callback=self.parse, headers=self.headers,dont_filter=True)