def parse_all(self, response):
try:
content_json = json.loads(response.body)
singleResult = content_json.get('singleResult')
if singleResult:
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item
except:
pass
def parse_nexttwomonth(self, response):
try:
content_json = json.loads(response.body)
singleResult = content_json.get('singleResult')
if singleResult:
list = singleResult.get('list')
if list:
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item
else:
pass
except:
pass
def parse_nextonemonth(self, response):
try:
content_json = json.loads(response.body)
singleResult = content_json.get('singleResult')
lastSellDay = response.meta.get['lastSellDay']
str_n_end_onemonthdate = response.meta.get[
'str_n_end_onemonthdate']
brand = response.meta.get['brand']
brandName = response.meta.get['brandName']
brandType = response.meta.get['brandType']
if singleResult:
list = singleResult.get('list')
if list:
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item
nexttwomonth = datetime.datetime.strptime(
str_n_end_onemonthdate, '%Y-%m-%d')
nexttwomonth = nexttwomonth + datetime.timedelta(days=30)
str_nexttwomonth = nexttwomonth.strftime('%Y-%m-%d')
url = 'http://compass.vis.vip.com/newGoods/details/getDetails?brandStoreName=' \
+ urllib.quote(
brand) + '&goodsCode=&sortColumn=goodsAmt&sortType=1&pageSize=200000&pageNumber=1&beginDate=' \
+ str_n_end_onemonthdate + '&endDate=' + str_nexttwomonth + '&brandName=' + urllib.quote(
brandName) + \
'&sumType=0&optGroupFlag=0&warehouseFlag=0&analysisType=1&selectedGoodsInfo=vipshopPrice%2CorderGoodsPrice&mixBrand=0&dateMode=0&dateType=D&detailType=D&brandType=' + \
urllib.quote(brandType) + '&goodsType=0&dateDim=0'
yield scrapy.Request(url,
callback=self.parse_nexttwomonth,
headers=header,
meta={
'brand': brand,
'dt': dt,
'lastSellDay': lastSellDay,
'str_n_end_onemonthdate':
str_n_end_onemonthdate,
'brand': brand,
'brandName': brandName,
'brandType': brandType,
},
cookies=cookie)
else:
pass
except:
pass
def parse_nextonemonth(self, response):
try:
content_json = json.loads(response.body)
singleResult = content_json.get('singleResult')
lastSellDay = response.meta.get['lastSellDay']
str_n_end_onemonthdate = response.meta.get[
'str_n_end_onemonthdate']
brand = response.meta.get['brand']
brandName = response.meta.get['brandName']
brandType = response.meta.get['brandType']
if singleResult:
list = singleResult.get('list')
if list:
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item
nexttwomonth = datetime.datetime.strptime(
str_n_end_onemonthdate, '%Y-%m-%d')
nexttwomonth = nexttwomonth + datetime.timedelta(days=30)
str_nexttwomonth = nexttwomonth.strftime('%Y-%m-%d')
url = 'http://compass.vis.vip.com/dangqi/details/getDangqiDetails?brandStoreName=' \
+ urllib.quote(brand) + '&brandType=' + urllib.quote(
brandType) + '&brandName=' + urllib.quote(brandName) + \
'&mixBrand=0&sortColumn=logDate&sortType=1&pageSize=20000&pageNumber=1&sumType=1&lv3CategoryFlag=0&optGroupFlag=0&warehouseFlag=0&analysisType=1&beginDate=' + \
str_n_end_onemonthdate + '&endDate=' + str_nexttwomonth + '&queryType=0'
print url
yield scrapy.Request(
url,
callback=self.parse_nextonemonth,
headers=header,
meta={
'brand': brand,
'updateTime': updateTime,
'lastSellDay': lastSellDay,
'str_n_end_onemonthdate': str_n_end_onemonthdate,
#'brand': brand,
'brandName': brandName,
'brandType': brandType,
},
cookies=cookie)
else:
pass
except:
pass
def parse_content(self, response):
try:
content_json = json.loads(response.body)
singleResult = content_json.get('singleResult')
if singleResult:
chart = singleResult.get('chart')
if chart:
data = chart.get('data')
if data:
flowUv = data.get('flowUv')
if flowUv:
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item
except:
pass
def parse_content(self, response):
if response.status == 200:
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item
else:
meta = response.meta
url = meta.get('url')
brand = meta.get('brand')
brandName = meta.get('brandName')
brandType = meta.get('brandType')
firstSellDay = meta.get('firstSellDay')
lastSellDay = meta.get('lastSellDay')
yield scrapy.Request(url, callback=self.parse_content, headers=header,
meta={'brand': brand, 'dt': dt, 'brandName': brandName,
'brandType': brandType,
'firstSellDay': firstSellDay, 'lastSellDay': lastSellDay, 'url': url},
cookies=cookie, dont_filter=True)
def parse_content(self, response):
item = WeipinItem()
item['content'] = response.body
item['meta'] = response.meta
yield item