def parse_two(self, response):
#传入上面的item1
item2 = response.meta['item1']
items = []
html = response.text
webSoup = BeautifulSoup(html)
div_soups = webSoup.find_all('div', 'item masonry_brick masonry-brick')
for div_soup in div_soups:
imageURL = div_soup.find_all('div',
'img')[0].find_all('img')[0]['src']
title = div_soup.find_all('div', 'title')[0].text
pageURL = div_soup.find_all('div',
'img')[0].find_all('a')[0]['href']
item = XiaohuaItem()
item['meizhiURL'] = pageURL
item['fileName'] = item2['fileName'] + '/' + title
item['title'] = title
items.append(item)
for i in range(1):
item = items[0]
fileName = item['fileName']
if not os.path.exists(fileName):
os.makedirs(fileName)
yield Request(url=item['meizhiURL'],
meta={'item2': item},
callback=self.parse_three)
def parse_one(self, response):
#创建一个大的list存储所有的item
base_url = "http://www.mmonly.cc"
items = []
websoup = BeautifulSoup(response.text)
a_dicts = websoup.find_all('div', "TagList")[0].find_all('a')
for a_text in a_dicts:
#创建实例,并转化为字典
main = (a_text['href'], a_text['title'])
item = XiaohuaItem()
item['siteURL'] = base_url + main[0]
item['title'] = main[1]
item['fileName'] = self.base + item['title']
items.append(item)
for i in range(1):
#创建文件夹
item = items[0]
fileName = item['fileName']
if not os.path.exists(fileName):
os.makedirs(fileName)
#用meta传入下一层
yield Request(url=item['siteURL'],
meta={'item1': item},
callback=self.parse_two)
def parse_two(self,response):
#传入上面的item1
# print(response)
item2=response.meta['item1']
source=requests.get(response.url)
source.encoding = source.apparent_encoding
html=source.text
#print(html)
pattern = re.compile('共(.*?)页',re.S)
# print(pattern)
Num=re.search(pattern,html).group(1)
# print(Num)
items=[]
for i in range(1,int(Num)+1):
item=XiaohuaItem()
item['fileName']=item2['fileName']
#构造每一个图片的存储路径
item['path']=item['fileName']+'/'+str(i)+'.jpg'
#构造每一个图片入口链接，以获取源码中的原图链接
item['pageURL']=response.url[:-5]+'_'+str(i)+'.html'
items.append(item)
for item in items:
fileName = item['fileName']
if not os.path.exists(fileName):
os.makedirs(fileName)
yield Request(url=item['pageURL'],meta={'item2':item},callback=self.parse_three)
def parse_three(self,response):
item=XiaohuaItem()
item3=response.meta['item2']
#传入上面的item2
pattern=re.compile(r'<li class="pic-down h-pic-down"><a target="_blank" class="down-btn" href=\'(.*?)\'>.*?</a>',re.S)
URL=re.search(pattern,response.text).group(1)
item['detailURL']=URL
item['title']=item3['title']
yield item
def parse_three(self, response):
item3 = response.meta['item2']
html = response.text.encode('utf-8')
# 用正则提取页数
pattern = re.compile(r'共(.*?)页', re.S)
Num = re.search(pattern, html).group(1)
for i in range(1, int(Num) + 1):
item = XiaohuaItem()
item['fileName'] = item3['fileName']
item['path'] = item['fileName'] + '/' + str(i) + '.jpg'
'''
def parse_one(self,response):
#创建一个大的list存储所有的item
items=[]
pattern=re.compile(r'<div class="title".*?<a.*?href="(.*?)">(.*?)</a></span></div>',re.S)
mains=re.findall(pattern,response.text)
for main in mains:
#创建实例,并转化为字典
item=XiaohuaItem()
item['siteURL']=main[0]
item['title']=main[1]
items.append(item)
for item in items:
#用meta传入下一层
yield Request(url=item['siteURL'],meta={'item1':item},callback=self.parse_two)
def parse_two(self,response):
#传入上面的item1
item2=response.meta['item1']
source=requests.get(response.url)
html=source.text.encode('utf-8')
pattern=re.compile(r'共(.*?)页',re.S)
Num=re.search(pattern,html.decode('utf-8')).group(1)
items=[]
for i in range(1,int(Num)+1):
item=XiaohuaItem()
#构造每一个图片入口链接，以获取源码中的原图链接
item['title']=item2['title']
item['pageURL']=response.url[:-5]+'_'+str(i)+'.html'
items.append(item)
for item in items:
yield Request(url=item['pageURL'],meta={'item2':item},callback=self.parse_three)