def parseCat(self, response):
parser = HtmlParser(response)
if 'Next' not in parser.xpath('//li/a/text()'):
for i in parser.xpath('//span[@class="bqQuoteLink"]/a//text()'):
self.mydb.quotes.insert({'quote': i})
else:
for i in parser.xpath('//span[@class="bqQuoteLink"]/a//text()'):
self.mydb.quotes.insert({'quote': i})
for url in parser.extract_urls('//li/a[contains(text(),"Next")]'):
yield Request(url, callback="parseCat")
def parseCat(self, response):
parser = HtmlParser(response)
if 'Next' not in parser.xpath('//li/a/text()'):
for i in parser.xpath('//span[@class="bqQuoteLink"]/a//text()'):
self.mydb.quotes.insert({'quote':i})
else:
for i in parser.xpath('//span[@class="bqQuoteLink"]/a//text()'):
self.mydb.quotes.insert({'quote':i})
for url in parser.extract_urls('//li/a[contains(text(),"Next")]'):
yield Request(url,callback="parseCat")
def parseCat(self, response):
parser = HtmlParser(response)
dbname= response.meta['u']
if not parser.xpath('//a[@class="next_page"]'):
for i in parser.xpath('//div[@class="quoteText"]'):
quote = i.text
for j in i.iterfind('a'):
author=j.text
self.mydb[dbname].insert({'quote':quote,'author':author})
else:
for i in parser.xpath('//div[@class="quoteText"]'):
quote = i.text
for j in i.iterfind('a'):
author=j.text
self.mydb[dbname].insert({'quote':quote,'author':author})
for url in parser.extract_urls('//a[@class="next_page"]'):
yield Request(url,callback="parseCat",meta={'u':dbname})
def parseCat(self, response):
parser = HtmlParser(response)
dbname = response.meta['u']
if not parser.xpath('//a[@class="next_page"]'):
for i in parser.xpath('//div[@class="quoteText"]'):
quote = i.text
for j in i.iterfind('a'):
author = j.text
self.mydb[dbname].insert({'quote': quote, 'author': author})
else:
for i in parser.xpath('//div[@class="quoteText"]'):
quote = i.text
for j in i.iterfind('a'):
author = j.text
self.mydb[dbname].insert({'quote': quote, 'author': author})
for url in parser.extract_urls('//a[@class="next_page"]'):
yield Request(url, callback="parseCat", meta={'u': dbname})
def parseAnimal(self, response):
print "parseAniml"
html = HtmlParser(response)
print "Downloading........"
for url in html.xpath('//div[@class="primary_photo"]/a/img/@src'):
urlretrieve(url, url.split("/")[-1])
def parseAnimal(self,response):
print 'parseAniml'
html = HtmlParser(response)
print 'Downloading........'
for url in html.xpath('//div[@class="primary_photo"]/a/img/@src'):
urlretrieve(url,url.split('/')[-1])