def parse2(self, response):
global companyCombo,urlCounter,urlCompany
sel = Selector(response)
jobs = sel.xpath('//div[contains(@type,"tuple")]').extract()
db = DbOperations()
for companyAttr in companyCombo:
if(companyAttr['url'] in response.url):
company = companyAttr
break
else:
return
db.insertJobList(jobs,company['ro_id'])
try:
jobNextUrl = sel.xpath('//div[contains(@class,"pagination")]/a/@href').extract()[1]
request = scrapy.Request(jobNextUrl , callback=self.parseNextUrl)
yield request
except:
print "Unexpected error:", sys.exc_info()[0]
