def parse2(self, response): global companyCombo,urlCounter,urlCompany sel = Selector(response) jobs = sel.xpath('//div[contains(@type,"tuple")]').extract() db = DbOperations() for companyAttr in companyCombo: if(companyAttr['url'] in response.url): company = companyAttr break else: return db.insertJobList(jobs,company['ro_id']) try: jobNextUrl = sel.xpath('//div[contains(@class,"pagination")]/a/@href').extract()[1] request = scrapy.Request(jobNextUrl , callback=self.parseNextUrl) yield request except: print "Unexpected error:", sys.exc_info()[0]