def visit_page(self, product_list_url, page_num): print "Page-{}, {}".format(page_num, product_list_url) response = urllib2.urlopen(product_list_url) if response.code is self.HTTP_STATUS_OK: page = Page() page.set_data(response.read()) page.set_url(product_list_url) page.set_out_file(self.output_filename + "-page-" + str(page_num) + "-") page.scrap() print "{} - Completed".format(product_list_url)