def getProductsByCat(self, subcat): for index, cat in enumerate(subcat): #print cat['url'] self.counter = 0 page = 0 while True: page += 1 url = "http://www.decathlon.pl/pl/getAjaxListProductNextPage?uriPage=/C-{}/I-Page{}_40".format(cat['subId'], page) #print url try: self.parse(cat['subId'], url) except urllib2.HTTPError as httpError: print httpError print url if str(httpError.code)[0] == '5': self.parse(cat['subId'], url) else: break except IndexError: if page == 1: urlCat = "{}/pl/getSubNavigationMenu?primaryCategoryId={}".format( Utils.getConfig()['siteURL'], cat['subId']) print "*** Let's try to seek deeper {}".format(url) dataCat = GetSubcategories.getThirdLevelCat([urlCat]) subcat[index+1:index+1] = dataCat self.getProductsByCat(dataCat) break print "{} {}".format(cat['url'].encode('utf-8'), self.counter)