def getProductsByCat(self, subcat):
for index, cat in enumerate(subcat):
#print cat['url']
self.counter = 0
page = 0
while True:
page += 1
url = "http://www.decathlon.pl/pl/getAjaxListProductNextPage?uriPage=/C-{}/I-Page{}_40".format(cat['subId'], page)
#print url
try:
self.parse(cat['subId'], url)
except urllib2.HTTPError as httpError:
print httpError
print url
if str(httpError.code)[0] == '5':
self.parse(cat['subId'], url)
else:
break
except IndexError:
if page == 1:
urlCat = "{}/pl/getSubNavigationMenu?primaryCategoryId={}".format(
Utils.getConfig()['siteURL'], cat['subId'])
print "*** Let's try to seek deeper {}".format(url)
dataCat = GetSubcategories.getThirdLevelCat([urlCat])
subcat[index+1:index+1] = dataCat
self.getProductsByCat(dataCat)
break
print "{} {}".format(cat['url'].encode('utf-8'), self.counter)
