while True:
try:
u = User(user["user_id"], user["auth_key"])
break
except TooManyRequests as e:
print "Maximum Request Reached! Wating for Next Hour..."
time.sleep(60) # retry after 1 min
continue
except BehanceException as e:
print "BehanceException: ", str(e)
break
pageNum = 1
while True:
try:
cols = u.get_collections(page=pageNum)
if len(cols) == 0:
break
for col in cols:
# avoid duplicate
if visitedCols.has_key(col["id"]):
continue
visitedCols[col["id"]] = col
dbcollection.insert(
json.loads(json.dumps(col), object_hook=remove_dot_key))
numCols += 1
print "PageNum (Total cols) = ", pageNum, ", ", numCols
pageNum += 1
if pageNum >= 400:
break
