#
# Begin mining...
max_crawl_id = dbw.get_friendships_max_crawl_id()
if max_crawl_id is None:
logging.debug( 'no previous crawls found.' )
crawl_id = 1
else:
crawl_id = max_crawl_id + 1
logging.info( 'crawl id = %s', crawl_id )
count_users = 0
sum_degree = 0
friend_rows_added = 0
all_users = dbw.get_all_users_with_checkins()
for indx, user_obj in enumerate( all_users ):
user_4sq_id = user_obj.foursq_id
logging.info( 'crawling user %s (%s of %s).', user_4sq_id, indx+1, len(all_users) )
try:
friends = call_wrapper( lambda: api.get_friends_of( user_4sq_id ) )
except HTTPError:
# Handle the case where a user is not found in the API
logging.info( "\t HTTPError for user %s. skipping.", user_4sq_id )
continue
logging.info( '\tfound %s friends.', len(friends) )
count_users += 1
sum_degree += len( friends )
