logging.basicConfig( filename="4sq.log", level=logging.DEBUG,
datefmt='%d/%m/%y|%H:%M:%S', format='|%(asctime)s|%(levelname)s| %(message)s' )
logging.info( 'initiating a friend crawl.' )
#
# Prep
import _credentials
access_tokens = _credentials.access_tokens
gateway = APIGateway( access_tokens=access_tokens, token_hourly_query_quota=500 )
api = APIWrapper( gateway )
dbw = DBWrapper()
if True: #~
from database import Friendship
if not dbw._get_engine().has_table('friendships'):
dbw._get_engine().create(Friendship.__table__)
#
# Begin mining...
max_crawl_id = dbw.get_friendships_max_crawl_id()
if max_crawl_id is None:
logging.debug( 'no previous crawls found.' )
crawl_id = 1
else:
crawl_id = max_crawl_id + 1
logging.info( 'crawl id = %s', crawl_id )
count_users = 0
sum_degree = 0
