# Set constants total_count = session.query(Tweet).count() current_limit = 100 window_time = 900 # Set initial variables current_offset = 500000 window_beginning = math.floor(time.time()) while current_offset < total_count: tweets = session.query(Tweet).offset(current_offset).limit(current_limit) ids = list() for tweet in tweets: ids.append(tweet.tid) try: statuses = api.lookup_statuses(ids) except TweepError: window_end = math.ceil(time.time()) time.sleep(window_time - (window_end - window_beginning) + 60) window_beginning = math.floor(time.time()) statuses = api.lookup_statuses(ids) for status in statuses: is_primetime = calc_primetime(status.created_at) ptweet = ProcessedTweet(tid=status.id, retweet_count=status.retweet_count, text=status.text, primetimetweet=is_primetime, num_followers=status.user.followers_count) session.add(ptweet) session.commit() stat_file = open('stat.txt', 'w') stat_file.write(str(current_offset)+'\n') stat_file.close()