#!/usr/bin/python

import os

import json

import urllib

import sys

import collections

import dateutil.parser

import subprocess

import httplib

import sqlite3

import datetime

import scraperwiki

import httplib

import random

import datetime

from secrets import *

MAX_TO_GET=100000

# Horrendous hack to work around some Twitter / Python incompatibility

# http://bobrochel.blogspot.co.nz/2010/11/bad-servers-chunked-encoding-and.html

def patch_http_response_read(func):

return inner

httplib.HTTPResponse.read = patch_http_response_read(httplib.HTTPResponse.read)

# Make sure you install this version of "twitter":

# http://pypi.python.org/pypi/twitter

# http://mike.verdone.ca/twitter/

# https://github.com/sixohsix/twitter

import twitter

#########################################################################

# Authentication to Twitter

# This is designed to, when good, be submitted as a patch to add to twitter.oauth_dance (which

# currently only has a function for PIN authentication, not redirect)

from twitter.api import Twitter

from twitter.oauth import OAuth, write_token_file, read_token_file

from twitter.oauth_dance import parse_oauth_tokens

def oauth_url_dance(consumer_key, consumer_secret, callback_url, oauth_verifier, pre_verify_token_filename, verified_token_filename):

return oauth_token, oauth_token_secret

def do_tool_oauth():

return tw

#########################################################################

# Helper functions

# Stores one Twitter user in the ScraperWiki database

def convert_user(batch, user):

return data

# After detecting an auth failed error mid work, call this

def clear_auth_and_restart():

sys.exit()

# Signal back to the calling Javascript, to the database, and custard's status API, our status

def set_status_and_exit(status, typ, message, extra = {}):

sys.exit()

# Store all our progress variables

def save_status():

scraperwiki.sql.save(['id'], data, table_name='__status')

# Load in all our progress variables

current_batch = 1

next_cursor = -1

batch_got = 0

batch_expected = 0

current_status = 'clean-slate'

def get_status():

current_status = data['current_status']

# http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python

def chunks(l, n):

return [l[i:i+n] for i in range(0, len(l), n)]

#########################################################################

# Main code

pages_got = 0

try:

# Rename old status table to new __status name.

# This can be removed after it has been active long enough to

# update all existing tools.

try :

scraperwiki.sql.execute("SELECT 1 FROM status")

except sqlite3.OperationalError:

pass

else:

scraperwiki.sql.execute("ALTER TABLE status RENAME TO __status")

# Parameters to this command vary:

# a. None: try and scrape Twitter followers

# b. callback_url oauth_verifier: have just come back from Twitter with these oauth tokens

# c. "clean-slate": wipe database and start again

if len(sys.argv) > 1 and sys.argv[1] == 'clean-slate':

scraperwiki.sql.execute("drop table if exists twitter_followers")

scraperwiki.sql.execute("drop table if exists __status")

os.system("crontab -r >/dev/null 2>&1")

set_status_and_exit('clean-slate', 'error', 'No user set')

sys.exit()

# Make the followers table *first* with dumb data, calling DumpTruck directly,

# so it appears before the status one in the list

scraperwiki.sql.dt.create_table({'id': 1, 'batch': 1}, 'twitter_followers')

scraperwiki.sql.execute("CREATE INDEX IF NOT EXISTS batch_index "

"ON twitter_followers (batch)")

# Get user we're working on from file we store it in

screen_name = open("user.txt").read().strip()

# Connect to Twitter

tw = do_tool_oauth()

# A batch is one scan through the list of followers - we have to scan as

# our API calls are limited. The cursor is Twitter's identifier of where

# in the current batch we are.

get_status()

# Note that each user is only in the most recent batch they've been found in

# (we don't keep all the history)

# Look up latest followers count

profile = tw.users.lookup(screen_name=screen_name)

batch_expected = profile[0]['followers_count']

# Things basically working, so make sure we run again by writing a crontab.

if not os.path.isfile("crontab"):

crontab = open("tool/crontab.template").read()

# ... run at a random minute to distribute load XXX platform should do this for us

crontab = crontab.replace("RANDOM", str(random.randint(0, 59)))

open("crontab", "w").write(crontab)

os.system("crontab crontab")

# Get as many pages in the batch as we can (most likely 15!)

onetime = 'ONETIME' in os.environ

live_dataset = 'LIVE_DATASET' in os.environ

while True:

#raise httplib.IncompleteRead('hi') # for testing

#print "getting", next_cursor

# get the identifiers of followers - one page worth (up to 5000 people)

if next_cursor == -1:

result = tw.followers.ids(screen_name=screen_name)

else:

result = tw.followers.ids(screen_name=screen_name, cursor=next_cursor)

ids = result['ids']

# and then the user details for all the ids

double_break = False

for chunk in chunks(ids, 100):

users = tw.users.lookup(user_id=(",".join(map(str, chunk))))

data = []

for user in users:

datum = convert_user(current_batch, user)

data.append(datum)

scraperwiki.sql.save(['id'], data, table_name="twitter_followers")

save_status()

# Don't allow more than a certain number

if batch_got >= MAX_TO_GET:

os.system("crontab -r >/dev/null 2>&1")

set_status_and_exit("ok-limit", 'ok', "Reached %d follower limit" % MAX_TO_GET)

# If being run from the user interface, return quickly after being

# sure we've got *something* (the Javascript will then spawn us

# again in the background to slowly get the rest)

if onetime:

double_break = True

break

if double_break:

break

# we have all the info for one page - record got and save it

pages_got += 1

next_cursor = result['next_cursor']

# While debugging, only do one page to avoid rate limits by uncommenting this:

# break

if next_cursor == 0:

# We've finished a batch

next_cursor = -1

current_batch += 1

if not live_dataset:

# Disable cron job, we're done

os.system("crontab -r >/dev/null 2>&1")

set_status_and_exit("ok-done", 'ok', "Finished")

break

except twitter.api.TwitterHTTPError, e:

if "Twitter sent status 401 for URL" in str(e):

clear_auth_and_restart()

# https://dev.twitter.com/docs/error-codes-responses

obj = json.loads(e.response_data)

code = obj['errors'][0]['code']

# authentication failure

if (code in [32, 89]):

clear_auth_and_restart()

# page not found

if code == 34:

set_status_and_exit('not-there', 'error', 'User not on Twitter')

# rate limit exceeded

if code == 88:

# provided we got at least one page, rate limit isn't an error but expected

if pages_got == 0:

set_status_and_exit('rate-limit', 'error', 'Twitter is rate limiting you')

else:

# anything else is an unexpected error - if ones occur a lot, add the above instead

raise

except httplib.IncompleteRead, e:

# I think this is effectively a rate limit error - so only count if it was first error

if pages_got == 0:

set_status_and_exit('rate-limit', 'error', 'Twitter broke the connection')

# Save progress message

set_status_and_exit("ok-updating", 'ok', "Running... %d/%d" % (batch_got, batch_expected))