def createManyTwitterAccounts():
items = range(0, 1000)
item_fun = functools.partial(createTwitterAccountWrapper)
monitor_script.runScript("create_twitter_accounts",
items=items,
item_fun=item_fun,
resolution=5)
def migrateCandidateFollowersFast(candidate, download_followers=False):
print "+II+: fast migrating followers for " + enc(candidate.name)
cursor = connections['old'].cursor()
cursor.execute("SELECT count(*) FROM followers_nov5 WHERE userid = '" +
candidate.twitter_screen_name + "'")
total_candidate_followers = cursor.fetchone()[0]
#item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers)
item_fun = functools.partial(migrateCandidateFollowerBlunt,
candidate=candidate)
window_size = 50000
total_processed = 0
num_windows = int(math.ceil(total_candidate_followers /
float(window_size)))
for c_window in range(0, num_windows):
query = "SELECT followersid " \
"FROM followers_nov5 " \
"WHERE userid = '" + candidate.twitter_screen_name + "'"\
+ " LIMIT " + str(window_size)\
+ " OFFSET " + str(total_processed)
cursor.execute(query)
window = cursor.fetchall()
total_processed += len(window)
title = "migrateCandidateFollowers " + str(
enc(candidate.twitter_screen_name)) + "(" + str(
c_window) + "/" + str(num_windows) + ")"
monitor_script.runScript(name=title,
items=window,
item_fun=item_fun,
resolution=5)
candidate.migrated_followers = True
candidate.save()
def downloadFollowersForStrayVoters(voters=None):
if not voters: voters = Voter.objects.all()
voters = voters.filter(downloaded_followers=False,
over_max_followers=False)
monitor_script.runScript("download_followers",
items=voters,
item_fun=downloadFollowersIDs,
resolution=5)
def migrateCandidateFollowers(candidate, download_followers=False):
print "+II+: migrating followers for " + enc(candidate.name)
cursor = connections['old'].cursor()
cursor.execute("SELECT followersid FROM followers_nov5 WHERE userid = '" + candidate.twitter_screen_name + "'")
candidate_followers = cursor.fetchall()
item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers)
monitor_script.runScript("migrateCandidateFollowers (" + str(enc(candidate.twitter_screen_name)) + ")", items=candidate_followers, item_fun=item_fun, resolution=5)
candidate.migrated_followers = True
candidate.save()
def parseCandidatesFromSpreadSheet(spreadsheet):
wb = open_workbook(spreadsheet)
sheet = wb.sheet_by_index(0)
items = range(1, sheet.nrows)
item_fun = functools.partial(parseCandidateFromSpreadSheet, sheet=sheet)
monitor_script.runScript(name="parseCandidates",
items=items,
item_fun=item_fun,
resolution=5)
def migrateFun():
candidates = Candidate.objects.filter(
candidate_type__in=[2, 3, 4,
5], migrated_followers=False).order_by("name")
item_fun = functools.partial(migrateCandidateFollowersFast,
download_followers=False)
monitor_script.runScript("migrateCandidateFollowers",
items=candidates,
item_fun=item_fun,
resolution=5)
def migrateCandidateFollowers(candidate, download_followers=False):
print "+II+: migrating followers for " + enc(candidate.name)
cursor = connections['old'].cursor()
cursor.execute("SELECT followersid FROM followers_nov5 WHERE userid = '" +
candidate.twitter_screen_name + "'")
candidate_followers = cursor.fetchall()
item_fun = functools.partial(migrateCandidateFollower,
candidate=candidate,
download_followers=download_followers)
monitor_script.runScript("migrateCandidateFollowers (" +
str(enc(candidate.twitter_screen_name)) + ")",
items=candidate_followers,
item_fun=item_fun,
resolution=5)
candidate.migrated_followers = True
candidate.save()
def partitionedDownloadFollowersForVoters(accounts_start_index, accounts_end_index, voter_start_id, voter_end_id):
"""
:param accounts_start_index: first twitter account to use
:param accounts_end_index: last twitter account to use
:param voter_start_id: id of voter marks edge of partition
"""
refreshTwitterAccounts(accounts_start_index, accounts_end_index)
# print "# twitter accounts: " + str(len(TWITTER_ACCOUNTS))
total_stray_voters = Voter.objects.filter(id__gte=voter_start_id, id__lte=voter_end_id, downloaded_followers=False, over_max_followers=False).count()
window_size = 1000
num_windows = int(math.ceil(total_stray_voters / float(window_size)))
for c_window in range(0,num_windows):
start_index = (c_window * window_size)
query = "SELECT id, twitter_id, twitter_screen_name,num_followee_attempts FROM twpol_voter where id>=%s AND id<=%s ORDER BY id LIMIT %s OFFSET %s"
voters = list(Voter.objects.raw(query, [voter_start_id, voter_end_id, window_size, start_index]))
title = "downloadFolloees " + str(voter_start_id) + "|" + str(voter_end_id) + " (" + str(c_window) + "/" + str(num_windows)+ ")"
monitor_script.runScript(name=title, items=voters, item_fun=downloadFolloweesIDs, resolution=5)
email_body = {"voter_start_id":voter_start_id, "voter_end_id":voter_end_id}
email = EmailMessage(subject='Heroku Worker Finished!', body=json.dumps(email_body), to=["*****@*****.**"])
email.send()
def partitionedDownloadFollowersForVoters(accounts_start_index,
accounts_end_index, voter_start_id,
voter_end_id):
"""
:param accounts_start_index: first twitter account to use
:param accounts_end_index: last twitter account to use
:param voter_start_id: id of voter marks edge of partition
"""
refreshTwitterAccounts(accounts_start_index, accounts_end_index)
# print "# twitter accounts: " + str(len(TWITTER_ACCOUNTS))
total_stray_voters = Voter.objects.filter(
id__gte=voter_start_id,
id__lte=voter_end_id,
downloaded_followers=False,
over_max_followers=False).count()
window_size = 1000
num_windows = int(math.ceil(total_stray_voters / float(window_size)))
for c_window in range(0, num_windows):
start_index = (c_window * window_size)
query = "SELECT id, twitter_id, twitter_screen_name,num_followee_attempts FROM twpol_voter where id>=%s AND id<=%s ORDER BY id LIMIT %s OFFSET %s"
voters = list(
Voter.objects.raw(
query,
[voter_start_id, voter_end_id, window_size, start_index]))
title = "downloadFolloees " + str(voter_start_id) + "|" + str(
voter_end_id) + " (" + str(c_window) + "/" + str(num_windows) + ")"
monitor_script.runScript(name=title,
items=voters,
item_fun=downloadFolloweesIDs,
resolution=5)
email_body = {
"voter_start_id": voter_start_id,
"voter_end_id": voter_end_id
}
email = EmailMessage(subject='Heroku Worker Finished!',
body=json.dumps(email_body),
to=["*****@*****.**"])
email.send()
def migrateCandidateFollowersFast(candidate,download_followers=False):
print "+II+: fast migrating followers for " + enc(candidate.name)
cursor = connections['old'].cursor()
cursor.execute("SELECT count(*) FROM followers_nov5 WHERE userid = '" + candidate.twitter_screen_name + "'")
total_candidate_followers = cursor.fetchone()[0]
#item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers)
item_fun = functools.partial(migrateCandidateFollowerBlunt, candidate=candidate)
window_size = 50000
total_processed = 0
num_windows = int(math.ceil(total_candidate_followers / float(window_size)))
for c_window in range(0,num_windows):
query = "SELECT followersid " \
"FROM followers_nov5 " \
"WHERE userid = '" + candidate.twitter_screen_name + "'"\
+ " LIMIT " + str(window_size)\
+ " OFFSET " + str(total_processed)
cursor.execute(query)
window = cursor.fetchall()
total_processed += len(window)
title = "migrateCandidateFollowers " + str(enc(candidate.twitter_screen_name)) + "(" + str(c_window) + "/" + str(num_windows)+ ")"
monitor_script.runScript(name=title, items=window, item_fun=item_fun, resolution=5)
candidate.migrated_followers = True
candidate.save()
def createManyTwitterAccounts():
items = range(0,1000)
item_fun = functools.partial(createTwitterAccountWrapper)
monitor_script.runScript("create_twitter_accounts", items=items, item_fun=item_fun, resolution=5)
def analyzeDistrict(district):
candidates = Candidate.objects.filter(district=district)
item_fun = functools.partial(migrateCandidateFollowers, download_followers=False)
monitor_script.runScript("analyzeDistrict ( " + str(district.district_number) + ")", items=candidates, item_fun=item_fun, resolution=5)
def analyzeState(state_code):
candidates = Candidate.objects.filter(district__state_code=state_code, migrated_followers=False)
item_fun = functools.partial(migrateCandidateFollowers, download_followers=True)
monitor_script.runScript("analyzeState (" + str(state_code) + ")", items=candidates, item_fun=item_fun, resolution=1)
def migrateFun():
candidates = Candidate.objects.filter(candidate_type__in=[2,3,4,5], migrated_followers=False).order_by("name")
item_fun = functools.partial(migrateCandidateFollowersFast, download_followers=False)
monitor_script.runScript("migrateCandidateFollowers", items=candidates, item_fun=item_fun, resolution=5)
def downloadFollowersForStrayVoters(voters=None):
if not voters: voters = Voter.objects.all()
voters = voters.filter(downloaded_followers=False, over_max_followers=False)
monitor_script.runScript("download_followers", items=voters, item_fun=downloadFollowersIDs, resolution=5)