def createManyTwitterAccounts(): items = range(0, 1000) item_fun = functools.partial(createTwitterAccountWrapper) monitor_script.runScript("create_twitter_accounts", items=items, item_fun=item_fun, resolution=5)
def migrateCandidateFollowersFast(candidate, download_followers=False): print "+II+: fast migrating followers for " + enc(candidate.name) cursor = connections['old'].cursor() cursor.execute("SELECT count(*) FROM followers_nov5 WHERE userid = '" + candidate.twitter_screen_name + "'") total_candidate_followers = cursor.fetchone()[0] #item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers) item_fun = functools.partial(migrateCandidateFollowerBlunt, candidate=candidate) window_size = 50000 total_processed = 0 num_windows = int(math.ceil(total_candidate_followers / float(window_size))) for c_window in range(0, num_windows): query = "SELECT followersid " \ "FROM followers_nov5 " \ "WHERE userid = '" + candidate.twitter_screen_name + "'"\ + " LIMIT " + str(window_size)\ + " OFFSET " + str(total_processed) cursor.execute(query) window = cursor.fetchall() total_processed += len(window) title = "migrateCandidateFollowers " + str( enc(candidate.twitter_screen_name)) + "(" + str( c_window) + "/" + str(num_windows) + ")" monitor_script.runScript(name=title, items=window, item_fun=item_fun, resolution=5) candidate.migrated_followers = True candidate.save()
def downloadFollowersForStrayVoters(voters=None): if not voters: voters = Voter.objects.all() voters = voters.filter(downloaded_followers=False, over_max_followers=False) monitor_script.runScript("download_followers", items=voters, item_fun=downloadFollowersIDs, resolution=5)
def migrateCandidateFollowers(candidate, download_followers=False): print "+II+: migrating followers for " + enc(candidate.name) cursor = connections['old'].cursor() cursor.execute("SELECT followersid FROM followers_nov5 WHERE userid = '" + candidate.twitter_screen_name + "'") candidate_followers = cursor.fetchall() item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers) monitor_script.runScript("migrateCandidateFollowers (" + str(enc(candidate.twitter_screen_name)) + ")", items=candidate_followers, item_fun=item_fun, resolution=5) candidate.migrated_followers = True candidate.save()
def parseCandidatesFromSpreadSheet(spreadsheet): wb = open_workbook(spreadsheet) sheet = wb.sheet_by_index(0) items = range(1, sheet.nrows) item_fun = functools.partial(parseCandidateFromSpreadSheet, sheet=sheet) monitor_script.runScript(name="parseCandidates", items=items, item_fun=item_fun, resolution=5)
def migrateFun(): candidates = Candidate.objects.filter( candidate_type__in=[2, 3, 4, 5], migrated_followers=False).order_by("name") item_fun = functools.partial(migrateCandidateFollowersFast, download_followers=False) monitor_script.runScript("migrateCandidateFollowers", items=candidates, item_fun=item_fun, resolution=5)
def migrateCandidateFollowers(candidate, download_followers=False): print "+II+: migrating followers for " + enc(candidate.name) cursor = connections['old'].cursor() cursor.execute("SELECT followersid FROM followers_nov5 WHERE userid = '" + candidate.twitter_screen_name + "'") candidate_followers = cursor.fetchall() item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers) monitor_script.runScript("migrateCandidateFollowers (" + str(enc(candidate.twitter_screen_name)) + ")", items=candidate_followers, item_fun=item_fun, resolution=5) candidate.migrated_followers = True candidate.save()
def partitionedDownloadFollowersForVoters(accounts_start_index, accounts_end_index, voter_start_id, voter_end_id): """ :param accounts_start_index: first twitter account to use :param accounts_end_index: last twitter account to use :param voter_start_id: id of voter marks edge of partition """ refreshTwitterAccounts(accounts_start_index, accounts_end_index) # print "# twitter accounts: " + str(len(TWITTER_ACCOUNTS)) total_stray_voters = Voter.objects.filter(id__gte=voter_start_id, id__lte=voter_end_id, downloaded_followers=False, over_max_followers=False).count() window_size = 1000 num_windows = int(math.ceil(total_stray_voters / float(window_size))) for c_window in range(0,num_windows): start_index = (c_window * window_size) query = "SELECT id, twitter_id, twitter_screen_name,num_followee_attempts FROM twpol_voter where id>=%s AND id<=%s ORDER BY id LIMIT %s OFFSET %s" voters = list(Voter.objects.raw(query, [voter_start_id, voter_end_id, window_size, start_index])) title = "downloadFolloees " + str(voter_start_id) + "|" + str(voter_end_id) + " (" + str(c_window) + "/" + str(num_windows)+ ")" monitor_script.runScript(name=title, items=voters, item_fun=downloadFolloweesIDs, resolution=5) email_body = {"voter_start_id":voter_start_id, "voter_end_id":voter_end_id} email = EmailMessage(subject='Heroku Worker Finished!', body=json.dumps(email_body), to=["*****@*****.**"]) email.send()
def partitionedDownloadFollowersForVoters(accounts_start_index, accounts_end_index, voter_start_id, voter_end_id): """ :param accounts_start_index: first twitter account to use :param accounts_end_index: last twitter account to use :param voter_start_id: id of voter marks edge of partition """ refreshTwitterAccounts(accounts_start_index, accounts_end_index) # print "# twitter accounts: " + str(len(TWITTER_ACCOUNTS)) total_stray_voters = Voter.objects.filter( id__gte=voter_start_id, id__lte=voter_end_id, downloaded_followers=False, over_max_followers=False).count() window_size = 1000 num_windows = int(math.ceil(total_stray_voters / float(window_size))) for c_window in range(0, num_windows): start_index = (c_window * window_size) query = "SELECT id, twitter_id, twitter_screen_name,num_followee_attempts FROM twpol_voter where id>=%s AND id<=%s ORDER BY id LIMIT %s OFFSET %s" voters = list( Voter.objects.raw( query, [voter_start_id, voter_end_id, window_size, start_index])) title = "downloadFolloees " + str(voter_start_id) + "|" + str( voter_end_id) + " (" + str(c_window) + "/" + str(num_windows) + ")" monitor_script.runScript(name=title, items=voters, item_fun=downloadFolloweesIDs, resolution=5) email_body = { "voter_start_id": voter_start_id, "voter_end_id": voter_end_id } email = EmailMessage(subject='Heroku Worker Finished!', body=json.dumps(email_body), to=["*****@*****.**"]) email.send()
def migrateCandidateFollowersFast(candidate,download_followers=False): print "+II+: fast migrating followers for " + enc(candidate.name) cursor = connections['old'].cursor() cursor.execute("SELECT count(*) FROM followers_nov5 WHERE userid = '" + candidate.twitter_screen_name + "'") total_candidate_followers = cursor.fetchone()[0] #item_fun = functools.partial(migrateCandidateFollower, candidate=candidate, download_followers=download_followers) item_fun = functools.partial(migrateCandidateFollowerBlunt, candidate=candidate) window_size = 50000 total_processed = 0 num_windows = int(math.ceil(total_candidate_followers / float(window_size))) for c_window in range(0,num_windows): query = "SELECT followersid " \ "FROM followers_nov5 " \ "WHERE userid = '" + candidate.twitter_screen_name + "'"\ + " LIMIT " + str(window_size)\ + " OFFSET " + str(total_processed) cursor.execute(query) window = cursor.fetchall() total_processed += len(window) title = "migrateCandidateFollowers " + str(enc(candidate.twitter_screen_name)) + "(" + str(c_window) + "/" + str(num_windows)+ ")" monitor_script.runScript(name=title, items=window, item_fun=item_fun, resolution=5) candidate.migrated_followers = True candidate.save()
def createManyTwitterAccounts(): items = range(0,1000) item_fun = functools.partial(createTwitterAccountWrapper) monitor_script.runScript("create_twitter_accounts", items=items, item_fun=item_fun, resolution=5)
def analyzeDistrict(district): candidates = Candidate.objects.filter(district=district) item_fun = functools.partial(migrateCandidateFollowers, download_followers=False) monitor_script.runScript("analyzeDistrict ( " + str(district.district_number) + ")", items=candidates, item_fun=item_fun, resolution=5)
def analyzeState(state_code): candidates = Candidate.objects.filter(district__state_code=state_code, migrated_followers=False) item_fun = functools.partial(migrateCandidateFollowers, download_followers=True) monitor_script.runScript("analyzeState (" + str(state_code) + ")", items=candidates, item_fun=item_fun, resolution=1)
def migrateFun(): candidates = Candidate.objects.filter(candidate_type__in=[2,3,4,5], migrated_followers=False).order_by("name") item_fun = functools.partial(migrateCandidateFollowersFast, download_followers=False) monitor_script.runScript("migrateCandidateFollowers", items=candidates, item_fun=item_fun, resolution=5)
def downloadFollowersForStrayVoters(voters=None): if not voters: voters = Voter.objects.all() voters = voters.filter(downloaded_followers=False, over_max_followers=False) monitor_script.runScript("download_followers", items=voters, item_fun=downloadFollowersIDs, resolution=5)