def load_resultcsv_to_resultmanual(electiondate_arg):
## queryset of objects to load
resultcsv_objects = ResultCsv.objects.filter(electiondate=electiondate_arg)
# resultcsv_objects = ResultCsv.objects.all()
counter = 0
for resultcsv in resultcsv_objects:
## include a newsroom-specific ID after statepostal to ensure uniqueness?
resultcsv_raceid = str(resultcsv.electiondate).replace(
"-", "") + state_fips[resultcsv.statepostal] + str(
resultcsv.raceid)
resultcsv_mapping = {
'dataentry': "manual",
# 'datasource': "",
'ballotorder': resultcsv.ballotorder,
# 'description': resultcsv.description,
'electiondate': resultcsv.electiondate,
'fipscode': resultcsv.fipscode,
'first': resultcsv.first,
'gdoc_import': True,
'id': resultcsv.id,
'incumbent': resultcsv.incumbent,
'is_ballot_measure': resultcsv.is_ballot_measure,
'last': resultcsv.last,
'lastupdated': resultcsv.lastupdated,
'level': resultcsv.level,
'note': resultcsv.note,
'officename': resultcsv.officename,
# 'nonpartisan': resultcsv.nonpartisan, ## needs to be uncommented in Result master model
'party': resultcsv.party,
'precinctsreporting': resultcsv.precinctsreporting,
'precinctsreportingpct': resultcsv.precinctsreportingpct,
'precinctstotal': resultcsv.precinctstotal,
'raceid': resultcsv_raceid,
'racetype': resultcsv.racetype,
'reportingunitname': resultcsv.reportingunitname,
'seatname': resultcsv.seatname,
'seatnum': resultcsv.seatnum,
'statepostal': resultcsv.statepostal,
# 'test': resultcsv.test, ## this isn't in the gdoc sheet, as of now
'uncontested': resultcsv.uncontested,
'votecount': resultcsv.votecount,
'votepct': resultcsv.votepct,
# 'votingsystem': result.votingsystem, ## needs to be uncommented in Result master model
'winner': resultcsv.winner
}
## load/create
obj, created = ResultManual.objects.update_or_create(
**resultcsv_mapping)
counter += 1
message = "\nLoaded:\t\t\t" + str(
counter) + " results loaded from ResultCsv to ResultManual"
slackbot(message)
def no_votes_yet(resultcheck_eval):
## after looping through the election tests, sleep and keep checking
if resultcheck_eval:
sleep_eval = 60
message = "No votes yet. The vote check loop will sleep for %s seconds before checking again.\n\n-------------------------------------------------\n" % (sleep_eval)
slackbot(message)
time.sleep(sleep_eval)
def handle(self, *args, **options):
if mccelectionsenv != "prod":
replay_resultmanual()
else:
message = "WARNING: You're on the production server, so this won't execute."
slackbot(message)
def snapshot(electiondate_string):
## *** need to have this on S3 or other external location to not fill up server ***
## NOTE: instead of saving anew, just use/copy the tmp file?
## should we snapshot manual results from gdoc? use separate parent dir to avoid conflict
if mccelectionsenv == "local":
# call_command('snapshot_results')
## copy and rename results.csv w/ cp command
file_path = os.environ["SAVER_PATH"]
orgin = file_path + "/tmp/results.csv"
now = timezone.localtime(timezone.now())
save_date = now.date()
save_date_string = str(save_date)
timestamp = now.strftime('%Y-%m-%d_%H-%M-%S')
snapshot_filename = "results%s.csv" % (timestamp)
destination_dir = "%s/%s/%s" % (file_path, electiondate_string, save_date_string)
destination = "%s/%s" % (destination_dir, snapshot_filename)
mkdir = "mkdir -p %s" % (destination_dir)
snapshot = "cp %s %s" % (origin, destination)
## making the dir, if it's not there
call(mkdir, shell=True)
message = "Making new directory, if needed:\n%s" % (destination_dir)
slackbot(message)
## actual snapshot executed
call(snapshot, shell=True)
message = "Snapshotting"
slackbot(message)
def snapshot_local(electiondate_string):
## for efficiency, we just copy the tmp file for local snapshots instead of saving anew
## this is only for local bc otherwise it would fill up the servers
## S3 snapshot would be needed for test/prod
## should we snapshot manual results from gdoc? use separate parent dir to avoid conflict
# call_command('snapshot_results')
## copy and rename results.csv w/ cp command
file_path = os.environ["SAVER_PATH"]
origin = file_path + "/tmp/results.csv"
now = timezone.localtime(timezone.now())
save_date = now.date()
save_date_string = str(save_date)
timestamp = now.strftime('%Y-%m-%d_%H-%M-%S')
snapshot_filename = "results%s.csv" % (timestamp)
destination_dir = "%s/%s/%s" % (file_path, electiondate_string,
save_date_string)
destination = "%s/%s" % (destination_dir, snapshot_filename)
message = "\nSNAPSHOTTING\nSaving to the following directory:\n%s\n" % (
destination_dir)
slackbot(message)
## making the dir, if it's not there
if not os.path.exists(destination_dir):
os.makedirs(destination_dir)
## copy the file
copyfile(origin, destination)
message = "File copied for snapshot\n"
slackbot(message)
def load_races_candidates():
# today = timezone.now().date() ## this gives GMT date
today = timezone.localtime(
timezone.now()).date() ## this gives actual NY date
elections_new = Election.objects.filter(created__gt=today,
title__contains="Automated")
number_of_elections = len(elections_new)
counter = 0
## loop through election imported today
for election in elections_new:
## set election date
electiondate = str(election.electiondate)
message = "\n---------- DOWNLOADING RACES/CANDIDATES ---------"
slackbot(message)
## load candidates and races
if str(electiondate) == str(today):
call_command('import_ap_elex', 'race', electiondate)
call_command('import_ap_elex', 'candidate', electiondate)
else:
call_command('import_ap_elex', 'race', electiondate, '-t')
call_command('import_ap_elex', 'candidate', electiondate, '-t')
counter += 1
if counter != number_of_elections:
## sleep this loop to avoid maxing out AP API calls
sleep_time = 30
message = "Sleeping for %s seconds" % (sleep_time)
slackbot(message)
time.sleep(sleep_time)
def load_data_races(electiondate_arg, live_arg, test_arg):
races = election_connection(electiondate_arg, live_arg, test_arg).races
# races = election_connection().races
counter = 0
## Race
for race_item in races:
## bc raceid is the same as id
# https://github.com/newsdev/elex/blob/master/elex/api/models.py#L718
state = race_item.statepostal
## use state FIPS to include in ID
state_id = state_fips[state]
mcc_id = int(
str(race_item.electiondate.replace("-", "")) + str(race_item.id) +
str(state_id))
mcc_unique_id = unicode(race_item.electiondate) + '-' + unicode(
race_item.raceid) + '-' + unicode(race_item.statename)
## map Elex fields to model fields
race_mapping = {
'dataentry': 'automated',
'datasource': 'Associated Press',
'description': race_item.description,
'electiondate': race_item.electiondate,
'id': mcc_id,
'is_ballot_measure': race_item.is_ballot_measure,
'unique_id': mcc_unique_id,
'initialization_data': race_item.initialization_data,
'national': race_item.national,
'officeid': race_item.officeid,
'officename': race_item.officename,
'party': race_item.party,
'raceid': race_item.raceid,
'racetype': race_item.racetype,
'racetypeid': race_item.racetypeid,
'seatname': race_item.seatname,
'seatnum': race_item.seatnum,
'statename': race_item.statename,
'statepostal': race_item.statepostal,
'test': race_item.test,
'uncontested': race_item.uncontested,
'lastupdated': race_item.lastupdated
# 'lastupdated': datetime_parser(race_item.lastupdated)
}
## if mccelectionid exists, update; else create
obj, created = Race.objects.update_or_create(**race_mapping)
counter += 1
message = "Loaded:\t\t\t" + str(counter) + " races"
slackbot(message)
def handle(self, *args, **options):
message = "\n-------------- gdoc import to ResultCsv --------------"
slackbot(message)
electiondate_arg = options['electiondate']
message = "\nElection date: \t\t" + electiondate_arg
slackbot(message)
import_csv(electiondate_arg)
def handle(self, *args, **options):
electiondate_arg = options['electiondate']
## load gdoc into ResultCsv
call_command("csv_import", electiondate_arg)
## load distinct races from ResultCsv into Races
# from results.election_loaders import load_resultcsv_to_race
message = "\n---------- ResultCsv import to Races ------------"
slackbot(message)
call_command("import_ap_elex", "resultcsv2race", electiondate_arg)
def handle(self, *args, **options):
## get election date argument
electiondate_arg = options['electiondate']
message = "\n------------ CALCULATE MANUAL ADMIN RESULTS ------------\n"
slackbot(message)
message = "Election date: \t\t" + electiondate_arg
slackbot(message)
update_vote_percent(electiondate_arg)
def clear_resultstage_resultcheck(resultstage_objects, resultcheck_objects):
## delete ResultStage data
resultstage_count = resultstage_objects.count()
resultstage_objects.delete()
message = "ResultStage items deleted:\t%s" % (resultstage_count)
slackbot(message)
## delete ResultCheck data
resultcheck_count = resultcheck_objects.count()
resultcheck_objects.delete()
message = "ResultCheck items deleted:\t%s" % (resultcheck_count)
slackbot(message)
def handle(self, *args, **options):
date_arg = options['date']
if date_arg:
message = "\nDate override: \t\t" + str(date_arg)
slackbot(message)
# electiondate_arg = options['electiondate']
# message = "Election date: \t\t" + electiondate_arg
# slackbot(message)
# loadresults_checkcount(electiondate=electiondate_arg)
loadresults_checkcount(date=date_arg)
def load_resultcsv_to_race(electiondate_arg):
## filtering on the election date shouldn't be necessary, but it's a data check
resultcsv_objects = ResultCsv.objects.filter(electiondate=electiondate_arg)
## determines which field to use for creating races
distinct_fields = ['raceid']
## queryset of objects to load unique by what's in values_list
resultcsv_objects = resultcsv_objects.order_by(*distinct_fields).distinct(
*distinct_fields)
counter = 0
for resultcsv in resultcsv_objects:
## include a newsroom-specific ID after statepostal to ensure uniqueness?
resultcsv_raceid = str(resultcsv.electiondate).replace(
"-", "") + state_fips[resultcsv.statepostal] + str(
resultcsv.raceid)
resultcsv_mapping = {
'dataentry': "manual",
# 'datasource': "",
# 'description': resultcsv.description,
'electiondate': resultcsv.electiondate,
'gdoc_import': True,
'id': resultcsv.id,
'is_ballot_measure': resultcsv.is_ballot_measure,
'lastupdated': resultcsv.lastupdated,
'note': resultcsv.note,
'officename': resultcsv.officename,
'nonpartisan': resultcsv.nonpartisan,
'party': resultcsv.party,
'raceid': resultcsv_raceid,
'racetype': resultcsv.racetype,
'seatname': resultcsv.seatname,
'seatnum': resultcsv.seatnum,
'statepostal': resultcsv.statepostal,
# 'test': resultcsv.test, ## this isn't in the gdoc sheet, as of now
'uncontested': resultcsv.uncontested,
# 'votingsystem': result.votingsystem, ## needs to be uncommented in Result master model
}
## load/create
obj, created = Race.objects.update_or_create(**resultcsv_mapping)
counter += 1
message = "\nLoaded:\t\t\t" + str(
counter) + " races loaded from ResultCsv to ResultRaces"
slackbot(message)
def handle(self, *args, **options):
## get election date argument
electiondate_arg = options['electiondate']
message = "\n--------------- CALCULATE GDOC RESULTS ---------------\n"
slackbot(message)
message = "Election date: \t\t" + electiondate_arg + "\n\n"
slackbot(message)
## call the functions
calculate_precinctsreportingpct(electiondate_arg)
calculate_votepct(electiondate_arg)
def handle(self, *args, **options):
electiondate_arg = options['electiondate']
model_arg = options['model']
start_time = timezone.localtime(timezone.now())
start_message = '\nStarted baking:\t\t' + str(
unicode(start_time)) + '\n'
message = start_message
slackbot(message)
bake_api_to_static(electiondate_arg, model_arg)
end_time = timezone.localtime(timezone.now())
end_message = '\nFinished baking:\t' + str(unicode(end_time)) + '\n\n'
bake_length = str(unicode(end_time - start_time))
message = end_message
slackbot(message)
message = 'Bake length:\t\t' + bake_length
slackbot(message)
message = '--------------------------------------------------\n'
slackbot(message)
def calculate_precinctsreportingpct(electiondate_arg):
gdoc_results = ResultManual.objects.filter(electiondate=electiondate_arg,
gdoc_import=True,
precinctsreporting__gt=0)
if gdoc_results > 0:
message = 'PRECINTS REPORTING PERCENT'
slackbot(message)
start_time = timezone.localtime(timezone.now())
start_message = 'Started calculations:\t' + str(unicode(start_time))
message = start_message
slackbot(message)
for result in gdoc_results:
precincts_reporting = result.precinctsreporting
precincts_total = result.precinctstotal
precincts_reporting_pct = result.precinctsreportingpct
if precincts_reporting and precincts_total:
precincts_reporting_pct = calculated_percent(
precincts_reporting, precincts_total)
elif result.precinctsreporting == None:
precincts_reporting_pct = None
result.save()
end_time = timezone.localtime(timezone.now())
end_message = '\nFinished calculations:\t' + str(
unicode(end_time)) + '\n'
import_length = str(unicode(end_time - start_time))
message = end_message
slackbot(message)
message = 'Calculation length:\t\t' + import_length
slackbot(message)
def handle(self, *args, **options):
date_arg = options['date']
message = "Election date: \t\t" + date_arg
slackbot(message)
## delete existing? nope, because the date here isn't a real election date, at least for now
# Result.objects.filter(dataentry="manual", election_date=date_arg)
## import gdoc to ResultCsv
call_command('csv_import', date_arg)
## import ResultCsv to Result archive
load_resultcsv_to_result()
def handle(self, *args, **options):
## or should this be called electiondate to be more specific?
date_arg = options['date']
if date_arg:
message = "\nDate override: \t\t" + str(date_arg)
slackbot(message)
all_arg = options['all']
message = "Download all? \t\t" + str(
all_arg) ## stringified bc it's a boolean
slackbot(message)
loadresults_checkcount(date=date_arg, all=all_arg)
def handle(self, *args, **options):
electiondate_arg = options['electiondate']
model_arg = options['model']
start_time = timezone.localtime(timezone.now())
start_message = '\nStarted generating:\t\t' + str(unicode(start_time)) + '\n'
message = start_message
slackbot(message)
generate_json(electiondate_arg, model_arg)
end_time = timezone.localtime(timezone.now())
end_message = '\nFinished generating:\t\t' + str(unicode(end_time)) + '\n\n'
generate_length = str(unicode(end_time - start_time))
message = end_message
slackbot(message)
message = 'Generate length:\t\t' + generate_length
slackbot(message)
message = '--------------------------------------------------\n'
slackbot(message)
def load_result_psql(model_arg):
from django.db import connection
import psycopg2
## vars, somewhat redundant with vars declared in save_results_csv() function
psql_connection = os.environ["PSQL_CONNECTION"]
file_path = os.environ["SAVER_PATH"]
csv_file = file_path + "/tmp/results.csv"
def model_count(model_arg):
if model_arg == "resultstage":
count = str(ResultStage.objects.all().count())
elif model_arg == "resultcheck":
count = str(ResultCheck.objects.all().count())
return count
## truncate table
if model_arg == "resultstage":
truncate_sql = "\"TRUNCATE TABLE results_resultstage CASCADE;\""
elif model_arg == "resultcheck":
truncate_sql = "\"TRUNCATE TABLE results_resultcheck CASCADE;\""
else:
message = "No model name match found"
slackbot(message)
truncate_command = psql_connection + " " + truncate_sql
## ?????????? combine TRUNCATE and COPY commands into one statement? ??????????
message = "About to remove %s from %s table..." % (model_count(model_arg),
model_arg)
slackbot(message)
## pass this to subprocess
call(truncate_command, shell=True)
## should we use the mgmt cmd instead? is one faster? if so, will need to be refactored to account for ResultCheck
# from django.core.management import call_command
# call_command('delete_result_psql', model_arg)
## load data via psql copy method
cursor = connection.cursor()
## should it be the way below or this way from a stackoverflow answer?
# COPY results_resultstage FROM stdin WITH CSV HEADER DELIMITER as ',';
message = "\nAbout to load..."
slackbot(message)
if model_arg == "resultstage":
copy_sql = '''
COPY results_resultstage FROM stdin DELIMITER ',' CSV HEADER;
'''
elif model_arg == "resultcheck":
copy_sql = '''
COPY results_resultcheck FROM stdin DELIMITER ',' CSV HEADER;
'''
with open(csv_file, 'r') as f:
cursor.copy_expert(sql=copy_sql, file=f)
connection.commit()
cursor.close()
message = "%s loaded:\t%s" % (model_arg, model_count(model_arg))
slackbot(message)
def load_data_candidates(electiondate_arg, live_arg, test_arg):
candidates = election_connection(electiondate_arg, live_arg,
test_arg).candidates
# candidates = election_connection().candidates
counter = 0
## Candidate
for candidate_item in candidates:
import datetime
# mcc_id = int(str(candidate_item.candidateid) + str(candidate_item.polid))
electiondate_int = electiondate_arg.replace("-", "")
# mcc_id = int((electiondate_int) + str(candidate_item.polnum))
# now = str(datetime.datetime.now()).replace(" ", "").replace("-", "").replace(".", "").replace(":", "")
now = '{:%H%M%S%f}'.format(datetime.datetime.now())
mcc_id = int(str(candidate_item.polnum) + now)
## map Elex fields to model fields
candidate_mapping = {
'dataentry': 'automated',
'datasource': 'Associated Press',
'ballotorder': candidate_item.ballotorder,
'candidateid': candidate_item.candidateid,
'electiondate': electiondate_arg,
'id': mcc_id,
'first': candidate_item.first,
'last': candidate_item.last,
'party': candidate_item.party,
'polid': candidate_item.polid,
'polnum': candidate_item.polnum,
'unique_id': candidate_item.unique_id,
}
## if mccelectionid exists, update; else create
obj, created = Candidate.objects.update_or_create(**candidate_mapping)
counter += 1
message = "Loaded:\t\t\t" + str(counter) + " candidates"
slackbot(message)
def handle(self, *args, **options):
run = True
while run:
## import new AP elections, if any
call_command('download_elections')
## check if there's an Election today; if so, start checking every minute whether to set live and start import
call_command('election_auto')
## Sleep this script
# if mccelectionsenv == "local":
# hours = 6 ## because if the next day is a test, we'll want to be able to capture that
## but how do we handle locally if a previous live election is still going and there's a test? need to stop and restart election_auto on correct day
hours = 1
election_sleep = 60 * 60 * hours ## seconds * minutes * hours
plural = "s" if hours > 1 else ""
message = "Sleeping download_elections and election_auto for %d hour%s" % (hours, plural)
slackbot(message)
# print message
time.sleep(election_sleep)
def calculate_votepct(electiondate_arg):
gdoc_results = ResultManual.objects.filter(electiondate=electiondate_arg,
gdoc_import=True,
votecount__gt=0)
if gdoc_results > 0:
calculated_total_votes_dict_list = gdoc_results.values(
'raceid').annotate(vote_count_total=Sum('votecount'))
message = '\n\nVOTE PERCENT'
slackbot(message)
start_time = timezone.localtime(timezone.now())
start_message = 'Started calculations:\t' + str(unicode(start_time))
message = start_message
slackbot(message)
## if there are any ResultManual objects that are from a gdoc import
if gdoc_results:
## set up a new empty dict
calculated_total_votes_dict = {}
## loop thru the list of dicts with raceids and vote totals to extract and add the relevant key/value pairs
for item in calculated_total_votes_dict_list:
calculated_total_votes_dict[
item['raceid']] = item['vote_count_total']
## loop thru the results
for resultmanual_obj in gdoc_results:
## get the raceid
resultmanual_obj_race_id = resultmanual_obj.raceid
## get the vote total
vote_total = calculated_total_votes_dict[
resultmanual_obj_race_id]
## get the vote count
vote_count = resultmanual_obj.votecount
## calculate the percent
percent_calculated = calculated_percent(vote_count, vote_total)
## get the object id
resultmanual_obj_id = resultmanual_obj.id
# print str(resultmanual_obj) + ": \t" + str(percent_calculated) + "\n"
## update the object
ResultManual.objects.filter(id=resultmanual_obj_id).update(
votepct=percent_calculated)
end_time = timezone.localtime(timezone.now())
end_message = '\nFinished calculations:\t' + str(
unicode(end_time)) + '\n'
import_length = str(unicode(end_time - start_time))
message = end_message
slackbot(message)
message = 'Calculation length:\t\t' + import_length + '\n'
slackbot(message)
def load_data_elections_all():
elections = Elections.get_elections(Elections(), datafile=None)
counter = 0
counter_not = 0
message = "Importing..."
slackbot(message)
for election_item in elections:
election_id = election_item.electiondate.replace("-", "")
election_mapping = {
'dataentry': 'automated',
'datasource': 'Associated Press',
'electiondate': election_item.electiondate,
'id': election_id,
'liveresults': election_item.liveresults,
'title':
'Automated election', ## e.g. extract state names and populate a /-separated string or something; or connect w/ FK like manual races
'testresults': election_item.testresults,
}
obj, created = Election.objects.update_or_create(**election_mapping)
# print obj.electiondate
counter += 1
message = "\nLoaded:\t\t\t" + str(counter) + " elections"
slackbot(message)
message = "Not loaded:\t\t" + str(counter_not) + " elections"
slackbot(message)
def delete_filtered(self, model_arg, electiondate_arg):
if model_arg == "race":
objects_filtered = Race.objects.filter(
electiondate=electiondate_arg).exclude(dataentry="manual")
elif model_arg == "result":
objects_filtered = Result.objects.filter(
electiondate=electiondate_arg).exclude(dataentry="manual")
elif model_arg == "candidate":
objects_filtered = Candidate.objects.filter(
electiondate=electiondate_arg).exclude(dataentry="manual")
# elif model_arg == "election":
# today_date = str(timezone.localtime(timezone.now()).date())
# today = int(today_date.replace("-", ""))
# objects_filtered = Election.objects.filter(id__gte=today).exclude(dataentry="manual") ## UPDATE: make sure deletes any election on today so that one can be loaded anew
elif model_arg == "resultmanual":
## this will only work if nothing is being entered into the admin for that electiondate; otherwise, it deletes everything and that would be a major problem
## does adding the gdoc_import=True solive that?
objects_filtered = ResultManual.objects.filter(
electiondate=electiondate_arg, gdoc_import=True)
elif model_arg == "resultcsv2race":
objects_filtered = Race.objects.filter(electiondate=electiondate_arg,
dataentry="manual",
gdoc_import=True)
elif model_arg == "resultcsv2check":
objects_filtered = ResultCheck.objects.filter(
electiondate=electiondate_arg, gdoc_import=True)
# elif model_arg == "resultcsv2result":
# objects_filtered = Result.objects.filter(electiondate=electiondate_arg, dataentry="manual") # gdoc_import=True, ## using that would require adding that field to Result and removing from ResultManual, then migrating
if model_arg != "election":
objects_filtered_count = objects_filtered.count()
message = "About to delete objects..."
slackbot(message)
objects_filtered.delete()
objects_deleted_message = "Deleted:\t\t%s %s objects" % (
objects_filtered_count, model_arg)
message = objects_deleted_message + "\n"
slackbot(message)
def handle(self, *args, **options):
electiondate_arg = options['electiondate']
model_arg = options['model']
test_arg = options['test']
override_arg = options['override']
if test_arg:
test = '(test)'
message = '\n------- JSON -> S3 for %s %s -------' % (model_arg, test)
slackbot(message)
print '\nElection date:\t\t%s' % (electiondate_arg)
print 'Model:\t\t\t%s' % (model_arg)
print 'Test?\t\t\t%s\n' % (test_arg)
start_time = timezone.localtime(timezone.now())
start_message = '\nStarted baking:\t\t' + str(
unicode(start_time)) + '\n'
message = start_message
slackbot(message)
## *** FUNCTION CALL ***
json_to_s3(electiondate_arg, model_arg, test_arg, override_arg)
end_time = timezone.localtime(timezone.now())
end_message = '\nFinished baking:\t' + str(unicode(end_time)) + '\n\n'
bake_length = str(unicode(end_time - start_time))
message = end_message
slackbot(message)
message = 'Bake length:\t\t' + bake_length
slackbot(message)
message = '--------------------------------------------------\n'
slackbot(message)
def update_vote_percent(electiondate_arg):
## all ResultManual objects that aren't zero (bc we can't divide with 0) or from a gdoc
resultmanual_objects = ResultManual.objects.filter(
electiondate=electiondate_arg,
votecount__gt=0).exclude(gdoc_import=True)
## all total vote count by race
calculated_total_votes_dict_list = resultmanual_objects.values(
'race_name_fk').annotate(vote_count_total=Sum('votecount'))
start_time = timezone.localtime(timezone.now())
start_message = '\nStarted calculations:\t' + str(unicode(start_time))
message = start_message
slackbot(message)
## if there are any ResultManual objects that aren't from a gdoc import
if resultmanual_objects:
## set up a new empty dict
calculated_total_votes_dict = {}
## loop thru the list of dicts with raceids and vote totals to extract and add the relevant key/value pairs
for item in calculated_total_votes_dict_list:
calculated_total_votes_dict[
item['race_name_fk']] = item['vote_count_total']
## loop thru the results
for resultmanual_obj in resultmanual_objects:
## get the race FK id
resultmanual_obj_raecid = resultmanual_obj.race_name_fk.id
## get the vote total
vote_total = calculated_total_votes_dict[resultmanual_obj_raecid]
## get the vote count
vote_count = resultmanual_obj.votecount
## calculate the percent
percent_calculated = calculated_percent(vote_count, vote_total)
## get the object id
resultmanual_obj_id = resultmanual_obj.id
## update the object
ResultManual.objects.filter(id=resultmanual_obj_id).update(
votepct=percent_calculated)
end_time = timezone.localtime(timezone.now())
end_message = '\nFinished calculations:\t' + str(unicode(end_time)) + '\n'
import_length = str(unicode(end_time - start_time))
message = end_message
slackbot(message)
message = 'Import length:\t\t' + import_length + '\n'
slackbot(message)
def handle(self, *args, **options):
id_arg = int(options['id'][0])
message = "Embed ID:\t\t" + str(id_arg)
slackbot(message)
## use queryset to filter object, check there's only 1 and save
embeds = Embed.objects.all()
embeds_filtered = embeds.filter(id=id_arg)
if embeds_filtered.count() == 1:
embed_object = embeds_filtered[0]
embed_object.save()
message = "Saved:\t\t%s\n\n" % (str(embed_object))
slackbot(message)
else:
message = "More than one object was found. Please fix and re-try."
slackbot(message)
def loadresults_checkcount(date="", all=""):
all_arg = str(all)
master_loop = True
while master_loop:
## current date/time whenever referenced
now = timezone.localtime(timezone.now()) ## gives actual NY date/time
## today's date
today = now.date()
## if the optional date arg is included, override the default
if date:
today_string = date
else:
today_string = str(today)
## election_obj function pulls the zero-indexed value
election = election_obj(today_string)
## is there an election?
if election:
election_today = True
else:
election_today = False
## all election tests for today
election_tests = Election.objects.filter(test=True,
testdate=today_string)
## if one or more test elections exist
if election_tests:
## how many?
election_tests_count = election_tests.count()
election_test_today = True
else:
election_test_today = False
## if there's an election or test today...
if election_today or election_test_today:
if election_today:
if election.dataentry == "manual" and not election.url:
message = "\nOh, hello! There's a manual election today: "
slackbot(message)
message = "- " + str(election)
slackbot(message)
## NEED TO ACCOUNT FOR when there's both an automated + manual/admin, automated + manual/gdoc elections
# elif election.dataentry == "manual" and election.url:
# message = "\nOh, hello! There's a manual gdoc election today: "
# slackbot(message)
# message = "- " + str(election)
# slackbot(message)
elif election.dataentry == "automated":
message = "\nOh, hello! There's an automated election today: "
slackbot(message)
message = "- " + str(election)
slackbot(message)
## NEED TO ACCOUNT FOR gdoc election tests
elif election_tests_count == 1:
election = election_tests[0]
message = "\nOh, hello! There's a test election today: "
slackbot(message)
message = "- " + str(election)
slackbot(message)
elif election_tests_count > 1:
message = "\nOh, hello! There are multiple test elections today: "
slackbot(message)
for election in election_tests:
message = "- " + str(election)
slackbot(message)
resultcheck_eval = True
counter_eval = 0
while resultcheck_eval:
## non-test election
if election_today:
# ## check vote count and if it returns False, then...
# if not check_vote_count(election, counter_eval, election_test_today):
# resultcheck_eval = False
# resultstage_import = True
## uses this form to match the correctly starttime value
now = timezone.now()
election_starttime = election.starttime
## if there's no starttime or the starttime is less than or equal to now
if not election_starttime or election_starttime <= now:
message = "\nChecking " + str(election) + "..."
slackbot(message)
## check vote count and if it returns False, then...
if not check_vote_count(election,
counter_eval,
election_test_today,
all=all_arg):
resultcheck_eval = False
resultstage_import = True
else:
message = "\nToday's election has a start time, so we won't check until then:\n%s\n%s GMT\n" % (
election, election_starttime)
slackbot(message)
## test election(s)
elif election_test_today:
from itertools import chain
## determine and set scope of what to loop thru; i.e. remaining election tests
endtime_null = election_tests.filter(
endtime__isnull=True
) # starttime__isnull=True can't go here bc it might filter out something that started, but never finished and had to be restarted
# print "\nNo endtime:\t\t\t" + str(endtime_null)
endtime_gt_now = election_tests.filter(endtime__gt=now)
# print "\nEndtime_gt_now:\t\t\t" + str(endtime_gt_now)
remaining_election_tests = list(
chain(endtime_null, endtime_gt_now))
# print "\nremaining_election_tests:\t" + str(remaining_election_tests)
election_tests_to_check = len(remaining_election_tests)
message = "\nHow many remaining test elections today?\n" + str(
election_tests_to_check
) + "\n" #+ str(remaining_election_tests)
slackbot(message)
# if there's more than one test election on this day
if election_tests_to_check > 0:
if election_tests_to_check > 1:
message = "\nBecause there are multiple election tests, we need to check each of them individually."
slackbot(message)
## if any exist, loop thru remaining election tests on this day
for election in remaining_election_tests:
## uses this form to match the correctly starttime value
now = timezone.now()
election_starttime = election.starttime
## if there's no starttime or the starttime is less than or equal to now
if not election_starttime or election_starttime <= now:
message = "\nChecking " + str(election) + "..."
slackbot(message)
## check vote count and see what it returns
if not check_vote_count(election,
counter_eval,
election_test_today,
all=all_arg):
resultcheck_eval = False
resultstage_import = True
break
else:
message = "\nThe following election has a start time, so we won't check until then:\n%s\n%s GMT\n" % (
election, election_starttime)
slackbot(message)
# ## after looping through the election tests, sleep and keep checking
# if resultcheck_eval:
# sleep_eval = 60
# message = "No votes yet. The vote check loop will sleep for %s seconds before checking again.\n\n-------------------------------------------------\n" % (sleep_eval)
# slackbot(message)
# time.sleep(sleep_eval)
## if there are no more election tests
elif not election_tests_to_check:
message = "\n @channel: Well, that's a wrap. No more election tests for today!\n"
slackbot(message)
resultcheck_eval = False
resultstage_import = False
master_loop = False
## after looking at the election/election test(s), sleep and keep checking
if resultcheck_eval:
sleep_eval = 60
message = "No votes yet. The vote check loop will sleep for %s seconds before checking again.\n\n-------------------------------------------------\n" % (
sleep_eval)
slackbot(message)
time.sleep(sleep_eval)
## everything here was moved to the check_vote_count function
## resultstage_import loop can only run on a single election (not all elections) specific to a day
counter_import = 0
## while resultstage_import var is True...
while resultstage_import:
## if conditions for continuing to to import to resultstage are False, then stop this loop
if not import_resultstage(election,
election_test_today,
counter_import,
now,
today_string,
all=all_arg):
resultstage_import = False
## or else keep it going
else:
resultstage_import = True
counter_import += 1
message = "\nNumber of imports:\t" + str(counter_import)
slackbot(message)
## !! everything here was moved to the import_resultstage function
## end of import while loop
else:
message = "\nNOTICE: There's no election today!\n"
slackbot(message)
master_loop = False
def import_resultstage(election_object,
election_test_today,
counter_import,
now,
today_string,
all=""):
all_arg = str(all)
message = "\n--------------- IMPORT RESULTS STAGE ---------------\n %s" % (
election_object)
slackbot(message)
election = election_object
electiondate_string = str(election.electiondate)
if election_test_today:
if all_arg:
call_command('import_ap_elex', 'resultstage', electiondate_string,
"-t", "-a")
else:
call_command('import_ap_elex', 'resultstage', electiondate_string,
"-t")
else:
if all_arg:
call_command('import_ap_elex', 'resultstage', electiondate_string,
"-a")
else:
call_command('import_ap_elex', 'resultstage', electiondate_string)
## snapshot
if mccelectionsenv == "local":
snapshot_local(electiondate_string)
# else:
## snapshot to S3
## this would likely be where we'd bake live results, but that process needs to happen at the end of import_ap_elex mgmt command due to possible later updates run independent of this election_auto script
## if this loop is the first import, set the resultcount var on the related Election object
if counter_import == 0:
## set resultcheck, get the count and print it
resultstage_count_int = resultstage_count(resultstage_objects())
election.resultcount = resultstage_count_int
election.save()
count = str(resultstage_count_int)
message = "\nSet result count field on Election model using ResultStage: %s" % (
count)
slackbot(message)
## set the election to live and set that as the start time
election.live = True
election.starttime = timezone.localtime(timezone.now())
election.save()
message = "Set election to live and set start time\n"
slackbot(message)
resultcount_changed = check_resultstage_count_change(
electiondate_string, resultstage_count(resultstage_objects()),
election)
## for the timer
hours = 12 ## ABSTRACT???
if election_test_today:
expiration_length = 60 ## ABSTRACT???
else:
expiration_length = 60 * hours
## QUESTION: do we want to do this? reduce import frequency if it's been more than XX hours?
if counter_import == expiration_length:
timer_expired = True
## set sleep time at 5 mins -- extended import time
sleep_import = 300 ## ABSTRACT???
else:
timer_expired = False
## set sleep time for 60 seconds
# sleep_import = 60
sleep_import = RESULTSTAGE_IMPORT_SLEEP_TIME
## returns a dict of the aggregate key and value
precinctsreportingpct_check = resultstage_objects().aggregate(
Avg("precinctsreportingpct"))
## pulls the value out of the dict
precinctsreportingpct_avg = precinctsreportingpct_check.values()[0] * 100
## for informational, not logic/check, purposes:
precinctsreportingpct_avg_state = resultstage_objects().filter(
level="state").aggregate(
Avg("precinctsreportingpct")).values()[0] * 100
precinctsreportingpct_avg_state_pres = resultstage_objects().filter(
level="state", officename="President").aggregate(
Avg("precinctsreportingpct")).values()[0] * 100
if precinctsreportingpct_avg and precinctsreportingpct_avg_state and precinctsreportingpct_avg_state_pres:
message = "PERCENT AVERAGES\nPercent avg total:\t\t\t%d \nPercent avg state-level:\t\t%d \nPercent avg state-level for prez:\t%d \n" % (
int(precinctsreportingpct_avg),
int(precinctsreportingpct_avg_state),
int(precinctsreportingpct_avg_state_pres))
slackbot(message)
elif precinctsreportingpct_avg_state and precinctsreportingpct_avg:
message = "PERCENT AVERAGES\nPercent avg total:\t\t\t%d \nPercent avg state-level:\t\t%d" % (
int(precinctsreportingpct_avg),
int(precinctsreportingpct_avg_state))
slackbot(message)
## if it's a test, set the % check as president avg
if election_test_today:
percent_check = precinctsreportingpct_avg_state_pres
## or else set it as overall % avg
else:
percent_check = precinctsreportingpct_avg
## if the following are true...
## there are fewer result objects now than there were initially
## the timer is expired
## the precincts reporting % avg is 100%
if resultcount_changed or timer_expired or percent_check == 100:
message = "\n----------------- IMPORT LOOP TERMINATION ------------------\n"
slackbot(message)
if resultcount_changed:
message = "Why is it ending?\tResult count changed"
slackbot(message)
elif timer_expired:
message = "Why is it ending?\tTimer expired"
slackbot(message)
elif precinctsreportingpct_avg:
message = "Why is it ending?\tAll precincts reporting"
slackbot(message)
## set the election "live" field as false and the end time
election.live = False
election.endtime = timezone.localtime(timezone.now())
election.save()
message = "\nSetting election live field to False and setting the end time.\n"
slackbot(message)
sleep_import = 0
if not election_test_today:
last_line = "- the script to sleep long enough to allow for the cache to clear\n"
else:
last_line = ""
message = "\nSet the following...\n - the load loop to stop \n - the \"live\" field for " + electiondate_string + " automated Election to False \n%s - the loop sleep time var to 0 to exit the loop immediately\n\nLoading the archive..." % (
last_line)
slackbot(message)
## load archive result data
if election_test_today:
if all_arg:
call_command('import_ap_elex', 'result', electiondate_string,
"-t", "-a")
else:
call_command('import_ap_elex', 'result', electiondate_string,
"-t")
else:
if all_arg:
call_command('import_ap_elex', 'result', electiondate_string,
"-a")
else:
call_command('import_ap_elex', 'result', electiondate_string)
message = "\n@channel: The import has completed for the election on %s\n" % (
electiondate_string)
slackbot(message)
## if it's not a test, sleep the script so the cache has time to reset with the archive data before clearing resultstage
# if not election_test_today:
# number_of_minutes = 15
# archive_cache_time = 60 * number_of_minutes
# time.sleep(archive_cache_time)
# message = "Sleeping for:\t%s minutes" % (number_of_minutes)
## delete ResultStage data
clear_resultstage_resultcheck(resultstage_objects(),
resultcheck_objects())
if election_test_today:
## all election tests for today
election_tests = Election.objects.filter(test=True,
testdate=today_string)
else:
election_tests = 0
resultstage_import = False
## this restarts the resultcheck loop if there's more than one test
if election_tests > 1:
message = "\nOh, hello! There's another test election today: "
slackbot(message)
message = election_test_today
slackbot(message)
message = "Restarting the resultcheck loop..."
slackbot(message)
resultstage_import = False ## this needs to be False bc we need to exit this loop and restart the check loop
# resultcheck_eval = True
## this stops the import loop
elif election_tests == 1 or election_tests == 0:
resultstage_import = False
sys.exit()
else:
resultstage_import = True
if sleep_import > 0:
message = "Sleeping the import loop for %s seconds" % (sleep_import)
slackbot(message)
time.sleep(sleep_import)
return resultstage_import
