def load_resultcsv_to_resultmanual(electiondate_arg): ## queryset of objects to load resultcsv_objects = ResultCsv.objects.filter(electiondate=electiondate_arg) # resultcsv_objects = ResultCsv.objects.all() counter = 0 for resultcsv in resultcsv_objects: ## include a newsroom-specific ID after statepostal to ensure uniqueness? resultcsv_raceid = str(resultcsv.electiondate).replace( "-", "") + state_fips[resultcsv.statepostal] + str( resultcsv.raceid) resultcsv_mapping = { 'dataentry': "manual", # 'datasource': "", 'ballotorder': resultcsv.ballotorder, # 'description': resultcsv.description, 'electiondate': resultcsv.electiondate, 'fipscode': resultcsv.fipscode, 'first': resultcsv.first, 'gdoc_import': True, 'id': resultcsv.id, 'incumbent': resultcsv.incumbent, 'is_ballot_measure': resultcsv.is_ballot_measure, 'last': resultcsv.last, 'lastupdated': resultcsv.lastupdated, 'level': resultcsv.level, 'note': resultcsv.note, 'officename': resultcsv.officename, # 'nonpartisan': resultcsv.nonpartisan, ## needs to be uncommented in Result master model 'party': resultcsv.party, 'precinctsreporting': resultcsv.precinctsreporting, 'precinctsreportingpct': resultcsv.precinctsreportingpct, 'precinctstotal': resultcsv.precinctstotal, 'raceid': resultcsv_raceid, 'racetype': resultcsv.racetype, 'reportingunitname': resultcsv.reportingunitname, 'seatname': resultcsv.seatname, 'seatnum': resultcsv.seatnum, 'statepostal': resultcsv.statepostal, # 'test': resultcsv.test, ## this isn't in the gdoc sheet, as of now 'uncontested': resultcsv.uncontested, 'votecount': resultcsv.votecount, 'votepct': resultcsv.votepct, # 'votingsystem': result.votingsystem, ## needs to be uncommented in Result master model 'winner': resultcsv.winner } ## load/create obj, created = ResultManual.objects.update_or_create( **resultcsv_mapping) counter += 1 message = "\nLoaded:\t\t\t" + str( counter) + " results loaded from ResultCsv to ResultManual" slackbot(message)
def no_votes_yet(resultcheck_eval): ## after looping through the election tests, sleep and keep checking if resultcheck_eval: sleep_eval = 60 message = "No votes yet. The vote check loop will sleep for %s seconds before checking again.\n\n-------------------------------------------------\n" % (sleep_eval) slackbot(message) time.sleep(sleep_eval)
def handle(self, *args, **options): if mccelectionsenv != "prod": replay_resultmanual() else: message = "WARNING: You're on the production server, so this won't execute." slackbot(message)
def snapshot(electiondate_string): ## *** need to have this on S3 or other external location to not fill up server *** ## NOTE: instead of saving anew, just use/copy the tmp file? ## should we snapshot manual results from gdoc? use separate parent dir to avoid conflict if mccelectionsenv == "local": # call_command('snapshot_results') ## copy and rename results.csv w/ cp command file_path = os.environ["SAVER_PATH"] orgin = file_path + "/tmp/results.csv" now = timezone.localtime(timezone.now()) save_date = now.date() save_date_string = str(save_date) timestamp = now.strftime('%Y-%m-%d_%H-%M-%S') snapshot_filename = "results%s.csv" % (timestamp) destination_dir = "%s/%s/%s" % (file_path, electiondate_string, save_date_string) destination = "%s/%s" % (destination_dir, snapshot_filename) mkdir = "mkdir -p %s" % (destination_dir) snapshot = "cp %s %s" % (origin, destination) ## making the dir, if it's not there call(mkdir, shell=True) message = "Making new directory, if needed:\n%s" % (destination_dir) slackbot(message) ## actual snapshot executed call(snapshot, shell=True) message = "Snapshotting" slackbot(message)
def snapshot_local(electiondate_string): ## for efficiency, we just copy the tmp file for local snapshots instead of saving anew ## this is only for local bc otherwise it would fill up the servers ## S3 snapshot would be needed for test/prod ## should we snapshot manual results from gdoc? use separate parent dir to avoid conflict # call_command('snapshot_results') ## copy and rename results.csv w/ cp command file_path = os.environ["SAVER_PATH"] origin = file_path + "/tmp/results.csv" now = timezone.localtime(timezone.now()) save_date = now.date() save_date_string = str(save_date) timestamp = now.strftime('%Y-%m-%d_%H-%M-%S') snapshot_filename = "results%s.csv" % (timestamp) destination_dir = "%s/%s/%s" % (file_path, electiondate_string, save_date_string) destination = "%s/%s" % (destination_dir, snapshot_filename) message = "\nSNAPSHOTTING\nSaving to the following directory:\n%s\n" % ( destination_dir) slackbot(message) ## making the dir, if it's not there if not os.path.exists(destination_dir): os.makedirs(destination_dir) ## copy the file copyfile(origin, destination) message = "File copied for snapshot\n" slackbot(message)
def load_races_candidates(): # today = timezone.now().date() ## this gives GMT date today = timezone.localtime( timezone.now()).date() ## this gives actual NY date elections_new = Election.objects.filter(created__gt=today, title__contains="Automated") number_of_elections = len(elections_new) counter = 0 ## loop through election imported today for election in elections_new: ## set election date electiondate = str(election.electiondate) message = "\n---------- DOWNLOADING RACES/CANDIDATES ---------" slackbot(message) ## load candidates and races if str(electiondate) == str(today): call_command('import_ap_elex', 'race', electiondate) call_command('import_ap_elex', 'candidate', electiondate) else: call_command('import_ap_elex', 'race', electiondate, '-t') call_command('import_ap_elex', 'candidate', electiondate, '-t') counter += 1 if counter != number_of_elections: ## sleep this loop to avoid maxing out AP API calls sleep_time = 30 message = "Sleeping for %s seconds" % (sleep_time) slackbot(message) time.sleep(sleep_time)
def load_data_races(electiondate_arg, live_arg, test_arg): races = election_connection(electiondate_arg, live_arg, test_arg).races # races = election_connection().races counter = 0 ## Race for race_item in races: ## bc raceid is the same as id # https://github.com/newsdev/elex/blob/master/elex/api/models.py#L718 state = race_item.statepostal ## use state FIPS to include in ID state_id = state_fips[state] mcc_id = int( str(race_item.electiondate.replace("-", "")) + str(race_item.id) + str(state_id)) mcc_unique_id = unicode(race_item.electiondate) + '-' + unicode( race_item.raceid) + '-' + unicode(race_item.statename) ## map Elex fields to model fields race_mapping = { 'dataentry': 'automated', 'datasource': 'Associated Press', 'description': race_item.description, 'electiondate': race_item.electiondate, 'id': mcc_id, 'is_ballot_measure': race_item.is_ballot_measure, 'unique_id': mcc_unique_id, 'initialization_data': race_item.initialization_data, 'national': race_item.national, 'officeid': race_item.officeid, 'officename': race_item.officename, 'party': race_item.party, 'raceid': race_item.raceid, 'racetype': race_item.racetype, 'racetypeid': race_item.racetypeid, 'seatname': race_item.seatname, 'seatnum': race_item.seatnum, 'statename': race_item.statename, 'statepostal': race_item.statepostal, 'test': race_item.test, 'uncontested': race_item.uncontested, 'lastupdated': race_item.lastupdated # 'lastupdated': datetime_parser(race_item.lastupdated) } ## if mccelectionid exists, update; else create obj, created = Race.objects.update_or_create(**race_mapping) counter += 1 message = "Loaded:\t\t\t" + str(counter) + " races" slackbot(message)
def handle(self, *args, **options): message = "\n-------------- gdoc import to ResultCsv --------------" slackbot(message) electiondate_arg = options['electiondate'] message = "\nElection date: \t\t" + electiondate_arg slackbot(message) import_csv(electiondate_arg)
def handle(self, *args, **options): electiondate_arg = options['electiondate'] ## load gdoc into ResultCsv call_command("csv_import", electiondate_arg) ## load distinct races from ResultCsv into Races # from results.election_loaders import load_resultcsv_to_race message = "\n---------- ResultCsv import to Races ------------" slackbot(message) call_command("import_ap_elex", "resultcsv2race", electiondate_arg)
def handle(self, *args, **options): ## get election date argument electiondate_arg = options['electiondate'] message = "\n------------ CALCULATE MANUAL ADMIN RESULTS ------------\n" slackbot(message) message = "Election date: \t\t" + electiondate_arg slackbot(message) update_vote_percent(electiondate_arg)
def clear_resultstage_resultcheck(resultstage_objects, resultcheck_objects): ## delete ResultStage data resultstage_count = resultstage_objects.count() resultstage_objects.delete() message = "ResultStage items deleted:\t%s" % (resultstage_count) slackbot(message) ## delete ResultCheck data resultcheck_count = resultcheck_objects.count() resultcheck_objects.delete() message = "ResultCheck items deleted:\t%s" % (resultcheck_count) slackbot(message)
def handle(self, *args, **options): date_arg = options['date'] if date_arg: message = "\nDate override: \t\t" + str(date_arg) slackbot(message) # electiondate_arg = options['electiondate'] # message = "Election date: \t\t" + electiondate_arg # slackbot(message) # loadresults_checkcount(electiondate=electiondate_arg) loadresults_checkcount(date=date_arg)
def load_resultcsv_to_race(electiondate_arg): ## filtering on the election date shouldn't be necessary, but it's a data check resultcsv_objects = ResultCsv.objects.filter(electiondate=electiondate_arg) ## determines which field to use for creating races distinct_fields = ['raceid'] ## queryset of objects to load unique by what's in values_list resultcsv_objects = resultcsv_objects.order_by(*distinct_fields).distinct( *distinct_fields) counter = 0 for resultcsv in resultcsv_objects: ## include a newsroom-specific ID after statepostal to ensure uniqueness? resultcsv_raceid = str(resultcsv.electiondate).replace( "-", "") + state_fips[resultcsv.statepostal] + str( resultcsv.raceid) resultcsv_mapping = { 'dataentry': "manual", # 'datasource': "", # 'description': resultcsv.description, 'electiondate': resultcsv.electiondate, 'gdoc_import': True, 'id': resultcsv.id, 'is_ballot_measure': resultcsv.is_ballot_measure, 'lastupdated': resultcsv.lastupdated, 'note': resultcsv.note, 'officename': resultcsv.officename, 'nonpartisan': resultcsv.nonpartisan, 'party': resultcsv.party, 'raceid': resultcsv_raceid, 'racetype': resultcsv.racetype, 'seatname': resultcsv.seatname, 'seatnum': resultcsv.seatnum, 'statepostal': resultcsv.statepostal, # 'test': resultcsv.test, ## this isn't in the gdoc sheet, as of now 'uncontested': resultcsv.uncontested, # 'votingsystem': result.votingsystem, ## needs to be uncommented in Result master model } ## load/create obj, created = Race.objects.update_or_create(**resultcsv_mapping) counter += 1 message = "\nLoaded:\t\t\t" + str( counter) + " races loaded from ResultCsv to ResultRaces" slackbot(message)
def handle(self, *args, **options): ## get election date argument electiondate_arg = options['electiondate'] message = "\n--------------- CALCULATE GDOC RESULTS ---------------\n" slackbot(message) message = "Election date: \t\t" + electiondate_arg + "\n\n" slackbot(message) ## call the functions calculate_precinctsreportingpct(electiondate_arg) calculate_votepct(electiondate_arg)
def handle(self, *args, **options): electiondate_arg = options['electiondate'] model_arg = options['model'] start_time = timezone.localtime(timezone.now()) start_message = '\nStarted baking:\t\t' + str( unicode(start_time)) + '\n' message = start_message slackbot(message) bake_api_to_static(electiondate_arg, model_arg) end_time = timezone.localtime(timezone.now()) end_message = '\nFinished baking:\t' + str(unicode(end_time)) + '\n\n' bake_length = str(unicode(end_time - start_time)) message = end_message slackbot(message) message = 'Bake length:\t\t' + bake_length slackbot(message) message = '--------------------------------------------------\n' slackbot(message)
def calculate_precinctsreportingpct(electiondate_arg): gdoc_results = ResultManual.objects.filter(electiondate=electiondate_arg, gdoc_import=True, precinctsreporting__gt=0) if gdoc_results > 0: message = 'PRECINTS REPORTING PERCENT' slackbot(message) start_time = timezone.localtime(timezone.now()) start_message = 'Started calculations:\t' + str(unicode(start_time)) message = start_message slackbot(message) for result in gdoc_results: precincts_reporting = result.precinctsreporting precincts_total = result.precinctstotal precincts_reporting_pct = result.precinctsreportingpct if precincts_reporting and precincts_total: precincts_reporting_pct = calculated_percent( precincts_reporting, precincts_total) elif result.precinctsreporting == None: precincts_reporting_pct = None result.save() end_time = timezone.localtime(timezone.now()) end_message = '\nFinished calculations:\t' + str( unicode(end_time)) + '\n' import_length = str(unicode(end_time - start_time)) message = end_message slackbot(message) message = 'Calculation length:\t\t' + import_length slackbot(message)
def handle(self, *args, **options): date_arg = options['date'] message = "Election date: \t\t" + date_arg slackbot(message) ## delete existing? nope, because the date here isn't a real election date, at least for now # Result.objects.filter(dataentry="manual", election_date=date_arg) ## import gdoc to ResultCsv call_command('csv_import', date_arg) ## import ResultCsv to Result archive load_resultcsv_to_result()
def handle(self, *args, **options): ## or should this be called electiondate to be more specific? date_arg = options['date'] if date_arg: message = "\nDate override: \t\t" + str(date_arg) slackbot(message) all_arg = options['all'] message = "Download all? \t\t" + str( all_arg) ## stringified bc it's a boolean slackbot(message) loadresults_checkcount(date=date_arg, all=all_arg)
def handle(self, *args, **options): electiondate_arg = options['electiondate'] model_arg = options['model'] start_time = timezone.localtime(timezone.now()) start_message = '\nStarted generating:\t\t' + str(unicode(start_time)) + '\n' message = start_message slackbot(message) generate_json(electiondate_arg, model_arg) end_time = timezone.localtime(timezone.now()) end_message = '\nFinished generating:\t\t' + str(unicode(end_time)) + '\n\n' generate_length = str(unicode(end_time - start_time)) message = end_message slackbot(message) message = 'Generate length:\t\t' + generate_length slackbot(message) message = '--------------------------------------------------\n' slackbot(message)
def load_result_psql(model_arg): from django.db import connection import psycopg2 ## vars, somewhat redundant with vars declared in save_results_csv() function psql_connection = os.environ["PSQL_CONNECTION"] file_path = os.environ["SAVER_PATH"] csv_file = file_path + "/tmp/results.csv" def model_count(model_arg): if model_arg == "resultstage": count = str(ResultStage.objects.all().count()) elif model_arg == "resultcheck": count = str(ResultCheck.objects.all().count()) return count ## truncate table if model_arg == "resultstage": truncate_sql = "\"TRUNCATE TABLE results_resultstage CASCADE;\"" elif model_arg == "resultcheck": truncate_sql = "\"TRUNCATE TABLE results_resultcheck CASCADE;\"" else: message = "No model name match found" slackbot(message) truncate_command = psql_connection + " " + truncate_sql ## ?????????? combine TRUNCATE and COPY commands into one statement? ?????????? message = "About to remove %s from %s table..." % (model_count(model_arg), model_arg) slackbot(message) ## pass this to subprocess call(truncate_command, shell=True) ## should we use the mgmt cmd instead? is one faster? if so, will need to be refactored to account for ResultCheck # from django.core.management import call_command # call_command('delete_result_psql', model_arg) ## load data via psql copy method cursor = connection.cursor() ## should it be the way below or this way from a stackoverflow answer? # COPY results_resultstage FROM stdin WITH CSV HEADER DELIMITER as ','; message = "\nAbout to load..." slackbot(message) if model_arg == "resultstage": copy_sql = ''' COPY results_resultstage FROM stdin DELIMITER ',' CSV HEADER; ''' elif model_arg == "resultcheck": copy_sql = ''' COPY results_resultcheck FROM stdin DELIMITER ',' CSV HEADER; ''' with open(csv_file, 'r') as f: cursor.copy_expert(sql=copy_sql, file=f) connection.commit() cursor.close() message = "%s loaded:\t%s" % (model_arg, model_count(model_arg)) slackbot(message)
def load_data_candidates(electiondate_arg, live_arg, test_arg): candidates = election_connection(electiondate_arg, live_arg, test_arg).candidates # candidates = election_connection().candidates counter = 0 ## Candidate for candidate_item in candidates: import datetime # mcc_id = int(str(candidate_item.candidateid) + str(candidate_item.polid)) electiondate_int = electiondate_arg.replace("-", "") # mcc_id = int((electiondate_int) + str(candidate_item.polnum)) # now = str(datetime.datetime.now()).replace(" ", "").replace("-", "").replace(".", "").replace(":", "") now = '{:%H%M%S%f}'.format(datetime.datetime.now()) mcc_id = int(str(candidate_item.polnum) + now) ## map Elex fields to model fields candidate_mapping = { 'dataentry': 'automated', 'datasource': 'Associated Press', 'ballotorder': candidate_item.ballotorder, 'candidateid': candidate_item.candidateid, 'electiondate': electiondate_arg, 'id': mcc_id, 'first': candidate_item.first, 'last': candidate_item.last, 'party': candidate_item.party, 'polid': candidate_item.polid, 'polnum': candidate_item.polnum, 'unique_id': candidate_item.unique_id, } ## if mccelectionid exists, update; else create obj, created = Candidate.objects.update_or_create(**candidate_mapping) counter += 1 message = "Loaded:\t\t\t" + str(counter) + " candidates" slackbot(message)
def handle(self, *args, **options): run = True while run: ## import new AP elections, if any call_command('download_elections') ## check if there's an Election today; if so, start checking every minute whether to set live and start import call_command('election_auto') ## Sleep this script # if mccelectionsenv == "local": # hours = 6 ## because if the next day is a test, we'll want to be able to capture that ## but how do we handle locally if a previous live election is still going and there's a test? need to stop and restart election_auto on correct day hours = 1 election_sleep = 60 * 60 * hours ## seconds * minutes * hours plural = "s" if hours > 1 else "" message = "Sleeping download_elections and election_auto for %d hour%s" % (hours, plural) slackbot(message) # print message time.sleep(election_sleep)
def calculate_votepct(electiondate_arg): gdoc_results = ResultManual.objects.filter(electiondate=electiondate_arg, gdoc_import=True, votecount__gt=0) if gdoc_results > 0: calculated_total_votes_dict_list = gdoc_results.values( 'raceid').annotate(vote_count_total=Sum('votecount')) message = '\n\nVOTE PERCENT' slackbot(message) start_time = timezone.localtime(timezone.now()) start_message = 'Started calculations:\t' + str(unicode(start_time)) message = start_message slackbot(message) ## if there are any ResultManual objects that are from a gdoc import if gdoc_results: ## set up a new empty dict calculated_total_votes_dict = {} ## loop thru the list of dicts with raceids and vote totals to extract and add the relevant key/value pairs for item in calculated_total_votes_dict_list: calculated_total_votes_dict[ item['raceid']] = item['vote_count_total'] ## loop thru the results for resultmanual_obj in gdoc_results: ## get the raceid resultmanual_obj_race_id = resultmanual_obj.raceid ## get the vote total vote_total = calculated_total_votes_dict[ resultmanual_obj_race_id] ## get the vote count vote_count = resultmanual_obj.votecount ## calculate the percent percent_calculated = calculated_percent(vote_count, vote_total) ## get the object id resultmanual_obj_id = resultmanual_obj.id # print str(resultmanual_obj) + ": \t" + str(percent_calculated) + "\n" ## update the object ResultManual.objects.filter(id=resultmanual_obj_id).update( votepct=percent_calculated) end_time = timezone.localtime(timezone.now()) end_message = '\nFinished calculations:\t' + str( unicode(end_time)) + '\n' import_length = str(unicode(end_time - start_time)) message = end_message slackbot(message) message = 'Calculation length:\t\t' + import_length + '\n' slackbot(message)
def load_data_elections_all(): elections = Elections.get_elections(Elections(), datafile=None) counter = 0 counter_not = 0 message = "Importing..." slackbot(message) for election_item in elections: election_id = election_item.electiondate.replace("-", "") election_mapping = { 'dataentry': 'automated', 'datasource': 'Associated Press', 'electiondate': election_item.electiondate, 'id': election_id, 'liveresults': election_item.liveresults, 'title': 'Automated election', ## e.g. extract state names and populate a /-separated string or something; or connect w/ FK like manual races 'testresults': election_item.testresults, } obj, created = Election.objects.update_or_create(**election_mapping) # print obj.electiondate counter += 1 message = "\nLoaded:\t\t\t" + str(counter) + " elections" slackbot(message) message = "Not loaded:\t\t" + str(counter_not) + " elections" slackbot(message)
def delete_filtered(self, model_arg, electiondate_arg): if model_arg == "race": objects_filtered = Race.objects.filter( electiondate=electiondate_arg).exclude(dataentry="manual") elif model_arg == "result": objects_filtered = Result.objects.filter( electiondate=electiondate_arg).exclude(dataentry="manual") elif model_arg == "candidate": objects_filtered = Candidate.objects.filter( electiondate=electiondate_arg).exclude(dataentry="manual") # elif model_arg == "election": # today_date = str(timezone.localtime(timezone.now()).date()) # today = int(today_date.replace("-", "")) # objects_filtered = Election.objects.filter(id__gte=today).exclude(dataentry="manual") ## UPDATE: make sure deletes any election on today so that one can be loaded anew elif model_arg == "resultmanual": ## this will only work if nothing is being entered into the admin for that electiondate; otherwise, it deletes everything and that would be a major problem ## does adding the gdoc_import=True solive that? objects_filtered = ResultManual.objects.filter( electiondate=electiondate_arg, gdoc_import=True) elif model_arg == "resultcsv2race": objects_filtered = Race.objects.filter(electiondate=electiondate_arg, dataentry="manual", gdoc_import=True) elif model_arg == "resultcsv2check": objects_filtered = ResultCheck.objects.filter( electiondate=electiondate_arg, gdoc_import=True) # elif model_arg == "resultcsv2result": # objects_filtered = Result.objects.filter(electiondate=electiondate_arg, dataentry="manual") # gdoc_import=True, ## using that would require adding that field to Result and removing from ResultManual, then migrating if model_arg != "election": objects_filtered_count = objects_filtered.count() message = "About to delete objects..." slackbot(message) objects_filtered.delete() objects_deleted_message = "Deleted:\t\t%s %s objects" % ( objects_filtered_count, model_arg) message = objects_deleted_message + "\n" slackbot(message)
def handle(self, *args, **options): electiondate_arg = options['electiondate'] model_arg = options['model'] test_arg = options['test'] override_arg = options['override'] if test_arg: test = '(test)' message = '\n------- JSON -> S3 for %s %s -------' % (model_arg, test) slackbot(message) print '\nElection date:\t\t%s' % (electiondate_arg) print 'Model:\t\t\t%s' % (model_arg) print 'Test?\t\t\t%s\n' % (test_arg) start_time = timezone.localtime(timezone.now()) start_message = '\nStarted baking:\t\t' + str( unicode(start_time)) + '\n' message = start_message slackbot(message) ## *** FUNCTION CALL *** json_to_s3(electiondate_arg, model_arg, test_arg, override_arg) end_time = timezone.localtime(timezone.now()) end_message = '\nFinished baking:\t' + str(unicode(end_time)) + '\n\n' bake_length = str(unicode(end_time - start_time)) message = end_message slackbot(message) message = 'Bake length:\t\t' + bake_length slackbot(message) message = '--------------------------------------------------\n' slackbot(message)
def update_vote_percent(electiondate_arg): ## all ResultManual objects that aren't zero (bc we can't divide with 0) or from a gdoc resultmanual_objects = ResultManual.objects.filter( electiondate=electiondate_arg, votecount__gt=0).exclude(gdoc_import=True) ## all total vote count by race calculated_total_votes_dict_list = resultmanual_objects.values( 'race_name_fk').annotate(vote_count_total=Sum('votecount')) start_time = timezone.localtime(timezone.now()) start_message = '\nStarted calculations:\t' + str(unicode(start_time)) message = start_message slackbot(message) ## if there are any ResultManual objects that aren't from a gdoc import if resultmanual_objects: ## set up a new empty dict calculated_total_votes_dict = {} ## loop thru the list of dicts with raceids and vote totals to extract and add the relevant key/value pairs for item in calculated_total_votes_dict_list: calculated_total_votes_dict[ item['race_name_fk']] = item['vote_count_total'] ## loop thru the results for resultmanual_obj in resultmanual_objects: ## get the race FK id resultmanual_obj_raecid = resultmanual_obj.race_name_fk.id ## get the vote total vote_total = calculated_total_votes_dict[resultmanual_obj_raecid] ## get the vote count vote_count = resultmanual_obj.votecount ## calculate the percent percent_calculated = calculated_percent(vote_count, vote_total) ## get the object id resultmanual_obj_id = resultmanual_obj.id ## update the object ResultManual.objects.filter(id=resultmanual_obj_id).update( votepct=percent_calculated) end_time = timezone.localtime(timezone.now()) end_message = '\nFinished calculations:\t' + str(unicode(end_time)) + '\n' import_length = str(unicode(end_time - start_time)) message = end_message slackbot(message) message = 'Import length:\t\t' + import_length + '\n' slackbot(message)
def handle(self, *args, **options): id_arg = int(options['id'][0]) message = "Embed ID:\t\t" + str(id_arg) slackbot(message) ## use queryset to filter object, check there's only 1 and save embeds = Embed.objects.all() embeds_filtered = embeds.filter(id=id_arg) if embeds_filtered.count() == 1: embed_object = embeds_filtered[0] embed_object.save() message = "Saved:\t\t%s\n\n" % (str(embed_object)) slackbot(message) else: message = "More than one object was found. Please fix and re-try." slackbot(message)
def loadresults_checkcount(date="", all=""): all_arg = str(all) master_loop = True while master_loop: ## current date/time whenever referenced now = timezone.localtime(timezone.now()) ## gives actual NY date/time ## today's date today = now.date() ## if the optional date arg is included, override the default if date: today_string = date else: today_string = str(today) ## election_obj function pulls the zero-indexed value election = election_obj(today_string) ## is there an election? if election: election_today = True else: election_today = False ## all election tests for today election_tests = Election.objects.filter(test=True, testdate=today_string) ## if one or more test elections exist if election_tests: ## how many? election_tests_count = election_tests.count() election_test_today = True else: election_test_today = False ## if there's an election or test today... if election_today or election_test_today: if election_today: if election.dataentry == "manual" and not election.url: message = "\nOh, hello! There's a manual election today: " slackbot(message) message = "- " + str(election) slackbot(message) ## NEED TO ACCOUNT FOR when there's both an automated + manual/admin, automated + manual/gdoc elections # elif election.dataentry == "manual" and election.url: # message = "\nOh, hello! There's a manual gdoc election today: " # slackbot(message) # message = "- " + str(election) # slackbot(message) elif election.dataentry == "automated": message = "\nOh, hello! There's an automated election today: " slackbot(message) message = "- " + str(election) slackbot(message) ## NEED TO ACCOUNT FOR gdoc election tests elif election_tests_count == 1: election = election_tests[0] message = "\nOh, hello! There's a test election today: " slackbot(message) message = "- " + str(election) slackbot(message) elif election_tests_count > 1: message = "\nOh, hello! There are multiple test elections today: " slackbot(message) for election in election_tests: message = "- " + str(election) slackbot(message) resultcheck_eval = True counter_eval = 0 while resultcheck_eval: ## non-test election if election_today: # ## check vote count and if it returns False, then... # if not check_vote_count(election, counter_eval, election_test_today): # resultcheck_eval = False # resultstage_import = True ## uses this form to match the correctly starttime value now = timezone.now() election_starttime = election.starttime ## if there's no starttime or the starttime is less than or equal to now if not election_starttime or election_starttime <= now: message = "\nChecking " + str(election) + "..." slackbot(message) ## check vote count and if it returns False, then... if not check_vote_count(election, counter_eval, election_test_today, all=all_arg): resultcheck_eval = False resultstage_import = True else: message = "\nToday's election has a start time, so we won't check until then:\n%s\n%s GMT\n" % ( election, election_starttime) slackbot(message) ## test election(s) elif election_test_today: from itertools import chain ## determine and set scope of what to loop thru; i.e. remaining election tests endtime_null = election_tests.filter( endtime__isnull=True ) # starttime__isnull=True can't go here bc it might filter out something that started, but never finished and had to be restarted # print "\nNo endtime:\t\t\t" + str(endtime_null) endtime_gt_now = election_tests.filter(endtime__gt=now) # print "\nEndtime_gt_now:\t\t\t" + str(endtime_gt_now) remaining_election_tests = list( chain(endtime_null, endtime_gt_now)) # print "\nremaining_election_tests:\t" + str(remaining_election_tests) election_tests_to_check = len(remaining_election_tests) message = "\nHow many remaining test elections today?\n" + str( election_tests_to_check ) + "\n" #+ str(remaining_election_tests) slackbot(message) # if there's more than one test election on this day if election_tests_to_check > 0: if election_tests_to_check > 1: message = "\nBecause there are multiple election tests, we need to check each of them individually." slackbot(message) ## if any exist, loop thru remaining election tests on this day for election in remaining_election_tests: ## uses this form to match the correctly starttime value now = timezone.now() election_starttime = election.starttime ## if there's no starttime or the starttime is less than or equal to now if not election_starttime or election_starttime <= now: message = "\nChecking " + str(election) + "..." slackbot(message) ## check vote count and see what it returns if not check_vote_count(election, counter_eval, election_test_today, all=all_arg): resultcheck_eval = False resultstage_import = True break else: message = "\nThe following election has a start time, so we won't check until then:\n%s\n%s GMT\n" % ( election, election_starttime) slackbot(message) # ## after looping through the election tests, sleep and keep checking # if resultcheck_eval: # sleep_eval = 60 # message = "No votes yet. The vote check loop will sleep for %s seconds before checking again.\n\n-------------------------------------------------\n" % (sleep_eval) # slackbot(message) # time.sleep(sleep_eval) ## if there are no more election tests elif not election_tests_to_check: message = "\n @channel: Well, that's a wrap. No more election tests for today!\n" slackbot(message) resultcheck_eval = False resultstage_import = False master_loop = False ## after looking at the election/election test(s), sleep and keep checking if resultcheck_eval: sleep_eval = 60 message = "No votes yet. The vote check loop will sleep for %s seconds before checking again.\n\n-------------------------------------------------\n" % ( sleep_eval) slackbot(message) time.sleep(sleep_eval) ## everything here was moved to the check_vote_count function ## resultstage_import loop can only run on a single election (not all elections) specific to a day counter_import = 0 ## while resultstage_import var is True... while resultstage_import: ## if conditions for continuing to to import to resultstage are False, then stop this loop if not import_resultstage(election, election_test_today, counter_import, now, today_string, all=all_arg): resultstage_import = False ## or else keep it going else: resultstage_import = True counter_import += 1 message = "\nNumber of imports:\t" + str(counter_import) slackbot(message) ## !! everything here was moved to the import_resultstage function ## end of import while loop else: message = "\nNOTICE: There's no election today!\n" slackbot(message) master_loop = False
def import_resultstage(election_object, election_test_today, counter_import, now, today_string, all=""): all_arg = str(all) message = "\n--------------- IMPORT RESULTS STAGE ---------------\n %s" % ( election_object) slackbot(message) election = election_object electiondate_string = str(election.electiondate) if election_test_today: if all_arg: call_command('import_ap_elex', 'resultstage', electiondate_string, "-t", "-a") else: call_command('import_ap_elex', 'resultstage', electiondate_string, "-t") else: if all_arg: call_command('import_ap_elex', 'resultstage', electiondate_string, "-a") else: call_command('import_ap_elex', 'resultstage', electiondate_string) ## snapshot if mccelectionsenv == "local": snapshot_local(electiondate_string) # else: ## snapshot to S3 ## this would likely be where we'd bake live results, but that process needs to happen at the end of import_ap_elex mgmt command due to possible later updates run independent of this election_auto script ## if this loop is the first import, set the resultcount var on the related Election object if counter_import == 0: ## set resultcheck, get the count and print it resultstage_count_int = resultstage_count(resultstage_objects()) election.resultcount = resultstage_count_int election.save() count = str(resultstage_count_int) message = "\nSet result count field on Election model using ResultStage: %s" % ( count) slackbot(message) ## set the election to live and set that as the start time election.live = True election.starttime = timezone.localtime(timezone.now()) election.save() message = "Set election to live and set start time\n" slackbot(message) resultcount_changed = check_resultstage_count_change( electiondate_string, resultstage_count(resultstage_objects()), election) ## for the timer hours = 12 ## ABSTRACT??? if election_test_today: expiration_length = 60 ## ABSTRACT??? else: expiration_length = 60 * hours ## QUESTION: do we want to do this? reduce import frequency if it's been more than XX hours? if counter_import == expiration_length: timer_expired = True ## set sleep time at 5 mins -- extended import time sleep_import = 300 ## ABSTRACT??? else: timer_expired = False ## set sleep time for 60 seconds # sleep_import = 60 sleep_import = RESULTSTAGE_IMPORT_SLEEP_TIME ## returns a dict of the aggregate key and value precinctsreportingpct_check = resultstage_objects().aggregate( Avg("precinctsreportingpct")) ## pulls the value out of the dict precinctsreportingpct_avg = precinctsreportingpct_check.values()[0] * 100 ## for informational, not logic/check, purposes: precinctsreportingpct_avg_state = resultstage_objects().filter( level="state").aggregate( Avg("precinctsreportingpct")).values()[0] * 100 precinctsreportingpct_avg_state_pres = resultstage_objects().filter( level="state", officename="President").aggregate( Avg("precinctsreportingpct")).values()[0] * 100 if precinctsreportingpct_avg and precinctsreportingpct_avg_state and precinctsreportingpct_avg_state_pres: message = "PERCENT AVERAGES\nPercent avg total:\t\t\t%d \nPercent avg state-level:\t\t%d \nPercent avg state-level for prez:\t%d \n" % ( int(precinctsreportingpct_avg), int(precinctsreportingpct_avg_state), int(precinctsreportingpct_avg_state_pres)) slackbot(message) elif precinctsreportingpct_avg_state and precinctsreportingpct_avg: message = "PERCENT AVERAGES\nPercent avg total:\t\t\t%d \nPercent avg state-level:\t\t%d" % ( int(precinctsreportingpct_avg), int(precinctsreportingpct_avg_state)) slackbot(message) ## if it's a test, set the % check as president avg if election_test_today: percent_check = precinctsreportingpct_avg_state_pres ## or else set it as overall % avg else: percent_check = precinctsreportingpct_avg ## if the following are true... ## there are fewer result objects now than there were initially ## the timer is expired ## the precincts reporting % avg is 100% if resultcount_changed or timer_expired or percent_check == 100: message = "\n----------------- IMPORT LOOP TERMINATION ------------------\n" slackbot(message) if resultcount_changed: message = "Why is it ending?\tResult count changed" slackbot(message) elif timer_expired: message = "Why is it ending?\tTimer expired" slackbot(message) elif precinctsreportingpct_avg: message = "Why is it ending?\tAll precincts reporting" slackbot(message) ## set the election "live" field as false and the end time election.live = False election.endtime = timezone.localtime(timezone.now()) election.save() message = "\nSetting election live field to False and setting the end time.\n" slackbot(message) sleep_import = 0 if not election_test_today: last_line = "- the script to sleep long enough to allow for the cache to clear\n" else: last_line = "" message = "\nSet the following...\n - the load loop to stop \n - the \"live\" field for " + electiondate_string + " automated Election to False \n%s - the loop sleep time var to 0 to exit the loop immediately\n\nLoading the archive..." % ( last_line) slackbot(message) ## load archive result data if election_test_today: if all_arg: call_command('import_ap_elex', 'result', electiondate_string, "-t", "-a") else: call_command('import_ap_elex', 'result', electiondate_string, "-t") else: if all_arg: call_command('import_ap_elex', 'result', electiondate_string, "-a") else: call_command('import_ap_elex', 'result', electiondate_string) message = "\n@channel: The import has completed for the election on %s\n" % ( electiondate_string) slackbot(message) ## if it's not a test, sleep the script so the cache has time to reset with the archive data before clearing resultstage # if not election_test_today: # number_of_minutes = 15 # archive_cache_time = 60 * number_of_minutes # time.sleep(archive_cache_time) # message = "Sleeping for:\t%s minutes" % (number_of_minutes) ## delete ResultStage data clear_resultstage_resultcheck(resultstage_objects(), resultcheck_objects()) if election_test_today: ## all election tests for today election_tests = Election.objects.filter(test=True, testdate=today_string) else: election_tests = 0 resultstage_import = False ## this restarts the resultcheck loop if there's more than one test if election_tests > 1: message = "\nOh, hello! There's another test election today: " slackbot(message) message = election_test_today slackbot(message) message = "Restarting the resultcheck loop..." slackbot(message) resultstage_import = False ## this needs to be False bc we need to exit this loop and restart the check loop # resultcheck_eval = True ## this stops the import loop elif election_tests == 1 or election_tests == 0: resultstage_import = False sys.exit() else: resultstage_import = True if sleep_import > 0: message = "Sleeping the import loop for %s seconds" % (sleep_import) slackbot(message) time.sleep(sleep_import) return resultstage_import