def handle(self, *args, **options):
sources = ResultSource.objects.filter(ready_to_build=True)
if sources:
retrieve = Retriever()
retrieve._build_and_move_results()
for src in sources:
logger.debug("Resetting %s to False in advance of next build" % (src.source_name))
src.ready_to_build = False
src.save(update_fields=["ready_to_build"])
else:
logger.debug("None of the results sources are ready to build just yet")
self.stdout.write("\nTask finished at %s\n" % str(datetime.datetime.now()))
def handle(self, *args, **options):
sources = ResultSource.objects.filter(ready_to_build=True)
if sources:
retrieve = Retriever()
retrieve._build_and_move_results()
for src in sources:
logger.debug("Resetting %s to False in advance of next build" %
(src.source_name))
src.ready_to_build = False
src.save(update_fields=["ready_to_build"])
else:
logger.debug(
"None of the results sources are ready to build just yet")
self.stdout.write("\nTask finished at %s\n" %
str(datetime.datetime.now()))
class BuildSbcResults(object):
"""
scaffolding to ingest la county registrar election results
"""
retrieve = Retriever()
data_directory = "%s/ballot_box/data_dump/" % (settings.BASE_DIR)
sources = ResultSource.objects.filter(source_short="sbc",
source_active=True)
elections = Election.objects.all().order_by("-election_date")
testing = elections[0].test_results
def _init(self, *args, **kwargs):
"""
"""
for src in self.sources:
self.get_results_file(src, self.data_directory)
self.parse_results_file(src, self.data_directory)
# self.retrieve._build_and_move_results()
def get_results_file(self, src, data_directory):
"""
"""
self.retrieve._request_results_and_save(src, data_directory)
self.retrieve._create_directory_for_latest_file(src, data_directory)
self.retrieve._copy_timestamped_file_to_latest(src, data_directory)
self.retrieve._archive_downloaded_file(src, data_directory)
self.retrieve._found_required_files(src, data_directory)
self.retrieve._unzip_latest_file(src, data_directory)
self.retrieve.log_message += "*** Ending Request ***\n"
logger.info(self.retrieve.log_message)
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
process = SbcProcessMethods()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(
electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
rows = ParseTabDelimited(latest_path).dictionaries
race_ids = process.get_race_ids_from(rows)
election_package = process.collate_and_fetch_records_for_race(
race_ids, rows)
races = election_package[0]
election_stats = election_package[1]["stats"]
file_timestring = None
file_timestamp = None
logger.info("\n*****\nUpdating...\n*****")
saver._update_result_timestamps(src, datetime.datetime.now())
election_info = process.compile_election_stats(election_stats)
# print election_info
for r in races:
contest_package = process.compile_contest_results(races[r])
# print contest_package
process.update_database(contest_package, election, src)
os.remove(latest_path)
logger.info("we've finished processing sbc results")
class BuildOcResults(object):
"""
scaffolding to ingest secretary of state election results
"""
retrieve = Retriever()
data_directory = "%s/ballot_box/data_dump/" % (settings.BASE_DIR)
sources = ResultSource.objects.filter(source_short="oc", source_active=True)
elections = Election.objects.all().order_by("-election_date")
testing = elections[0].test_results
def _init(self, *args, **kwargs):
"""
"""
for src in self.sources:
self.get_results_file(src, self.data_directory)
self.parse_results_file(src, self.data_directory)
src.ready_to_build = True
src.save(update_fields=["ready_to_build"])
def get_results_file(self, src, data_directory):
"""
"""
self.retrieve._request_results_and_save(src, data_directory)
self.retrieve._create_directory_for_latest_file(src, data_directory)
self.retrieve._copy_timestamped_file_to_latest(src, data_directory)
self.retrieve._archive_downloaded_file(src, data_directory)
self.retrieve._found_required_files(src, data_directory)
self.retrieve._unzip_latest_file(src, data_directory)
self.retrieve.log_message += "*** Ending Request ***\n"
logger.info(self.retrieve.log_message)
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
compiler = BuildResults()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
soup = BeautifulSoup(open(latest_path), "xml")
file_timestring = unicode(soup.CumulativeRptHeader.ReportHeader.attrs["run_date"])
file_timestamp = parse(file_timestring, dayfirst=False).datetime
if self.testing == True:
update_this = self.testing
else:
update_this = saver._eval_timestamps(file_timestamp, src.source_latest)
if update_this == False:
logger.info("\n*****\nwe have newer data in the database so let's delete these files\n*****")
os.remove(latest_path)
else:
logger.info("\n*****\nwe have new data to save and we'll update timestamps in the database\n*****")
saver._update_result_timestamps(src, file_timestamp)
races = soup.find_all("ContestTotal")
race_log = "\n"
for race in races:
if race.attrs["contest_type"] == "MS":
"""
this is a proposition
"""
result = compiler._compile_measure(race, election, src)
race_log += saver.make_office(result.office)
race_log += saver.make_contest(result.office, result.contest)
race_log += saver.make_measure(result.contest, result.measure)
# elif race.attrs["contest_type"] == "??":
# """
# this is a judicial race
# """
# pass
else:
"""
this is a non-judicial candidate
"""
result = compiler._compile_candidate(race, election, src)
race_log += saver.make_office(result.office)
race_log += saver.make_contest(result.office, result.contest)
for candidate in result.candidates:
race_log += saver.make_candidate(result.contest, candidate)
#logger.info(race_log)
os.remove(latest_path)
logger.info("\n*****\nwe've finished processing orange county results\n*****")
else:
logger.error("XML file to parse is not at expected location")
class BuildLacResults(object):
"""
scaffolding to ingest la county registrar election results
"""
retrieve = Retriever()
data_directory = "%s/ballot_box/data_dump/" % (settings.BASE_DIR)
sources = ResultSource.objects.filter(source_short="lac",
source_active=True)
elections = Election.objects.all().order_by("-election_date")
testing = elections[0].test_results
def _init(self, *args, **kwargs):
"""
"""
for src in self.sources:
self.get_results_file(src, self.data_directory)
self.parse_results_file(src, self.data_directory)
src.ready_to_build = True
src.save(update_fields=["ready_to_build"])
def get_results_file(self, src, data_directory):
"""
"""
self.retrieve._request_results_and_save(src, data_directory)
self.retrieve._create_directory_for_latest_file(src, data_directory)
self.retrieve._copy_timestamped_file_to_latest(src, data_directory)
self.retrieve._archive_downloaded_file(src, data_directory)
self.retrieve._found_required_files(src, data_directory)
self.retrieve._unzip_latest_file(src, data_directory)
self.retrieve.log_message += "*** Ending Request ***\n"
logger.info(self.retrieve.log_message)
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
process = LacProcessMethods()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(
electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
rows = process.open_results_file(latest_path)
race_ids = process.get_race_ids_from(rows)
election_package = process.collate_and_fetch_records_for_race(
race_ids, rows)
races = election_package[0]
election_title = election_package[1]["title"]
election_stats = election_package[1]["stats"]
file_timestring = None
file_timestamp = None
pacific = pytz.timezone("US/Pacific")
for t in election_title:
if t[3:5] == "TD":
parser = TD_parser()
parsed = parser.parse_line(t)
timestring = parsed["date"] + " " + parsed["time"]
file_timestring = timestring
if file_timestring:
file_timestamp = parse(file_timestring,
dayfirst=False).datetime
file_timestamp = file_timestamp.replace(tzinfo=pacific)
if self.testing == True:
update_this = self.testing
else:
update_this = saver._eval_timestamps(
file_timestamp, src.source_latest)
if update_this == False:
logger.info(
"\n*****\nwe have newer data in the database so let's delete these files\n*****"
)
os.remove(latest_path)
else:
logger.info(
"\n*****\nwe have new data to save and we'll update timestamps in the database\n*****"
)
saver._update_result_timestamps(src, file_timestamp)
title = process.dictify_records_and_return(
election_title)
stats = process.dictify_records_and_return(
election_stats)
election_info = process.compile_election_stats(
title, stats)
for r in races:
records = process.dictify_records_and_return(
races[r])
"""
checks to see if this is a recall contest or a nonpartisan contest
for now, it's unclear how to store or display these contests
in future, however, we may want to parse and return their results
"""
skip = process.check_if_recall_or_nonpartisan(
records)
if skip:
pass
else:
contest_package = process.compile_contest_results(
records)
process.update_database(
contest_package, election, src)
logger.info("we've finished processing lac results")
else:
logger.error(
"unable to determine whether this data is newer than what we already have."
)