def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
process = SbcProcessMethods()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
rows = ParseTabDelimited(latest_path).dictionaries
race_ids = process.get_race_ids_from(rows)
election_package = process.collate_and_fetch_records_for_race(race_ids, rows)
races = election_package[0]
election_stats = election_package[1]["stats"]
file_timestring = None
file_timestamp = None
logger.info("\n*****\nUpdating...\n*****")
saver._update_result_timestamps(src, datetime.datetime.now())
election_info = process.compile_election_stats(election_stats)
# print election_info
for r in races:
contest_package = process.compile_contest_results(races[r])
# print contest_package
process.update_database(contest_package, election, src)
os.remove(latest_path)
logger.info("we've finished processing sbc results")
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
process = SbcProcessMethods()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(
electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
rows = ParseTabDelimited(latest_path).dictionaries
race_ids = process.get_race_ids_from(rows)
election_package = process.collate_and_fetch_records_for_race(
race_ids, rows)
races = election_package[0]
election_stats = election_package[1]["stats"]
file_timestring = None
file_timestamp = None
logger.info("\n*****\nUpdating...\n*****")
saver._update_result_timestamps(src, datetime.datetime.now())
election_info = process.compile_election_stats(election_stats)
# print election_info
for r in races:
contest_package = process.compile_contest_results(races[r])
# print contest_package
process.update_database(contest_package, election, src)
os.remove(latest_path)
logger.info("we've finished processing sbc results")
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
process = LacProcessMethods()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
rows = process.open_results_file(latest_path)
race_ids = process.get_race_ids_from(rows)
election_package = process.collate_and_fetch_records_for_race(race_ids, rows)
races = election_package[0]
election_title = election_package[1]["title"]
election_stats = election_package[1]["stats"]
file_timestring = None
file_timestamp = None
pacific = pytz.timezone("US/Pacific")
for t in election_title:
if t[3:5] == "TD":
parser = TD_parser()
parsed = parser.parse_line(t)
timestring = parsed["date"] + " " + parsed["time"]
file_timestring = timestring
if file_timestring:
file_timestamp = parse(file_timestring, dayfirst=False).datetime
file_timestamp = file_timestamp.replace(tzinfo=pacific)
if self.testing == True:
update_this = self.testing
else:
update_this = saver._eval_timestamps(file_timestamp, src.source_latest)
if update_this == False:
logger.info("\n*****\nwe have newer data in the database so let's delete these files\n*****")
os.remove(latest_path)
else:
logger.info("\n*****\nwe have new data to save and we'll update timestamps in the database\n*****")
saver._update_result_timestamps(src, file_timestamp)
title = process.dictify_records_and_return(election_title)
stats = process.dictify_records_and_return(election_stats)
election_info = process.compile_election_stats(title, stats)
for r in races:
records = process.dictify_records_and_return(races[r])
"""
checks to see if this is a recall contest or a nonpartisan contest
for now, it's unclear how to store or display these contests
in future, however, we may want to parse and return their results
"""
skip = process.check_if_recall_or_nonpartisan(records)
if skip:
pass
else:
contest_package = process.compile_contest_results(records)
process.update_database(contest_package, election, src)
logger.info("we've finished processing lac results")
else:
logger.error("unable to determine whether this data is newer than what we already have.")
def update_database(self, contest_package, election, src):
""" import candidates, measures, office and contest info from compiled data """
saver = Saver()
framer = Framer()
fixer = Namefixer()
county_name = "San Bernardino County"
contest = contest_package['contest_details']
candidates = contest_package['candidates']
measures = contest_package['measures']
judges = contest_package['judges']
race_log = "\n"
if framer._to_num(contest['CONTEST_TOTAL'])["convert"] == True:
total_contest_votes = framer._to_num(contest['CONTEST_TOTAL'])["value"]
else:
total_contest_votes = None
raise Exception("total_contest_votes is not a number")
# Check level of contest (i.e. local, statewide)
if "United States" in contest['CONTEST_FULL_NAME'] or "STATE" in contest['CONTEST_FULL_NAME']:
level = "california"
framer.contest["is_statewide"] = True
else:
level = "county"
framer.contest["is_statewide"] = False
framer.contest["level"] = level
""" Need to deal with judicial contests in future elections.. None in 2016 general
if contest['is_judicial_contest']:
# This is a judicial appointee
if "SUPREME COURT" in contest["contest_title"]:
contestname = "Supreme Court"
elif "APPELLATE COURT" in contest["contest_title"]:
contestname = "Courts of Appeal District 02"
else:
contestname = contest["contest_title"]
officename = contestname
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_statewide"] = True
framer.contest["level"] = "california"
# need to determine appropriate level
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = True
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest['total_precincts'])["convert"] == True:
pt = framer._to_num(contest['total_precincts'])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest['precincts_reporting'])["convert"] == True:
pr = framer._to_num(contest['precincts_reporting'])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)
if framer._to_num(contest['registration'])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest['registration'])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = fixer._fix(contestname) # framer.office["officename"]
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
# election.electionid,
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for judge in judges:
fullname = judge["judicial_name"].title()
framer.judicial["firstname"] = None
framer.judicial["lastname"] = None
framer.judicial["ballotorder"] = None
framer.judicial["fullname"] = fullname
framer.judicial["judicialslug"] = slugify(fullname)
framer.judicial["description"] = judge['judicial_text'].title()
framer.judicial["poss_error"] = False
if framer._to_num(judge['yes_votes'])["convert"] == True:
yescount = framer._to_num(judge['yes_votes'])["value"]
framer.judicial["yescount"] = yescount
else:
framer.judicial["yescount"] = None
raise Exception("yescount is not a number")
if framer._to_num(judge['yes_percent'])["convert"] == True:
yespct = framer._to_num(judge['yes_percent'])["value"]
framer.judicial["yespct"] = yespct
else:
framer.judicial["yespct"] = None
raise Exception("yespct is not a number")
if framer._to_num(judge['no_votes'])["convert"] == True:
nocount = framer._to_num(judge['no_votes'])["value"]
framer.judicial["nocount"] = nocount
else:
framer.judicial["nocount"] = None
raise Exception("nocount is not a number")
if framer._to_num(judge['no_percent'])["convert"] == True:
nopct = framer._to_num(judge['no_percent'])["value"]
framer.judicial["nopct"] = nopct
else:
framer.judicial["nopct"] = None
raise Exception("nopct is not a number")
framer.judicial["judgeid"] = saver._make_this_id(
"judicial",
framer.contest["contestid"],
framer.judicial["judicialslug"],
)
race_log += saver.make_judicial(framer.contest, framer.judicial)
"""
if contest['is_ballot_measure']:
""" this is a ballot measure """
framer.contest["level"] = "county"
framer.contest["is_statewide"] = False
contestname = contest['CONTEST_FULL_NAME']
if ("Proposition" in contestname) or ("Recall" in contestname):
this_type = "Proposition"
contestname = contestname.replace("State ", "")
measurename = contestname
else:
this_type = "Measure"
contestname = fixer._fix(contestname)
split_name = contestname.split(",")
contestname = split_name[0]
measurename = split_name[1]
officename = framer._concat(
# this_type,
contestname,
delimiter="-",
)
description = "%s Measure" % officename
fullname = measurename
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = True
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest['TOTAL_PRECINCTS'])["convert"] == True:
pt = framer._to_num(contest['TOTAL_PRECINCTS'])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest['PROCESSED_DONE'])["convert"] == True:
pr = framer._to_num(contest['PROCESSED_DONE'])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)*100
# if framer._to_num(contest['registration'])["convert"] == True:
# framer.contest["votersregistered"] = framer._to_num(
# contest['registration'])["value"]
# else:
# framer.contest["votersregistered"] = None
# raise Exception(
# "votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = contestname
framer.contest["contestdescription"] = description
framer.contest["contestid"] = saver._make_contest_id(
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for measure in measures:
framer.measure["ballotorder"] = measure["measure_order"]
framer.measure["fullname"] = fullname
framer.measure["measureslug"] = slugify(fullname)
framer.measure["description"] = ""
framer.measure["poss_error"] = False
if framer._to_num(measure['yes_votes'])["convert"] == True:
yescount = framer._to_num(measure['yes_votes'])["value"]
framer.measure["yescount"] = yescount
else:
framer.measure["yescount"] = None
raise Exception("yescount is not a number")
framer.measure["yespct"] = framer._calc_pct(
framer.measure["yescount"],
total_contest_votes
)*100
if framer._to_num(measure['no_votes'])["convert"] == True:
nocount = framer._to_num(measure['no_votes'])["value"]
framer.measure["nocount"] = nocount
else:
framer.measure["nocount"] = None
raise Exception("nocount is not a number")
framer.measure["nopct"] = framer._calc_pct(
framer.measure['nocount'],
total_contest_votes
)*100
framer.measure["measureid"] = saver._make_this_id(
"measure",
framer.contest["contestid"],
framer.measure["measureslug"],
)
race_log += saver.make_measure(framer.contest, framer.measure)
else:
""" this is a candidate for elected office """
# strip_district = contest['district'].lstrip("0")
framer.contest["seatnum"] = "1010101"
contestname = contest['CONTEST_FULL_NAME']
if "Member of the State Assembly" in contestname:
contestname = contestname.replace("Member of the", "")
elif "Supervisor" in contestname:
contestname = "%s %s" % (county_name, contestname)
elif "United States Representative" in contestname:
contestname = contestname.replace("United States Representative,", "US House of Representatives")
elif "State Senator" in contestname:
contestname = contestname.replace("State Senator,", "State Senate")
elif "United States Senator" in contestname:
contestname = "US Senate"
elif "Judge of the Superior Court" in contestname:
contestname = county_name + " " + contestname
framer.office["officename"] = contestname.replace(".", "")
framer.office["officeslug"] = slugify(framer.office["officename"])
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
if len(candidates) < 2:
framer.contest["is_uncontested"] = True
else:
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest['TOTAL_PRECINCTS'])["convert"] == True:
pt = framer._to_num(contest['TOTAL_PRECINCTS'])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest['PROCESSED_DONE'])["convert"] == True:
pr = framer._to_num(contest['PROCESSED_DONE'])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)*100
# if framer._to_num(contest['registration'])["convert"] == True:
# framer.contest["votersregistered"] = framer._to_num(
# contest['registration'])["value"]
# else:
# framer.contest["votersregistered"] = None
# raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = fixer._fix(contestname) # framer.office["officename"]
if int(contest["VOTE_FOR"]) > 1:
framer.contest["contestdescription"] = "Vote for " + contest["VOTE_FOR"]
else:
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
# election.electionid,
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for candidate in candidates:
fullname = candidate['CANDIDATE_FULL_NAME']
if " - " in fullname:
fullname = fullname.split(" - ")[1]
fullname = fullname.decode('iso-8859-1').encode('utf8')
fixer = Namefixer()
fullname = fixer._titlecase_with_accents(fullname)
party_id = candidate['CANDIDATE_PARTY_ID']
if party_id == "2":
party = "Democratic"
elif party_id == "3":
party = "Green"
elif party_id == "4":
party = "Libertarian"
elif party_id == "5":
party = "Peace and Freedom"
elif party_id == "6":
party = "Republican"
elif party_id == "8":
party = "Republican"
else:
party = ""
framer.candidate["ballotorder"] = candidate["CANDIDATE_ORDER"]
framer.candidate["firstname"] = None
framer.candidate["lastname"] = None
framer.candidate["fullname"] = fullname
framer.candidate["candidateslug"] = slugify(fullname)
framer.candidate["party"] = party
framer.candidate["incumbent"] = False
framer.candidate["poss_error"] = False
if framer._to_num(candidate['TOTAL'])["convert"] == True:
framer.candidate["votecount"] = framer._to_num(candidate['TOTAL'])[
"value"]
else:
framer.candidate["votecount"] = None
raise Exception("votecount is not a number")
if total_contest_votes > 0:
framer.candidate["votepct"] = framer._calc_pct(
framer.candidate["votecount"],
total_contest_votes
)*100
else:
framer.candidate["votepct"] = 0.00
logger.debug("No votes have been cast in " + contestname + " or votepct is not a number")
framer.candidate["candidateid"] = saver._make_this_id(
"candidate",
framer.contest["contestid"],
framer.candidate["candidateslug"],
)
race_log += saver.make_candidate(framer.contest, framer.candidate)
def update_database(self, contest_package, election, src):
""" import candidates, measures, office and contest info from compiled data """
saver = Saver()
framer = Framer()
fixer = Namefixer()
county_name = "San Bernardino County"
contest = contest_package['contest_details']
candidates = contest_package['candidates']
measures = contest_package['measures']
judges = contest_package['judges']
race_log = "\n"
if framer._to_num(contest['CONTEST_TOTAL'])["convert"] == True:
total_contest_votes = framer._to_num(
contest['CONTEST_TOTAL'])["value"]
else:
total_contest_votes = None
raise Exception("total_contest_votes is not a number")
# Check level of contest (i.e. local, statewide)
if "United States" in contest[
'CONTEST_FULL_NAME'] or "STATE" in contest['CONTEST_FULL_NAME']:
level = "california"
framer.contest["is_statewide"] = True
else:
level = "county"
framer.contest["is_statewide"] = False
framer.contest["level"] = level
""" Need to deal with judicial contests in future elections.. None in 2016 general
if contest['is_judicial_contest']:
# This is a judicial appointee
if "SUPREME COURT" in contest["contest_title"]:
contestname = "Supreme Court"
elif "APPELLATE COURT" in contest["contest_title"]:
contestname = "Courts of Appeal District 02"
else:
contestname = contest["contest_title"]
officename = contestname
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_statewide"] = True
framer.contest["level"] = "california"
# need to determine appropriate level
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = True
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest['total_precincts'])["convert"] == True:
pt = framer._to_num(contest['total_precincts'])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest['precincts_reporting'])["convert"] == True:
pr = framer._to_num(contest['precincts_reporting'])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)
if framer._to_num(contest['registration'])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest['registration'])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = fixer._fix(contestname) # framer.office["officename"]
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
# election.electionid,
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for judge in judges:
fullname = judge["judicial_name"].title()
framer.judicial["firstname"] = None
framer.judicial["lastname"] = None
framer.judicial["ballotorder"] = None
framer.judicial["fullname"] = fullname
framer.judicial["judicialslug"] = slugify(fullname)
framer.judicial["description"] = judge['judicial_text'].title()
framer.judicial["poss_error"] = False
if framer._to_num(judge['yes_votes'])["convert"] == True:
yescount = framer._to_num(judge['yes_votes'])["value"]
framer.judicial["yescount"] = yescount
else:
framer.judicial["yescount"] = None
raise Exception("yescount is not a number")
if framer._to_num(judge['yes_percent'])["convert"] == True:
yespct = framer._to_num(judge['yes_percent'])["value"]
framer.judicial["yespct"] = yespct
else:
framer.judicial["yespct"] = None
raise Exception("yespct is not a number")
if framer._to_num(judge['no_votes'])["convert"] == True:
nocount = framer._to_num(judge['no_votes'])["value"]
framer.judicial["nocount"] = nocount
else:
framer.judicial["nocount"] = None
raise Exception("nocount is not a number")
if framer._to_num(judge['no_percent'])["convert"] == True:
nopct = framer._to_num(judge['no_percent'])["value"]
framer.judicial["nopct"] = nopct
else:
framer.judicial["nopct"] = None
raise Exception("nopct is not a number")
framer.judicial["judgeid"] = saver._make_this_id(
"judicial",
framer.contest["contestid"],
framer.judicial["judicialslug"],
)
race_log += saver.make_judicial(framer.contest, framer.judicial)
"""
if contest['is_ballot_measure']:
""" this is a ballot measure """
framer.contest["level"] = "county"
framer.contest["is_statewide"] = False
contestname = contest['CONTEST_FULL_NAME']
if ("Proposition" in contestname) or ("Recall" in contestname):
this_type = "Proposition"
contestname = contestname.replace("State ", "")
measurename = contestname
else:
this_type = "Measure"
contestname = fixer._fix(contestname)
split_name = contestname.split(",")
contestname = split_name[0]
measurename = split_name[1]
officename = framer._concat(
# this_type,
contestname,
delimiter="-",
)
description = "%s Measure" % officename
fullname = measurename
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = True
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest['TOTAL_PRECINCTS'])["convert"] == True:
pt = framer._to_num(contest['TOTAL_PRECINCTS'])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest['PROCESSED_DONE'])["convert"] == True:
pr = framer._to_num(contest['PROCESSED_DONE'])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]) * 100
# if framer._to_num(contest['registration'])["convert"] == True:
# framer.contest["votersregistered"] = framer._to_num(
# contest['registration'])["value"]
# else:
# framer.contest["votersregistered"] = None
# raise Exception(
# "votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = contestname
framer.contest["contestdescription"] = description
framer.contest["contestid"] = saver._make_contest_id(
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for measure in measures:
framer.measure["ballotorder"] = measure["measure_order"]
framer.measure["fullname"] = fullname
framer.measure["measureslug"] = slugify(fullname)
framer.measure["description"] = ""
framer.measure["poss_error"] = False
if framer._to_num(measure['yes_votes'])["convert"] == True:
yescount = framer._to_num(measure['yes_votes'])["value"]
framer.measure["yescount"] = yescount
else:
framer.measure["yescount"] = None
raise Exception("yescount is not a number")
framer.measure["yespct"] = framer._calc_pct(
framer.measure["yescount"], total_contest_votes) * 100
if framer._to_num(measure['no_votes'])["convert"] == True:
nocount = framer._to_num(measure['no_votes'])["value"]
framer.measure["nocount"] = nocount
else:
framer.measure["nocount"] = None
raise Exception("nocount is not a number")
framer.measure["nopct"] = framer._calc_pct(
framer.measure['nocount'], total_contest_votes) * 100
framer.measure["measureid"] = saver._make_this_id(
"measure",
framer.contest["contestid"],
framer.measure["measureslug"],
)
race_log += saver.make_measure(framer.contest, framer.measure)
else:
""" this is a candidate for elected office """
# strip_district = contest['district'].lstrip("0")
framer.contest["seatnum"] = "1010101"
contestname = contest['CONTEST_FULL_NAME']
if "Member of the State Assembly" in contestname:
contestname = contestname.replace("Member of the", "")
elif "Supervisor" in contestname:
contestname = "%s %s" % (county_name, contestname)
elif "United States Representative" in contestname:
contestname = contestname.replace(
"United States Representative,",
"US House of Representatives")
elif "State Senator" in contestname:
contestname = contestname.replace("State Senator,",
"State Senate")
elif "United States Senator" in contestname:
contestname = "US Senate"
elif "Judge of the Superior Court" in contestname:
contestname = county_name + " " + contestname
framer.office["officename"] = contestname.replace(".", "")
framer.office["officeslug"] = slugify(framer.office["officename"])
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
if len(candidates) < 2:
framer.contest["is_uncontested"] = True
else:
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest['TOTAL_PRECINCTS'])["convert"] == True:
pt = framer._to_num(contest['TOTAL_PRECINCTS'])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest['PROCESSED_DONE'])["convert"] == True:
pr = framer._to_num(contest['PROCESSED_DONE'])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]) * 100
# if framer._to_num(contest['registration'])["convert"] == True:
# framer.contest["votersregistered"] = framer._to_num(
# contest['registration'])["value"]
# else:
# framer.contest["votersregistered"] = None
# raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = fixer._fix(
contestname) # framer.office["officename"]
if int(contest["VOTE_FOR"]) > 1:
framer.contest[
"contestdescription"] = "Vote for " + contest["VOTE_FOR"]
else:
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
# election.electionid,
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for candidate in candidates:
fullname = candidate['CANDIDATE_FULL_NAME']
if " - " in fullname:
fullname = fullname.split(" - ")[1]
fullname = fullname.decode('iso-8859-1').encode('utf8')
fixer = Namefixer()
fullname = fixer._titlecase_with_accents(fullname)
party_id = candidate['CANDIDATE_PARTY_ID']
if party_id == "2":
party = "Democratic"
elif party_id == "3":
party = "Green"
elif party_id == "4":
party = "Libertarian"
elif party_id == "5":
party = "Peace and Freedom"
elif party_id == "6":
party = "Republican"
elif party_id == "8":
party = "Republican"
else:
party = ""
framer.candidate["ballotorder"] = candidate["CANDIDATE_ORDER"]
framer.candidate["firstname"] = None
framer.candidate["lastname"] = None
framer.candidate["fullname"] = fullname
framer.candidate["candidateslug"] = slugify(fullname)
framer.candidate["party"] = party
framer.candidate["incumbent"] = False
framer.candidate["poss_error"] = False
if framer._to_num(candidate['TOTAL'])["convert"] == True:
framer.candidate["votecount"] = framer._to_num(
candidate['TOTAL'])["value"]
else:
framer.candidate["votecount"] = None
raise Exception("votecount is not a number")
if total_contest_votes > 0:
framer.candidate["votepct"] = framer._calc_pct(
framer.candidate["votecount"],
total_contest_votes) * 100
else:
framer.candidate["votepct"] = 0.00
logger.debug("No votes have been cast in " + contestname +
" or votepct is not a number")
framer.candidate["candidateid"] = saver._make_this_id(
"candidate",
framer.contest["contestid"],
framer.candidate["candidateslug"],
)
race_log += saver.make_candidate(framer.contest,
framer.candidate)
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
compiler = BuildResults()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
soup = BeautifulSoup(open(latest_path), "xml")
file_timestring = unicode(soup.CumulativeRptHeader.ReportHeader.attrs["run_date"])
file_timestamp = parse(file_timestring, dayfirst=False).datetime
if self.testing == True:
update_this = self.testing
else:
update_this = saver._eval_timestamps(file_timestamp, src.source_latest)
if update_this == False:
logger.info("\n*****\nwe have newer data in the database so let's delete these files\n*****")
os.remove(latest_path)
else:
logger.info("\n*****\nwe have new data to save and we'll update timestamps in the database\n*****")
saver._update_result_timestamps(src, file_timestamp)
races = soup.find_all("ContestTotal")
race_log = "\n"
for race in races:
if race.attrs["contest_type"] == "MS":
"""
this is a proposition
"""
result = compiler._compile_measure(race, election, src)
race_log += saver.make_office(result.office)
race_log += saver.make_contest(result.office, result.contest)
race_log += saver.make_measure(result.contest, result.measure)
# elif race.attrs["contest_type"] == "??":
# """
# this is a judicial race
# """
# pass
else:
"""
this is a non-judicial candidate
"""
result = compiler._compile_candidate(race, election, src)
race_log += saver.make_office(result.office)
race_log += saver.make_contest(result.office, result.contest)
for candidate in result.candidates:
race_log += saver.make_candidate(result.contest, candidate)
#logger.info(race_log)
os.remove(latest_path)
logger.info("\n*****\nwe've finished processing orange county results\n*****")
else:
logger.error("XML file to parse is not at expected location")
def update_database(self, contest_package, election, src):
""" import candidates, measures, office and contest info from compiled data """
saver = Saver()
framer = Framer()
fixer = Namefixer()
county_name = "Los Angeles County"
contest = contest_package["contest_details"]
candidates = contest_package["candidates"]
measures = contest_package["measures"]
judges = contest_package["judges"]
race_log = "\n"
# """
# Check level of contest (i.e. local, statewide)
# """
if "U.S." in contest["contest_title"] or "STATE" in contest["contest_title"]:
level = "county"
framer.contest["is_statewide"] = True
else:
level = "county"
framer.contest["is_statewide"] = False
framer.contest["level"] = level
if contest["is_judicial_contest"]:
"""
This is a judicial appointee
"""
if "SUPREME COURT" in contest["contest_title"]:
contestname = "Supreme Court"
elif "APPELLATE COURT" in contest["contest_title"]:
contestname = "Courts of Appeal District 02"
else:
contestname = contest["contest_title"]
officename = contestname
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_statewide"] = True
framer.contest["level"] = "county"
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = True
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest["total_precincts"])["convert"] == True:
pt = framer._to_num(contest["total_precincts"])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest["precincts_reporting"])["convert"] == True:
pr = framer._to_num(contest["precincts_reporting"])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)
if framer._to_num(contest["registration"])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest["registration"])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = fixer._fix(contestname)
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for judge in judges:
fullname = fixer._titlecase_with_accents(judge["judicial_name"].encode('utf8'))
framer.judicial["firstname"] = None
framer.judicial["lastname"] = None
framer.judicial["ballotorder"] = None
framer.judicial["fullname"] = fullname
framer.judicial["judicialslug"] = slugify(fullname)
framer.judicial["description"] = judge["judicial_text"].title()
framer.judicial["poss_error"] = False
if framer._to_num(judge["yes_votes"])["convert"] == True:
yescount = framer._to_num(judge["yes_votes"])["value"]
framer.judicial["yescount"] = yescount
else:
framer.judicial["yescount"] = None
raise Exception("yescount is not a number")
if framer._to_num(judge["yes_percent"])["convert"] == True:
yespct = framer._to_num(judge["yes_percent"])["value"]
framer.judicial["yespct"] = yespct
else:
framer.judicial["yespct"] = None
raise Exception("yespct is not a number")
if framer._to_num(judge["no_votes"])["convert"] == True:
nocount = framer._to_num(judge["no_votes"])["value"]
framer.judicial["nocount"] = nocount
else:
framer.judicial["nocount"] = None
raise Exception("nocount is not a number")
if framer._to_num(judge["no_percent"])["convert"] == True:
nopct = framer._to_num(judge["no_percent"])["value"]
framer.judicial["nopct"] = nopct
else:
framer.judicial["nopct"] = None
raise Exception("nopct is not a number")
framer.judicial["judgeid"] = saver._make_this_id(
"judicial",
framer.contest["contestid"],
framer.judicial["judicialslug"],
)
race_log += saver.make_judicial(framer.contest, framer.judicial)
elif contest["is_ballot_measure"]:
"""
this is a ballot measure
"""
framer.contest["level"] = "county"
framer.contest["is_statewide"] = False
contestname = contest["contest_title"]
if contest["contest_id"] == "00":
if contestname == "COUNTY MEASURE A":
this_type = "Measure"
contestname = "%swide" % (county_name)
contest["contest_title_cont"] = "Measure A"
else:
this_type = "Proposition"
contestname = contestname.replace("STATE MEASURE", "Proposition")
else:
this_type = "Measure"
contestname = fixer._fix(contestname)
contestname = contestname.title()
if contestname == "Metro Transportation Authority":
contestname = "%swide" % (county_name)
if contest["contest_title_cont"]:
fullname = (contest["contest_title_cont"]).replace("MEASURE", "Measure")
else:
fullname = contestname
if contest["contest_id"] == "00":
description = None
officename = framer._concat(
# this_type,
contestname,
delimiter="-",
)
else:
description = "%s %s" % (contestname, this_type)
officename = framer._concat(
# this_type,
contestname,
delimiter="-",
)
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = True
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest["total_precincts"])["convert"] == True:
pt = framer._to_num(contest["total_precincts"])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest["precincts_reporting"])["convert"] == True:
pr = framer._to_num(contest["precincts_reporting"])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)
if framer._to_num(contest["registration"])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest["registration"])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception(
"votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = contestname
framer.contest["contestdescription"] = description
framer.contest["contestid"] = saver._make_contest_id(
# election.electionid,
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for measure in measures:
framer.measure["ballotorder"] = None
framer.measure["fullname"] = fullname
framer.measure["measureslug"] = slugify(fullname)
framer.measure["description"] = measure["measure_text"].title()
framer.measure["poss_error"] = False
if framer._to_num(measure["yes_votes"])["convert"] == True:
yescount = framer._to_num(measure["yes_votes"])["value"]
framer.measure["yescount"] = yescount
else:
framer.measure["yescount"] = None
raise Exception("yescount is not a number")
if framer._to_num(measure["yes_percent"])["convert"] == True:
yespct = framer._to_num(measure["yes_percent"])["value"]
framer.measure["yespct"] = yespct
else:
framer.measure["yespct"] = None
raise Exception("yespct is not a number")
if framer._to_num(measure["no_votes"])["convert"] == True:
nocount = framer._to_num(measure["no_votes"])["value"]
framer.measure["nocount"] = nocount
else:
framer.measure["nocount"] = None
raise Exception("nocount is not a number")
if framer._to_num(measure["no_percent"])["convert"] == True:
nopct = framer._to_num(measure["no_percent"])["value"]
framer.measure["nopct"] = nopct
else:
framer.measure["nopct"] = None
raise Exception("nopct is not a number")
if this_type == "Proposition":
framer.measure["measureid"] = saver._make_this_id(
"measure",
"%s-%s" % (src.source_short, framer.contest["level"]),
"%s-%s" % (this_type.lower(), measure["measure_id"].lower())
)
else:
framer.measure["measureid"] = saver._make_this_id(
"measure",
framer.contest["contestid"],
"%s-%s" % (measure["measure_id"].lower(), slugify(framer.measure["description"].lower()))
)
race_log += saver.make_measure(framer.contest, framer.measure)
else:
"""
this is a candidate for elected office
"""
strip_district = contest["district"].lstrip("0")
framer.contest["seatnum"] = "1010101"
if contest["contest_title"] == "MEMBER OF THE ASSEMBLY":
contestname = "State Assembly District %s" % (strip_district)
elif "SUPERVISOR" in contest["contest_title"]:
contestname = "Supervisor District %s" % (strip_district)
contestname = fixer._affix_county(county_name,contestname)
elif "U.S. REPRESENTATIVE" in contest["contest_title"]:
contestname = "U.S. House of Representatives District %s" % (strip_district)
elif "DELEGATES" in contest["contest_title"]:
designation = contest["contest_title_cont"].replace("CONGRESSIONAL DISTRICT-", "")
if designation == "REP":
this_desig = "Republican"
elif designation == "DEM":
this_desig = "Democratic"
else:
this_desig = None
contestname = "%s Presidential Primary Delegates - CD%s" % (this_desig, strip_district)
elif "STATE SENATOR" in contest["contest_title"]:
contestname = "State Senate District %s" % (strip_district)
elif "UNITED STATES SENATOR" in contest["contest_title"]:
contestname = "US Senate"
elif "PRESIDENTIAL PREFERENCE" in contest["contest_title"]:
designation = contest["contest_title_cont"].upper()
if designation == "AI":
this_desig = "American Independent"
elif designation == "REP":
this_desig = "Republican"
elif designation == "DEM":
this_desig = "Democratic"
elif designation == "PF":
this_desig = "Peace And Freedom"
elif designation == "LIB":
this_desig = "Libertarian"
elif designation == "GRN":
this_desig = "Green"
else:
this_desig = None
contestname = "%s Presidential Primary" % (this_desig)
elif "PARTY COUNTY COMMITTEE" in contest["contest_title"]:
contestname = "%s Party County Committee District %s" % (
contest["party_name"].capitalize(), strip_district)
elif "JUDGE-SUPERIOR COURT" in contest["contest_title"]:
contestname = "Judge Superior Court %s" % (contest["contest_title_cont"].title())
contestname = fixer._affix_county(county_name, contestname)
else:
contestname = "%s %s" % (contest["contest_title"].title(), contest["contest_title_cont"].title())
contestname = fixer._fix(contestname)
contestname = contestname.title()
contestname = framer._convert_district_number(contestname)
framer.office["officename"] = contestname.replace(".", "")
framer.office["officeslug"] = slugify(framer.office["officename"])
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
if len(candidates) < 2:
framer.contest["is_uncontested"] = True
else:
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest["total_precincts"])["convert"] == True:
pt = framer._to_num(contest["total_precincts"])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(contest["precincts_reporting"])["convert"] == True:
pr = framer._to_num(contest["precincts_reporting"])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"]
)
if framer._to_num(contest["registration"])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest["registration"])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = contestname
try:
framer.contest["contestdescription"] = contest[
"vote_for_text"].capitalize() + " " + contest["vote_for_number"]
except:
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for candidate in candidates:
fullname = fixer._titlecase_with_accents(candidate["candidate_name"].encode('utf8'))
party = candidate["party_short"]
if party == "REP":
party = "Republican"
elif party == "DEM":
party = "Democrat"
elif party == "AI":
party = "American Independent"
elif party == "PF":
party = "Peace And Freedom"
elif party == "LIB":
party = "Libertarian"
elif party == "GRN":
party = "Green"
elif party == "NP":
party = "No Party Preference"
framer.candidate["ballotorder"] = None
framer.candidate["firstname"] = None
framer.candidate["lastname"] = None
framer.candidate["fullname"] = fullname
framer.candidate["candidateslug"] = slugify(fullname)
framer.candidate["party"] = party
framer.candidate["incumbent"] = False
framer.candidate["poss_error"] = False
if framer._to_num(candidate["votes"])["convert"] == True:
framer.candidate["votecount"] = framer._to_num(candidate["votes"])["value"]
else:
framer.candidate["votecount"] = None
raise Exception("votecount is not a number")
if framer._to_num(candidate["percent_of_vote"])["convert"] == True:
framer.candidate["votepct"] = framer._to_num(candidate["percent_of_vote"])["value"]
else:
framer.candidate["votepct"] = None
raise Exception("votepct is not a number")
framer.candidate["candidateid"] = saver._make_this_id(
"candidate",
framer.contest["contestid"],
framer.candidate["candidateslug"],
)
race_log += saver.make_candidate(framer.contest, framer.candidate)
class BuildResults(object):
"""
"""
saver = Saver()
framer = Framer()
fixer = Namefixer()
state_measures = [
"1656",
"1682",
"1683",
"1684",
"1685",
"1687"
]
state_recall = [
"1655"
]
local_measures = [
"1687",
"1688",
"1689",
"1690",
"1691"
]
# def _compile_judicial(self, race, race_id, election, src):
# """
# """
# r = race.find("TotalVotes")
# this_type = "judicial"
# contestname = unicode(" ".join(race.ContestName.stripped_strings))
# if race_id == "140":
# officename_idx = self.framer._find_nth(contestname, " - ", 1)
# officename = unicode(contestname[:officename_idx].replace(".", ""))
# officename = unicode(officename.replace(" Justice", ""))
# fullname_idx = self.framer._find_nth(contestname, " - ", 1) + 3
# elif race_id == "150":
# officename_idx = self.framer._find_nth(contestname, " - ", 2)
# officename = unicode(contestname[:officename_idx].replace(" - ", " "))
# fullname_idx = self.framer._find_nth(contestname, " - ", 2) + 3
# fullname = unicode(contestname[fullname_idx:])
# level = "county"
# seatnum = None
# is_statewide = True
# precinctstotal = r.find(attrs={"Id": "TP"}).contents[0]
# precinctsreport = r.find(attrs={"Id": "PR"}).contents[0]
# reporttype = r.find(attrs={"Id": "RT"}).contents[0]
# yescount = self.framer._to_num(r.find_all("Selection")[0].ValidVotes.contents[0])["value"]
# yespct = self.framer._to_num(r.find(attrs={"Id": "PYV"}).contents[0])["value"]
# nocount = self.framer._to_num(r.find_all("Selection")[1].ValidVotes.contents[0])["value"]
# nopct = self.framer._to_num(r.find(attrs={"Id": "PNV"}).contents[0])["value"]
# self.framer.office["officename"] = officename
# self.framer.office["officeslug"] = slugify(officename)
# self.framer.office["active"] = True
# self.framer.office["officeid"] = self.framer.office["officeslug"]
# self.framer.contest["election_id"] = election.id
# self.framer.contest["resultsource_id"] = src.id
# self.framer.contest["seatnum"] = seatnum
# self.framer.contest["is_uncontested"] = False
# self.framer.contest["is_national"] = False
# self.framer.contest["is_statewide"] = True
# self.framer.contest["level"] = level
# self.framer.contest["is_ballot_measure"] = False
# self.framer.contest["is_judicial"] = True
# self.framer.contest["is_runoff"] = False
# self.framer.contest["reporttype"] = None
# self.framer.contest["poss_error"] = False
# if self.framer._to_num(precinctstotal)["convert"] == True:
# pt = self.framer._to_num(precinctstotal)["value"]
# self.framer.contest["precinctstotal"] = pt
# else:
# self.framer.contest["precinctstotal"] = None
# raise Exception("precinctstotal is not a number")
# if self.framer._to_num(precinctsreport)["convert"] == True:
# pr = self.framer._to_num(precinctsreport)["value"]
# self.framer.contest["precinctsreporting"] = pr
# else:
# self.framer.contest["precinctsreporting"] = None
# raise Exception("precinctsreporting is not a number")
# self.framer.contest["precinctsreportingpct"] = self.framer._calc_pct(
# self.framer.contest["precinctsreporting"], self.framer.contest["precinctstotal"])
# self.framer.contest["votersregistered"] = self.framer._to_num(None)["value"]
# self.framer.contest["votersturnout"] = self.framer._to_num(None)["value"]
# self.framer.contest["contestname"] = self.framer.office["officename"]
# self.framer.contest["contestdescription"] = None
# if race_id == "140":
# self.framer.contest["contestid"] = self.saver._make_contest_id(
# # election.electionid,
# src.source_short,
# self.framer.contest["level"],
# self.framer.office["officeslug"],
# )
# self.framer.judicial["ballotorder"] = None
# self.framer.judicial["firstname"] = None
# self.framer.judicial["lastname"] = None
# self.framer.judicial["fullname"] = fullname
# self.framer.judicial["judicialslug"] = slugify(fullname)
# self.framer.judicial["yescount"] = yescount
# self.framer.judicial["yespct"] = yespct
# self.framer.judicial["nocount"] = nocount
# self.framer.judicial["nopct"] = nopct
# self.framer.judicial["poss_error"] = False
# self.framer.judicial["judgeid"] = self.saver._make_this_id(
# "judicial",
# self.framer.contest["contestid"],
# self.framer.judicial["judicialslug"],
# )
# elif race_id == "150":
# self.framer.contest["contestid"] = self.saver._make_contest_id(
# # election.electionid,
# src.source_short,
# self.framer.contest["level"],
# self.framer.office["officeslug"],
# )
# self.framer.judicial["ballotorder"] = None
# self.framer.judicial["firstname"] = None
# self.framer.judicial["lastname"] = None
# self.framer.judicial["fullname"] = fullname
# self.framer.judicial["judicialslug"] = slugify(fullname)
# self.framer.judicial["yescount"] = yescount
# self.framer.judicial["yespct"] = yespct
# self.framer.judicial["nocount"] = nocount
# self.framer.judicial["nopct"] = nopct
# self.framer.judicial["poss_error"] = False
# self.framer.judicial["judgeid"] = self.saver._make_this_id(
# "judicial",
# self.framer.contest["contestid"],
# self.framer.judicial["judicialslug"],
# )
# return self.framer
def _compile_measure(self, race, election, src):
"""
"""
r = race
if r.attrs["race_id"] in self.local_measures:
this_type = "Advisory Vote"
elif race.attrs["race_id"] in self.state_measures:
this_type = "Proposition"
elif race.attrs["race_id"] in self.state_recall:
this_type = "Recall"
else:
this_type = "Measure"
fixed = self.fixer._fix_oc_props(this_type, race.attrs["contest_title"])
prop_num = fixed["prop_num"]
measure_fullname = fixed["measure_fullname"]
description = fixed["description"]
this_type = fixed["type"]
if this_type == "Proposition":
officename = self.framer._concat(
this_type,
prop_num,
delimiter=" ",
)
elif this_type == "Measure" or this_type == "Advisory Vote":
officename = self.framer._concat(
fixed["district"],
delimiter=" ",
)
else:
officename = self.framer._concat(
fixed["district"],
delimiter=" ",
)
level = "county"
seatnum = None
precinctstotal = race.attrs["total_precincts"]
precinctsreport = race.attrs["counted_precincts"]
vote_counts = race.find_all("ChoiceTotal")
yescount = self.framer._to_num(vote_counts[0].attrs["c_total_votes"])["value"]
nocount = self.framer._to_num(vote_counts[1].attrs["c_total_votes"])["value"]
total = yescount + nocount
yespct = self.framer._calc_pct(yescount, total)
nopct = self.framer._calc_pct(nocount, total)
self.framer.office["officename"] = officename
self.framer.office["officeslug"] = slugify(officename)
self.framer.office["active"] = True
self.framer.office["officeid"] = self.framer.office["officeslug"]
self.framer.contest["election_id"] = election.id
self.framer.contest["resultsource_id"] = src.id
self.framer.contest["seatnum"] = seatnum
self.framer.contest["is_uncontested"] = False
self.framer.contest["is_national"] = False
self.framer.contest["is_statewide"] = False
self.framer.contest["level"] = level
self.framer.contest["is_ballot_measure"] = True
self.framer.contest["is_judicial"] = False
self.framer.contest["is_runoff"] = False
self.framer.contest["reporttype"] = None
self.framer.contest["poss_error"] = False
if self.framer._to_num(precinctstotal)["convert"] == True:
pt = self.framer._to_num(precinctstotal)["value"]
self.framer.contest["precinctstotal"] = pt
else:
self.framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if self.framer._to_num(precinctsreport)["convert"] == True:
pr = self.framer._to_num(precinctsreport)["value"]
self.framer.contest["precinctsreporting"] = pr
else:
self.framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
self.framer.contest["precinctsreportingpct"] = self.framer._calc_pct(
self.framer.contest["precinctsreporting"],
self.framer.contest["precinctstotal"]
)
self.framer.contest["votersregistered"] = self.framer._to_num(race.attrs["reg_voters"])["value"]
self.framer.contest["votersturnout"] = self.framer._to_num(None)["value"]
self.framer.measure["ballotorder"] = None
self.framer.measure["fullname"] = measure_fullname
self.framer.measure["description"] = description
self.framer.measure["yescount"] = yescount
self.framer.measure["yespct"] = yespct
self.framer.measure["nocount"] = nocount
self.framer.measure["nopct"] = nopct
self.framer.measure["poss_error"] = False
if this_type == "Proposition":
self.framer.contest["contestname"] = self.framer._concat(
officename,
delimiter=" ",
)
self.framer.measure["measureid"] = self.saver._make_this_id(
"measure",
"%s-%s" % (src.source_short, self.framer.contest["level"]),
slugify(measure_fullname),
)
elif this_type == "":
self.framer.contest["contestname"] = self.framer._concat(
officename,
delimiter=" ",
)
self.framer.measure["measureid"] = self.saver._make_this_id(
"measure",
"%s-%s-%s" % (src.source_short, self.framer.contest["level"], fixed["district"]),
slugify(measure_fullname),
)
else:
self.framer.contest["contestname"] = self.framer._concat(
officename,
delimiter=" ",
)
self.framer.measure["measureid"] = self.saver._make_this_id(
"measure",
"%s-%s-%s" % (src.source_short, self.framer.contest["level"], fixed["district"]),
slugify(measure_fullname),
)
self.framer.contest["contestdescription"] = self.framer._concat(
officename,
this_type,
delimiter=" ",
)
self.framer.contest["contestid"] = self.saver._make_contest_id(
src.source_short,
self.framer.contest["level"],
self.framer.office["officeslug"],
)
return self.framer
def _compile_candidate(self, race, election, src):
"""
"""
this_type = "candidate"
officename = unicode(race.attrs["contest_title"].title())
if "United States Representative" in officename:
cleaning_name = officename.replace("United States Representative", "US House of Representatives")
cleaning_name = cleaning_name.replace(" District", "")
cleaning_name = cleaning_name.replace("US House of Representatives ", "")
district_seat = cleaning_name[0:2]
officename = "US House of Representatives District %s" % (district_seat)
if "Member Of The State Assembly" in officename:
cleaning_name = officename.replace("Member Of The State Assembly", "State Assembly")
cleaning_name = cleaning_name.replace(" District", "")
cleaning_name = cleaning_name.replace("State Assembly ", "")
district_seat = cleaning_name[0:2]
officename = "State Assembly District %s" % (district_seat)
if "State Senator" in officename:
cleaning_name = officename.replace("State Senator", "State Senate")
cleaning_name = cleaning_name.replace(" District", "")
cleaning_name = cleaning_name.replace("State Senate ", "")
district_seat = cleaning_name[0:2]
officename = "State Senate District %s" % (district_seat)
if "United States Senator" in officename:
officename = "US Senate"
if "President And Vice President" in officename:
officename = "President-Vice President"
description = unicode(officename)
prop_num = None
level = "county"
seatnum = None
precinctstotal = race.attrs["total_precincts"]
precinctsreport = race.attrs["counted_precincts"]
reporttype = None
poss_error = False
self.framer.office["officename"] = officename
self.framer.office["officeslug"] = slugify(officename)
self.framer.office["active"] = True
self.framer.office["officeid"] = self.framer.office["officeslug"]
self.framer.contest["election_id"] = election.id
self.framer.contest["resultsource_id"] = src.id
self.framer.contest["seatnum"] = seatnum
self.framer.contest["is_uncontested"] = False
self.framer.contest["is_national"] = False
self.framer.contest["is_statewide"] = False
self.framer.contest["level"] = level
self.framer.contest["is_ballot_measure"] = False
self.framer.contest["is_judicial"] = False
self.framer.contest["is_runoff"] = False
self.framer.contest["reporttype"] = reporttype
self.framer.contest["poss_error"] = poss_error
if self.framer._to_num(precinctstotal)["convert"] == True:
pt = self.framer._to_num(precinctstotal)["value"]
self.framer.contest["precinctstotal"] = pt
else:
self.framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if self.framer._to_num(precinctsreport)["convert"] == True:
pr = self.framer._to_num(precinctsreport)["value"]
self.framer.contest["precinctsreporting"] = pr
else:
self.framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
self.framer.contest["precinctsreportingpct"] = self.framer._calc_pct(
self.framer.contest["precinctsreporting"],
self.framer.contest["precinctstotal"]
)
self.framer.contest["votersregistered"] = self.framer._to_num(race.attrs["reg_voters"])["value"]
self.framer.contest["votersturnout"] = self.framer._to_num(None)["value"]
self.framer.contest["contestdescription"] = None
self.framer.contest["contestid"] = self.saver._make_contest_id(
# election.electionid,
src.source_short,
self.framer.contest["level"],
self.framer.office["officeslug"],
)
if self.framer.contest["contestid"] == "oc-county-county-supervisor-1st-district":
self.framer.contest["contestname"] = "Orange County Supervisor District 5"
else:
self.framer.contest["contestname"] = self.framer.office["officename"]
self.framer.candidates = []
for candidate in race.find_all("ChoiceTotal"):
this_candidate = {}
fullname = candidate.attrs["candidate_name"].encode('utf8')
fullname = self.fixer._titlecase_with_accents(fullname)
party = unicode(candidate.attrs["party"].title())
if party == "Democratic" or party == "Dem":
party = "Democrat"
elif party == "Rep":
party = "Republican"
elif party == "Grn":
party = "Green"
elif party == "P-F":
party = "Peace and Freedom"
elif party == "Lib":
party = "Libertarian"
elif party == "AI" or party == "A-I" or party == "Ind":
party = "American Independent"
votecount = self.framer._to_num(candidate.attrs["c_total_votes"])["value"]
votepct = 0
this_candidate["ballotorder"] = candidate.attrs["cand_Seq_nbr"]
this_candidate["firstname"] = None
this_candidate["lastname"] = None
this_candidate["fullname"] = fullname.replace("* ", "")
this_candidate["candidateslug"] = slugify(fullname)
this_candidate["party"] = party
this_candidate["incumbent"] = False
this_candidate["votecount"] = votecount
this_candidate["votepct"] = votepct
this_candidate["poss_error"] = False
this_candidate["candidateid"] = self.saver._make_this_id(
"candidate",
self.framer.contest["contestid"],
this_candidate["candidateslug"],
)
self.framer.candidates.append(this_candidate)
return self.framer
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
compiler = BuildResults()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
soup = BeautifulSoup(open(latest_path), "xml")
file_timestring = unicode(soup.find("IssueDate").contents[0])
file_timestamp = parse(file_timestring, dayfirst=False).datetime
if self.testing == True:
update_this = self.testing
else:
update_this = saver._eval_timestamps(file_timestamp, src.source_latest)
if update_this == False:
logger.info("\n*****\nwe have newer data in the database so let's delete these files\n*****")
os.remove(latest_path)
else:
logger.info("\n*****\nwe have new data so we'll update timestamps in the database\n*****")
saver._update_result_timestamps(src, file_timestamp)
races = soup.find_all("Contest")
race_log = "\n"
proposition_ids = [
"190000000068",
"190000000069",
"190000000070",
"190000000071",
"190000000072"
]
for race in races:
# if race.ContestIdentifier.attrs["IdNumber"][0:3] == "140":
# """
# this is a judicial candidate
# """
# result = compiler._compile_judicial(race, "140", election, src)
# race_log += saver.make_office(result.office)
# race_log += saver.make_contest(result.office, result.contest)
# race_log += saver.make_judicial(result.contest, result.judicial)
# elif race.ContestIdentifier.attrs["IdNumber"][0:3] == "150":
# """
# this is a judicial candidate
# """
# result = compiler._compile_judicial(race, "150", election, src)
# race_log += saver.make_office(result.office)
# race_log += saver.make_contest(result.office, result.contest)
# race_log += saver.make_judicial(result.contest, result.judicial)
if race.ContestIdentifier.attrs["IdNumber"] in proposition_ids:
"""
this is a proposition
"""
result = compiler._compile_measure(race, election, src)
race_log += saver.make_office(result.office)
race_log += saver.make_contest(result.office, result.contest)
race_log += saver.make_measure(result.contest, result.measure)
else:
"""
this is a non-judicial candidate
"""
result = compiler._compile_candidate(race, election, src)
race_log += saver.make_office(result.office)
race_log += saver.make_contest(result.office, result.contest)
for candidate in result.candidates:
race_log += saver.make_candidate(result.contest, candidate)
#logger.info(race_log)
os.remove(latest_path)
logger.info("\n*****\nwe've finished processing sos results\n*****")
else:
logger.error("XML file to parse is not at expected location")
class BuildResults(object):
"""
"""
saver = Saver()
framer = Framer()
fixer = Namefixer()
def _compile_judicial(self, race, race_id, election, src):
"""
"""
r = race.find("TotalVotes")
this_type = "judicial"
contestname = unicode(" ".join(race.ContestName.stripped_strings))
if race_id == "140":
officename_idx = self.framer._find_nth(contestname, " - ", 1)
officename = unicode(contestname[:officename_idx].replace(".", ""))
officename = unicode(officename.replace(" Justice", ""))
fullname_idx = self.framer._find_nth(contestname, " - ", 1) + 3
elif race_id == "150":
officename_idx = self.framer._find_nth(contestname, " - ", 2)
officename = unicode(
contestname[:officename_idx].replace(" - ", " "))
fullname_idx = self.framer._find_nth(contestname, " - ", 2) + 3
fullname = unicode(contestname[fullname_idx:])
level = "california"
seatnum = None
is_statewide = True
precinctstotal = r.find(attrs={"Id": "TP"}).contents[0]
precinctsreport = r.find(attrs={"Id": "PR"}).contents[0]
reporttype = r.find(attrs={"Id": "RT"}).contents[0]
yescount = self.framer._to_num(r.find_all("Selection")[0].ValidVotes.contents[0])["value"]
yespct = self.framer._to_num(r.find(attrs={"Id": "PYV"}).contents[0])["value"]
nocount = self.framer._to_num(r.find_all("Selection")[1].ValidVotes.contents[0])["value"]
nopct = self.framer._to_num(r.find(attrs={"Id": "PNV"}).contents[0])["value"]
self.framer.office["officename"] = officename
self.framer.office["officeslug"] = slugify(officename)
self.framer.office["active"] = True
self.framer.office["officeid"] = self.framer.office["officeslug"]
self.framer.contest["election_id"] = election.id
self.framer.contest["resultsource_id"] = src.id
self.framer.contest["seatnum"] = seatnum
self.framer.contest["is_uncontested"] = False
self.framer.contest["is_national"] = False
self.framer.contest["is_statewide"] = True
self.framer.contest["level"] = level
self.framer.contest["is_ballot_measure"] = False
self.framer.contest["is_judicial"] = True
self.framer.contest["is_runoff"] = False
self.framer.contest["reporttype"] = None
self.framer.contest["poss_error"] = False
if self.framer._to_num(precinctstotal)["convert"] == True:
pt = self.framer._to_num(precinctstotal)["value"]
self.framer.contest["precinctstotal"] = pt
else:
self.framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if self.framer._to_num(precinctsreport)["convert"] == True:
pr = self.framer._to_num(precinctsreport)["value"]
self.framer.contest["precinctsreporting"] = pr
else:
self.framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
self.framer.contest["precinctsreportingpct"] = self.framer._calc_pct(self.framer.contest["precinctsreporting"], self.framer.contest["precinctstotal"])
self.framer.contest["votersregistered"] = self.framer._to_num(None)["value"]
self.framer.contest["votersturnout"] = self.framer._to_num(None)["value"]
self.framer.contest["contestname"] = self.framer.office["officename"]
self.framer.contest["contestdescription"] = None
if race_id == "140":
self.framer.contest["contestid"] = self.saver._make_contest_id(
# election.electionid,
src.source_short,
self.framer.contest["level"],
self.framer.office["officeslug"],
)
self.framer.judicial["ballotorder"] = None
self.framer.judicial["firstname"] = None
self.framer.judicial["lastname"] = None
self.framer.judicial["fullname"] = fullname
self.framer.judicial["judicialslug"] = slugify(fullname)
self.framer.judicial["yescount"] = yescount
self.framer.judicial["yespct"] = yespct
self.framer.judicial["nocount"] = nocount
self.framer.judicial["nopct"] = nopct
self.framer.judicial["poss_error"] = False
self.framer.judicial["judgeid"] = self.saver._make_this_id(
"judicial",
self.framer.contest["contestid"],
self.framer.judicial["judicialslug"],
)
elif race_id == "150":
self.framer.contest["contestid"] = self.saver._make_contest_id(
# election.electionid,
src.source_short,
self.framer.contest["level"],
self.framer.office["officeslug"],
)
self.framer.judicial["ballotorder"] = None
self.framer.judicial["firstname"] = None
self.framer.judicial["lastname"] = None
self.framer.judicial["fullname"] = fullname
self.framer.judicial["judicialslug"] = slugify(fullname)
self.framer.judicial["yescount"] = yescount
self.framer.judicial["yespct"] = yespct
self.framer.judicial["nocount"] = nocount
self.framer.judicial["nopct"] = nopct
self.framer.judicial["poss_error"] = False
self.framer.judicial["judgeid"] = self.saver._make_this_id(
"judicial",
self.framer.contest["contestid"],
self.framer.judicial["judicialslug"],
)
return self.framer
def _compile_measure(self, race, election, src):
"""
"""
r = race.find("TotalVotes")
this_type = "Proposition"
fullname = unicode(race.ContestName.contents[0].replace(".", ""))
description = unicode(" ".join(race.ContestName.stripped_strings).replace(".", ""))
prop_num = self.framer._get_prop_number(race.ContestIdentifier.attrs["IdNumber"], "190")
officename = self.framer._concat(
this_type,
prop_num,
delimiter=" ",
)
level = None
seatnum = None
precinctstotal = r.find(attrs={"Id": "TP"}).contents[0]
precinctsreport = r.find(attrs={"Id": "PR"}).contents[0]
reporttype = r.find(attrs={"Id": "RT"}).contents[0]
yescount = self.framer._to_num(r.find_all("Selection")[0].ValidVotes.contents[0])["value"]
yespct = self.framer._to_num(r.find(attrs={"Id": "PYV"}).contents[0])["value"]
nocount = self.framer._to_num(r.find_all("Selection")[1].ValidVotes.contents[0])["value"]
nopct = self.framer._to_num(r.find(attrs={"Id": "PNV"}).contents[0])["value"]
self.framer.office["officename"] = officename
self.framer.office["officeslug"] = slugify(officename)
self.framer.office["active"] = True
self.framer.office["officeid"] = self.framer.office["officeslug"]
self.framer.contest["election_id"] = election.id
self.framer.contest["resultsource_id"] = src.id
self.framer.contest["seatnum"] = seatnum
self.framer.contest["is_uncontested"] = False
self.framer.contest["is_national"] = False
self.framer.contest["is_statewide"] = True
self.framer.contest["level"] = "california"
self.framer.contest["is_ballot_measure"] = True
self.framer.contest["is_judicial"] = False
self.framer.contest["is_runoff"] = False
self.framer.contest["reporttype"] = None
self.framer.contest["poss_error"] = False
if self.framer._to_num(precinctstotal)["convert"] == True:
pt = self.framer._to_num(precinctstotal)["value"]
self.framer.contest["precinctstotal"] = pt
else:
self.framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if self.framer._to_num(precinctsreport)["convert"] == True:
pr = self.framer._to_num(precinctsreport)["value"]
self.framer.contest["precinctsreporting"] = pr
else:
self.framer.contest["precinctsreporting"] = None
raise Exception(
"precinctsreporting is not a number")
self.framer.contest["precinctsreportingpct"] = self.framer._calc_pct(
self.framer.contest["precinctsreporting"],
self.framer.contest["precinctstotal"]
)
self.framer.contest["votersregistered"] = self.framer._to_num(None)["value"]
self.framer.contest["votersturnout"] = self.framer._to_num(None)["value"]
self.framer.contest["contestname"] = self.framer._concat(
officename,
fullname,
delimiter=" ",
)
self.framer.contest["contestdescription"] = description
self.framer.contest["contestid"] = self.saver._make_contest_id(
# election.electionid,
src.source_short,
self.framer.contest["level"],
self.framer.office["officeslug"],
)
self.framer.measure["ballotorder"] = None
self.framer.measure["fullname"] = officename
self.framer.measure["measureslug"] = slugify(self.framer.contest["contestname"])
self.framer.measure["description"] = description
self.framer.measure["yescount"] = yescount
self.framer.measure["yespct"] = yespct
self.framer.measure["nocount"] = nocount
self.framer.measure["nopct"] = nopct
self.framer.measure["poss_error"] = False
self.framer.measure["measureid"] = self.saver._make_this_id(
"measure",
self.framer.contest["contestid"],
slugify(self.framer.measure["description"]),
)
return self.framer
def _compile_candidate(self, race, election, src):
"""
"""
r = race.find("TotalVotes")
this_type = "candidate"
contestname = unicode(" ".join(race.ContestName.stripped_strings))
officename_idx = self.framer._find_nth(contestname, " - ", 1)
officename = unicode(contestname[:officename_idx].replace(".", ""))
if "United States Representative" in officename:
officename = officename.replace("United States Representative", "US House of Representatives")
if "State Assembly Member" in officename:
officename = officename.replace("State Assembly Member", "State Assembly")
level_idx = self.framer._find_nth(contestname, " - ", 1) + 3
level = unicode(contestname[level_idx:].replace(" Results", "").lower())
seatnum = None
precinctstotal = r.find(attrs={"Id": "TP"}).contents[0]
precinctsreport = r.find(attrs={"Id": "PR"}).contents[0]
reporttype = r.find(attrs={"Id": "RT"}).contents[0]
self.framer.office["officename"] = officename
self.framer.office["officeslug"] = slugify(officename)
self.framer.office["active"] = True
self.framer.office["officeid"] = self.framer.office["officeslug"]
self.framer.contest["election_id"] = election.id
self.framer.contest["resultsource_id"] = src.id
self.framer.contest["seatnum"] = seatnum
self.framer.contest["is_uncontested"] = False
self.framer.contest["is_national"] = False
self.framer.contest["is_statewide"] = True
self.framer.contest["level"] = level
self.framer.contest["is_ballot_measure"] = False
self.framer.contest["is_judicial"] = False
self.framer.contest["is_runoff"] = False
self.framer.contest["reporttype"] = None
self.framer.contest["poss_error"] = False
if self.framer._to_num(precinctstotal)["convert"] == True:
pt = self.framer._to_num(precinctstotal)["value"]
self.framer.contest["precinctstotal"] = pt
else:
self.framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if self.framer._to_num(precinctsreport)["convert"] == True:
pr = self.framer._to_num(precinctsreport)["value"]
self.framer.contest["precinctsreporting"] = pr
else:
self.framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
self.framer.contest["precinctsreportingpct"] = self.framer._calc_pct(
self.framer.contest["precinctsreporting"],
self.framer.contest["precinctstotal"]
)
self.framer.contest["votersregistered"] = self.framer._to_num(None)["value"]
self.framer.contest["votersturnout"] = self.framer._to_num(None)["value"]
self.framer.contest["contestname"] = self.framer.office["officename"]
self.framer.contest["contestdescription"] = None
self.framer.contest["contestid"] = self.saver._make_contest_id(
# election.electionid,
src.source_short,
self.framer.contest["level"],
self.framer.office["officeslug"],
)
self.framer.candidates = []
for candidate in r.find_all("Selection"):
this_candidate = {}
fullname = candidate.Candidate.CandidateFullName.PersonFullName.contents[0].encode('utf8')
fullname = self.fixer._titlecase_with_accents(fullname)
party = unicode(candidate.AffiliationIdentifier.RegisteredName.contents[0])
if party == "Democratic":
party = "Democrat"
votecount = self.framer._to_num(candidate.ValidVotes.contents[0])["value"]
votepct = self.framer._to_num(candidate.CountMetric.contents[0])["value"]
this_candidate["ballotorder"] = None
this_candidate["firstname"] = None
this_candidate["lastname"] = None
this_candidate["fullname"] = fullname
this_candidate["candidateslug"] = slugify(fullname)
this_candidate["party"] = party
this_candidate["incumbent"] = False
this_candidate["votecount"] = votecount
this_candidate["votepct"] = votepct
this_candidate["poss_error"] = False
this_candidate["candidateid"] = self.saver._make_this_id(
"candidate",
self.framer.contest["contestid"],
this_candidate["candidateslug"],
)
self.framer.candidates.append(this_candidate)
return self.framer
def parse_results_file(self, src, data_directory):
"""
"""
saver = Saver()
process = LacProcessMethods()
latest_directory = "%s%s_latest" % (data_directory, src.source_short)
election = Election.objects.filter(
electionid=src.election.electionid).first()
for file in src.source_files.split(", "):
latest_path = os.path.join(latest_directory, file)
file_exists = os.path.isfile(latest_path)
file_has_size = os.path.getsize(latest_path)
if file_exists == True and file_has_size > 0:
rows = process.open_results_file(latest_path)
race_ids = process.get_race_ids_from(rows)
election_package = process.collate_and_fetch_records_for_race(
race_ids, rows)
races = election_package[0]
election_title = election_package[1]["title"]
election_stats = election_package[1]["stats"]
file_timestring = None
file_timestamp = None
pacific = pytz.timezone("US/Pacific")
for t in election_title:
if t[3:5] == "TD":
parser = TD_parser()
parsed = parser.parse_line(t)
timestring = parsed["date"] + " " + parsed["time"]
file_timestring = timestring
if file_timestring:
file_timestamp = parse(file_timestring,
dayfirst=False).datetime
file_timestamp = file_timestamp.replace(tzinfo=pacific)
if self.testing == True:
update_this = self.testing
else:
update_this = saver._eval_timestamps(
file_timestamp, src.source_latest)
if update_this == False:
logger.info(
"\n*****\nwe have newer data in the database so let's delete these files\n*****"
)
os.remove(latest_path)
else:
logger.info(
"\n*****\nwe have new data to save and we'll update timestamps in the database\n*****"
)
saver._update_result_timestamps(src, file_timestamp)
title = process.dictify_records_and_return(
election_title)
stats = process.dictify_records_and_return(
election_stats)
election_info = process.compile_election_stats(
title, stats)
for r in races:
records = process.dictify_records_and_return(
races[r])
"""
checks to see if this is a recall contest or a nonpartisan contest
for now, it's unclear how to store or display these contests
in future, however, we may want to parse and return their results
"""
skip = process.check_if_recall_or_nonpartisan(
records)
if skip:
pass
else:
contest_package = process.compile_contest_results(
records)
process.update_database(
contest_package, election, src)
logger.info("we've finished processing lac results")
else:
logger.error(
"unable to determine whether this data is newer than what we already have."
)
def update_database(self, contest_package, election, src):
""" import candidates, measures, office and contest info from compiled data """
saver = Saver()
framer = Framer()
fixer = Namefixer()
county_name = "Los Angeles County"
contest = contest_package["contest_details"]
candidates = contest_package["candidates"]
measures = contest_package["measures"]
judges = contest_package["judges"]
race_log = "\n"
# """
# Check level of contest (i.e. local, statewide)
# """
if "U.S." in contest["contest_title"] or "STATE" in contest[
"contest_title"]:
level = "county"
framer.contest["is_statewide"] = True
else:
level = "county"
framer.contest["is_statewide"] = False
framer.contest["level"] = level
if contest["is_judicial_contest"]:
"""
This is a judicial appointee
"""
if "SUPREME COURT" in contest["contest_title"]:
contestname = "Supreme Court"
elif "APPELLATE COURT" in contest["contest_title"]:
contestname = "Courts of Appeal District 02"
else:
contestname = contest["contest_title"]
officename = contestname
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_statewide"] = True
framer.contest["level"] = "county"
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = True
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest["total_precincts"])["convert"] == True:
pt = framer._to_num(contest["total_precincts"])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(
contest["precincts_reporting"])["convert"] == True:
pr = framer._to_num(contest["precincts_reporting"])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"])
if framer._to_num(contest["registration"])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest["registration"])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = fixer._fix(contestname)
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for judge in judges:
fullname = fixer._titlecase_with_accents(
judge["judicial_name"].encode('utf8'))
framer.judicial["firstname"] = None
framer.judicial["lastname"] = None
framer.judicial["ballotorder"] = None
framer.judicial["fullname"] = fullname
framer.judicial["judicialslug"] = slugify(fullname)
framer.judicial["description"] = judge["judicial_text"].title()
framer.judicial["poss_error"] = False
if framer._to_num(judge["yes_votes"])["convert"] == True:
yescount = framer._to_num(judge["yes_votes"])["value"]
framer.judicial["yescount"] = yescount
else:
framer.judicial["yescount"] = None
raise Exception("yescount is not a number")
if framer._to_num(judge["yes_percent"])["convert"] == True:
yespct = framer._to_num(judge["yes_percent"])["value"]
framer.judicial["yespct"] = yespct
else:
framer.judicial["yespct"] = None
raise Exception("yespct is not a number")
if framer._to_num(judge["no_votes"])["convert"] == True:
nocount = framer._to_num(judge["no_votes"])["value"]
framer.judicial["nocount"] = nocount
else:
framer.judicial["nocount"] = None
raise Exception("nocount is not a number")
if framer._to_num(judge["no_percent"])["convert"] == True:
nopct = framer._to_num(judge["no_percent"])["value"]
framer.judicial["nopct"] = nopct
else:
framer.judicial["nopct"] = None
raise Exception("nopct is not a number")
framer.judicial["judgeid"] = saver._make_this_id(
"judicial",
framer.contest["contestid"],
framer.judicial["judicialslug"],
)
race_log += saver.make_judicial(framer.contest,
framer.judicial)
elif contest["is_ballot_measure"]:
"""
this is a ballot measure
"""
framer.contest["level"] = "county"
framer.contest["is_statewide"] = False
contestname = contest["contest_title"]
if contest["contest_id"] == "00":
if contestname == "COUNTY MEASURE A":
this_type = "Measure"
contestname = "%swide" % (county_name)
contest["contest_title_cont"] = "Measure A"
else:
this_type = "Proposition"
contestname = contestname.replace("STATE MEASURE",
"Proposition")
else:
this_type = "Measure"
contestname = fixer._fix(contestname)
contestname = contestname.title()
if contestname == "Metro Transportation Authority":
contestname = "%swide" % (county_name)
if contest["contest_title_cont"]:
fullname = (contest["contest_title_cont"]).replace(
"MEASURE", "Measure")
else:
fullname = contestname
if contest["contest_id"] == "00":
description = None
officename = framer._concat(
# this_type,
contestname,
delimiter="-",
)
else:
description = "%s %s" % (contestname, this_type)
officename = framer._concat(
# this_type,
contestname,
delimiter="-",
)
framer.office["officename"] = officename
framer.office["officeslug"] = slugify(officename)
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
framer.contest["seatnum"] = None
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = True
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest["total_precincts"])["convert"] == True:
pt = framer._to_num(contest["total_precincts"])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(
contest["precincts_reporting"])["convert"] == True:
pr = framer._to_num(contest["precincts_reporting"])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"])
if framer._to_num(contest["registration"])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest["registration"])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = contestname
framer.contest["contestdescription"] = description
framer.contest["contestid"] = saver._make_contest_id(
# election.electionid,
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for measure in measures:
framer.measure["ballotorder"] = None
framer.measure["fullname"] = fullname
framer.measure["measureslug"] = slugify(fullname)
framer.measure["description"] = measure["measure_text"].title()
framer.measure["poss_error"] = False
if framer._to_num(measure["yes_votes"])["convert"] == True:
yescount = framer._to_num(measure["yes_votes"])["value"]
framer.measure["yescount"] = yescount
else:
framer.measure["yescount"] = None
raise Exception("yescount is not a number")
if framer._to_num(measure["yes_percent"])["convert"] == True:
yespct = framer._to_num(measure["yes_percent"])["value"]
framer.measure["yespct"] = yespct
else:
framer.measure["yespct"] = None
raise Exception("yespct is not a number")
if framer._to_num(measure["no_votes"])["convert"] == True:
nocount = framer._to_num(measure["no_votes"])["value"]
framer.measure["nocount"] = nocount
else:
framer.measure["nocount"] = None
raise Exception("nocount is not a number")
if framer._to_num(measure["no_percent"])["convert"] == True:
nopct = framer._to_num(measure["no_percent"])["value"]
framer.measure["nopct"] = nopct
else:
framer.measure["nopct"] = None
raise Exception("nopct is not a number")
if this_type == "Proposition":
framer.measure["measureid"] = saver._make_this_id(
"measure",
"%s-%s" % (src.source_short, framer.contest["level"]),
"%s-%s" %
(this_type.lower(), measure["measure_id"].lower()))
else:
framer.measure["measureid"] = saver._make_this_id(
"measure", framer.contest["contestid"], "%s-%s" %
(measure["measure_id"].lower(),
slugify(framer.measure["description"].lower())))
race_log += saver.make_measure(framer.contest, framer.measure)
else:
"""
this is a candidate for elected office
"""
strip_district = contest["district"].lstrip("0")
framer.contest["seatnum"] = "1010101"
if contest["contest_title"] == "MEMBER OF THE ASSEMBLY":
contestname = "State Assembly District %s" % (strip_district)
elif "SUPERVISOR" in contest["contest_title"]:
contestname = "Supervisor District %s" % (strip_district)
contestname = fixer._affix_county(county_name, contestname)
elif "U.S. REPRESENTATIVE" in contest["contest_title"]:
contestname = "U.S. House of Representatives District %s" % (
strip_district)
elif "DELEGATES" in contest["contest_title"]:
designation = contest["contest_title_cont"].replace(
"CONGRESSIONAL DISTRICT-", "")
if designation == "REP":
this_desig = "Republican"
elif designation == "DEM":
this_desig = "Democratic"
else:
this_desig = None
contestname = "%s Presidential Primary Delegates - CD%s" % (
this_desig, strip_district)
elif "STATE SENATOR" in contest["contest_title"]:
contestname = "State Senate District %s" % (strip_district)
elif "UNITED STATES SENATOR" in contest["contest_title"]:
contestname = "US Senate"
elif "PRESIDENTIAL PREFERENCE" in contest["contest_title"]:
designation = contest["contest_title_cont"].upper()
if designation == "AI":
this_desig = "American Independent"
elif designation == "REP":
this_desig = "Republican"
elif designation == "DEM":
this_desig = "Democratic"
elif designation == "PF":
this_desig = "Peace And Freedom"
elif designation == "LIB":
this_desig = "Libertarian"
elif designation == "GRN":
this_desig = "Green"
else:
this_desig = None
contestname = "%s Presidential Primary" % (this_desig)
elif "PARTY COUNTY COMMITTEE" in contest["contest_title"]:
contestname = "%s Party County Committee District %s" % (
contest["party_name"].capitalize(), strip_district)
elif "JUDGE-SUPERIOR COURT" in contest["contest_title"]:
contestname = "Judge Superior Court %s" % (
contest["contest_title_cont"].title())
contestname = fixer._affix_county(county_name, contestname)
else:
contestname = "%s %s" % (contest["contest_title"].title(),
contest["contest_title_cont"].title())
contestname = fixer._fix(contestname)
contestname = contestname.title()
contestname = framer._convert_district_number(contestname)
framer.office["officename"] = contestname.replace(".", "")
framer.office["officeslug"] = slugify(framer.office["officename"])
framer.office["active"] = True
framer.office["officeid"] = framer.office["officeslug"]
framer.contest["election_id"] = election.id
framer.contest["resultsource_id"] = src.id
if len(candidates) < 2:
framer.contest["is_uncontested"] = True
else:
framer.contest["is_uncontested"] = False
framer.contest["is_national"] = False
framer.contest["is_ballot_measure"] = False
framer.contest["is_judicial"] = False
framer.contest["is_runoff"] = False
framer.contest["reporttype"] = None
framer.contest["poss_error"] = False
if framer._to_num(contest["total_precincts"])["convert"] == True:
pt = framer._to_num(contest["total_precincts"])["value"]
framer.contest["precinctstotal"] = pt
else:
framer.contest["precinctstotal"] = None
raise Exception("precinctstotal is not a number")
if framer._to_num(
contest["precincts_reporting"])["convert"] == True:
pr = framer._to_num(contest["precincts_reporting"])["value"]
framer.contest["precinctsreporting"] = pr
else:
framer.contest["precinctsreporting"] = None
raise Exception("precinctsreporting is not a number")
framer.contest["precinctsreportingpct"] = framer._calc_pct(
framer.contest["precinctsreporting"],
framer.contest["precinctstotal"])
if framer._to_num(contest["registration"])["convert"] == True:
framer.contest["votersregistered"] = framer._to_num(
contest["registration"])["value"]
else:
framer.contest["votersregistered"] = None
raise Exception("votersregistered is not a number")
framer.contest["votersturnout"] = None
framer.contest["contestname"] = contestname
try:
framer.contest["contestdescription"] = contest[
"vote_for_text"].capitalize(
) + " " + contest["vote_for_number"]
except:
framer.contest["contestdescription"] = None
framer.contest["contestid"] = saver._make_contest_id(
src.source_short,
framer.contest["level"],
framer.office["officeslug"],
)
race_log += saver.make_office(framer.office)
race_log += saver.make_contest(framer.office, framer.contest)
for candidate in candidates:
fullname = fixer._titlecase_with_accents(
candidate["candidate_name"].encode('utf8'))
party = candidate["party_short"]
if party == "REP":
party = "Republican"
elif party == "DEM":
party = "Democrat"
elif party == "AI":
party = "American Independent"
elif party == "PF":
party = "Peace And Freedom"
elif party == "LIB":
party = "Libertarian"
elif party == "GRN":
party = "Green"
elif party == "NP":
party = "No Party Preference"
framer.candidate["ballotorder"] = None
framer.candidate["firstname"] = None
framer.candidate["lastname"] = None
framer.candidate["fullname"] = fullname
framer.candidate["candidateslug"] = slugify(fullname)
framer.candidate["party"] = party
framer.candidate["incumbent"] = False
framer.candidate["poss_error"] = False
if framer._to_num(candidate["votes"])["convert"] == True:
framer.candidate["votecount"] = framer._to_num(
candidate["votes"])["value"]
else:
framer.candidate["votecount"] = None
raise Exception("votecount is not a number")
if framer._to_num(
candidate["percent_of_vote"])["convert"] == True:
framer.candidate["votepct"] = framer._to_num(
candidate["percent_of_vote"])["value"]
else:
framer.candidate["votepct"] = None
raise Exception("votepct is not a number")
framer.candidate["candidateid"] = saver._make_this_id(
"candidate",
framer.contest["contestid"],
framer.candidate["candidateslug"],
)
race_log += saver.make_candidate(framer.contest,
framer.candidate)