def process_county_files(c_url, outfile, prefix):
"""
Data files must be prefected and named <prefix><county>.xml
Processes each file, extracts vote totals, rolls up by vote type,
and returns a list of items
"""
jurisdiction = clarify.Jurisdiction(url=c_url, level='state')
allitems = []
for j in jurisdiction.get_subjurisdictions():
county = j.name
if j is not None:
try:
if j.report_url('xml'):
filename = prefix + j.name + ".xml"
items = extract_data_from_file(filename)
grouped_items = rollup_by_vote_type(items)
allitems = allitems + grouped_items
except Exception as ex:
print("FAILED: Processing ", county, filename, j.report_url('xml'))
if hasattr(ex, 'message'):
print(ex.message)
else:
print(ex)
output_file(outfile, allitems)
def process_county_files(c_url, outfile, prefix):
# Should be state, not county
jurisdiction = clarify.Jurisdiction(url=c_url, level='state')
allitems = []
allcandidates = []
for j in jurisdiction.get_subjurisdictions():
county = j.name
if j is not None:
try:
if j.report_url('xml'):
filename = prefix + j.name + ".xml"
# Calls Helper Functions
items = extract_data_from_file(filename)
grouped_items = rollup_by_vote_type(items)
allitems = allitems + grouped_items
except Exception as ex:
print("FAILED: Processing ", county, filename,
j.report_url('xml'))
if hasattr(ex, 'message'):
print(ex.message)
else:
print(ex)
output_file(outfile, allitems)
def statewide_results(url):
j = clarify.Jurisdiction(url=url, level="state")
r = requests.get(
"http://results.enr.clarityelections.com/WV/74487/207685/reports/detailxml.zip",
stream=True)
z = zipfile.ZipFile(BytesIO(r.content))
z.extractall()
p = clarify.Parser()
p.parse("detail.xml")
results = []
for result in p.results:
candidate = result.choice.text
office, district = parse_office(result.contest.text)
party = parse_party(result.contest.text)
if '(' in candidate and party is None:
if '(I)' in candidate:
if '(I)(I)' in candidate:
candidate = candidate.split('(I)')[0]
party = 'I'
else:
candidate, party = candidate.split('(I)')
candidate = candidate.strip() + ' (I)'
else:
print(candidate)
candidate, party = candidate.split('(', 1)
candidate = candidate.strip()
party = party.replace(')', '').strip()
if result.jurisdiction:
county = result.jurisdiction.name
else:
county = None
r = [
x for x in results if x['county'] == county
and x['office'] == office and x['district'] == district
and x['party'] == party and x['candidate'] == candidate
]
if r:
r[0][result.vote_type] = result.votes
else:
results.append({
'county': county,
'office': office,
'district': district,
'party': party,
'candidate': candidate,
result.vote_type: result.votes
})
with open("20180508__wv__general.csv", "wt") as csvfile:
w = csv.writer(csvfile)
w.writerow(
['county', 'office', 'district', 'party', 'candidate', 'votes'])
for row in results:
total_votes = row[
'Election Day'] # + row['Absentee by Mail'] + row['Advance in Person'] + row['Provisional']
w.writerow([
row['county'], row['office'], row['district'], row['party'],
row['candidate'], total_votes
])
def link_to_xml(link):
"""a function that takes a link and returns an xml file"""
s = clarify.Jurisdiction(url=link, level='county')
r = requests.get(s.report_url('xml'), stream=True)
z = ZipFile(ioDuder(r.content))
return z.open('detail.xml') # return an accessed detail.xml file
def get_detail_xml_files(county_url, logger):
j = clarify.Jurisdiction(url=county_url, level='state')
subs = j.get_subjurisdictions()
num_counties = len(subs)
logger.info(f'Found {num_counties} counties to process')
for s in subs:
county = s.name
url = s.report_url('xml')
if not url:
logger.error(f'No detail XML URL found for {county} country')
else:
helpers.download_detail_xml_file(url, county, logger)
def statewide_results(url):
j = clarify.Jurisdiction(url=url, level="state")
r = requests.get(j.report_url('xml'), stream=True)
z = zipfile.ZipFile(StringIO.StringIO(r.content))
z.extractall()
p = clarify.Parser()
p.parse("detail.xml")
results = []
for result in p.results:
candidate = result.choice.text
office, district = parse_office(result.contest.text)
party = parse_party(result.contest.text)
if party is None and '(' in candidate:
candidate, party = candidate.split('(')
candidate = candidate.strip()
party = party.replace(')', '').strip()
if result.jurisdiction:
county = result.jurisdiction.name
else:
county = None
r = [
x for x in results if x['county'] == county
and x['office'] == office and x['district'] == district
and x['party'] == party and x['candidate'] == candidate
]
if r:
r[0][result.vote_type] = result.votes
else:
results.append({
'county': county,
'office': office,
'district': district,
'party': party,
'candidate': candidate,
result.vote_type: result.votes
})
with open("20121204__ga__general__runoff.csv", "wb") as csvfile:
w = unicodecsv.writer(csvfile, encoding='utf-8')
w.writerow([
'county', 'office', 'district', 'party', 'candidate', 'votes',
'election_day', 'absentee', 'early_voting', 'provisional'
])
for row in results:
total_votes = row['Election Day'] + row['Absentee by Mail'] + row[
'Advance in Person'] + row['Provisional']
w.writerow([
row['county'], row['office'], row['district'], row['party'],
row['candidate'], total_votes, row['Election Day'],
row['Absentee by Mail'], row['Advance in Person'],
row['Provisional']
])
def process_county_files():
c_url = "http://results.enr.clarityelections.com/AR/42843/113233/en/summary.html"
jurisdiction = clarify.Jurisdiction(url=c_url, level='county')
allitems = []
for j in jurisdiction.get_subjurisdictions():
print("j ", j.name, j.report_url('xml'))
if j.report_url('xml'):
filename = j.name + ".xml"
items = go(filename)
allitems = allitems + items
outfile = "20121106__ar__general__precinct.csv"
output_file(outfile, allitems)
def download_county_files(url, filename):
no_xml = []
j = clarify.Jurisdiction(url=url, level="state")
subs = j.get_subjurisdictions()
for sub in subs:
try:
r = requests.get(sub.report_url('xml'), stream=True)
z = zipfile.ZipFile(BytesIO(r.content))
z.extractall()
precinct_results(sub.name.replace(' ', '_').lower(), filename)
except:
no_xml.append(sub.name)
print(no_xml)
def get_county_files():
c_url = "http://results.enr.clarityelections.com/AR/42843/113233/en/summary.html"
jurisdiction = clarify.Jurisdiction(url=c_url, level='state')
for j in jurisdiction.get_subjurisdictions():
print("j ", j.name, j.report_url('xml'))
if j.report_url('xml'):
detail_url = j.report_url('xml').replace("http:", "https:")
r = requests.get(detail_url, allow_redirects=True)
zip_filename = j.name + "_detailxml.zip"
open(zip_filename, 'wb').write(r.content)
zip = zipfile.ZipFile(zip_filename)
zip.extractall()
shutil.move("detail.xml", j.name + ".xml")
def _build_metadata(self, year, elections):
meta = []
year_int = int(year)
for election in elections:
if election['special'] == True:
results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
for result in results:
ocd_id = 'ocd-division/country:us/state:sc'
generated_filename = self._generates_special_filename(result)
pre_processed_url = result['url']
meta.append({
"generated_filename": generated_filename,
"raw_url": election['direct_links'][0],
"pre_processed_url": None,
"ocd_id": ocd_id,
"name": 'South Carolina',
"election": election['slug']
})
elif 'summary.html' in election['direct_links'][0]:
# county-level results
j = clarify.Jurisdiction(url=election['direct_links'][0], level='state') # check for clarity url
ocd_id = 'ocd-division/country:us/state:sc'
generated_filename = self._generate_county_filename(election)
meta.append({
"generated_filename": generated_filename,
"pre_processed_url": None,
"raw_url": j.report_url('xml'),
"ocd_id": ocd_id,
"name": 'South Carolina',
"election": election['slug']
})
# precinct-level results, one file per county
subs = j.get_subjurisdictions()
for county in self._jurisdictions():
try:
subj = [s for s in subs if s.name.strip() == county['name'].strip()][0]
generated_filename = self._generate_precinct_filename(election, county)
meta.append({
"generated_filename": generated_filename,
"pre_processed_url": None,
"raw_url": subj.report_url('xml'),
"ocd_id": county['ocd_id'],
"name": county['name'],
"election": election['slug']
})
except IndexError:
continue
return meta
def get_county_files(c_url, prefix):
# state should be here, not county (in 2016)
jurisdiction = clarify.Jurisdiction(url=c_url, level='state')
for j in jurisdiction.get_subjurisdictions():
print("County: ", j.name, j.report_url('xml'))
if j.report_url('xml'):
detail_url = j.report_url('xml').replace("http:", "https:")
r = requests.get(detail_url, allow_redirects=True)
zip_filename = j.name + "_detailxml.zip"
open(zip_filename, 'wb').write(r.content)
zip = zipfile.ZipFile(zip_filename)
zip.extractall()
shutil.move("detail.xml", prefix + j.name + ".xml")
def get_county_files(c_url, prefix):
"""
using the summary url, downloads all the detail xml files , unzips them,
renames the detail to prefix_county.xml
"""
jurisdiction = clarify.Jurisdiction(url=c_url, level='state')
for j in jurisdiction.get_subjurisdictions():
print("County: ", j.name, j.report_url('xml'))
if j.report_url('xml'):
detail_url = j.report_url('xml').replace("http:", "https:")
r = requests.get(detail_url, allow_redirects=True)
zip_filename = j.name + "_detailxml.zip"
open(zip_filename, 'wb').write(r.content)
zip = zipfile.ZipFile(zip_filename)
zip.extractall()
shutil.move("detail.xml", prefix + j.name + ".xml")
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Georgia uses a newer version of clarity. Install clarify from git with below command
# pip install git+https://github.com/openelections/clarify.git#egg=clarify
import clarify, subprocess, os, wget, shutil
from zipfile import ZipFile
j = clarify.Jurisdiction(url='https://results.enr.clarityelections.com/GA/105369/web.264614/#/summary', level='state')
subs = j.get_subjurisdictions()
working_directory = os.getcwd()
xml_directory = os.path.join(working_directory, "xml_data")
default_zip = os.path.join(working_directory, "detailxml.zip")
default_xml = os.path.join(working_directory, "detail.xml")
# Never edit anything in the xml_data directory. This directory is automatically deleted at the start of every new run.
if os.path.exists(xml_directory):
shutil.rmtree(xml_directory)
## Now create the xml_directory
if not os.path.exists(xml_directory):
os.makedirs(xml_directory)
#subs[0].name is Appling. subs[158].name is Worth.
# Final range = 159 (the total number of counties)
for i in range(159):
# Remember that all downloaded data is zipped!
url = subs[i].report_url('xml')
def clarify_sarpy():
'''
1. Fetch zipped XML file.
2. Unzip in memory.
3. Load into Clarify.
4. Loop over results, write to file.
'''
# discover path to zipfile and fetch
s = clarify.Jurisdiction(url=url, level='county')
r = requests.get(s.report_url('xml'), stream=True)
z = ZipFile(ioDuder(r.content))
# hand off to clarify
p = clarify.Parser()
p.parse(z.open('detail.xml'))
results = []
# According to the Sarpy County election commissioner, results with a
# `vote_type` of "underVotes" -- e.g., only 1 vote cast in a "pick 2"
# race -- or "overVotes" -- e.g., 3 votes cast in a "pick 2" race --
# are just context information and should not be included in results
excluded_vote_types = ['underVotes', 'overVotes']
for result in p.results:
if result.vote_type not in excluded_vote_types:
candidate = result.choice.text
office, district = parse_office(result.contest.text)
party = result.choice.party
try:
precinct = result.jurisdiction.name
except AttributeError:
precinct = None
r = [x for x in results if x['precinct'] == precinct and \
x['office'] == office and x['district'] == district and \
x['party'] == party and x['candidate'] == candidate]
if r:
r[0][result.vote_type] = result.votes
else:
record = {
'county': county,
'precinct': precinct,
'office': office,
'district': district,
'party': party,
'candidate': candidate,
result.vote_type: result.votes
}
results.append(record)
# build filename
election_date = p.election_date.strftime('%Y%m%d')
filename = '__'.join([
election_date,
'ne',
election_type,
county.lower(),
'precinct.csv'
])
# to filter out "total" rows, uncomment the next line
# results = [x for x in results if x['precinct']]
with open(filename, 'wb') as outfile:
f = unicodecsv.writer(outfile, encoding='utf-8')
# headers
f.writerow(['county', 'precinct', 'office', 'district', 'party',
'candidate', 'votes', 'election_day', 'early_voting',
'provisional'])
for row in results:
total_votes = row['Early Voting'] + row['Provisionals'] \
+ row['Election Day']
f.writerow([row['county'], row['precinct'], row['office'],
row['district'], row['party'], row['candidate'],
total_votes, row['Election Day'],
row['Early Voting'], row['Provisionals']])
