def process_county_files(c_url, outfile, prefix): """ Data files must be prefected and named <prefix><county>.xml Processes each file, extracts vote totals, rolls up by vote type, and returns a list of items """ jurisdiction = clarify.Jurisdiction(url=c_url, level='state') allitems = [] for j in jurisdiction.get_subjurisdictions(): county = j.name if j is not None: try: if j.report_url('xml'): filename = prefix + j.name + ".xml" items = extract_data_from_file(filename) grouped_items = rollup_by_vote_type(items) allitems = allitems + grouped_items except Exception as ex: print("FAILED: Processing ", county, filename, j.report_url('xml')) if hasattr(ex, 'message'): print(ex.message) else: print(ex) output_file(outfile, allitems)
def process_county_files(c_url, outfile, prefix): # Should be state, not county jurisdiction = clarify.Jurisdiction(url=c_url, level='state') allitems = [] allcandidates = [] for j in jurisdiction.get_subjurisdictions(): county = j.name if j is not None: try: if j.report_url('xml'): filename = prefix + j.name + ".xml" # Calls Helper Functions items = extract_data_from_file(filename) grouped_items = rollup_by_vote_type(items) allitems = allitems + grouped_items except Exception as ex: print("FAILED: Processing ", county, filename, j.report_url('xml')) if hasattr(ex, 'message'): print(ex.message) else: print(ex) output_file(outfile, allitems)
def statewide_results(url): j = clarify.Jurisdiction(url=url, level="state") r = requests.get( "http://results.enr.clarityelections.com/WV/74487/207685/reports/detailxml.zip", stream=True) z = zipfile.ZipFile(BytesIO(r.content)) z.extractall() p = clarify.Parser() p.parse("detail.xml") results = [] for result in p.results: candidate = result.choice.text office, district = parse_office(result.contest.text) party = parse_party(result.contest.text) if '(' in candidate and party is None: if '(I)' in candidate: if '(I)(I)' in candidate: candidate = candidate.split('(I)')[0] party = 'I' else: candidate, party = candidate.split('(I)') candidate = candidate.strip() + ' (I)' else: print(candidate) candidate, party = candidate.split('(', 1) candidate = candidate.strip() party = party.replace(')', '').strip() if result.jurisdiction: county = result.jurisdiction.name else: county = None r = [ x for x in results if x['county'] == county and x['office'] == office and x['district'] == district and x['party'] == party and x['candidate'] == candidate ] if r: r[0][result.vote_type] = result.votes else: results.append({ 'county': county, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes }) with open("20180508__wv__general.csv", "wt") as csvfile: w = csv.writer(csvfile) w.writerow( ['county', 'office', 'district', 'party', 'candidate', 'votes']) for row in results: total_votes = row[ 'Election Day'] # + row['Absentee by Mail'] + row['Advance in Person'] + row['Provisional'] w.writerow([ row['county'], row['office'], row['district'], row['party'], row['candidate'], total_votes ])
def link_to_xml(link): """a function that takes a link and returns an xml file""" s = clarify.Jurisdiction(url=link, level='county') r = requests.get(s.report_url('xml'), stream=True) z = ZipFile(ioDuder(r.content)) return z.open('detail.xml') # return an accessed detail.xml file
def get_detail_xml_files(county_url, logger): j = clarify.Jurisdiction(url=county_url, level='state') subs = j.get_subjurisdictions() num_counties = len(subs) logger.info(f'Found {num_counties} counties to process') for s in subs: county = s.name url = s.report_url('xml') if not url: logger.error(f'No detail XML URL found for {county} country') else: helpers.download_detail_xml_file(url, county, logger)
def statewide_results(url): j = clarify.Jurisdiction(url=url, level="state") r = requests.get(j.report_url('xml'), stream=True) z = zipfile.ZipFile(StringIO.StringIO(r.content)) z.extractall() p = clarify.Parser() p.parse("detail.xml") results = [] for result in p.results: candidate = result.choice.text office, district = parse_office(result.contest.text) party = parse_party(result.contest.text) if party is None and '(' in candidate: candidate, party = candidate.split('(') candidate = candidate.strip() party = party.replace(')', '').strip() if result.jurisdiction: county = result.jurisdiction.name else: county = None r = [ x for x in results if x['county'] == county and x['office'] == office and x['district'] == district and x['party'] == party and x['candidate'] == candidate ] if r: r[0][result.vote_type] = result.votes else: results.append({ 'county': county, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes }) with open("20121204__ga__general__runoff.csv", "wb") as csvfile: w = unicodecsv.writer(csvfile, encoding='utf-8') w.writerow([ 'county', 'office', 'district', 'party', 'candidate', 'votes', 'election_day', 'absentee', 'early_voting', 'provisional' ]) for row in results: total_votes = row['Election Day'] + row['Absentee by Mail'] + row[ 'Advance in Person'] + row['Provisional'] w.writerow([ row['county'], row['office'], row['district'], row['party'], row['candidate'], total_votes, row['Election Day'], row['Absentee by Mail'], row['Advance in Person'], row['Provisional'] ])
def process_county_files(): c_url = "http://results.enr.clarityelections.com/AR/42843/113233/en/summary.html" jurisdiction = clarify.Jurisdiction(url=c_url, level='county') allitems = [] for j in jurisdiction.get_subjurisdictions(): print("j ", j.name, j.report_url('xml')) if j.report_url('xml'): filename = j.name + ".xml" items = go(filename) allitems = allitems + items outfile = "20121106__ar__general__precinct.csv" output_file(outfile, allitems)
def download_county_files(url, filename): no_xml = [] j = clarify.Jurisdiction(url=url, level="state") subs = j.get_subjurisdictions() for sub in subs: try: r = requests.get(sub.report_url('xml'), stream=True) z = zipfile.ZipFile(BytesIO(r.content)) z.extractall() precinct_results(sub.name.replace(' ', '_').lower(), filename) except: no_xml.append(sub.name) print(no_xml)
def get_county_files(): c_url = "http://results.enr.clarityelections.com/AR/42843/113233/en/summary.html" jurisdiction = clarify.Jurisdiction(url=c_url, level='state') for j in jurisdiction.get_subjurisdictions(): print("j ", j.name, j.report_url('xml')) if j.report_url('xml'): detail_url = j.report_url('xml').replace("http:", "https:") r = requests.get(detail_url, allow_redirects=True) zip_filename = j.name + "_detailxml.zip" open(zip_filename, 'wb').write(r.content) zip = zipfile.ZipFile(zip_filename) zip.extractall() shutil.move("detail.xml", j.name + ".xml")
def _build_metadata(self, year, elections): meta = [] year_int = int(year) for election in elections: if election['special'] == True: results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True] for result in results: ocd_id = 'ocd-division/country:us/state:sc' generated_filename = self._generates_special_filename(result) pre_processed_url = result['url'] meta.append({ "generated_filename": generated_filename, "raw_url": election['direct_links'][0], "pre_processed_url": None, "ocd_id": ocd_id, "name": 'South Carolina', "election": election['slug'] }) elif 'summary.html' in election['direct_links'][0]: # county-level results j = clarify.Jurisdiction(url=election['direct_links'][0], level='state') # check for clarity url ocd_id = 'ocd-division/country:us/state:sc' generated_filename = self._generate_county_filename(election) meta.append({ "generated_filename": generated_filename, "pre_processed_url": None, "raw_url": j.report_url('xml'), "ocd_id": ocd_id, "name": 'South Carolina', "election": election['slug'] }) # precinct-level results, one file per county subs = j.get_subjurisdictions() for county in self._jurisdictions(): try: subj = [s for s in subs if s.name.strip() == county['name'].strip()][0] generated_filename = self._generate_precinct_filename(election, county) meta.append({ "generated_filename": generated_filename, "pre_processed_url": None, "raw_url": subj.report_url('xml'), "ocd_id": county['ocd_id'], "name": county['name'], "election": election['slug'] }) except IndexError: continue return meta
def get_county_files(c_url, prefix): # state should be here, not county (in 2016) jurisdiction = clarify.Jurisdiction(url=c_url, level='state') for j in jurisdiction.get_subjurisdictions(): print("County: ", j.name, j.report_url('xml')) if j.report_url('xml'): detail_url = j.report_url('xml').replace("http:", "https:") r = requests.get(detail_url, allow_redirects=True) zip_filename = j.name + "_detailxml.zip" open(zip_filename, 'wb').write(r.content) zip = zipfile.ZipFile(zip_filename) zip.extractall() shutil.move("detail.xml", prefix + j.name + ".xml")
def get_county_files(c_url, prefix): """ using the summary url, downloads all the detail xml files , unzips them, renames the detail to prefix_county.xml """ jurisdiction = clarify.Jurisdiction(url=c_url, level='state') for j in jurisdiction.get_subjurisdictions(): print("County: ", j.name, j.report_url('xml')) if j.report_url('xml'): detail_url = j.report_url('xml').replace("http:", "https:") r = requests.get(detail_url, allow_redirects=True) zip_filename = j.name + "_detailxml.zip" open(zip_filename, 'wb').write(r.content) zip = zipfile.ZipFile(zip_filename) zip.extractall() shutil.move("detail.xml", prefix + j.name + ".xml")
#!/usr/bin/python # -*- coding: utf-8 -*- # Georgia uses a newer version of clarity. Install clarify from git with below command # pip install git+https://github.com/openelections/clarify.git#egg=clarify import clarify, subprocess, os, wget, shutil from zipfile import ZipFile j = clarify.Jurisdiction(url='https://results.enr.clarityelections.com/GA/105369/web.264614/#/summary', level='state') subs = j.get_subjurisdictions() working_directory = os.getcwd() xml_directory = os.path.join(working_directory, "xml_data") default_zip = os.path.join(working_directory, "detailxml.zip") default_xml = os.path.join(working_directory, "detail.xml") # Never edit anything in the xml_data directory. This directory is automatically deleted at the start of every new run. if os.path.exists(xml_directory): shutil.rmtree(xml_directory) ## Now create the xml_directory if not os.path.exists(xml_directory): os.makedirs(xml_directory) #subs[0].name is Appling. subs[158].name is Worth. # Final range = 159 (the total number of counties) for i in range(159): # Remember that all downloaded data is zipped! url = subs[i].report_url('xml')
def clarify_sarpy(): ''' 1. Fetch zipped XML file. 2. Unzip in memory. 3. Load into Clarify. 4. Loop over results, write to file. ''' # discover path to zipfile and fetch s = clarify.Jurisdiction(url=url, level='county') r = requests.get(s.report_url('xml'), stream=True) z = ZipFile(ioDuder(r.content)) # hand off to clarify p = clarify.Parser() p.parse(z.open('detail.xml')) results = [] # According to the Sarpy County election commissioner, results with a # `vote_type` of "underVotes" -- e.g., only 1 vote cast in a "pick 2" # race -- or "overVotes" -- e.g., 3 votes cast in a "pick 2" race -- # are just context information and should not be included in results excluded_vote_types = ['underVotes', 'overVotes'] for result in p.results: if result.vote_type not in excluded_vote_types: candidate = result.choice.text office, district = parse_office(result.contest.text) party = result.choice.party try: precinct = result.jurisdiction.name except AttributeError: precinct = None r = [x for x in results if x['precinct'] == precinct and \ x['office'] == office and x['district'] == district and \ x['party'] == party and x['candidate'] == candidate] if r: r[0][result.vote_type] = result.votes else: record = { 'county': county, 'precinct': precinct, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes } results.append(record) # build filename election_date = p.election_date.strftime('%Y%m%d') filename = '__'.join([ election_date, 'ne', election_type, county.lower(), 'precinct.csv' ]) # to filter out "total" rows, uncomment the next line # results = [x for x in results if x['precinct']] with open(filename, 'wb') as outfile: f = unicodecsv.writer(outfile, encoding='utf-8') # headers f.writerow(['county', 'precinct', 'office', 'district', 'party', 'candidate', 'votes', 'election_day', 'early_voting', 'provisional']) for row in results: total_votes = row['Early Voting'] + row['Provisionals'] \ + row['Election Day'] f.writerow([row['county'], row['precinct'], row['office'], row['district'], row['party'], row['candidate'], total_votes, row['Election Day'], row['Early Voting'], row['Provisionals']])