def load(self): self._common_kwargs = self._build_common_election_kwargs() self._common_kwargs['reporting_level'] = 'county' # Store result instances for bulk loading results = [] p = clarify.Parser() p.parse(self._file_handle) for result in p.results: if self._skip_row(result.contest): continue if row['county'].strip() == '': total_votes = int(row['votes'].strip()) else: rr_kwargs = self._common_kwargs.copy() rr_kwargs.update(self._build_contest_kwargs(row)) rr_kwargs.update(self._build_candidate_kwargs(row)) jurisdiction = row['county'].strip() rr_kwargs.update({ 'jurisdiction': jurisdiction, 'ocd_id': "{}/county:{}".format(self.mapping['ocd_id'], ocd_type_id(jurisdiction)), 'office': row['office'].strip(), 'district': row['district'].strip(), 'votes': int(row['votes'].strip()) }) results.append(RawResult(**rr_kwargs)) RawResult.objects.insert(results)
def statewide_results(url): j = clarify.Jurisdiction(url=url, level="state") r = requests.get( "http://results.enr.clarityelections.com/WV/74487/207685/reports/detailxml.zip", stream=True) z = zipfile.ZipFile(BytesIO(r.content)) z.extractall() p = clarify.Parser() p.parse("detail.xml") results = [] for result in p.results: candidate = result.choice.text office, district = parse_office(result.contest.text) party = parse_party(result.contest.text) if '(' in candidate and party is None: if '(I)' in candidate: if '(I)(I)' in candidate: candidate = candidate.split('(I)')[0] party = 'I' else: candidate, party = candidate.split('(I)') candidate = candidate.strip() + ' (I)' else: print(candidate) candidate, party = candidate.split('(', 1) candidate = candidate.strip() party = party.replace(')', '').strip() if result.jurisdiction: county = result.jurisdiction.name else: county = None r = [ x for x in results if x['county'] == county and x['office'] == office and x['district'] == district and x['party'] == party and x['candidate'] == candidate ] if r: r[0][result.vote_type] = result.votes else: results.append({ 'county': county, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes }) with open("20180508__wv__general.csv", "wt") as csvfile: w = csv.writer(csvfile) w.writerow( ['county', 'office', 'district', 'party', 'candidate', 'votes']) for row in results: total_votes = row[ 'Election Day'] # + row['Absentee by Mail'] + row['Advance in Person'] + row['Provisional'] w.writerow([ row['county'], row['office'], row['district'], row['party'], row['candidate'], total_votes ])
def precinct_results(county_name, filename): f = filename + '__' + county_name + '__precinct.csv' p = clarify.Parser() p.parse("detail.xml") results = [] vote_types = [] for result in [x for x in p.results if not 'Number of Precincts' in x.vote_type]: vote_types.append(result.vote_type) if result.choice is None: continue candidate = result.choice.text office, district = parse_office(result.contest.text) party = result.choice.party if '(' in candidate and party is None: if '(I)' in candidate: if '(I)(I)' in candidate: candidate = candidate.split('(I)')[0] party = 'I' else: candidate, party = candidate.split('(I)') else: candidate, party = candidate.split('(', 1) candidate = candidate.strip() party = party.replace(')','').strip() county = p.region if result.jurisdiction: precinct = result.jurisdiction.name else: precinct = None if precinct == None: continue r = [x for x in results if x['county'] == county and x['precinct'] == precinct and x['office'] == office and x['district'] == district and x['party'] == party and x['candidate'] == candidate] if r: r[0][result.vote_type] = result.votes else: results.append({ 'county': county, 'precinct': precinct, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes}) vote_types = list(set(vote_types)) if 'overVotes' in vote_types: vote_types.remove('overVotes') if 'Overvotes' in vote_types: vote_types.remove('Overvotes') if 'underVotes' in vote_types: vote_types.remove('underVotes') if 'Undervotes' in vote_types: vote_types.remove('Undervotes') with open(f, "wt") as csvfile: w = csv.writer(csvfile) headers = ['county', 'precinct', 'office', 'district', 'party', 'candidate', 'votes'] #+ [x.replace(' ','_').lower() for x in vote_types] w.writerow(headers) for row in results: if 'Republican' in row['office']: row['party'] = 'REP' elif 'Democrat' in row['office']: row['party'] = 'DEM' total_votes = sum([row[k] for k in vote_types if k in row]) w.writerow([row['county'], row['precinct'], row['office'], row['district'], row['party'], row['candidate'], total_votes])# + [row[k] for k in vote_types])
def statewide_results(url): j = clarify.Jurisdiction(url=url, level="state") r = requests.get(j.report_url('xml'), stream=True) z = zipfile.ZipFile(StringIO.StringIO(r.content)) z.extractall() p = clarify.Parser() p.parse("detail.xml") results = [] for result in p.results: candidate = result.choice.text office, district = parse_office(result.contest.text) party = parse_party(result.contest.text) if party is None and '(' in candidate: candidate, party = candidate.split('(') candidate = candidate.strip() party = party.replace(')', '').strip() if result.jurisdiction: county = result.jurisdiction.name else: county = None r = [ x for x in results if x['county'] == county and x['office'] == office and x['district'] == district and x['party'] == party and x['candidate'] == candidate ] if r: r[0][result.vote_type] = result.votes else: results.append({ 'county': county, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes }) with open("20121204__ga__general__runoff.csv", "wb") as csvfile: w = unicodecsv.writer(csvfile, encoding='utf-8') w.writerow([ 'county', 'office', 'district', 'party', 'candidate', 'votes', 'election_day', 'absentee', 'early_voting', 'provisional' ]) for row in results: total_votes = row['Election Day'] + row['Absentee by Mail'] + row[ 'Advance in Person'] + row['Provisional'] w.writerow([ row['county'], row['office'], row['district'], row['party'], row['candidate'], total_votes, row['Election Day'], row['Absentee by Mail'], row['Advance in Person'], row['Provisional'] ])
def go(filename): items = [] p = clarify.Parser() p.parse(filename) print("Processing ", filename, p.election_name, p.region) county = p.region for r in p.results: if r.votes > 0: if r.choice is not None: item = buildLine(county, r) if item is not None: items.append(item) return (items)
def extract_data_from_file(filename): items = [] p = clarify.Parser() p.parse(filename) print("Processing ", filename, p.election_name, p.region) county = p.region for r in p.results: if r.votes > 0: if r.choice is not None: # This is called to build a line from results item = buildLine(county, r) if item is not None: items.append(item) return (items)
def extract_data_from_file(filename): "process a file and return list of items" items = [] p = clarify.Parser() p.parse(filename) print( "Processing ", filename, p.election_name, p.region ) county = p.region for r in p.results: if r.votes > 0: if r.choice is not None: item = buildLine(county, r) if item is not None: items.append( item ) return(items)
def clarify_sarpy(): ''' 1. Fetch zipped XML file. 2. Unzip in memory. 3. Load into Clarify. 4. Loop over results, write to file. ''' # discover path to zipfile and fetch s = clarify.Jurisdiction(url=url, level='county') r = requests.get(s.report_url('xml'), stream=True) z = ZipFile(ioDuder(r.content)) # hand off to clarify p = clarify.Parser() p.parse(z.open('detail.xml')) results = [] # According to the Sarpy County election commissioner, results with a # `vote_type` of "underVotes" -- e.g., only 1 vote cast in a "pick 2" # race -- or "overVotes" -- e.g., 3 votes cast in a "pick 2" race -- # are just context information and should not be included in results excluded_vote_types = ['underVotes', 'overVotes'] for result in p.results: if result.vote_type not in excluded_vote_types: candidate = result.choice.text office, district = parse_office(result.contest.text) party = result.choice.party try: precinct = result.jurisdiction.name except AttributeError: precinct = None r = [x for x in results if x['precinct'] == precinct and \ x['office'] == office and x['district'] == district and \ x['party'] == party and x['candidate'] == candidate] if r: r[0][result.vote_type] = result.votes else: record = { 'county': county, 'precinct': precinct, 'office': office, 'district': district, 'party': party, 'candidate': candidate, result.vote_type: result.votes } results.append(record) # build filename election_date = p.election_date.strftime('%Y%m%d') filename = '__'.join([ election_date, 'ne', election_type, county.lower(), 'precinct.csv' ]) # to filter out "total" rows, uncomment the next line # results = [x for x in results if x['precinct']] with open(filename, 'wb') as outfile: f = unicodecsv.writer(outfile, encoding='utf-8') # headers f.writerow(['county', 'precinct', 'office', 'district', 'party', 'candidate', 'votes', 'election_day', 'early_voting', 'provisional']) for row in results: total_votes = row['Early Voting'] + row['Provisionals'] \ + row['Election Day'] f.writerow([row['county'], row['precinct'], row['office'], row['district'], row['party'], row['candidate'], total_votes, row['Election Day'], row['Early Voting'], row['Provisionals']])
def main(): with get_westmoreland_xml_file() as f_in: parser = clarify.Parser() parser.parse(f_in) clarity_to_csv(parser)