def load_2002_file(self, mapping, contest): headers = [ 'office', 'fill', 'jurisdiction', 'last', 'middle', 'first', 'party', 'winner', 'vote_type', 'votes', 'fill2' ] # Store result instances for bulk loading results = [] with open(join(self.cache_dir, mapping['generated_filename']), 'rU') as csvfile: reader = unicodecsv.DictReader(csvfile, fieldnames = headers, delimiter='|', encoding='latin-1') reporting_level = 'county' #TODO: replace jurisidctions with datasource.mappings #jurisdictions = self.jurisdiction_mappings(('ocd','fips','urlname','name')) for row in reader: if row['jurisdiction'].strip() == 'Baltimore City': jurisdiction = [x for x in jurisdictions if x['name'] == "Baltimore City"][0] else: jurisdiction = [x for x in jurisdictions if x['name']+" County" == row['jurisdiction'].strip()][0] if row['winner'] == '1': winner = True else: winner = False if row['last'].strip() == 'zz998': name = row['last'].strip() candidate = Candidate(state=self.state.upper(), family_name=name) write_in=True else: cand_name = self.combine_name_parts([row['first'], row['middle'], row['last']]) name = self.parse_name(cand_name) write_in=False candidate = Candidate(state=self.state.upper(), given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name) result_kwargs = { 'ocd_id': ocd_id + "/precinct:" + str(row['Election District']) + "-" + str(row['Election Precinct']), 'jurisdiction': jurisdiction, 'raw_office': row['Office Name'] + ' ' + row['Office District'], 'reporting_level': reporting_level, 'candidate': candidate, 'party': row['Party'], 'write_in': write_in, 'total_votes': total_votes, 'vote_breakdowns': vote_breakdowns } result = Result(**result_kwargs) #self.load_county_2002(row, jurisdiction['ocd'], jurisdiction['name'], candidate, reporting_level, write_in, winner))) Result.insert(results)
def _get_or_instantiate_result(self, fields): slug = Result.make_slug(election_id=fields['election_id'], contest_slug=fields['contest'].slug, candidate_slug=fields['candidate'].slug, reporting_level=fields['reporting_level'], jurisdiction=fields['jurisdiction']) try: return self._results_cache[slug], False except KeyError: result = Result(**fields) self._results_cache[slug] = result return result, True
def test_get_list_filter_by_level(self): level = 'precinct' # Make sure there's at least one result at this level if Result.objects(reporting_level=level).count() == 0: candidate = Candidate.objects()[0] contest = candidate.contest ResultFactory(candidate=candidate, contest=contest, reporting_level=level) data = self.roller.get_list(state="MD", reporting_level=level) self.assertEqual(len(data), Result.objects(state="MD", reporting_level=level).count())
def test_get_list_filter_by_level(self): level = 'precinct' # Make sure there's at least one result at this level if Result.objects(reporting_level=level).count() == 0: candidate = Candidate.objects()[0] contest = candidate.contest ResultFactory(candidate=candidate, contest=contest, reporting_level=level) data = self.roller.get_list(state="MD", reporting_level=level) self.assertEqual( len(data), Result.objects(state="MD", reporting_level=level).count())
def _prep_non2002_precinct_result(self, row, mapping): kwargs = self._result_kwargs_non2002(row, mapping) vote_breakdowns = { 'election_night_total': int(float(row['Election Night Votes'])) } raw_district = str(row['Election District']) raw_precinct = str(row['Election Precinct']) kwargs.update({ #'ocd_id': mapping['ocd_id'], 'reporting_level': 'precinct', 'raw_jurisdiction': raw_precinct, 'jurisdiction': mapping['name'] + ' ' + raw_district + "-" + raw_precinct, 'party': row['Party'].strip(), 'raw_total_votes': int(float(row['Election Night Votes'])), 'raw_winner': self._non2002_winner(row), 'raw_write_in': self._non2002_writein(row), 'raw_vote_breakdowns': vote_breakdowns, #TODO: Move total votes to transform step? #'total_votes': int(float(row['Election Night Votes'])), #'winner': self._non2002_winner(row), #'write_in': self._non2002_writein(row), }) return Result(**kwargs)
def load_county(self, row, ocd_id, jurisdiction, candidate, reporting_level, write_in, winner): try: total_votes = int(float(row['Total Votes'])) except: print row vote_breakdowns = { 'election_night_total': int(float(row['Election Night Votes'])), 'absentee_total': int(float(row['Absentees Votes'])), 'provisional_total': int(float(row['Provisional Votes'])), 'second_absentee_total': int(float(row['2nd Absentees Votes']))} return Result(ocd_id=ocd_id, jurisdiction=jurisdiction, raw_office=row['Office Name']+' '+row['Office District'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes, vote_breakdowns=vote_breakdowns)
def _prep_non2002_state_leg_results(self, row, mapping): kwargs = self._result_kwargs_non2002(row, mapping) kwargs.update({ 'reporting_level': 'state_legislative', 'raw_winner': self._non2002_winner(row), # at the contest-level 'write_in': self._non2002_writein(row), }) try: kwargs['raw_write_in'] = row['Write-In?'].strip( ), # at the contest-level except KeyError as e: pass self._update_non2002_candidate_parties(row, kwargs['candidate']) results = [] for field, val in row.items(): # Legislative fields prefixed with LEGS if not field.startswith('LEGS'): continue district = field.split()[1].strip() kwargs.update({ 'ocd_id': "ocd-division/country:us/state:md/sldl:%s" % district, 'jurisdiction': district, 'raw_jurisdiction': field, 'raw_total_votes': self._non2002_total_votes(val), }) results.append(Result(**kwargs)) return results
def test_get_list(self): data = self.roller.get_list(state='md', datefilter='20121106', type='general') self.assertNotEqual(len(data), 0) self.assertEqual( len(data), Result.objects( election_id__contains='md-2012-11-06-general').count())
def load_state_legislative(self, row, districts, candidate, reporting_level, write_in, winner): results = [] for district in districts: if row[district].strip() == '': total_votes = 0 else: total_votes = int(float(row[district])) if row['Party'] not in candidate.raw_parties: candidate.raw_parties.append(row['Party']) ocd = "ocd-division/country:us/state:md/sldl:%s" % district.strip().split(' ')[1] results.append(Result(ocd_id=ocd, jurisdiction=district.strip(), raw_office=row['Office Name'].strip()+' '+row['Office District'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes, winner=winner)) return results
def load_county_2002(self, row): total_votes = int(float(row['votes'])) #TODO: replace above 2 lines with below reporting_level = 'precinct' ocd_id = mapping['ocd_id'] total_votes = int(float(row['votes'])) kwargs = { #TODO: generate ocd_id #'ocd_id': ocd_id, #'jurisdiction': jurisdiction, 'reporting_level': reporting_level, 'raw_office': row['office'].strip(), 'candidate': candidate, 'party': row['party'].strip(), 'write_in': write_in, 'raw_total_votes': total_votes, 'vote_breakdowns': {} } return Result(**kwargs)
def __call__(self): results = self._create_results_collection() for rr in self.get_raw_results(): fields = self._get_fields(rr, result_fields) fields['contest'] = self.get_contest(rr) fields['candidate'] = self.get_candidate(rr, extra={ 'contest': fields['contest'], }) fields['contest'] = fields['candidate'].contest fields['raw_result'] = rr party = self.get_party(rr) if party: fields['party'] = party.abbrev #fields['winner'] = self._parse_winner(rr) fields['jurisdiction'] = self._strip_leading_zeros(rr.jurisdiction) fields = self._alter_result_fields(fields, rr) result = Result(**fields) results.append(result) self._create_results(results)
def _prep_non2002_county_result(self, row, mapping): kwargs = self._result_kwargs_non2002(row, mapping) vote_brkdwon_fields = [ ('election_night_total', 'Election Night Votes'), ('absentee_total', 'Absentees Votes'), ('provisional_total', 'Provisional Votes'), ('second_absentee_total', '2nd Absentees Votes'), ] vote_breakdowns = {} for field, key in vote_brkdwon_fields: try: vote_breakdowns[field] = row[key].strip() except KeyError: pass kwargs.update({ 'ocd_id': mapping['ocd_id'], 'reporting_level': 'county', 'jurisdiction': mapping['name'], 'party': row['Party'].strip(), 'raw_total_votes': self._non2002_total_votes(row['Total Votes']), 'raw_winner': self._non2002_winner(row), 'raw_write_in': self._non2002_writein(row), 'raw_vote_breakdowns': vote_breakdowns, #TODO: Move to transforms step? #'total_votes': self._non2002_total_votes(row['Total Votes']), #'winner': self._non2002_winner(row), #'write_in': self._non2002_writein(row), }) return Result(**kwargs)
def load_2002_file(self, mapping): headers = [ 'office', 'district', 'jurisdiction', 'last', 'middle', 'first', 'party', 'winner', 'vote_type', 'votes', 'fill2' ] # Store result instances for bulk loading results = [] with self._file_handle as csvfile: reader = unicodecsv.DictReader(csvfile, fieldnames=headers, delimiter='|', encoding='latin-1') reporting_level = 'county' #TODO: replace jurisidctions with datasource.mappings #geographies = self.jurisdiction_mappings(('ocd','fips','urlname','name')) geographies = self.mappings() for row in reader: if row['jurisdiction'].strip() == 'Baltimore City': geo_obj = [ x for x in geographies if x['name'] == "Baltimore City" ][0] else: geo_obj = [ x for x in geographies if x['name'] + " County" == row['jurisdiction'].strip() ][0] # Winner #TODO: make this raw_winner #TODO: push this to transform step raw_winner = row['winner'] #if row['winner'] == '1': # raw_winner = True #else: # raw_winner = False #TODO: push name parsing to transform step # zz998 is for write-ins if row['last'].strip() == 'zz998': # name = row['last'].strip() # #candidate = Candidate(state=self.state.upper(), family_name=name) write_in = True else: # cand_name = self.combine_name_parts([row['first'], row['middle'], row['last']]) # name = self.parse_name(cand_name) write_in = False # candidate = Candidate(state=self.state.upper(), given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name) #TODO: handle below using the self._get_candidate_whatever(row) #TODO: need a new function/method to handle 2002? cand_kwargs = { 'raw_family_name': row['last'].strip(), 'raw_given_name': row['first'].strip(), 'raw_additional_name': row['middle'].strip(), } #TODO: add raw_party to candidate.raw_parties result_kwargs = { #'election': #'candidate': candidate, 'ocd_id': geo_obj['ocd_id'], #TODO: make this the county number from file 'raw_jurisdiction': row['jurisdiction'], 'jurisdiction': geo_obj['name'], 'raw_office': row['office'].strip(), 'raw_district': row['district'].strip(), 'reporting_level': 'county', 'raw_party': row['party'], # needs to be a member of a list 'write_in': write_in, 'raw_total_votes': row['votes'], } result = Result(**result_kwargs) Result.objects.insert(results)
def test_get_list(self): data = self.roller.get_list(state='md', datefilter='20121106', type='general') self.assertNotEqual(len(data), 0) self.assertEqual(len(data), Result.objects(election_id__contains='md-2012-11-06-general').count())
def load_2002_file(self, mapping, contest): headers = [ "office", "fill", "jurisdiction", "last", "middle", "first", "party", "winner", "vote_type", "votes", "fill2", ] # Store result instances for bulk loading results = [] with open(join(self.cache_dir, mapping["generated_filename"]), "rU") as csvfile: reader = unicodecsv.DictReader(csvfile, fieldnames=headers, delimiter="|", encoding="latin-1") reporting_level = "county" # TODO: replace jurisidctions with datasource.mappings # jurisdictions = self.jurisdiction_mappings(('ocd','fips','urlname','name')) for row in reader: if row["jurisdiction"].strip() == "Baltimore City": jurisdiction = [x for x in jurisdictions if x["name"] == "Baltimore City"][0] else: jurisdiction = [x for x in jurisdictions if x["name"] + " County" == row["jurisdiction"].strip()][0] if row["winner"] == "1": winner = True else: winner = False if row["last"].strip() == "zz998": name = row["last"].strip() candidate = Candidate(state=self.state.upper(), family_name=name) write_in = True else: cand_name = self.combine_name_parts([row["first"], row["middle"], row["last"]]) name = self.parse_name(cand_name) write_in = False candidate = Candidate( state=self.state.upper(), given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name, ) result_kwargs = { "ocd_id": ocd_id + "/precinct:" + str(row["Election District"]) + "-" + str(row["Election Precinct"]), "jurisdiction": jurisdiction, "raw_office": row["Office Name"] + " " + row["Office District"], "reporting_level": reporting_level, "candidate": candidate, "party": row["Party"], "write_in": write_in, "total_votes": total_votes, "vote_breakdowns": vote_breakdowns, } result = Result(**result_kwargs) # self.load_county_2002(row, jurisdiction['ocd'], jurisdiction['name'], candidate, reporting_level, write_in, winner))) Result.insert(results)
def load_2000_primary_file(self, mapping, contest): candidates = {} results = [] with open(join(self.cache_dir, mapping['generated_filename']), 'rU') as csvfile: lines = csvfile.readlines() #TODO: use Datasource.mappings instead of jurisdiction_mappings #import ipdb;ipdb.set_trace() #jurisdictions = self.jurisdiction_mappings(('ocd','fips','urlname','name')) #jurisdictions = self.datasource.mappings() for line in lines[0:377]: if line.strip() == '': continue # skip blank lines else: # determine if this is a row with an office cols = line.strip().split(',') if cols[0][1:29] == 'President and Vice President': office_name = 'President - Vice Pres' if 'Democratic' in cols[0]: party = 'DEM' else: party = 'REP' continue elif cols[0][1:13] == 'U.S. Senator': office_name = 'U.S. Senator' if 'Democratic' in cols[0]: party = 'DEM' else: party = 'REP' continue elif cols[0][1:27] == 'Representative in Congress': district = int(cols[0][71:73]) office_name = 'U.S. Congress ' + str(district) if 'Democratic' in cols[0]: party = 'DEM' else: party = 'REP' continue # skip offices we don't want elif cols[0][1:21] == 'Judge of the Circuit': continue elif cols[0][1:31] == 'Female Delegates and Alternate': continue elif cols[0][1:29] == 'Male Delegates and Alternate': continue elif cols[0][1:28] == 'Delegates to the Republican': continue elif cols[0] == '""': candidates = [x.replace('"','').strip() for x in cols if x.replace('"','') != ''] winner = [i for i, j in enumerate(candidates) if 'Winner' in j][0] # index of winning candidate candidates[winner] = candidates[winner].split(' Winner')[0] # handle name_parse and Uncommitted candidate else: # county results result = [x.replace('"','').strip() for x in cols if x != ''] #juris = [j for j in jurisdictions if j['name'] == result[0].strip()][0] juris = [j for j in self.datasource.mappings() if j['name'] == result[0].strip()][0] cand_results = zip(candidates, result[1:]) for cand, votes in cand_results: name = self.parse_name(cand) #cand_kwargs = { #'given_name': name.first, #'additional_name': name.middle, #'family_name': name.last, #'suffix': name.suffix, #'name': name.full_name #} if office_name == 'President - Vice Pres': cand_kwargs['state'] = 'US' else: cand_kwargs['state'] = self.state.upper() #TODO Get or create candidate candidate = Candidate(**cand_kwargs) result_kwargs = { 'candidate': candidate, 'ocd_id': juris['ocd'], 'jurisdiction': juris['name'], 'raw_office': office_name, 'reporting_level': 'county', 'party': party, 'write_in': False, 'total_votes': int(votes), 'vote_breakdowns': {} } result = Result(**result_kwargs) results.append(result)
def load_precinct(self, row, ocd_id, jurisdiction, candidate, reporting_level, write_in, winner): jurisdiction = jurisdiction+' '+str(row['Election District'])+"-"+str(row['Election Precinct']) total_votes = int(float(row['Election Night Votes'])) vote_breakdowns = { 'election_night_total': int(float(row['Election Night Votes']))} return Result(ocd_id=ocd_id+"/precinct:"+str(row['Election District'])+"-"+str(row['Election Precinct']), jurisdiction=jurisdiction, raw_office=row['Office Name']+' '+row['Office District'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes, vote_breakdowns=vote_breakdowns)
def load_county_2002(self, row, ocd_id, jurisdiction, candidate, reporting_level, write_in, winner): total_votes = int(float(row['votes'])) return Result(ocd_id=ocd_id, jurisdiction=jurisdiction, raw_office=row['office'].strip(), reporting_level=reporting_level, candidate=candidate, party=row['party'].strip(), write_in=write_in, total_votes=total_votes, vote_breakdowns={})