def load_2002_file(self, mapping, contest):
headers = [
'office',
'fill',
'jurisdiction',
'last',
'middle',
'first',
'party',
'winner',
'vote_type',
'votes',
'fill2'
]
# Store result instances for bulk loading
results = []
with open(join(self.cache_dir, mapping['generated_filename']), 'rU') as csvfile:
reader = unicodecsv.DictReader(csvfile, fieldnames = headers, delimiter='|', encoding='latin-1')
reporting_level = 'county'
#TODO: replace jurisidctions with datasource.mappings
#jurisdictions = self.jurisdiction_mappings(('ocd','fips','urlname','name'))
for row in reader:
if row['jurisdiction'].strip() == 'Baltimore City':
jurisdiction = [x for x in jurisdictions if x['name'] == "Baltimore City"][0]
else:
jurisdiction = [x for x in jurisdictions if x['name']+" County" == row['jurisdiction'].strip()][0]
if row['winner'] == '1':
winner = True
else:
winner = False
if row['last'].strip() == 'zz998':
name = row['last'].strip()
candidate = Candidate(state=self.state.upper(), family_name=name)
write_in=True
else:
cand_name = self.combine_name_parts([row['first'], row['middle'], row['last']])
name = self.parse_name(cand_name)
write_in=False
candidate = Candidate(state=self.state.upper(), given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name)
result_kwargs = {
'ocd_id': ocd_id + "/precinct:" + str(row['Election District']) + "-" + str(row['Election Precinct']),
'jurisdiction': jurisdiction,
'raw_office': row['Office Name'] + ' ' + row['Office District'],
'reporting_level': reporting_level,
'candidate': candidate,
'party': row['Party'],
'write_in': write_in,
'total_votes': total_votes,
'vote_breakdowns': vote_breakdowns
}
result = Result(**result_kwargs)
#self.load_county_2002(row, jurisdiction['ocd'], jurisdiction['name'], candidate, reporting_level, write_in, winner)))
Result.insert(results)
def _get_or_instantiate_result(self, fields):
slug = Result.make_slug(election_id=fields['election_id'],
contest_slug=fields['contest'].slug,
candidate_slug=fields['candidate'].slug,
reporting_level=fields['reporting_level'],
jurisdiction=fields['jurisdiction'])
try:
return self._results_cache[slug], False
except KeyError:
result = Result(**fields)
self._results_cache[slug] = result
return result, True
def test_get_list_filter_by_level(self):
level = 'precinct'
# Make sure there's at least one result at this level
if Result.objects(reporting_level=level).count() == 0:
candidate = Candidate.objects()[0]
contest = candidate.contest
ResultFactory(candidate=candidate, contest=contest,
reporting_level=level)
data = self.roller.get_list(state="MD", reporting_level=level)
self.assertEqual(len(data),
Result.objects(state="MD", reporting_level=level).count())
def test_get_list_filter_by_level(self):
level = 'precinct'
# Make sure there's at least one result at this level
if Result.objects(reporting_level=level).count() == 0:
candidate = Candidate.objects()[0]
contest = candidate.contest
ResultFactory(candidate=candidate,
contest=contest,
reporting_level=level)
data = self.roller.get_list(state="MD", reporting_level=level)
self.assertEqual(
len(data),
Result.objects(state="MD", reporting_level=level).count())
def _prep_non2002_precinct_result(self, row, mapping):
kwargs = self._result_kwargs_non2002(row, mapping)
vote_breakdowns = {
'election_night_total': int(float(row['Election Night Votes']))
}
raw_district = str(row['Election District'])
raw_precinct = str(row['Election Precinct'])
kwargs.update({
#'ocd_id': mapping['ocd_id'],
'reporting_level':
'precinct',
'raw_jurisdiction':
raw_precinct,
'jurisdiction':
mapping['name'] + ' ' + raw_district + "-" + raw_precinct,
'party':
row['Party'].strip(),
'raw_total_votes':
int(float(row['Election Night Votes'])),
'raw_winner':
self._non2002_winner(row),
'raw_write_in':
self._non2002_writein(row),
'raw_vote_breakdowns':
vote_breakdowns,
#TODO: Move total votes to transform step?
#'total_votes': int(float(row['Election Night Votes'])),
#'winner': self._non2002_winner(row),
#'write_in': self._non2002_writein(row),
})
return Result(**kwargs)
def load_county(self, row, ocd_id, jurisdiction, candidate, reporting_level, write_in, winner):
try:
total_votes = int(float(row['Total Votes']))
except:
print row
vote_breakdowns = { 'election_night_total': int(float(row['Election Night Votes'])), 'absentee_total': int(float(row['Absentees Votes'])), 'provisional_total': int(float(row['Provisional Votes'])), 'second_absentee_total': int(float(row['2nd Absentees Votes']))}
return Result(ocd_id=ocd_id, jurisdiction=jurisdiction, raw_office=row['Office Name']+' '+row['Office District'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes, vote_breakdowns=vote_breakdowns)
def _prep_non2002_state_leg_results(self, row, mapping):
kwargs = self._result_kwargs_non2002(row, mapping)
kwargs.update({
'reporting_level': 'state_legislative',
'raw_winner': self._non2002_winner(row), # at the contest-level
'write_in': self._non2002_writein(row),
})
try:
kwargs['raw_write_in'] = row['Write-In?'].strip(
), # at the contest-level
except KeyError as e:
pass
self._update_non2002_candidate_parties(row, kwargs['candidate'])
results = []
for field, val in row.items():
# Legislative fields prefixed with LEGS
if not field.startswith('LEGS'):
continue
district = field.split()[1].strip()
kwargs.update({
'ocd_id':
"ocd-division/country:us/state:md/sldl:%s" % district,
'jurisdiction':
district,
'raw_jurisdiction':
field,
'raw_total_votes':
self._non2002_total_votes(val),
})
results.append(Result(**kwargs))
return results
def test_get_list(self):
data = self.roller.get_list(state='md',
datefilter='20121106',
type='general')
self.assertNotEqual(len(data), 0)
self.assertEqual(
len(data),
Result.objects(
election_id__contains='md-2012-11-06-general').count())
def _get_or_instantiate_result(self, fields):
slug = Result.make_slug(election_id=fields['election_id'],
contest_slug=fields['contest'].slug,
candidate_slug=fields['candidate'].slug,
reporting_level=fields['reporting_level'],
jurisdiction=fields['jurisdiction'])
try:
return self._results_cache[slug], False
except KeyError:
result = Result(**fields)
self._results_cache[slug] = result
return result, True
def load_state_legislative(self, row, districts, candidate, reporting_level, write_in, winner):
results = []
for district in districts:
if row[district].strip() == '':
total_votes = 0
else:
total_votes = int(float(row[district]))
if row['Party'] not in candidate.raw_parties:
candidate.raw_parties.append(row['Party'])
ocd = "ocd-division/country:us/state:md/sldl:%s" % district.strip().split(' ')[1]
results.append(Result(ocd_id=ocd, jurisdiction=district.strip(), raw_office=row['Office Name'].strip()+' '+row['Office District'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes, winner=winner))
return results
def load_county_2002(self, row):
total_votes = int(float(row['votes']))
#TODO: replace above 2 lines with below
reporting_level = 'precinct'
ocd_id = mapping['ocd_id']
total_votes = int(float(row['votes']))
kwargs = {
#TODO: generate ocd_id
#'ocd_id': ocd_id,
#'jurisdiction': jurisdiction,
'reporting_level': reporting_level,
'raw_office': row['office'].strip(),
'candidate': candidate,
'party': row['party'].strip(),
'write_in': write_in,
'raw_total_votes': total_votes,
'vote_breakdowns': {}
}
return Result(**kwargs)
def __call__(self):
results = self._create_results_collection()
for rr in self.get_raw_results():
fields = self._get_fields(rr, result_fields)
fields['contest'] = self.get_contest(rr)
fields['candidate'] = self.get_candidate(rr,
extra={
'contest':
fields['contest'],
})
fields['contest'] = fields['candidate'].contest
fields['raw_result'] = rr
party = self.get_party(rr)
if party:
fields['party'] = party.abbrev
#fields['winner'] = self._parse_winner(rr)
fields['jurisdiction'] = self._strip_leading_zeros(rr.jurisdiction)
fields = self._alter_result_fields(fields, rr)
result = Result(**fields)
results.append(result)
self._create_results(results)
def _prep_non2002_county_result(self, row, mapping):
kwargs = self._result_kwargs_non2002(row, mapping)
vote_brkdwon_fields = [
('election_night_total', 'Election Night Votes'),
('absentee_total', 'Absentees Votes'),
('provisional_total', 'Provisional Votes'),
('second_absentee_total', '2nd Absentees Votes'),
]
vote_breakdowns = {}
for field, key in vote_brkdwon_fields:
try:
vote_breakdowns[field] = row[key].strip()
except KeyError:
pass
kwargs.update({
'ocd_id':
mapping['ocd_id'],
'reporting_level':
'county',
'jurisdiction':
mapping['name'],
'party':
row['Party'].strip(),
'raw_total_votes':
self._non2002_total_votes(row['Total Votes']),
'raw_winner':
self._non2002_winner(row),
'raw_write_in':
self._non2002_writein(row),
'raw_vote_breakdowns':
vote_breakdowns,
#TODO: Move to transforms step?
#'total_votes': self._non2002_total_votes(row['Total Votes']),
#'winner': self._non2002_winner(row),
#'write_in': self._non2002_writein(row),
})
return Result(**kwargs)
def load_2002_file(self, mapping):
headers = [
'office', 'district', 'jurisdiction', 'last', 'middle', 'first',
'party', 'winner', 'vote_type', 'votes', 'fill2'
]
# Store result instances for bulk loading
results = []
with self._file_handle as csvfile:
reader = unicodecsv.DictReader(csvfile,
fieldnames=headers,
delimiter='|',
encoding='latin-1')
reporting_level = 'county'
#TODO: replace jurisidctions with datasource.mappings
#geographies = self.jurisdiction_mappings(('ocd','fips','urlname','name'))
geographies = self.mappings()
for row in reader:
if row['jurisdiction'].strip() == 'Baltimore City':
geo_obj = [
x for x in geographies if x['name'] == "Baltimore City"
][0]
else:
geo_obj = [
x for x in geographies if x['name'] +
" County" == row['jurisdiction'].strip()
][0]
# Winner
#TODO: make this raw_winner
#TODO: push this to transform step
raw_winner = row['winner']
#if row['winner'] == '1':
# raw_winner = True
#else:
# raw_winner = False
#TODO: push name parsing to transform step
# zz998 is for write-ins
if row['last'].strip() == 'zz998':
# name = row['last'].strip()
# #candidate = Candidate(state=self.state.upper(), family_name=name)
write_in = True
else:
# cand_name = self.combine_name_parts([row['first'], row['middle'], row['last']])
# name = self.parse_name(cand_name)
write_in = False
# candidate = Candidate(state=self.state.upper(), given_name=name.first, additional_name=name.middle, family_name=name.last, suffix=name.suffix, name=name.full_name)
#TODO: handle below using the self._get_candidate_whatever(row)
#TODO: need a new function/method to handle 2002?
cand_kwargs = {
'raw_family_name': row['last'].strip(),
'raw_given_name': row['first'].strip(),
'raw_additional_name': row['middle'].strip(),
}
#TODO: add raw_party to candidate.raw_parties
result_kwargs = {
#'election':
#'candidate': candidate,
'ocd_id': geo_obj['ocd_id'],
#TODO: make this the county number from file
'raw_jurisdiction': row['jurisdiction'],
'jurisdiction': geo_obj['name'],
'raw_office': row['office'].strip(),
'raw_district': row['district'].strip(),
'reporting_level': 'county',
'raw_party':
row['party'], # needs to be a member of a list
'write_in': write_in,
'raw_total_votes': row['votes'],
}
result = Result(**result_kwargs)
Result.objects.insert(results)
def test_get_list(self):
data = self.roller.get_list(state='md', datefilter='20121106',
type='general')
self.assertNotEqual(len(data), 0)
self.assertEqual(len(data),
Result.objects(election_id__contains='md-2012-11-06-general').count())
def load_2002_file(self, mapping, contest):
headers = [
"office",
"fill",
"jurisdiction",
"last",
"middle",
"first",
"party",
"winner",
"vote_type",
"votes",
"fill2",
]
# Store result instances for bulk loading
results = []
with open(join(self.cache_dir, mapping["generated_filename"]), "rU") as csvfile:
reader = unicodecsv.DictReader(csvfile, fieldnames=headers, delimiter="|", encoding="latin-1")
reporting_level = "county"
# TODO: replace jurisidctions with datasource.mappings
# jurisdictions = self.jurisdiction_mappings(('ocd','fips','urlname','name'))
for row in reader:
if row["jurisdiction"].strip() == "Baltimore City":
jurisdiction = [x for x in jurisdictions if x["name"] == "Baltimore City"][0]
else:
jurisdiction = [x for x in jurisdictions if x["name"] + " County" == row["jurisdiction"].strip()][0]
if row["winner"] == "1":
winner = True
else:
winner = False
if row["last"].strip() == "zz998":
name = row["last"].strip()
candidate = Candidate(state=self.state.upper(), family_name=name)
write_in = True
else:
cand_name = self.combine_name_parts([row["first"], row["middle"], row["last"]])
name = self.parse_name(cand_name)
write_in = False
candidate = Candidate(
state=self.state.upper(),
given_name=name.first,
additional_name=name.middle,
family_name=name.last,
suffix=name.suffix,
name=name.full_name,
)
result_kwargs = {
"ocd_id": ocd_id
+ "/precinct:"
+ str(row["Election District"])
+ "-"
+ str(row["Election Precinct"]),
"jurisdiction": jurisdiction,
"raw_office": row["Office Name"] + " " + row["Office District"],
"reporting_level": reporting_level,
"candidate": candidate,
"party": row["Party"],
"write_in": write_in,
"total_votes": total_votes,
"vote_breakdowns": vote_breakdowns,
}
result = Result(**result_kwargs)
# self.load_county_2002(row, jurisdiction['ocd'], jurisdiction['name'], candidate, reporting_level, write_in, winner)))
Result.insert(results)
def load_2000_primary_file(self, mapping, contest):
candidates = {}
results = []
with open(join(self.cache_dir, mapping['generated_filename']), 'rU') as csvfile:
lines = csvfile.readlines()
#TODO: use Datasource.mappings instead of jurisdiction_mappings
#import ipdb;ipdb.set_trace()
#jurisdictions = self.jurisdiction_mappings(('ocd','fips','urlname','name'))
#jurisdictions = self.datasource.mappings()
for line in lines[0:377]:
if line.strip() == '':
continue # skip blank lines
else: # determine if this is a row with an office
cols = line.strip().split(',')
if cols[0][1:29] == 'President and Vice President':
office_name = 'President - Vice Pres'
if 'Democratic' in cols[0]:
party = 'DEM'
else:
party = 'REP'
continue
elif cols[0][1:13] == 'U.S. Senator':
office_name = 'U.S. Senator'
if 'Democratic' in cols[0]:
party = 'DEM'
else:
party = 'REP'
continue
elif cols[0][1:27] == 'Representative in Congress':
district = int(cols[0][71:73])
office_name = 'U.S. Congress ' + str(district)
if 'Democratic' in cols[0]:
party = 'DEM'
else:
party = 'REP'
continue
# skip offices we don't want
elif cols[0][1:21] == 'Judge of the Circuit':
continue
elif cols[0][1:31] == 'Female Delegates and Alternate':
continue
elif cols[0][1:29] == 'Male Delegates and Alternate':
continue
elif cols[0][1:28] == 'Delegates to the Republican':
continue
elif cols[0] == '""':
candidates = [x.replace('"','').strip() for x in cols if x.replace('"','') != '']
winner = [i for i, j in enumerate(candidates) if 'Winner' in j][0] # index of winning candidate
candidates[winner] = candidates[winner].split(' Winner')[0]
# handle name_parse and Uncommitted candidate
else: # county results
result = [x.replace('"','').strip() for x in cols if x != '']
#juris = [j for j in jurisdictions if j['name'] == result[0].strip()][0]
juris = [j for j in self.datasource.mappings() if j['name'] == result[0].strip()][0]
cand_results = zip(candidates, result[1:])
for cand, votes in cand_results:
name = self.parse_name(cand)
#cand_kwargs = {
#'given_name': name.first,
#'additional_name': name.middle,
#'family_name': name.last,
#'suffix': name.suffix,
#'name': name.full_name
#}
if office_name == 'President - Vice Pres':
cand_kwargs['state'] = 'US'
else:
cand_kwargs['state'] = self.state.upper()
#TODO Get or create candidate
candidate = Candidate(**cand_kwargs)
result_kwargs = {
'candidate': candidate,
'ocd_id': juris['ocd'],
'jurisdiction': juris['name'],
'raw_office': office_name,
'reporting_level': 'county',
'party': party,
'write_in': False,
'total_votes': int(votes),
'vote_breakdowns': {}
}
result = Result(**result_kwargs)
results.append(result)
def load_precinct(self, row, ocd_id, jurisdiction, candidate, reporting_level, write_in, winner):
jurisdiction = jurisdiction+' '+str(row['Election District'])+"-"+str(row['Election Precinct'])
total_votes = int(float(row['Election Night Votes']))
vote_breakdowns = { 'election_night_total': int(float(row['Election Night Votes']))}
return Result(ocd_id=ocd_id+"/precinct:"+str(row['Election District'])+"-"+str(row['Election Precinct']), jurisdiction=jurisdiction, raw_office=row['Office Name']+' '+row['Office District'], reporting_level=reporting_level, candidate=candidate, party=row['Party'], write_in=write_in, total_votes=total_votes, vote_breakdowns=vote_breakdowns)
def load_county_2002(self, row, ocd_id, jurisdiction, candidate, reporting_level, write_in, winner):
total_votes = int(float(row['votes']))
return Result(ocd_id=ocd_id, jurisdiction=jurisdiction, raw_office=row['office'].strip(), reporting_level=reporting_level, candidate=candidate, party=row['party'].strip(), write_in=write_in, total_votes=total_votes, vote_breakdowns={})
