def vote_details(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = list(vote.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None],
vote.created)
# load the role for the VP, since load_roles_at_date only loads
# MoC roles
has_vp_vote = False
for voter in voters:
if voter.voter_type == VoterType.vice_president:
from person.types import RoleType
has_vp_vote = True
try:
voter.person.role = voter.person.roles.get(
role_type=RoleType.vicepresident,
startdate__lte=vote.created,
enddate__gte=vote.created)
except:
raise
pass # wahtever
# Test if we have a diagram for this vote. The only
# way to test is to try to make it.
try:
vote_thumbnail_image(request, congress, session, chamber_code, number,
"diagram")
has_diagram = True
except Http404:
has_diagram = False
# sorting by party actually sorts by party first and by ideology score
# second.
congress = int(congress)
global ideology_scores
load_ideology_scores(congress)
if ideology_scores[congress]:
for voter in voters:
voter.ideolog_score = ideology_scores[congress].get(
voter.person.id if voter.person else 0,
ideology_scores[congress].get(
"MEDIAN:" + (voter.person.role.party if voter.person
and voter.person.role else ""),
ideology_scores[congress]["MEDIAN"]))
# perform an initial sort for display
voters.sort(
key=lambda x: (x.option.key, x.person.role.party if x.person and x.
person.role else "", x.person.name_no_details_lastfirst
if x.person else x.get_voter_type_display()))
return {
'vote': vote,
'voters': voters,
'CongressChamber': CongressChamber,
"VoterType": VoterType,
"VoteCategory": VoteCategory._items,
'has_vp_vote': has_vp_vote,
'has_diagram': has_diagram,
}
def possible_reconsideration_votes(self, voters=None):
# Identify possible voters who voted against their view in order to be on the winning
# side so they may make a motion to reconsider later. Senate only. Since we don't
# know which option represents the winning option, we just look at party leaders who
# voted against their party.
if self.chamber != CongressChamber.senate: return []
# Get vote totals by party.
if voters == None:
voters = list(self.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
by_party = { }
for voter in voters:
if not voter.person or not voter.person.role: continue
if voter.option.key not in ("+", "-"): continue
by_party.setdefault(voter.person.role.party, {}).setdefault(voter.option_id, set()).add(voter)
# Find the plurality option by party.
for party in by_party:
by_party[party] = max(by_party[party], key = lambda x : len(by_party[party][x]))
# See if any party leaders voted against their party.
candidates = []
for voter in voters:
if voter.person and voter.person.role and voter.person.role.leadership_title:
if voter.option.key in ("+", "-") and voter.option_id != by_party[voter.person.role.party]:
candidates.append(voter)
return candidates
def vote_export_csv(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = vote.voters.all().select_related('person', 'option')
load_roles_at_date([x.person for x in voters if x.person], vote.created)
outfile = StringIO()
writer = csv.writer(outfile)
for voter in voters:
writer.writerow([
voter.person.pk if voter.person else "--",
voter.person.role.state
if voter.person and voter.person.role else "--",
voter.person.role.district
if voter.person and voter.person.role else "--",
voter.option.value,
voter.person.name_no_district().encode('utf-8')
if voter.person else voter.get_voter_type_display(),
voter.person.role.party
if voter.person and voter.person.role else "--",
])
output = outfile.getvalue()
firstline = '%s Vote #%d %s - %s\n' % (
vote.get_chamber_display(), vote.number, vote.created.isoformat(),
vote.question) # strftime doesn't work on dates before 1900
firstline = firstline.encode('utf-8')
r = HttpResponse(firstline + output, content_type='text/csv')
r['Content-Disposition'] = 'attachment; filename=' + vote.get_absolute_url(
)[1:].replace("/", "_") + ".csv"
return r
def possible_reconsideration_votes(self, voters=None):
# Identify possible voters who voted against their view in order to be on the winning
# side so they may make a motion to reconsider later. Senate only. Since we don't
# know which option represents the winning option, we just look at party leaders who
# voted against their party.
if self.chamber != CongressChamber.senate: return []
# Get vote totals by party.
if voters == None:
voters = list(self.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None],
vote.created)
by_party = {}
for voter in voters:
if not voter.person or not voter.person.role: continue
if voter.option.key not in ("+", "-"): continue
by_party.setdefault(voter.person.role.party,
{}).setdefault(voter.option_id,
set()).add(voter)
# Find the plurality option by party.
for party in by_party:
by_party[party] = max(by_party[party],
key=lambda x: len(by_party[party][x]))
# See if any party leaders voted against their party.
candidates = []
for voter in voters:
if voter.person and voter.person.role and voter.person.role.leadership_title:
if voter.option.key in (
"+", "-"
) and voter.option_id != by_party[voter.person.role.party]:
candidates.append(voter)
return candidates
def vote_details(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = list(vote.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
# load the role for the VP, since load_roles_at_date only loads
# MoC roles
has_vp_vote = False
for voter in voters:
if voter.voter_type == VoterType.vice_president:
from person.types import RoleType
has_vp_vote = True
try:
voter.person.role = voter.person.roles.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created)
except:
raise
pass # wahtever
# Test if we have a diagram for this vote. The only
# way to test is to try to make it.
try:
vote_thumbnail_image(request, congress, session, chamber_code, number, "diagram")
has_diagram = True
except Http404:
has_diagram = False
# sorting by party actually sorts by party first and by ideology score
# second.
congress = int(congress)
global ideology_scores
load_ideology_scores(congress)
if ideology_scores[congress]:
for voter in voters:
voter.ideolog_score = ideology_scores[congress].get(
voter.person.id if voter.person else 0,
ideology_scores[congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""),
ideology_scores[congress]["MEDIAN"]))
# perform an initial sort for display
voters.sort(key = lambda x : (x.option.key, x.person.role.party if x.person and x.person.role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display()))
# did any Senate leaders switch their vote for a motion to reconsider?
reconsiderers = vote.possible_reconsideration_votes(voters)
reconsiderers_titles = "/".join(v.person.role.leadership_title for v in reconsiderers)
# compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
get_vote_outliers(voters)
return {'vote': vote,
'voters': voters,
'CongressChamber': CongressChamber,
"VoterType": VoterType,
"VoteCategory": VoteCategory._items,
'has_vp_vote': has_vp_vote,
'has_diagram': has_diagram,
'reconsiderers': (reconsiderers, reconsiderers_titles),
}
def simple_record(self):
# load people and their roles
all_voters = list(self.voters.all().select_related('person', 'option'))
persons = [x.person for x in all_voters if x.person != None]
load_roles_at_date(persons, self.created)
return [
{ "vote": v.option.value, "moc": v.person.role.simple_record() }
for v in all_voters
if v.voter_type_is_member and v.person is not None and v.person.role is not None
]
def simple_record(self):
# load people and their roles
all_voters = list(self.voters.all().select_related('person', 'option'))
persons = [x.person for x in all_voters if x.person != None]
load_roles_at_date(persons, self.created)
return [{
"vote": v.option.value,
"moc": v.person.role.simple_record()
} for v in all_voters if v.voter_type_is_member
and v.person is not None and v.person.role is not None]
def vote_export_csv(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = vote.voters.all().select_related('person', 'option')
load_roles_at_date([x.person for x in voters if x.person], vote.created)
outfile = StringIO()
writer = csv.writer(outfile)
for voter in voters:
writer.writerow([
voter.person.pk if voter.person else "--",
voter.person.role.state if voter.person and voter.person.role else "--",
voter.person.role.district if voter.person and voter.person.role else "--",
voter.option.value,
voter.person.name_no_district().encode('utf-8') if voter.person else voter.get_voter_type_display(),
voter.person.role.party if voter.person and voter.person.role else "--",])
output = outfile.getvalue()
firstline = '%s Vote #%d %s - %s\n' % (vote.get_chamber_display(), vote.number,
vote.created.isoformat(), vote.question) # strftime doesn't work on dates before 1900
firstline = firstline.encode('utf-8')
r = HttpResponse(firstline + output, content_type='text/csv')
r['Content-Disposition'] = 'attachment; filename=' + vote.get_absolute_url()[1:].replace("/", "_") + ".csv"
return r
def vote_details(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = list(vote.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
# load the role for the VP, since load_roles_at_date only loads
# MoC roles
has_vp_vote = False
for voter in voters:
if voter.voter_type == VoterType.vice_president:
from person.types import RoleType
has_vp_vote = True
try:
voter.person.role = voter.person.roles.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created)
except:
raise
pass # wahtever
# sorting by party actually sorts by party first and by ideology score
# second.
congress = int(congress)
global ideology_scores
load_ideology_scores(congress)
if ideology_scores[congress]:
for voter in voters:
voter.ideolog_score = ideology_scores[congress].get(
voter.person.id if voter.person else 0,
ideology_scores[congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""),
ideology_scores[congress]["MEDIAN"]))
voters.sort(key = lambda x : (x.option.key, x.person.role.party if x.person and x.person.role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display()))
return {'vote': vote,
'voters': voters,
'CongressChamber': CongressChamber,
"VoterType": VoterType,
"VoteCategory": VoteCategory._items,
'has_vp_vote': has_vp_vote,
}
def totals(self):
# If cached value exists then return it
if hasattr(self, '_cached_totals'):
return self._cached_totals
# else do all these things:
items = []
# Extract all voters, find their role at the time
# the vote was
all_voters = list(self.voters.all().select_related('person', 'option'))
voters_by_option = {}
for option in self.options.all():
voters_by_option[option] = [x for x in all_voters if x.option == option]
total_count = len(all_voters)
persons = [x.person for x in all_voters if x.person != None]
load_roles_at_date(persons, self.created)
# Find all parties which participated in vote
# and sort them in order which they should be displayed
def cmp_party(x):
"""
Sort the parties by the number of voters in that party.
"""
return -len([p for p in all_voters if p.person and p.person.role and p.person.role.party == x])
def get_party(voter):
if voter.voter_type != VoterType.vice_president:
if voter.person and voter.person.role:
return voter.person.role.party
else:
return "Unknown"
else:
return "Vice President"
all_parties = list(set(get_party(x) for x in all_voters))
all_parties.sort(key=cmp_party)
total_party_stats = dict((x, {'yes': 0, 'no': 0, 'other': 0, 'total': 0})\
for x in all_parties)
# For each option find party break down,
# total vote count and percentage in total count
details = []
for option in self.options.all():
voters = voters_by_option.get(option, [])
percent = round(len(voters) / float(total_count) * 100.0)
party_stats = dict((x, 0) for x in all_parties)
for voter in voters:
party = get_party(voter)
party_stats[party] += 1
total_party_stats[party]['total'] += 1
if option.key == '+':
total_party_stats[party]['yes'] += 1
elif option.key == '-':
total_party_stats[party]['no'] += 1
else:
total_party_stats[party]['other'] += 1
party_counts = [party_stats.get(x, 0) for x in all_parties]
party_counts = [{"party": all_parties[i], "count": c, 'chart_width': 190 * c / total_count} for i, c in enumerate(party_counts)]
detail = {'option': option, 'count': len(voters),
'percent': int(percent), 'party_counts': party_counts,
'chart_width': 190 * int(percent) / 100}
if option.key == '+':
detail['yes'] = True
if option.key == '-':
detail['no'] = True
if option.key in ('0', 'P'):
detail['hide_if_empty'] = True
details.append(detail)
party_counts = [total_party_stats[x] for x in all_parties]
# sort options by well-known keys, then by total number of votes
option_sort_order = {"+":0, "-":1, "P":2, "0":3}
details.sort(key = lambda d : (option_sort_order.get(d['option'].key, None), -d['count']))
# hide Present/Not Voting if no one voted that way
details = [d for d in details if d["count"] > 0 or "hide_if_empty" not in d]
totals = {'options': details, 'total_count': total_count,
'party_counts': party_counts, 'parties': all_parties,
}
self._cached_totals = totals
return totals
def main(options):
"""
Parse rolls.
"""
# Setup XML processors
vote_processor = VoteProcessor()
option_processor = VoteOptionProcessor()
voter_processor = VoterProcessor()
voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all())
# The pattern which the roll file matches
# Filename contains info which should be placed to DB
# along with info extracted from the XML file
re_path = re.compile("data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml")
chamber_mapping = {"s": CongressChamber.senate, "h": CongressChamber.house}
if options.filter:
files = glob.glob(options.filter)
log.info("Parsing rolls matching %s" % options.filter)
elif options.congress:
files = glob.glob("data/us/%s/rolls/*.xml" % options.congress)
log.info("Parsing rolls of only congress#%s" % options.congress)
else:
files = glob.glob("data/us/*/rolls/*.xml")
log.info("Processing votes: %d files" % len(files))
total = len(files)
progress = Progress(total=total, name="files", step=10)
def log_delete_qs(qs):
if qs.count() == 0:
return
print "Deleting obsoleted records: ", qs
# if qs.count() > 3:
# print "Delete skipped..."
# return
qs.delete()
seen_obj_ids = set()
had_error = False
for fname in files:
progress.tick()
match = re_path.search(fname)
try:
existing_vote = Vote.objects.get(
congress=match.group(1),
chamber=chamber_mapping[match.group(2)],
session=match.group(3),
number=match.group(4),
)
except Vote.DoesNotExist:
existing_vote = None
if (
not File.objects.is_changed(fname)
and not options.force
and existing_vote != None
and not existing_vote.missing_data
):
seen_obj_ids.add(existing_vote.id)
continue
try:
tree = etree.parse(fname)
## Look for votes with VP tie breakers.
# if len(tree.xpath("/roll/voter[@VP='1']")) == 0:
# had_error = True # prevent delete at the end
# continue
# Process role object
for roll_node in tree.xpath("/roll"):
vote = vote_processor.process(Vote(), roll_node)
if existing_vote:
vote.id = existing_vote.id
match = re_path.search(fname)
vote.congress = int(match.group(1))
vote.chamber = chamber_mapping[match.group(2)]
vote.session = match.group(3)
vote.number = int(match.group(4))
# Get related bill & amendment.
for bill_node in roll_node.xpath("bill"):
related_bill_num = bill_node.get("number")
if 9 <= vote.congress <= 42 and vote.session in ("1", "2"):
# Bill numbering from the American Memory colletion is different. The number combines
# the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to
# the 9th congress numbering seems to be wholly assigned by us and not related to
# actual numbering, so we skip matching those bills.
related_bill_num = "%d%04d%d" % (int(vote.session), int(bill_node.get("number")), 0)
try:
vote.related_bill = Bill.objects.get(
congress=bill_node.get("session"),
bill_type=BillType.by_xml_code(bill_node.get("type")),
number=related_bill_num,
)
except Bill.DoesNotExist:
if vote.congress >= 93:
vote.missing_data = True
for amdt_node in roll_node.xpath("amendment"):
if amdt_node.get("ref") == "regular" and vote.related_bill is not None:
try:
vote.related_amendment = Amendment.objects.get(
congress=vote.related_bill.congress,
amendment_type=AmendmentType.by_slug(amdt_node.get("number")[0]),
number=amdt_node.get("number")[1:],
)
except Amendment.DoesNotExist:
if vote.congress >= 93:
print "Missing amendment", fname
vote.missing_data = True
elif amdt_node.get("ref") == "bill-serial":
# It is impossible to associate House votes with amendments just from the House
# vote XML because the amendment-num might correspond either with the A___ number
# or with the "An amendment, numbered ____" number from the amendment purpose,
# and there's really no way to figure out which. Maybe we can use the amendment
# sponsor instead?
# vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number"))
# Instead, we set related_amendment from the amendment parser. Here, we have to
# preserve the related_amendment if it is set.
if existing_vote:
vote.related_amendment = existing_vote.related_amendment
# clean up some question text and use the question_details field
if (
vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override)
and vote.related_bill
):
# For passage votes, set the question to the bill title and put the question
# details in the details field.
vote.question = truncatewords(vote.related_bill.title, 20)
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display()
elif vote.category == VoteCategory.amendment and vote.related_amendment:
# For votes on amendments, make a better title/explanation.
vote.question = truncatewords(vote.related_amendment.title, 20)
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display()
elif vote.related_bill and vote.question.startswith(
"On the Cloture Motion " + vote.related_bill.display_number
):
vote.question = "Cloture on " + truncatewords(vote.related_bill.title, 20)
elif vote.related_bill and vote.question.startswith(
"On Cloture on the Motion to Proceed " + vote.related_bill.display_number
):
vote.question = "Cloture on " + truncatewords(vote.related_bill.title, 20)
vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display()
elif vote.related_bill and vote.question.startswith(
"On the Motion to Proceed " + vote.related_bill.display_number
):
vote.question = "Motion to Proceed on " + truncatewords(vote.related_bill.title, 20)
elif vote.related_amendment and vote.question.startswith(
"On the Cloture Motion "
+ vote.related_amendment.get_amendment_type_display()
+ " "
+ str(vote.related_amendment.number)
):
vote.question = "Cloture on " + truncatewords(vote.related_amendment.title, 20)
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display()
# weird House foratting of bill numbers ("H RES 123 Blah blah")
if vote.related_bill:
vote.question = re.sub(
"(On [^:]+): "
+ vote.related_bill.display_number.replace(". ", " ").replace(".", " ").upper()
+ " .*",
r"\1: " + truncatewords(vote.related_bill.title, 15),
vote.question,
)
vote.save()
seen_obj_ids.add(vote.id) # don't delete me later
# Process roll options, overwrite existing options where possible.
seen_option_ids = set()
roll_options = {}
for option_node in roll_node.xpath("./option"):
option = option_processor.process(VoteOption(), option_node)
option.vote = vote
if existing_vote:
try:
option.id = VoteOption.objects.filter(vote=vote, key=option.key)[
0
].id # get is better, but I had the database corruption problem
except IndexError:
pass
option.save()
roll_options[option.key] = option
seen_option_ids.add(option.id)
log_delete_qs(
VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids)
) # may cascade and delete the Voters too?
# Process roll voters, overwriting existing voters where possible.
if existing_vote:
existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id"))
seen_voter_ids = set()
voters = list()
for voter_node in roll_node.xpath("./voter"):
voter = voter_processor.process(roll_options, Voter(), voter_node)
voter.vote = vote
voter.created = vote.created
# for VP votes, load the actual person...
if voter.voter_type == VoterType.vice_president:
try:
r = PersonRole.objects.get(
role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created
)
voter.person_role = r
voter.person = r.person
except:
# overlapping roles? missing data?
log.error("Could not resolve vice president in %s" % fname, exc_info=ex)
if existing_vote and voter.person:
try:
voter.id = existing_voters[voter.person.id]
except KeyError:
pass
voters.append(voter)
if voter.voter_type == VoterType.unknown and not vote.missing_data:
vote.missing_data = True
vote.save()
# pre-fetch the role of each voter
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
for voter in voters:
voter.person_role = voter.person.role
if voter.person_role is None:
log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created))
vote.missing_data = True
vote.save()
# save all of the records (inserting/updating)
for voter in voters:
voter.save()
seen_voter_ids.add(voter.id)
# remove obsolete voter records
log_delete_qs(
Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids)
) # possibly already deleted by cascade above
# pre-calculate totals
vote.calculate_totals()
if not options.disable_events:
vote.create_event()
File.objects.save_file(fname)
except Exception, ex:
log.error("Error in processing %s" % fname, exc_info=ex)
had_error = True
#!script
import csv, sys
from django.conf import settings
from vote.models import *
from person.util import load_roles_at_date
w = csv.writer(sys.stdout)
w.writerow([
"congress", "session", "number", "date", "question", "link", "person", "vote"
])
for vote in Vote.objects.filter(congress=113, chamber=CongressChamber.house).order_by('created'):
voters = list(vote.voters.all().select_related('person', 'option'))
voters = sorted(voters, key = lambda v : v.person.sortname)
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
for voter in voters:
if voter.person.role.state != "AZ": continue
row = [
vote.congress,
vote.session,
vote.number,
vote.created.strftime("%x %X"),
vote.question,
settings.SITE_ROOT_URL + vote.get_absolute_url(),
voter.person.sortname,
voter.option.value,
]
w.writerow([unicode(v).encode("utf8") for v in row])
total_has_companion = 0
total_companion_bipartisan = 0
for bill in Bill.objects.filter(congress=congress, bill_type__in=(BillType.house_bill, BillType.senate_bill)).select_related("sponsor"):
#.filter(introduced_date__gte=date(2011, 8, 1)):
total += 1
# People we care about. Batch load party information for sponsors
# and cosponsors to be fast. Load roles at the bill introduced date.
# Only look at cosponsors who joined on the introduced date (otherwise
# they may have changed party between those two dates).
persons = []
if not bill.sponsor: continue
persons.append(bill.sponsor)
persons.extend([c.person for c in Cosponsor.objects.filter(bill=bill, joined=bill.introduced_date).select_related("person")])
load_roles_at_date(persons, bill.introduced_date)
if len(persons) > 1: has_initial_cosponsors += 1
# How bipartisan_cosp is this bill?
parties = set()
for p in persons:
if p.role:
parties.add(p.role.party)
if "Democrat" in parties and "Republican" in parties:
bipartisan_cosp += 1
if bill.bill_type == BillType.senate_bill:
related_bill = None
try:
relation = bill.relatedbills.filter(relation='identical', related_bill__bill_type=BillType.house_bill).select_related("related_bill", "related_bill__sponsor").get()
def totals(self):
# If cached value exists then return it
if hasattr(self, '_cached_totals'):
return self._cached_totals
# else do all these things:
items = []
# Extract all voters, find their role at the time
# the vote was
all_voters = list(self.voters.all().select_related('person', 'option'))
voters_by_option = {}
for option in self.options.all():
voters_by_option[option] = [
x for x in all_voters if x.option == option
]
total_count = len(all_voters)
persons = [x.person for x in all_voters if x.person != None]
load_roles_at_date(persons, self.created)
# Find all parties which participated in vote
# and sort them in order which they should be displayed
def cmp_party(x):
"""
Sort the parties by the number of voters in that party.
"""
return -len([
p for p in all_voters
if p.person and p.person.role and p.person.role.party == x
])
def get_party(voter):
if voter.voter_type != VoterType.vice_president:
if voter.person and voter.person.role:
return voter.person.role.party
else:
return "Unknown"
else:
return "Vice President"
all_parties = list(set(get_party(x) for x in all_voters))
all_parties.sort(key=cmp_party)
total_party_stats = dict((x, {'yes': 0, 'no': 0, 'other': 0, 'total': 0})\
for x in all_parties)
# For each option find party break down,
# total vote count and percentage in total count
details = []
for option in self.options.all():
voters = voters_by_option.get(option, [])
percent = round(len(voters) / float(total_count) * 100.0)
party_stats = dict((x, 0) for x in all_parties)
for voter in voters:
party = get_party(voter)
party_stats[party] += 1
total_party_stats[party]['total'] += 1
if option.key == '+':
total_party_stats[party]['yes'] += 1
elif option.key == '-':
total_party_stats[party]['no'] += 1
else:
total_party_stats[party]['other'] += 1
party_counts = [party_stats.get(x, 0) for x in all_parties]
party_counts = [{
"party": all_parties[i],
"count": c,
'chart_width': 190 * c / total_count
} for i, c in enumerate(party_counts)]
detail = {
'option': option,
'count': len(voters),
'percent': int(percent),
'party_counts': party_counts,
'chart_width': 190 * int(percent) / 100
}
if option.key == '+':
detail['yes'] = True
if option.key == '-':
detail['no'] = True
if option.key in ('0', 'P'):
detail['hide_if_empty'] = True
details.append(detail)
party_counts = [total_party_stats[x] for x in all_parties]
# sort options by well-known keys, then by total number of votes
option_sort_order = {"+": 0, "-": 1, "P": 2, "0": 3}
details.sort(key=lambda d: (option_sort_order.get(
d['option'].key, None), -d['count']))
# hide Present/Not Voting if no one voted that way
details = [
d for d in details if d["count"] > 0 or "hide_if_empty" not in d
]
totals = {
'options': details,
'total_count': total_count,
'party_counts': party_counts,
'parties': all_parties,
}
self._cached_totals = totals
return totals
def main(options):
"""
Parse rolls.
"""
# Setup XML processors
vote_processor = VoteProcessor()
option_processor = VoteOptionProcessor()
voter_processor = VoterProcessor()
voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all())
chamber_mapping = {'s': CongressChamber.senate,
'h': CongressChamber.house}
if options.filter:
files = glob.glob(options.filter)
log.info('Parsing rolls matching %s' % options.filter)
elif options.congress:
files = glob.glob(settings.CONGRESS_DATA_PATH + '/%s/votes/*/*/data.xml' % options.congress)
log.info('Parsing rolls of only congress#%s' % options.congress)
else:
files = glob.glob('data/congress/*/votes/*/*/data.xml')
log.info('Processing votes: %d files' % len(files))
total = len(files)
progress = Progress(total=total, name='files', step=10)
def log_delete_qs(qs):
if qs.count() == 0: return
print("Deleting obsoleted records: ", qs)
#if qs.count() > 3:
# print "Delete skipped..."
# return
qs.delete()
seen_obj_ids = set()
had_error = False
for fname in files:
progress.tick()
match = re.match(r"data/congress/(?P<congress>\d+)/votes/(?P<session>[ABC0-9]+)/(?P<chamber>[hs])(?P<number>\d+)/data.xml$", fname)
try:
existing_vote = Vote.objects.get(congress=int(match.group("congress")), chamber=chamber_mapping[match.group("chamber")], session=match.group("session"), number=int(match.group("number")))
except Vote.DoesNotExist:
existing_vote = None
if not File.objects.is_changed(fname) and not options.force and existing_vote != None and not existing_vote.missing_data:
seen_obj_ids.add(existing_vote.id)
continue
try:
tree = etree.parse(fname)
## Look for votes with VP tie breakers.
#if len(tree.xpath("/roll/voter[@VP='1']")) == 0:
# had_error = True # prevent delete at the end
# continue
# Process role object
roll_node = tree.xpath('/roll')[0]
# Sqlite is much faster when lots of saves are wrapped in a transaction,
# and we do a lot of saves because it's a lot of voters.
from django.db import transaction
with transaction.atomic():
vote = vote_processor.process(Vote(), roll_node)
if existing_vote: vote.id = existing_vote.id
vote.congress = int(match.group("congress"))
vote.chamber = chamber_mapping[match.group("chamber")]
vote.session = match.group("session")
vote.number = int(match.group("number"))
# Get related bill & amendment.
for bill_node in roll_node.xpath("bill"):
related_bill_num = bill_node.get("number")
if 9 <= vote.congress <= 42 and vote.session in ('1', '2'):
# Bill numbering from the American Memory colletion is different. The number combines
# the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to
# the 9th congress numbering seems to be wholly assigned by us and not related to
# actual numbering, so we skip matching those bills.
related_bill_num = "%d%04d%d" % (int(vote.session), int(bill_node.get("number")), 0)
try:
vote.related_bill = Bill.objects.get(congress=bill_node.get("session"), bill_type=BillType.by_xml_code(bill_node.get("type")), number=related_bill_num)
except Bill.DoesNotExist:
if vote.congress >= 93:
vote.missing_data = True
for amdt_node in roll_node.xpath("amendment"):
if amdt_node.get("ref") == "regular" and vote.related_bill is not None:
try:
vote.related_amendment = Amendment.objects.get(congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug(amdt_node.get("number")[0]+"amdt"), number=amdt_node.get("number")[1:])
except Amendment.DoesNotExist:
if vote.congress >= 93:
print("Missing amendment", fname)
vote.missing_data = True
elif amdt_node.get("ref") == "bill-serial":
# It is impossible to associate House votes with amendments just from the House
# vote XML because the amendment-num might correspond either with the A___ number
# or with the "An amendment, numbered ____" number from the amendment purpose,
# and there's really no way to figure out which. Maybe we can use the amendment
# sponsor instead?
#vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number"))
# Instead, we set related_amendment from the amendment parser. Here, we have to
# preserve the related_amendment if it is set.
if existing_vote: vote.related_amendment = existing_vote.related_amendment
# clean up some question text and use the question_details field
if vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill:
# For passage votes, set the question to the bill title and put the question
# details in the details field.
vote.question = vote.related_bill.title
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display()
elif vote.category == VoteCategory.amendment and vote.related_amendment:
# For votes on amendments, make a better title/explanation.
vote.question = vote.related_amendment.title
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display()
elif vote.related_bill and vote.question.startswith("On the Cloture Motion " + vote.related_bill.display_number):
vote.question = "Cloture on " + vote.related_bill.title
elif vote.related_bill and vote.question.startswith("On Cloture on the Motion to Proceed " + vote.related_bill.display_number):
vote.question = "Cloture on " + vote.related_bill.title
vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display()
elif vote.related_bill and vote.question.startswith("On the Motion to Proceed " + vote.related_bill.display_number):
vote.question = "Motion to Proceed on " + vote.related_bill.title
elif vote.related_amendment and vote.question.startswith("On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number)):
vote.question = "Cloture on " + vote.related_amendment.title
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display()
# weird House foratting of bill numbers ("H RES 123 Blah blah")
if vote.related_bill:
vote.question = re.sub(
"(On [^:]+): " + vote.related_bill.display_number.replace(". ", " ").replace(".", " ").upper() + " .*",
r"\1: " + truncatewords(vote.related_bill.title, 15),
vote.question)
vote.save()
seen_obj_ids.add(vote.id) # don't delete me later
# Process roll options, overwrite existing options where possible.
seen_option_ids = set()
roll_options = {}
for option_node in roll_node.xpath('./option'):
option = option_processor.process(VoteOption(), option_node)
option.vote = vote
if existing_vote:
try:
option.id = VoteOption.objects.filter(vote=vote, key=option.key)[0].id # get is better, but I had the database corruption problem
except IndexError:
pass
option.save()
roll_options[option.key] = option
seen_option_ids.add(option.id)
log_delete_qs(VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids)) # may cascade and delete the Voters too?
# Process roll voters, overwriting existing voters where possible.
if existing_vote:
existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id"))
seen_voter_ids = set()
voters = list()
for voter_node in roll_node.xpath('./voter'):
voter = voter_processor.process(roll_options, Voter(), voter_node)
voter.vote = vote
voter.created = vote.created
# for VP votes, load the actual person & role...
if voter.voter_type == VoterType.vice_president:
try:
r = PersonRole.objects.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created)
voter.person_role = r
voter.person = r.person
except PersonRole.DoesNotExist:
# overlapping roles? missing data?
log.error('Could not resolve vice president in %s' % fname)
if existing_vote and voter.person:
try:
voter.id = existing_voters[voter.person.id]
except KeyError:
pass
voters.append(voter)
if voter.voter_type == VoterType.unknown and not vote.missing_data:
vote.missing_data = True
vote.save()
# pre-fetch the role of each voter
load_roles_at_date([x.person for x in voters if x.person != None], vote.created, vote.congress)
for voter in list(voters):
if voter.voter_type != VoterType.vice_president:
voter.person_role = voter.person.role
# If we couldn't match a role for this person on the date of the vote, and if the voter was Not Voting,
# and we're looking at historical data, then this is probably a data error --- the voter wasn't even in office.
# At the start of each Congress, the House does a Call by States and Election of the Speaker, before swearing
# in. In the 116th Congress, these votes had a Not Voting for Walter Jones who had not yet made it to DC, and
# then omitted Jones in the votes after swearing in. In those cases, look for a role coming up.
if voter.person_role is None and voter.option.key == "0" and vote.question in ("Call by States", "Election of the Speaker"):
voter.person_role = voter.person.roles.filter(startdate__gt=vote.created, startdate__lt=vote.created+timedelta(days=30)).first()
if voter.person_role is None:
if vote.source == VoteSource.keithpoole and voter.option.key == "0":
# Drop this record.
voters.remove(voter)
else:
log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created))
vote.missing_data = True
vote.save()
# save all of the records (inserting/updating)
for voter in voters:
voter.save()
seen_voter_ids.add(voter.id)
# remove obsolete voter records
log_delete_qs(Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids)) # possibly already deleted by cascade above
# pre-calculate totals
vote.calculate_totals()
if not options.disable_events:
vote.create_event()
File.objects.save_file(fname)
except Exception as ex:
log.error('Error in processing %s' % fname, exc_info=ex)
had_error = True
# delete vote objects that are no longer represented on disk
if options.congress and not options.filter and not had_error:
log_delete_qs(Vote.objects.filter(congress=options.congress).exclude(id__in = seen_obj_ids))
def main(options):
"""
Parse rolls.
"""
# Setup XML processors
vote_processor = VoteProcessor()
option_processor = VoteOptionProcessor()
voter_processor = VoterProcessor()
voter_processor.PERSON_CACHE = dict(
(x.pk, x) for x in Person.objects.all())
# The pattern which the roll file matches
# Filename contains info which should be placed to DB
# along with info extracted from the XML file
re_path = re.compile('data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml')
chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house}
if options.filter:
files = glob.glob(options.filter)
log.info('Parsing rolls matching %s' % options.filter)
elif options.congress:
files = glob.glob('data/us/%s/rolls/*.xml' % options.congress)
log.info('Parsing rolls of only congress#%s' % options.congress)
else:
files = glob.glob('data/us/*/rolls/*.xml')
log.info('Processing votes: %d files' % len(files))
total = len(files)
progress = Progress(total=total, name='files', step=10)
def log_delete_qs(qs):
if qs.count() == 0: return
print "Deleting obsoleted records: ", qs
#if qs.count() > 3:
# print "Delete skipped..."
# return
qs.delete()
seen_obj_ids = set()
had_error = False
for fname in files:
progress.tick()
match = re_path.search(fname)
try:
existing_vote = Vote.objects.get(
congress=match.group(1),
chamber=chamber_mapping[match.group(2)],
session=match.group(3),
number=match.group(4))
except Vote.DoesNotExist:
existing_vote = None
if not File.objects.is_changed(
fname
) and not options.force and existing_vote != None and not existing_vote.missing_data:
seen_obj_ids.add(existing_vote.id)
continue
try:
tree = etree.parse(fname)
## Look for votes with VP tie breakers.
#if len(tree.xpath("/roll/voter[@VP='1']")) == 0:
# had_error = True # prevent delete at the end
# continue
# Process role object
roll_node = tree.xpath('/roll')[0]
# Sqlite is much faster when lots of saves are wrapped in a transaction,
# and we do a lot of saves because it's a lot of voters.
from django.db import transaction
with transaction.atomic():
vote = vote_processor.process(Vote(), roll_node)
if existing_vote: vote.id = existing_vote.id
match = re_path.search(fname)
vote.congress = int(match.group(1))
vote.chamber = chamber_mapping[match.group(2)]
vote.session = match.group(3)
vote.number = int(match.group(4))
# Get related bill & amendment.
for bill_node in roll_node.xpath("bill"):
related_bill_num = bill_node.get("number")
if 9 <= vote.congress <= 42 and vote.session in ('1', '2'):
# Bill numbering from the American Memory colletion is different. The number combines
# the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to
# the 9th congress numbering seems to be wholly assigned by us and not related to
# actual numbering, so we skip matching those bills.
related_bill_num = "%d%04d%d" % (int(
vote.session), int(bill_node.get("number")), 0)
try:
vote.related_bill = Bill.objects.get(
congress=bill_node.get("session"),
bill_type=BillType.by_xml_code(
bill_node.get("type")),
number=related_bill_num)
except Bill.DoesNotExist:
if vote.congress >= 93:
vote.missing_data = True
for amdt_node in roll_node.xpath("amendment"):
if amdt_node.get(
"ref"
) == "regular" and vote.related_bill is not None:
try:
vote.related_amendment = Amendment.objects.get(
congress=vote.related_bill.congress,
amendment_type=AmendmentType.by_slug(
amdt_node.get("number")[0]),
number=amdt_node.get("number")[1:])
except Amendment.DoesNotExist:
if vote.congress >= 93:
print "Missing amendment", fname
vote.missing_data = True
elif amdt_node.get("ref") == "bill-serial":
# It is impossible to associate House votes with amendments just from the House
# vote XML because the amendment-num might correspond either with the A___ number
# or with the "An amendment, numbered ____" number from the amendment purpose,
# and there's really no way to figure out which. Maybe we can use the amendment
# sponsor instead?
#vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number"))
# Instead, we set related_amendment from the amendment parser. Here, we have to
# preserve the related_amendment if it is set.
if existing_vote:
vote.related_amendment = existing_vote.related_amendment
# clean up some question text and use the question_details field
if vote.category in (
VoteCategory.passage, VoteCategory.passage_suspension,
VoteCategory.veto_override) and vote.related_bill:
# For passage votes, set the question to the bill title and put the question
# details in the details field.
vote.question = vote.related_bill.title
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display(
)
elif vote.category == VoteCategory.amendment and vote.related_amendment:
# For votes on amendments, make a better title/explanation.
vote.question = vote.related_amendment.title
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display(
)
elif vote.related_bill and vote.question.startswith(
"On the Cloture Motion " +
vote.related_bill.display_number):
vote.question = "Cloture on " + vote.related_bill.title
elif vote.related_bill and vote.question.startswith(
"On Cloture on the Motion to Proceed " +
vote.related_bill.display_number):
vote.question = "Cloture on " + vote.related_bill.title
vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display(
)
elif vote.related_bill and vote.question.startswith(
"On the Motion to Proceed " +
vote.related_bill.display_number):
vote.question = "Motion to Proceed on " + vote.related_bill.title
elif vote.related_amendment and vote.question.startswith(
"On the Cloture Motion " +
vote.related_amendment.get_amendment_type_display() +
" " + str(vote.related_amendment.number)):
vote.question = "Cloture on " + vote.related_amendment.title
vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display(
)
# weird House foratting of bill numbers ("H RES 123 Blah blah")
if vote.related_bill:
vote.question = re.sub(
"(On [^:]+): " +
vote.related_bill.display_number.replace(
". ", " ").replace(".", " ").upper() + " .*",
r"\1: " + truncatewords(vote.related_bill.title, 15),
vote.question)
vote.save()
seen_obj_ids.add(vote.id) # don't delete me later
# Process roll options, overwrite existing options where possible.
seen_option_ids = set()
roll_options = {}
for option_node in roll_node.xpath('./option'):
option = option_processor.process(VoteOption(),
option_node)
option.vote = vote
if existing_vote:
try:
option.id = VoteOption.objects.filter(
vote=vote, key=option.key
)[0].id # get is better, but I had the database corruption problem
except IndexError:
pass
option.save()
roll_options[option.key] = option
seen_option_ids.add(option.id)
log_delete_qs(
VoteOption.objects.filter(vote=vote).exclude(
id__in=seen_option_ids)
) # may cascade and delete the Voters too?
# Process roll voters, overwriting existing voters where possible.
if existing_vote:
existing_voters = dict(
Voter.objects.filter(vote=vote).values_list(
"person", "id"))
seen_voter_ids = set()
voters = list()
for voter_node in roll_node.xpath('./voter'):
voter = voter_processor.process(roll_options, Voter(),
voter_node)
voter.vote = vote
voter.created = vote.created
# for VP votes, load the actual person & role...
if voter.voter_type == VoterType.vice_president:
try:
r = PersonRole.objects.get(
role_type=RoleType.vicepresident,
startdate__lte=vote.created,
enddate__gte=vote.created)
voter.person_role = r
voter.person = r.person
except PersonRole.DoesNotExist:
# overlapping roles? missing data?
log.error(
'Could not resolve vice president in %s' %
fname)
if existing_vote and voter.person:
try:
voter.id = existing_voters[voter.person.id]
except KeyError:
pass
voters.append(voter)
if voter.voter_type == VoterType.unknown and not vote.missing_data:
vote.missing_data = True
vote.save()
# pre-fetch the role of each voter
load_roles_at_date(
[x.person for x in voters if x.person != None],
vote.created, vote.congress)
for voter in list(voters):
if voter.voter_type != VoterType.vice_president:
voter.person_role = voter.person.role
# If we couldn't match a role for this person on the date of the vote, and if the voter was Not Voting,
# and we're looking at historical data, then this is probably a data error --- the voter wasn't even in office.
if voter.person_role is None:
if vote.source == VoteSource.keithpoole and voter.option.key == "0":
# Drop this record.
voters.remove(voter)
else:
log.error("%s: Could not find role for %s on %s." %
(fname, voter.person, vote.created))
vote.missing_data = True
vote.save()
# save all of the records (inserting/updating)
for voter in voters:
voter.save()
seen_voter_ids.add(voter.id)
# remove obsolete voter records
log_delete_qs(
Voter.objects.filter(vote=vote).exclude(
id__in=seen_voter_ids)
) # possibly already deleted by cascade above
# pre-calculate totals
vote.calculate_totals()
if not options.disable_events:
vote.create_event()
File.objects.save_file(fname)
except Exception, ex:
log.error('Error in processing %s' % fname, exc_info=ex)
had_error = True
def vote_thumbnail_image(request, congress, session, chamber_code, number, image_type):
vote = load_vote(congress, session, chamber_code, number)
import cairo, re, math
from StringIO import StringIO
# general image properties
font_face = "DejaVu Serif Condensed"
image_width = 300
image_height = 250 if image_type == "thumbnail" else 170
# format text to print on the image
vote_title = re.sub(r"^On the Motion to ", "To ", vote.question)
if re.match(r"Cloture .*Rejected", vote.result):
vote_result_2 = "Filibustered"
elif re.match(r"Cloture .*Agreed to", vote.result):
vote_result_2 = "Proceed"
else:
vote_result_2 = re.sub("^(Bill|Amendment|Resolution|Conference Report|Nomination|Motion|Motion to \S+) ", "", vote.result)
if vote_result_2 == "unknown": vote_result_2 = ""
vote_date = vote.created.strftime("%x") if vote.created.year > 1900 else vote.created.isoformat().split("T")[0]
vote_citation = vote.get_chamber_display() + " Vote #" + str(vote.number) + " -- " + vote_date
# get vote totals by option and by party
totals = vote.totals()
total_count = 0
total_counts = { } # key: total ({ "+": 123 }, etc.)
yea_counts_by_party = [0,0,0] # D, I, R (+ votes totals)
nay_counts_by_party = [0,0,0] # D, I, R (+/- votes totals)
nonvoting_members_totals = [0,0,0] # D, I, R
party_index = { "Democrat": 0, "Republican": 2 }
for opt in totals["options"]:
total_counts[opt["option"].key] = opt["count"]
for i in xrange(len(totals["parties"])):
j = party_index.get(totals["parties"][i], 1)
if opt["option"].key not in ("+", "-"):
# most votes are by proportion of those voting (not some cloture etc.),
# so put present/not-voting tallies in a separate group
nonvoting_members_totals[j] += opt["party_counts"][i]["count"]
continue
total_count += opt["party_counts"][i]["count"]
if opt["option"].key == "+":
yea_counts_by_party[j] += opt["party_counts"][i]["count"]
else:
nay_counts_by_party[j] += opt["party_counts"][i]["count"]
if total_count == 0 or "+" not in total_counts or "-" not in total_counts: raise Http404() # no thumbnail for other sorts of votes
vote_result_1 = "%d-%d" % (total_counts["+"], total_counts["-"])
def show_text_centered(ctx, text, max_width=None):
while True:
(x_bearing, y_bearing, width, height, x_advance, y_advance) = ctx.text_extents(text)
if max_width is not None and width > max_width:
text2 = re.sub(r" \S+(\.\.\.)?$", "...", text)
if text2 != text:
text = text2
continue
break
ctx.rel_move_to(-width/2, height)
ctx.show_text(text)
im = cairo.ImageSurface(cairo.FORMAT_ARGB32, image_width, image_height)
ctx = cairo.Context(im)
ctx.select_font_face(font_face, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
# clear background
ctx.set_source_rgb(1,1,1)
ctx.new_path()
ctx.line_to(0, 0)
ctx.line_to(image_width, 0)
ctx.line_to(image_width, image_height)
ctx.line_to(0, image_height)
ctx.fill()
chart_top = 0
if image_type == "thumbnail":
# Title
ctx.set_font_size(20)
ctx.set_source_rgb(.2,.2,.2)
ctx.move_to(150,10)
show_text_centered(ctx, vote_title, max_width=.95*image_width)
chart_top = 50
# Vote Tally
font_size = 26 if len(vote_result_2) < 10 else 22
ctx.set_font_size(font_size)
ctx.set_source_rgb(.1, .1, .1)
ctx.move_to(150,chart_top)
show_text_centered(ctx, vote_result_1)
# Vote Result
ctx.move_to(150,chart_top+12+font_size)
show_text_centered(ctx, vote_result_2)
w = max(ctx.text_extents(vote_result_1)[2], ctx.text_extents(vote_result_2)[2])
# Line
ctx.set_line_width(1)
ctx.new_path()
ctx.line_to(150-w/2, chart_top+5+font_size)
ctx.rel_line_to(w, 0)
ctx.stroke()
if image_type == "thumbnail":
# Vote Chamber/Date/Number
ctx.select_font_face(font_face, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
ctx.set_font_size(14)
ctx.move_to(150,image_height-25)
show_text_centered(ctx, vote_citation, max_width=.98*image_width)
# Seats
# Construct an array of rows of seats, where each entry maps to a particular
# voter.
# How many rows of seats? That is hard coded by chamber.
seating_rows = 8 if vote.chamber == CongressChamber.house else 4
# 4 for Senate (http://www.senate.gov/artandhistory/art/special/Desks/chambermap.cfm)
# about 8 for the House
# Long ago Congress had very few people.
seating_rows = min(total_count / 8 + 1, seating_rows)
# Determine the seating chart dimensions: the radius of the inside row of
# seats and the radius of the outside row of seats.
inner_r = w/2 * 1.25 + 5 # wrap closely around the text in the middle
if seating_rows <= 4: inner_r = max(inner_r, 75) # don't make the inner radius too small
outer_r = image_width * .45 # end close to the image width
# If we assume the absolute spacing of seats is constant from row to row, then
# the number of seats per row grows linearly with the radius, following the
# circumference. If s0 is the number of seats on the inner row, then
# floor(s0 * outer_r/inner_r) is the number of seats on the outer row. The total
# number of seats is found by the sum of the arithmetic sequence (n/2 * (a_1+a_n)):
# n = (seating_rows/2)*(s0 + s0*outer_r/inner_r)
# We want exactly total_count seats, so solving for s0...
seats_per_row = 2.0 * total_count / (seating_rows*(1.0 + outer_r/inner_r))
# How wide to draw a seat?
seat_size = min(.8 * (outer_r-inner_r) / seating_rows, .35 * (2*3.14159*inner_r) / seats_per_row)
# Determine how many seats on each row.
seats_so_far = 0
rowcounts = []
for row in xrange(seating_rows):
# What's the radius of this row?
if seating_rows > 1:
r = inner_r + (outer_r-inner_r) * row / float(seating_rows-1)
else:
r = inner_r
# How many seats should we put on this row?
if row < seating_rows-1:
# Start with seats_per_row on the inner row and grow linearly.
# Round to an integer. Alternate rounding down and up.
n_seats = seats_per_row * r/inner_r
n_seats = int(math.floor(n_seats) if (row % 2 == 0) else math.ceil(n_seats))
else:
# On the outermost row, just put in how many left we need
# so we always have exactly the right number of seats.
n_seats = total_count - seats_so_far
rowcounts.append(n_seats)
seats_so_far += n_seats
# Make a list of all of the seats as a list of tuples of the
# form (rownum, seatnum) where seatnum is an index from the
# left side.
seats = []
for row, count in enumerate(rowcounts):
for i in xrange(count):
seats.append( (row, i) )
# Sort the seats in the order we will fill them from left to right,
# bottom to top.
seats.sort(key = lambda seat : (seat[1]/float(rowcounts[seat[0]]), -seat[0]) )
# We can draw in two modes. In one mode, we don't care which actual
# person corresponds to which seat. We just draw the groups of voters
# in blocks. Or we can draw the actual people in seats we assign
# according to their ideology score, from left to right.
# See if we have ideology scores.
voter_details = None
if True:
global ideology_scores
load_ideology_scores(vote.congress)
if ideology_scores[vote.congress]:
voter_details = [ ]
# Load the voters, getting their role at the time they voted.
voters = list(vote.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
# Store ideology scores
for voter in voters:
if voter.option.key not in ("+", "-"): continue
party = party_index.get(voter.person.role.party if voter.person and voter.person.role else "Unknown", 1)
option = 0 if voter.option.key == "+" else 1
coord = ideology_scores[vote.congress].get(voter.person.id if voter.person else "UNKNOWN",
ideology_scores[vote.congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""),
ideology_scores[vote.congress]["MEDIAN"]))
voter_details.append( (coord, (party, option)) )
# Sort voters by party, then by ideology score, then by vote.
voter_details.sort(key = lambda x : (x[1][0], x[0], x[1][1]))
if len(voter_details) != len(seats):
raise ValueError("Gotta check this.")
voter_details = None # abort
if not voter_details:
# Just assign voters to seats in blocks.
#
# We're fill the seats from left to right different groups of voters:
# D+, D-, I+, I-, R-, R+
# For each group, for each voter in the group, pop off a seat and
# draw him in that seat.
# for each group of voters...
seat_idx = 0
for (party, vote) in [ (0, 0), (0, 1), (1, 0), (1, 1), (2, 1), (2, 0) ]:
# how many votes in this group?
n_voters = (yea_counts_by_party if vote == 0 else nay_counts_by_party)[party]
# for each voter...
for i in xrange(n_voters):
seats[seat_idx] = (seats[seat_idx], (party, vote))
seat_idx += 1
else:
# Assign voters to seats in order.
for i in xrange(len(voter_details)):
seats[i] = (seats[i], voter_details[i][1])
# Draw the seats.
group_colors = {
(0, 0): (0.05, 0.24, 0.63), # D+
(0, 1): (0.85, 0.85, 1.0), # D-
(1, 0): (0.07, 0.05, 0.07), # I+
(1, 1): (0.85, 0.85, 0.85), # I-
(2, 0): (0.90, 0.05, 0.07), # R+
(2, 1): (1.0, 0.85, 0.85), # R-
}
for ((row, seat_pos), (party, vote)) in seats:
# radius of this row (again, code dup)
if seating_rows > 1:
r = inner_r + (outer_r-inner_r) * row / float(seating_rows-1)
else:
r = inner_r
# draw
ctx.set_source_rgb(*group_colors[(party, vote)])
ctx.identity_matrix()
ctx.translate(image_width/2, chart_top+25)
ctx.rotate(3.14159 - 3.14159 * seat_pos/float(rowcounts[row]-1))
ctx.translate(r, 0)
ctx.rectangle(-seat_size/2, -seat_size/2, seat_size, seat_size)
ctx.fill()
# Convert the image buffer to raw PNG bytes.
buf = StringIO()
im.write_to_png(buf)
v = buf.getvalue()
# Form the response.
r = HttpResponse(v, content_type='image/png')
r["Content-Length"] = len(v)
return r
def vote_thumbnail_image(request, congress, session, chamber_code, number, image_type):
vote = load_vote(congress, session, chamber_code, number)
import cairo, re, math
from StringIO import StringIO
# general image properties
font_face = "DejaVu Serif Condensed"
image_width = 300
image_height = 250 if image_type == "thumbnail" else 170
# format text to print on the image
vote_title = re.sub(r"^On the Motion to ", "To ", vote.question)
if re.match(r"Cloture .*Rejected", vote.result):
vote_result_2 = "Filibustered"
elif re.match(r"Cloture .*Agreed to", vote.result):
vote_result_2 = "Proceed"
else:
vote_result_2 = re.sub("^(Bill|Amendment|Resolution|Conference Report|Nomination|Motion|Motion to \S+) ", "", vote.result)
if vote_result_2 == "unknown": vote_result_2 = ""
vote_date = vote.created.strftime("%x") if vote.created.year > 1900 else vote.created.isoformat().split("T")[0]
vote_citation = vote.get_chamber_display() + " Vote #" + str(vote.number) + " -- " + vote_date
# get vote totals by option and by party
totals = vote.totals()
total_count = 0
total_counts = { } # key: total ({ "+": 123 }, etc.)
yea_counts_by_party = [0,0,0] # D, I, R (+ votes totals)
nay_counts_by_party = [0,0,0] # D, I, R (+/- votes totals)
nonvoting_members_totals = [0,0,0] # D, I, R
party_index = { "Democrat": 0, "Republican": 2 }
for opt in totals["options"]:
total_counts[opt["option"].key] = opt["count"]
for i in xrange(len(totals["parties"])):
j = party_index.get(totals["parties"][i], 1)
if opt["option"].key not in ("+", "-"):
# most votes are by proportion of those voting (not some cloture etc.),
# so put present/not-voting tallies in a separate group
nonvoting_members_totals[j] += opt["party_counts"][i]["count"]
continue
total_count += opt["party_counts"][i]["count"]
if opt["option"].key == "+":
yea_counts_by_party[j] += opt["party_counts"][i]["count"]
else:
nay_counts_by_party[j] += opt["party_counts"][i]["count"]
if total_count == 0 or "+" not in total_counts or "-" not in total_counts: raise Http404() # no thumbnail for other sorts of votes
vote_result_1 = "%d-%d" % (total_counts["+"], total_counts["-"])
def show_text_centered(ctx, text, max_width=None):
while True:
(x_bearing, y_bearing, width, height, x_advance, y_advance) = ctx.text_extents(text)
if max_width is not None and width > max_width:
text2 = re.sub(r" \S+(\.\.\.)?$", "...", text)
if text2 != text:
text = text2
continue
break
ctx.rel_move_to(-width/2, height)
ctx.show_text(text)
im = cairo.ImageSurface(cairo.FORMAT_ARGB32, image_width, image_height)
ctx = cairo.Context(im)
ctx.select_font_face(font_face, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD)
# clear background
ctx.set_source_rgb(1,1,1)
ctx.new_path()
ctx.line_to(0, 0)
ctx.line_to(image_width, 0)
ctx.line_to(image_width, image_height)
ctx.line_to(0, image_height)
ctx.fill()
chart_top = 0
if image_type == "thumbnail":
# Title
ctx.set_font_size(20)
ctx.set_source_rgb(.2,.2,.2)
ctx.move_to(150,10)
show_text_centered(ctx, vote_title, max_width=.95*image_width)
chart_top = 50
# Vote Tally
font_size = 26 if len(vote_result_2) < 10 else 22
ctx.set_font_size(font_size)
ctx.set_source_rgb(.1, .1, .1)
ctx.move_to(150,chart_top)
show_text_centered(ctx, vote_result_1)
# Vote Result
ctx.move_to(150,chart_top+12+font_size)
show_text_centered(ctx, vote_result_2)
w = max(ctx.text_extents(vote_result_1)[2], ctx.text_extents(vote_result_2)[2])
# Line
ctx.set_line_width(1)
ctx.new_path()
ctx.line_to(150-w/2, chart_top+5+font_size)
ctx.rel_line_to(w, 0)
ctx.stroke()
if image_type == "thumbnail":
# Vote Chamber/Date/Number
ctx.select_font_face(font_face, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
ctx.set_font_size(14)
ctx.move_to(150,image_height-25)
show_text_centered(ctx, vote_citation, max_width=.98*image_width)
# Seats
# Construct an array of rows of seats, where each entry maps to a particular
# voter.
# How many rows of seats? That is hard coded by chamber.
seating_rows = 8 if vote.chamber == CongressChamber.house else 4
# 4 for Senate (http://www.senate.gov/artandhistory/art/special/Desks/chambermap.cfm)
# about 8 for the House
# Long ago Congress had very few people.
seating_rows = min(total_count / 8 + 1, seating_rows)
# Determine the seating chart dimensions: the radius of the inside row of
# seats and the radius of the outside row of seats.
inner_r = w/2 * 1.25 + 5 # wrap closely around the text in the middle
if seating_rows <= 4: inner_r = max(inner_r, 75) # don't make the inner radius too small
outer_r = image_width * .45 # end close to the image width
# If we assume the absolute spacing of seats is constant from row to row, then
# the number of seats per row grows linearly with the radius, following the
# circumference. If s0 is the number of seats on the inner row, then
# floor(s0 * outer_r/inner_r) is the number of seats on the outer row. The total
# number of seats is found by the sum of the arithmetic sequence (n/2 * (a_1+a_n)):
# n = (seating_rows/2)*(s0 + s0*outer_r/inner_r)
# We want exactly total_count seats, so solving for s0...
seats_per_row = 2.0 * total_count / (seating_rows*(1.0 + outer_r/inner_r))
# How wide to draw a seat?
seat_size = min(.8 * (outer_r-inner_r) / seating_rows, .35 * (2*3.14159*inner_r) / seats_per_row)
# Determine how many seats on each row.
seats_so_far = 0
rowcounts = []
for row in xrange(seating_rows):
# What's the radius of this row?
if seating_rows > 1:
r = inner_r + (outer_r-inner_r) * row / float(seating_rows-1)
else:
r = inner_r
# How many seats should we put on this row?
if row < seating_rows-1:
# Start with seats_per_row on the inner row and grow linearly.
# Round to an integer. Alternate rounding down and up.
n_seats = seats_per_row * r/inner_r
n_seats = int(math.floor(n_seats) if (row % 2 == 0) else math.ceil(n_seats))
else:
# On the outermost row, just put in how many left we need
# so we always have exactly the right number of seats.
n_seats = total_count - seats_so_far
rowcounts.append(n_seats)
seats_so_far += n_seats
# Make a list of all of the seats as a list of tuples of the
# form (rownum, seatnum) where seatnum is an index from the
# left side.
seats = []
for row, count in enumerate(rowcounts):
for i in xrange(count):
seats.append( (row, i) )
# Sort the seats in the order we will fill them from left to right,
# bottom to top.
seats.sort(key = lambda seat : (seat[1]/float(rowcounts[seat[0]]), -seat[0]) )
# We can draw in two modes. In one mode, we don't care which actual
# person corresponds to which seat. We just draw the groups of voters
# in blocks. Or we can draw the actual people in seats we assign
# according to their ideology score, from left to right.
# See if we have ideology scores.
voter_details = None
if True:
global ideology_scores
load_ideology_scores(vote.congress)
if ideology_scores[vote.congress]:
voter_details = [ ]
# Load the voters, getting their role at the time they voted.
voters = list(vote.voters.all().select_related('person', 'option'))
load_roles_at_date([x.person for x in voters if x.person != None], vote.created)
# Store ideology scores
for voter in voters:
if voter.option.key not in ("+", "-"): continue
party = party_index.get(voter.person.role.party if voter.person and voter.person.role else "Unknown", 1)
option = 0 if voter.option.key == "+" else 1
coord = ideology_scores[vote.congress].get(voter.person.id if voter.person else "UNKNOWN",
ideology_scores[vote.congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""),
ideology_scores[vote.congress]["MEDIAN"]))
voter_details.append( (coord, (party, option)) )
# Sort voters by party, then by ideology score, then by vote.
voter_details.sort(key = lambda x : (x[1][0], x[0], x[1][1]))
if len(voter_details) != len(seats):
raise ValueError("Gotta check this.")
voter_details = None # abort
if not voter_details:
# Just assign voters to seats in blocks.
#
# We're fill the seats from left to right different groups of voters:
# D+, D-, I+, I-, R-, R+
# For each group, for each voter in the group, pop off a seat and
# draw him in that seat.
# for each group of voters...
seat_idx = 0
for (party, vote) in [ (0, 0), (0, 1), (1, 0), (1, 1), (2, 1), (2, 0) ]:
# how many votes in this group?
n_voters = (yea_counts_by_party if vote == 0 else nay_counts_by_party)[party]
# for each voter...
for i in xrange(n_voters):
seats[seat_idx] = (seats[seat_idx], (party, vote))
seat_idx += 1
else:
# Assign voters to seats in order.
for i in xrange(len(voter_details)):
seats[i] = (seats[i], voter_details[i][1])
# Draw the seats.
group_colors = {
(0, 0): (0.05, 0.24, 0.63), # D+
(0, 1): (0.85, 0.85, 1.0), # D-
(1, 0): (0.07, 0.05, 0.07), # I+
(1, 1): (0.85, 0.85, 0.85), # I-
(2, 0): (0.90, 0.05, 0.07), # R+
(2, 1): (1.0, 0.85, 0.85), # R-
}
for ((row, seat_pos), (party, vote)) in seats:
# radius of this row (again, code dup)
if seating_rows > 1:
r = inner_r + (outer_r-inner_r) * row / float(seating_rows-1)
else:
r = inner_r
# draw
ctx.set_source_rgb(*group_colors[(party, vote)])
ctx.identity_matrix()
ctx.translate(image_width/2, chart_top+25)
ctx.rotate(3.14159 - 3.14159 * seat_pos/float(rowcounts[row]-1))
ctx.translate(r, 0)
ctx.rectangle(-seat_size/2, -seat_size/2, seat_size, seat_size)
ctx.fill()
# Convert the image buffer to raw PNG bytes.
buf = StringIO()
im.write_to_png(buf)
v = buf.getvalue()
# Form the response.
r = HttpResponse(v, content_type='image/png')
r["Content-Length"] = len(v)
return r
#.filter(introduced_date__gte=date(2011, 8, 1)):
total += 1
# People we care about. Batch load party information for sponsors
# and cosponsors to be fast. Load roles at the bill introduced date.
# Only look at cosponsors who joined on the introduced date (otherwise
# they may have changed party between those two dates).
persons = []
if not bill.sponsor: continue
persons.append(bill.sponsor)
persons.extend([
c.person for c in Cosponsor.objects.filter(
bill=bill, joined=bill.introduced_date).select_related(
"person")
])
load_roles_at_date(persons, bill.introduced_date)
if len(persons) > 1: has_initial_cosponsors += 1
# How bipartisan_cosp is this bill?
parties = set()
for p in persons:
if p.role:
parties.add(p.role.party)
if "Democrat" in parties and "Republican" in parties:
bipartisan_cosp += 1
if bill.bill_type == BillType.senate_bill:
related_bill = None
try:
relation = bill.relatedbills.filter(
