def vote_details(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = list(vote.voters.all().select_related('person', 'option')) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) # load the role for the VP, since load_roles_at_date only loads # MoC roles has_vp_vote = False for voter in voters: if voter.voter_type == VoterType.vice_president: from person.types import RoleType has_vp_vote = True try: voter.person.role = voter.person.roles.get( role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) except: raise pass # wahtever # Test if we have a diagram for this vote. The only # way to test is to try to make it. try: vote_thumbnail_image(request, congress, session, chamber_code, number, "diagram") has_diagram = True except Http404: has_diagram = False # sorting by party actually sorts by party first and by ideology score # second. congress = int(congress) global ideology_scores load_ideology_scores(congress) if ideology_scores[congress]: for voter in voters: voter.ideolog_score = ideology_scores[congress].get( voter.person.id if voter.person else 0, ideology_scores[congress].get( "MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""), ideology_scores[congress]["MEDIAN"])) # perform an initial sort for display voters.sort( key=lambda x: (x.option.key, x.person.role.party if x.person and x. person.role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display())) return { 'vote': vote, 'voters': voters, 'CongressChamber': CongressChamber, "VoterType": VoterType, "VoteCategory": VoteCategory._items, 'has_vp_vote': has_vp_vote, 'has_diagram': has_diagram, }
def possible_reconsideration_votes(self, voters=None): # Identify possible voters who voted against their view in order to be on the winning # side so they may make a motion to reconsider later. Senate only. Since we don't # know which option represents the winning option, we just look at party leaders who # voted against their party. if self.chamber != CongressChamber.senate: return [] # Get vote totals by party. if voters == None: voters = list(self.voters.all().select_related('person', 'option')) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) by_party = { } for voter in voters: if not voter.person or not voter.person.role: continue if voter.option.key not in ("+", "-"): continue by_party.setdefault(voter.person.role.party, {}).setdefault(voter.option_id, set()).add(voter) # Find the plurality option by party. for party in by_party: by_party[party] = max(by_party[party], key = lambda x : len(by_party[party][x])) # See if any party leaders voted against their party. candidates = [] for voter in voters: if voter.person and voter.person.role and voter.person.role.leadership_title: if voter.option.key in ("+", "-") and voter.option_id != by_party[voter.person.role.party]: candidates.append(voter) return candidates
def vote_export_csv(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = vote.voters.all().select_related('person', 'option') load_roles_at_date([x.person for x in voters if x.person], vote.created) outfile = StringIO() writer = csv.writer(outfile) for voter in voters: writer.writerow([ voter.person.pk if voter.person else "--", voter.person.role.state if voter.person and voter.person.role else "--", voter.person.role.district if voter.person and voter.person.role else "--", voter.option.value, voter.person.name_no_district().encode('utf-8') if voter.person else voter.get_voter_type_display(), voter.person.role.party if voter.person and voter.person.role else "--", ]) output = outfile.getvalue() firstline = '%s Vote #%d %s - %s\n' % ( vote.get_chamber_display(), vote.number, vote.created.isoformat(), vote.question) # strftime doesn't work on dates before 1900 firstline = firstline.encode('utf-8') r = HttpResponse(firstline + output, content_type='text/csv') r['Content-Disposition'] = 'attachment; filename=' + vote.get_absolute_url( )[1:].replace("/", "_") + ".csv" return r
def possible_reconsideration_votes(self, voters=None): # Identify possible voters who voted against their view in order to be on the winning # side so they may make a motion to reconsider later. Senate only. Since we don't # know which option represents the winning option, we just look at party leaders who # voted against their party. if self.chamber != CongressChamber.senate: return [] # Get vote totals by party. if voters == None: voters = list(self.voters.all().select_related('person', 'option')) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) by_party = {} for voter in voters: if not voter.person or not voter.person.role: continue if voter.option.key not in ("+", "-"): continue by_party.setdefault(voter.person.role.party, {}).setdefault(voter.option_id, set()).add(voter) # Find the plurality option by party. for party in by_party: by_party[party] = max(by_party[party], key=lambda x: len(by_party[party][x])) # See if any party leaders voted against their party. candidates = [] for voter in voters: if voter.person and voter.person.role and voter.person.role.leadership_title: if voter.option.key in ( "+", "-" ) and voter.option_id != by_party[voter.person.role.party]: candidates.append(voter) return candidates
def vote_details(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = list(vote.voters.all().select_related('person', 'option')) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) # load the role for the VP, since load_roles_at_date only loads # MoC roles has_vp_vote = False for voter in voters: if voter.voter_type == VoterType.vice_president: from person.types import RoleType has_vp_vote = True try: voter.person.role = voter.person.roles.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) except: raise pass # wahtever # Test if we have a diagram for this vote. The only # way to test is to try to make it. try: vote_thumbnail_image(request, congress, session, chamber_code, number, "diagram") has_diagram = True except Http404: has_diagram = False # sorting by party actually sorts by party first and by ideology score # second. congress = int(congress) global ideology_scores load_ideology_scores(congress) if ideology_scores[congress]: for voter in voters: voter.ideolog_score = ideology_scores[congress].get( voter.person.id if voter.person else 0, ideology_scores[congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""), ideology_scores[congress]["MEDIAN"])) # perform an initial sort for display voters.sort(key = lambda x : (x.option.key, x.person.role.party if x.person and x.person.role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display())) # did any Senate leaders switch their vote for a motion to reconsider? reconsiderers = vote.possible_reconsideration_votes(voters) reconsiderers_titles = "/".join(v.person.role.leadership_title for v in reconsiderers) # compute statistical outliers (this marks the Voter instances with an is_outlier attribute) get_vote_outliers(voters) return {'vote': vote, 'voters': voters, 'CongressChamber': CongressChamber, "VoterType": VoterType, "VoteCategory": VoteCategory._items, 'has_vp_vote': has_vp_vote, 'has_diagram': has_diagram, 'reconsiderers': (reconsiderers, reconsiderers_titles), }
def simple_record(self): # load people and their roles all_voters = list(self.voters.all().select_related('person', 'option')) persons = [x.person for x in all_voters if x.person != None] load_roles_at_date(persons, self.created) return [ { "vote": v.option.value, "moc": v.person.role.simple_record() } for v in all_voters if v.voter_type_is_member and v.person is not None and v.person.role is not None ]
def simple_record(self): # load people and their roles all_voters = list(self.voters.all().select_related('person', 'option')) persons = [x.person for x in all_voters if x.person != None] load_roles_at_date(persons, self.created) return [{ "vote": v.option.value, "moc": v.person.role.simple_record() } for v in all_voters if v.voter_type_is_member and v.person is not None and v.person.role is not None]
def vote_export_csv(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = vote.voters.all().select_related('person', 'option') load_roles_at_date([x.person for x in voters if x.person], vote.created) outfile = StringIO() writer = csv.writer(outfile) for voter in voters: writer.writerow([ voter.person.pk if voter.person else "--", voter.person.role.state if voter.person and voter.person.role else "--", voter.person.role.district if voter.person and voter.person.role else "--", voter.option.value, voter.person.name_no_district().encode('utf-8') if voter.person else voter.get_voter_type_display(), voter.person.role.party if voter.person and voter.person.role else "--",]) output = outfile.getvalue() firstline = '%s Vote #%d %s - %s\n' % (vote.get_chamber_display(), vote.number, vote.created.isoformat(), vote.question) # strftime doesn't work on dates before 1900 firstline = firstline.encode('utf-8') r = HttpResponse(firstline + output, content_type='text/csv') r['Content-Disposition'] = 'attachment; filename=' + vote.get_absolute_url()[1:].replace("/", "_") + ".csv" return r
def vote_details(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = list(vote.voters.all().select_related('person', 'option')) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) # load the role for the VP, since load_roles_at_date only loads # MoC roles has_vp_vote = False for voter in voters: if voter.voter_type == VoterType.vice_president: from person.types import RoleType has_vp_vote = True try: voter.person.role = voter.person.roles.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) except: raise pass # wahtever # sorting by party actually sorts by party first and by ideology score # second. congress = int(congress) global ideology_scores load_ideology_scores(congress) if ideology_scores[congress]: for voter in voters: voter.ideolog_score = ideology_scores[congress].get( voter.person.id if voter.person else 0, ideology_scores[congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""), ideology_scores[congress]["MEDIAN"])) voters.sort(key = lambda x : (x.option.key, x.person.role.party if x.person and x.person.role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display())) return {'vote': vote, 'voters': voters, 'CongressChamber': CongressChamber, "VoterType": VoterType, "VoteCategory": VoteCategory._items, 'has_vp_vote': has_vp_vote, }
def totals(self): # If cached value exists then return it if hasattr(self, '_cached_totals'): return self._cached_totals # else do all these things: items = [] # Extract all voters, find their role at the time # the vote was all_voters = list(self.voters.all().select_related('person', 'option')) voters_by_option = {} for option in self.options.all(): voters_by_option[option] = [x for x in all_voters if x.option == option] total_count = len(all_voters) persons = [x.person for x in all_voters if x.person != None] load_roles_at_date(persons, self.created) # Find all parties which participated in vote # and sort them in order which they should be displayed def cmp_party(x): """ Sort the parties by the number of voters in that party. """ return -len([p for p in all_voters if p.person and p.person.role and p.person.role.party == x]) def get_party(voter): if voter.voter_type != VoterType.vice_president: if voter.person and voter.person.role: return voter.person.role.party else: return "Unknown" else: return "Vice President" all_parties = list(set(get_party(x) for x in all_voters)) all_parties.sort(key=cmp_party) total_party_stats = dict((x, {'yes': 0, 'no': 0, 'other': 0, 'total': 0})\ for x in all_parties) # For each option find party break down, # total vote count and percentage in total count details = [] for option in self.options.all(): voters = voters_by_option.get(option, []) percent = round(len(voters) / float(total_count) * 100.0) party_stats = dict((x, 0) for x in all_parties) for voter in voters: party = get_party(voter) party_stats[party] += 1 total_party_stats[party]['total'] += 1 if option.key == '+': total_party_stats[party]['yes'] += 1 elif option.key == '-': total_party_stats[party]['no'] += 1 else: total_party_stats[party]['other'] += 1 party_counts = [party_stats.get(x, 0) for x in all_parties] party_counts = [{"party": all_parties[i], "count": c, 'chart_width': 190 * c / total_count} for i, c in enumerate(party_counts)] detail = {'option': option, 'count': len(voters), 'percent': int(percent), 'party_counts': party_counts, 'chart_width': 190 * int(percent) / 100} if option.key == '+': detail['yes'] = True if option.key == '-': detail['no'] = True if option.key in ('0', 'P'): detail['hide_if_empty'] = True details.append(detail) party_counts = [total_party_stats[x] for x in all_parties] # sort options by well-known keys, then by total number of votes option_sort_order = {"+":0, "-":1, "P":2, "0":3} details.sort(key = lambda d : (option_sort_order.get(d['option'].key, None), -d['count'])) # hide Present/Not Voting if no one voted that way details = [d for d in details if d["count"] > 0 or "hide_if_empty" not in d] totals = {'options': details, 'total_count': total_count, 'party_counts': party_counts, 'parties': all_parties, } self._cached_totals = totals return totals
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all()) # The pattern which the roll file matches # Filename contains info which should be placed to DB # along with info extracted from the XML file re_path = re.compile("data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml") chamber_mapping = {"s": CongressChamber.senate, "h": CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info("Parsing rolls matching %s" % options.filter) elif options.congress: files = glob.glob("data/us/%s/rolls/*.xml" % options.congress) log.info("Parsing rolls of only congress#%s" % options.congress) else: files = glob.glob("data/us/*/rolls/*.xml") log.info("Processing votes: %d files" % len(files)) total = len(files) progress = Progress(total=total, name="files", step=10) def log_delete_qs(qs): if qs.count() == 0: return print "Deleting obsoleted records: ", qs # if qs.count() > 3: # print "Delete skipped..." # return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re_path.search(fname) try: existing_vote = Vote.objects.get( congress=match.group(1), chamber=chamber_mapping[match.group(2)], session=match.group(3), number=match.group(4), ) except Vote.DoesNotExist: existing_vote = None if ( not File.objects.is_changed(fname) and not options.force and existing_vote != None and not existing_vote.missing_data ): seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. # if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object for roll_node in tree.xpath("/roll"): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id match = re_path.search(fname) vote.congress = int(match.group(1)) vote.chamber = chamber_mapping[match.group(2)] vote.session = match.group(3) vote.number = int(match.group(4)) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): related_bill_num = bill_node.get("number") if 9 <= vote.congress <= 42 and vote.session in ("1", "2"): # Bill numbering from the American Memory colletion is different. The number combines # the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to # the 9th congress numbering seems to be wholly assigned by us and not related to # actual numbering, so we skip matching those bills. related_bill_num = "%d%04d%d" % (int(vote.session), int(bill_node.get("number")), 0) try: vote.related_bill = Bill.objects.get( congress=bill_node.get("session"), bill_type=BillType.by_xml_code(bill_node.get("type")), number=related_bill_num, ) except Bill.DoesNotExist: if vote.congress >= 93: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get("ref") == "regular" and vote.related_bill is not None: try: vote.related_amendment = Amendment.objects.get( congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug(amdt_node.get("number")[0]), number=amdt_node.get("number")[1:], ) except Amendment.DoesNotExist: if vote.congress >= 93: print "Missing amendment", fname vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? # vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if ( vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill ): # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = truncatewords(vote.related_bill.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = truncatewords(vote.related_amendment.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith( "On the Cloture Motion " + vote.related_bill.display_number ): vote.question = "Cloture on " + truncatewords(vote.related_bill.title, 20) elif vote.related_bill and vote.question.startswith( "On Cloture on the Motion to Proceed " + vote.related_bill.display_number ): vote.question = "Cloture on " + truncatewords(vote.related_bill.title, 20) vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith( "On the Motion to Proceed " + vote.related_bill.display_number ): vote.question = "Motion to Proceed on " + truncatewords(vote.related_bill.title, 20) elif vote.related_amendment and vote.question.startswith( "On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number) ): vote.question = "Cloture on " + truncatewords(vote.related_amendment.title, 20) vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace(". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question, ) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath("./option"): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter(vote=vote, key=option.key)[ 0 ].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs( VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids) ) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id")) seen_voter_ids = set() voters = list() for voter_node in roll_node.xpath("./voter"): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get( role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created ) voter.person_role = r voter.person = r.person except: # overlapping roles? missing data? log.error("Could not resolve vice president in %s" % fname, exc_info=ex) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voters.append(voter) if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() # pre-fetch the role of each voter load_roles_at_date([x.person for x in voters if x.person != None], vote.created) for voter in voters: voter.person_role = voter.person.role if voter.person_role is None: log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created)) vote.missing_data = True vote.save() # save all of the records (inserting/updating) for voter in voters: voter.save() seen_voter_ids.add(voter.id) # remove obsolete voter records log_delete_qs( Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids) ) # possibly already deleted by cascade above # pre-calculate totals vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception, ex: log.error("Error in processing %s" % fname, exc_info=ex) had_error = True
#!script import csv, sys from django.conf import settings from vote.models import * from person.util import load_roles_at_date w = csv.writer(sys.stdout) w.writerow([ "congress", "session", "number", "date", "question", "link", "person", "vote" ]) for vote in Vote.objects.filter(congress=113, chamber=CongressChamber.house).order_by('created'): voters = list(vote.voters.all().select_related('person', 'option')) voters = sorted(voters, key = lambda v : v.person.sortname) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) for voter in voters: if voter.person.role.state != "AZ": continue row = [ vote.congress, vote.session, vote.number, vote.created.strftime("%x %X"), vote.question, settings.SITE_ROOT_URL + vote.get_absolute_url(), voter.person.sortname, voter.option.value, ] w.writerow([unicode(v).encode("utf8") for v in row])
total_has_companion = 0 total_companion_bipartisan = 0 for bill in Bill.objects.filter(congress=congress, bill_type__in=(BillType.house_bill, BillType.senate_bill)).select_related("sponsor"): #.filter(introduced_date__gte=date(2011, 8, 1)): total += 1 # People we care about. Batch load party information for sponsors # and cosponsors to be fast. Load roles at the bill introduced date. # Only look at cosponsors who joined on the introduced date (otherwise # they may have changed party between those two dates). persons = [] if not bill.sponsor: continue persons.append(bill.sponsor) persons.extend([c.person for c in Cosponsor.objects.filter(bill=bill, joined=bill.introduced_date).select_related("person")]) load_roles_at_date(persons, bill.introduced_date) if len(persons) > 1: has_initial_cosponsors += 1 # How bipartisan_cosp is this bill? parties = set() for p in persons: if p.role: parties.add(p.role.party) if "Democrat" in parties and "Republican" in parties: bipartisan_cosp += 1 if bill.bill_type == BillType.senate_bill: related_bill = None try: relation = bill.relatedbills.filter(relation='identical', related_bill__bill_type=BillType.house_bill).select_related("related_bill", "related_bill__sponsor").get()
def totals(self): # If cached value exists then return it if hasattr(self, '_cached_totals'): return self._cached_totals # else do all these things: items = [] # Extract all voters, find their role at the time # the vote was all_voters = list(self.voters.all().select_related('person', 'option')) voters_by_option = {} for option in self.options.all(): voters_by_option[option] = [ x for x in all_voters if x.option == option ] total_count = len(all_voters) persons = [x.person for x in all_voters if x.person != None] load_roles_at_date(persons, self.created) # Find all parties which participated in vote # and sort them in order which they should be displayed def cmp_party(x): """ Sort the parties by the number of voters in that party. """ return -len([ p for p in all_voters if p.person and p.person.role and p.person.role.party == x ]) def get_party(voter): if voter.voter_type != VoterType.vice_president: if voter.person and voter.person.role: return voter.person.role.party else: return "Unknown" else: return "Vice President" all_parties = list(set(get_party(x) for x in all_voters)) all_parties.sort(key=cmp_party) total_party_stats = dict((x, {'yes': 0, 'no': 0, 'other': 0, 'total': 0})\ for x in all_parties) # For each option find party break down, # total vote count and percentage in total count details = [] for option in self.options.all(): voters = voters_by_option.get(option, []) percent = round(len(voters) / float(total_count) * 100.0) party_stats = dict((x, 0) for x in all_parties) for voter in voters: party = get_party(voter) party_stats[party] += 1 total_party_stats[party]['total'] += 1 if option.key == '+': total_party_stats[party]['yes'] += 1 elif option.key == '-': total_party_stats[party]['no'] += 1 else: total_party_stats[party]['other'] += 1 party_counts = [party_stats.get(x, 0) for x in all_parties] party_counts = [{ "party": all_parties[i], "count": c, 'chart_width': 190 * c / total_count } for i, c in enumerate(party_counts)] detail = { 'option': option, 'count': len(voters), 'percent': int(percent), 'party_counts': party_counts, 'chart_width': 190 * int(percent) / 100 } if option.key == '+': detail['yes'] = True if option.key == '-': detail['no'] = True if option.key in ('0', 'P'): detail['hide_if_empty'] = True details.append(detail) party_counts = [total_party_stats[x] for x in all_parties] # sort options by well-known keys, then by total number of votes option_sort_order = {"+": 0, "-": 1, "P": 2, "0": 3} details.sort(key=lambda d: (option_sort_order.get( d['option'].key, None), -d['count'])) # hide Present/Not Voting if no one voted that way details = [ d for d in details if d["count"] > 0 or "hide_if_empty" not in d ] totals = { 'options': details, 'total_count': total_count, 'party_counts': party_counts, 'parties': all_parties, } self._cached_totals = totals return totals
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict((x.pk, x) for x in Person.objects.all()) chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info('Parsing rolls matching %s' % options.filter) elif options.congress: files = glob.glob(settings.CONGRESS_DATA_PATH + '/%s/votes/*/*/data.xml' % options.congress) log.info('Parsing rolls of only congress#%s' % options.congress) else: files = glob.glob('data/congress/*/votes/*/*/data.xml') log.info('Processing votes: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=10) def log_delete_qs(qs): if qs.count() == 0: return print("Deleting obsoleted records: ", qs) #if qs.count() > 3: # print "Delete skipped..." # return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re.match(r"data/congress/(?P<congress>\d+)/votes/(?P<session>[ABC0-9]+)/(?P<chamber>[hs])(?P<number>\d+)/data.xml$", fname) try: existing_vote = Vote.objects.get(congress=int(match.group("congress")), chamber=chamber_mapping[match.group("chamber")], session=match.group("session"), number=int(match.group("number"))) except Vote.DoesNotExist: existing_vote = None if not File.objects.is_changed(fname) and not options.force and existing_vote != None and not existing_vote.missing_data: seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. #if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object roll_node = tree.xpath('/roll')[0] # Sqlite is much faster when lots of saves are wrapped in a transaction, # and we do a lot of saves because it's a lot of voters. from django.db import transaction with transaction.atomic(): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id vote.congress = int(match.group("congress")) vote.chamber = chamber_mapping[match.group("chamber")] vote.session = match.group("session") vote.number = int(match.group("number")) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): related_bill_num = bill_node.get("number") if 9 <= vote.congress <= 42 and vote.session in ('1', '2'): # Bill numbering from the American Memory colletion is different. The number combines # the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to # the 9th congress numbering seems to be wholly assigned by us and not related to # actual numbering, so we skip matching those bills. related_bill_num = "%d%04d%d" % (int(vote.session), int(bill_node.get("number")), 0) try: vote.related_bill = Bill.objects.get(congress=bill_node.get("session"), bill_type=BillType.by_xml_code(bill_node.get("type")), number=related_bill_num) except Bill.DoesNotExist: if vote.congress >= 93: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get("ref") == "regular" and vote.related_bill is not None: try: vote.related_amendment = Amendment.objects.get(congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug(amdt_node.get("number")[0]+"amdt"), number=amdt_node.get("number")[1:]) except Amendment.DoesNotExist: if vote.congress >= 93: print("Missing amendment", fname) vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? #vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if vote.category in (VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill: # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = vote.related_bill.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith("On the Cloture Motion " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title elif vote.related_bill and vote.question.startswith("On Cloture on the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display() elif vote.related_bill and vote.question.startswith("On the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Motion to Proceed on " + vote.related_bill.title elif vote.related_amendment and vote.question.startswith("On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number)): vote.question = "Cloture on " + vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display() # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace(". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath('./option'): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter(vote=vote, key=option.key)[0].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs(VoteOption.objects.filter(vote=vote).exclude(id__in=seen_option_ids)) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict(Voter.objects.filter(vote=vote).values_list("person", "id")) seen_voter_ids = set() voters = list() for voter_node in roll_node.xpath('./voter'): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person & role... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get(role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) voter.person_role = r voter.person = r.person except PersonRole.DoesNotExist: # overlapping roles? missing data? log.error('Could not resolve vice president in %s' % fname) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voters.append(voter) if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() # pre-fetch the role of each voter load_roles_at_date([x.person for x in voters if x.person != None], vote.created, vote.congress) for voter in list(voters): if voter.voter_type != VoterType.vice_president: voter.person_role = voter.person.role # If we couldn't match a role for this person on the date of the vote, and if the voter was Not Voting, # and we're looking at historical data, then this is probably a data error --- the voter wasn't even in office. # At the start of each Congress, the House does a Call by States and Election of the Speaker, before swearing # in. In the 116th Congress, these votes had a Not Voting for Walter Jones who had not yet made it to DC, and # then omitted Jones in the votes after swearing in. In those cases, look for a role coming up. if voter.person_role is None and voter.option.key == "0" and vote.question in ("Call by States", "Election of the Speaker"): voter.person_role = voter.person.roles.filter(startdate__gt=vote.created, startdate__lt=vote.created+timedelta(days=30)).first() if voter.person_role is None: if vote.source == VoteSource.keithpoole and voter.option.key == "0": # Drop this record. voters.remove(voter) else: log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created)) vote.missing_data = True vote.save() # save all of the records (inserting/updating) for voter in voters: voter.save() seen_voter_ids.add(voter.id) # remove obsolete voter records log_delete_qs(Voter.objects.filter(vote=vote).exclude(id__in=seen_voter_ids)) # possibly already deleted by cascade above # pre-calculate totals vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception as ex: log.error('Error in processing %s' % fname, exc_info=ex) had_error = True # delete vote objects that are no longer represented on disk if options.congress and not options.filter and not had_error: log_delete_qs(Vote.objects.filter(congress=options.congress).exclude(id__in = seen_obj_ids))
def main(options): """ Parse rolls. """ # Setup XML processors vote_processor = VoteProcessor() option_processor = VoteOptionProcessor() voter_processor = VoterProcessor() voter_processor.PERSON_CACHE = dict( (x.pk, x) for x in Person.objects.all()) # The pattern which the roll file matches # Filename contains info which should be placed to DB # along with info extracted from the XML file re_path = re.compile('data/us/(\d+)/rolls/([hs])(\w+)-(\d+)\.xml') chamber_mapping = {'s': CongressChamber.senate, 'h': CongressChamber.house} if options.filter: files = glob.glob(options.filter) log.info('Parsing rolls matching %s' % options.filter) elif options.congress: files = glob.glob('data/us/%s/rolls/*.xml' % options.congress) log.info('Parsing rolls of only congress#%s' % options.congress) else: files = glob.glob('data/us/*/rolls/*.xml') log.info('Processing votes: %d files' % len(files)) total = len(files) progress = Progress(total=total, name='files', step=10) def log_delete_qs(qs): if qs.count() == 0: return print "Deleting obsoleted records: ", qs #if qs.count() > 3: # print "Delete skipped..." # return qs.delete() seen_obj_ids = set() had_error = False for fname in files: progress.tick() match = re_path.search(fname) try: existing_vote = Vote.objects.get( congress=match.group(1), chamber=chamber_mapping[match.group(2)], session=match.group(3), number=match.group(4)) except Vote.DoesNotExist: existing_vote = None if not File.objects.is_changed( fname ) and not options.force and existing_vote != None and not existing_vote.missing_data: seen_obj_ids.add(existing_vote.id) continue try: tree = etree.parse(fname) ## Look for votes with VP tie breakers. #if len(tree.xpath("/roll/voter[@VP='1']")) == 0: # had_error = True # prevent delete at the end # continue # Process role object roll_node = tree.xpath('/roll')[0] # Sqlite is much faster when lots of saves are wrapped in a transaction, # and we do a lot of saves because it's a lot of voters. from django.db import transaction with transaction.atomic(): vote = vote_processor.process(Vote(), roll_node) if existing_vote: vote.id = existing_vote.id match = re_path.search(fname) vote.congress = int(match.group(1)) vote.chamber = chamber_mapping[match.group(2)] vote.session = match.group(3) vote.number = int(match.group(4)) # Get related bill & amendment. for bill_node in roll_node.xpath("bill"): related_bill_num = bill_node.get("number") if 9 <= vote.congress <= 42 and vote.session in ('1', '2'): # Bill numbering from the American Memory colletion is different. The number combines # the session, bill number, and a 0 or 5 for regular or 'half' numbering. Prior to # the 9th congress numbering seems to be wholly assigned by us and not related to # actual numbering, so we skip matching those bills. related_bill_num = "%d%04d%d" % (int( vote.session), int(bill_node.get("number")), 0) try: vote.related_bill = Bill.objects.get( congress=bill_node.get("session"), bill_type=BillType.by_xml_code( bill_node.get("type")), number=related_bill_num) except Bill.DoesNotExist: if vote.congress >= 93: vote.missing_data = True for amdt_node in roll_node.xpath("amendment"): if amdt_node.get( "ref" ) == "regular" and vote.related_bill is not None: try: vote.related_amendment = Amendment.objects.get( congress=vote.related_bill.congress, amendment_type=AmendmentType.by_slug( amdt_node.get("number")[0]), number=amdt_node.get("number")[1:]) except Amendment.DoesNotExist: if vote.congress >= 93: print "Missing amendment", fname vote.missing_data = True elif amdt_node.get("ref") == "bill-serial": # It is impossible to associate House votes with amendments just from the House # vote XML because the amendment-num might correspond either with the A___ number # or with the "An amendment, numbered ____" number from the amendment purpose, # and there's really no way to figure out which. Maybe we can use the amendment # sponsor instead? #vote.related_amendment = Amendment.objects.get(bill=vote.related_bill, sequence=amdt_node.get("number")) # Instead, we set related_amendment from the amendment parser. Here, we have to # preserve the related_amendment if it is set. if existing_vote: vote.related_amendment = existing_vote.related_amendment # clean up some question text and use the question_details field if vote.category in ( VoteCategory.passage, VoteCategory.passage_suspension, VoteCategory.veto_override) and vote.related_bill: # For passage votes, set the question to the bill title and put the question # details in the details field. vote.question = vote.related_bill.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) elif vote.category == VoteCategory.amendment and vote.related_amendment: # For votes on amendments, make a better title/explanation. vote.question = vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) elif vote.related_bill and vote.question.startswith( "On the Cloture Motion " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title elif vote.related_bill and vote.question.startswith( "On Cloture on the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Cloture on " + vote.related_bill.title vote.question_details = "On Cloture on the Motion to Proceed in the " + vote.get_chamber_display( ) elif vote.related_bill and vote.question.startswith( "On the Motion to Proceed " + vote.related_bill.display_number): vote.question = "Motion to Proceed on " + vote.related_bill.title elif vote.related_amendment and vote.question.startswith( "On the Cloture Motion " + vote.related_amendment.get_amendment_type_display() + " " + str(vote.related_amendment.number)): vote.question = "Cloture on " + vote.related_amendment.title vote.question_details = vote.vote_type + " in the " + vote.get_chamber_display( ) # weird House foratting of bill numbers ("H RES 123 Blah blah") if vote.related_bill: vote.question = re.sub( "(On [^:]+): " + vote.related_bill.display_number.replace( ". ", " ").replace(".", " ").upper() + " .*", r"\1: " + truncatewords(vote.related_bill.title, 15), vote.question) vote.save() seen_obj_ids.add(vote.id) # don't delete me later # Process roll options, overwrite existing options where possible. seen_option_ids = set() roll_options = {} for option_node in roll_node.xpath('./option'): option = option_processor.process(VoteOption(), option_node) option.vote = vote if existing_vote: try: option.id = VoteOption.objects.filter( vote=vote, key=option.key )[0].id # get is better, but I had the database corruption problem except IndexError: pass option.save() roll_options[option.key] = option seen_option_ids.add(option.id) log_delete_qs( VoteOption.objects.filter(vote=vote).exclude( id__in=seen_option_ids) ) # may cascade and delete the Voters too? # Process roll voters, overwriting existing voters where possible. if existing_vote: existing_voters = dict( Voter.objects.filter(vote=vote).values_list( "person", "id")) seen_voter_ids = set() voters = list() for voter_node in roll_node.xpath('./voter'): voter = voter_processor.process(roll_options, Voter(), voter_node) voter.vote = vote voter.created = vote.created # for VP votes, load the actual person & role... if voter.voter_type == VoterType.vice_president: try: r = PersonRole.objects.get( role_type=RoleType.vicepresident, startdate__lte=vote.created, enddate__gte=vote.created) voter.person_role = r voter.person = r.person except PersonRole.DoesNotExist: # overlapping roles? missing data? log.error( 'Could not resolve vice president in %s' % fname) if existing_vote and voter.person: try: voter.id = existing_voters[voter.person.id] except KeyError: pass voters.append(voter) if voter.voter_type == VoterType.unknown and not vote.missing_data: vote.missing_data = True vote.save() # pre-fetch the role of each voter load_roles_at_date( [x.person for x in voters if x.person != None], vote.created, vote.congress) for voter in list(voters): if voter.voter_type != VoterType.vice_president: voter.person_role = voter.person.role # If we couldn't match a role for this person on the date of the vote, and if the voter was Not Voting, # and we're looking at historical data, then this is probably a data error --- the voter wasn't even in office. if voter.person_role is None: if vote.source == VoteSource.keithpoole and voter.option.key == "0": # Drop this record. voters.remove(voter) else: log.error("%s: Could not find role for %s on %s." % (fname, voter.person, vote.created)) vote.missing_data = True vote.save() # save all of the records (inserting/updating) for voter in voters: voter.save() seen_voter_ids.add(voter.id) # remove obsolete voter records log_delete_qs( Voter.objects.filter(vote=vote).exclude( id__in=seen_voter_ids) ) # possibly already deleted by cascade above # pre-calculate totals vote.calculate_totals() if not options.disable_events: vote.create_event() File.objects.save_file(fname) except Exception, ex: log.error('Error in processing %s' % fname, exc_info=ex) had_error = True
def vote_thumbnail_image(request, congress, session, chamber_code, number, image_type): vote = load_vote(congress, session, chamber_code, number) import cairo, re, math from StringIO import StringIO # general image properties font_face = "DejaVu Serif Condensed" image_width = 300 image_height = 250 if image_type == "thumbnail" else 170 # format text to print on the image vote_title = re.sub(r"^On the Motion to ", "To ", vote.question) if re.match(r"Cloture .*Rejected", vote.result): vote_result_2 = "Filibustered" elif re.match(r"Cloture .*Agreed to", vote.result): vote_result_2 = "Proceed" else: vote_result_2 = re.sub("^(Bill|Amendment|Resolution|Conference Report|Nomination|Motion|Motion to \S+) ", "", vote.result) if vote_result_2 == "unknown": vote_result_2 = "" vote_date = vote.created.strftime("%x") if vote.created.year > 1900 else vote.created.isoformat().split("T")[0] vote_citation = vote.get_chamber_display() + " Vote #" + str(vote.number) + " -- " + vote_date # get vote totals by option and by party totals = vote.totals() total_count = 0 total_counts = { } # key: total ({ "+": 123 }, etc.) yea_counts_by_party = [0,0,0] # D, I, R (+ votes totals) nay_counts_by_party = [0,0,0] # D, I, R (+/- votes totals) nonvoting_members_totals = [0,0,0] # D, I, R party_index = { "Democrat": 0, "Republican": 2 } for opt in totals["options"]: total_counts[opt["option"].key] = opt["count"] for i in xrange(len(totals["parties"])): j = party_index.get(totals["parties"][i], 1) if opt["option"].key not in ("+", "-"): # most votes are by proportion of those voting (not some cloture etc.), # so put present/not-voting tallies in a separate group nonvoting_members_totals[j] += opt["party_counts"][i]["count"] continue total_count += opt["party_counts"][i]["count"] if opt["option"].key == "+": yea_counts_by_party[j] += opt["party_counts"][i]["count"] else: nay_counts_by_party[j] += opt["party_counts"][i]["count"] if total_count == 0 or "+" not in total_counts or "-" not in total_counts: raise Http404() # no thumbnail for other sorts of votes vote_result_1 = "%d-%d" % (total_counts["+"], total_counts["-"]) def show_text_centered(ctx, text, max_width=None): while True: (x_bearing, y_bearing, width, height, x_advance, y_advance) = ctx.text_extents(text) if max_width is not None and width > max_width: text2 = re.sub(r" \S+(\.\.\.)?$", "...", text) if text2 != text: text = text2 continue break ctx.rel_move_to(-width/2, height) ctx.show_text(text) im = cairo.ImageSurface(cairo.FORMAT_ARGB32, image_width, image_height) ctx = cairo.Context(im) ctx.select_font_face(font_face, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD) # clear background ctx.set_source_rgb(1,1,1) ctx.new_path() ctx.line_to(0, 0) ctx.line_to(image_width, 0) ctx.line_to(image_width, image_height) ctx.line_to(0, image_height) ctx.fill() chart_top = 0 if image_type == "thumbnail": # Title ctx.set_font_size(20) ctx.set_source_rgb(.2,.2,.2) ctx.move_to(150,10) show_text_centered(ctx, vote_title, max_width=.95*image_width) chart_top = 50 # Vote Tally font_size = 26 if len(vote_result_2) < 10 else 22 ctx.set_font_size(font_size) ctx.set_source_rgb(.1, .1, .1) ctx.move_to(150,chart_top) show_text_centered(ctx, vote_result_1) # Vote Result ctx.move_to(150,chart_top+12+font_size) show_text_centered(ctx, vote_result_2) w = max(ctx.text_extents(vote_result_1)[2], ctx.text_extents(vote_result_2)[2]) # Line ctx.set_line_width(1) ctx.new_path() ctx.line_to(150-w/2, chart_top+5+font_size) ctx.rel_line_to(w, 0) ctx.stroke() if image_type == "thumbnail": # Vote Chamber/Date/Number ctx.select_font_face(font_face, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL) ctx.set_font_size(14) ctx.move_to(150,image_height-25) show_text_centered(ctx, vote_citation, max_width=.98*image_width) # Seats # Construct an array of rows of seats, where each entry maps to a particular # voter. # How many rows of seats? That is hard coded by chamber. seating_rows = 8 if vote.chamber == CongressChamber.house else 4 # 4 for Senate (http://www.senate.gov/artandhistory/art/special/Desks/chambermap.cfm) # about 8 for the House # Long ago Congress had very few people. seating_rows = min(total_count / 8 + 1, seating_rows) # Determine the seating chart dimensions: the radius of the inside row of # seats and the radius of the outside row of seats. inner_r = w/2 * 1.25 + 5 # wrap closely around the text in the middle if seating_rows <= 4: inner_r = max(inner_r, 75) # don't make the inner radius too small outer_r = image_width * .45 # end close to the image width # If we assume the absolute spacing of seats is constant from row to row, then # the number of seats per row grows linearly with the radius, following the # circumference. If s0 is the number of seats on the inner row, then # floor(s0 * outer_r/inner_r) is the number of seats on the outer row. The total # number of seats is found by the sum of the arithmetic sequence (n/2 * (a_1+a_n)): # n = (seating_rows/2)*(s0 + s0*outer_r/inner_r) # We want exactly total_count seats, so solving for s0... seats_per_row = 2.0 * total_count / (seating_rows*(1.0 + outer_r/inner_r)) # How wide to draw a seat? seat_size = min(.8 * (outer_r-inner_r) / seating_rows, .35 * (2*3.14159*inner_r) / seats_per_row) # Determine how many seats on each row. seats_so_far = 0 rowcounts = [] for row in xrange(seating_rows): # What's the radius of this row? if seating_rows > 1: r = inner_r + (outer_r-inner_r) * row / float(seating_rows-1) else: r = inner_r # How many seats should we put on this row? if row < seating_rows-1: # Start with seats_per_row on the inner row and grow linearly. # Round to an integer. Alternate rounding down and up. n_seats = seats_per_row * r/inner_r n_seats = int(math.floor(n_seats) if (row % 2 == 0) else math.ceil(n_seats)) else: # On the outermost row, just put in how many left we need # so we always have exactly the right number of seats. n_seats = total_count - seats_so_far rowcounts.append(n_seats) seats_so_far += n_seats # Make a list of all of the seats as a list of tuples of the # form (rownum, seatnum) where seatnum is an index from the # left side. seats = [] for row, count in enumerate(rowcounts): for i in xrange(count): seats.append( (row, i) ) # Sort the seats in the order we will fill them from left to right, # bottom to top. seats.sort(key = lambda seat : (seat[1]/float(rowcounts[seat[0]]), -seat[0]) ) # We can draw in two modes. In one mode, we don't care which actual # person corresponds to which seat. We just draw the groups of voters # in blocks. Or we can draw the actual people in seats we assign # according to their ideology score, from left to right. # See if we have ideology scores. voter_details = None if True: global ideology_scores load_ideology_scores(vote.congress) if ideology_scores[vote.congress]: voter_details = [ ] # Load the voters, getting their role at the time they voted. voters = list(vote.voters.all().select_related('person', 'option')) load_roles_at_date([x.person for x in voters if x.person != None], vote.created) # Store ideology scores for voter in voters: if voter.option.key not in ("+", "-"): continue party = party_index.get(voter.person.role.party if voter.person and voter.person.role else "Unknown", 1) option = 0 if voter.option.key == "+" else 1 coord = ideology_scores[vote.congress].get(voter.person.id if voter.person else "UNKNOWN", ideology_scores[vote.congress].get("MEDIAN:" + (voter.person.role.party if voter.person and voter.person.role else ""), ideology_scores[vote.congress]["MEDIAN"])) voter_details.append( (coord, (party, option)) ) # Sort voters by party, then by ideology score, then by vote. voter_details.sort(key = lambda x : (x[1][0], x[0], x[1][1])) if len(voter_details) != len(seats): raise ValueError("Gotta check this.") voter_details = None # abort if not voter_details: # Just assign voters to seats in blocks. # # We're fill the seats from left to right different groups of voters: # D+, D-, I+, I-, R-, R+ # For each group, for each voter in the group, pop off a seat and # draw him in that seat. # for each group of voters... seat_idx = 0 for (party, vote) in [ (0, 0), (0, 1), (1, 0), (1, 1), (2, 1), (2, 0) ]: # how many votes in this group? n_voters = (yea_counts_by_party if vote == 0 else nay_counts_by_party)[party] # for each voter... for i in xrange(n_voters): seats[seat_idx] = (seats[seat_idx], (party, vote)) seat_idx += 1 else: # Assign voters to seats in order. for i in xrange(len(voter_details)): seats[i] = (seats[i], voter_details[i][1]) # Draw the seats. group_colors = { (0, 0): (0.05, 0.24, 0.63), # D+ (0, 1): (0.85, 0.85, 1.0), # D- (1, 0): (0.07, 0.05, 0.07), # I+ (1, 1): (0.85, 0.85, 0.85), # I- (2, 0): (0.90, 0.05, 0.07), # R+ (2, 1): (1.0, 0.85, 0.85), # R- } for ((row, seat_pos), (party, vote)) in seats: # radius of this row (again, code dup) if seating_rows > 1: r = inner_r + (outer_r-inner_r) * row / float(seating_rows-1) else: r = inner_r # draw ctx.set_source_rgb(*group_colors[(party, vote)]) ctx.identity_matrix() ctx.translate(image_width/2, chart_top+25) ctx.rotate(3.14159 - 3.14159 * seat_pos/float(rowcounts[row]-1)) ctx.translate(r, 0) ctx.rectangle(-seat_size/2, -seat_size/2, seat_size, seat_size) ctx.fill() # Convert the image buffer to raw PNG bytes. buf = StringIO() im.write_to_png(buf) v = buf.getvalue() # Form the response. r = HttpResponse(v, content_type='image/png') r["Content-Length"] = len(v) return r
#.filter(introduced_date__gte=date(2011, 8, 1)): total += 1 # People we care about. Batch load party information for sponsors # and cosponsors to be fast. Load roles at the bill introduced date. # Only look at cosponsors who joined on the introduced date (otherwise # they may have changed party between those two dates). persons = [] if not bill.sponsor: continue persons.append(bill.sponsor) persons.extend([ c.person for c in Cosponsor.objects.filter( bill=bill, joined=bill.introduced_date).select_related( "person") ]) load_roles_at_date(persons, bill.introduced_date) if len(persons) > 1: has_initial_cosponsors += 1 # How bipartisan_cosp is this bill? parties = set() for p in persons: if p.role: parties.add(p.role.party) if "Democrat" in parties and "Republican" in parties: bipartisan_cosp += 1 if bill.bill_type == BillType.senate_bill: related_bill = None try: relation = bill.relatedbills.filter(