def fetch_statements(committee): from person.views import http_rest_json from parser.processor import Processor # only full committee statements are available if committee.committee: # has a parent committee return [] # load statements from ProPublica API, ignoring any network errors try: statements = http_rest_json( "https://api.propublica.org/congress/v1/statements/committees/{committee_id}.json".format( committee_id=committee.code, ), headers={ 'X-API-Key': settings.PROPUBLICA_CONGRESS_API_KEY, 'Content-Type': 'application/json', 'Accept': 'application/json' }) if statements["status"] != "OK": raise Exception() statements = statements["results"] except: return [] # make simplified statements records statements = [{ "date": Processor.parse_datetime(s["date"]).date(), "type": s["statement_type"], "title": s["title"], "url": s["url"], } for s in statements if s["date"]] # # downcase all-caps titles # for s in statements: # if s["title"] != s["title"].upper(): continue # s["title"] = s["title"].lower() # if s["person"]: s["title"] = s["title"].replace(s["person"].lastname.lower(), s["person"].lastname) # common easy case fix return statements
def fetch_statements(bill): from person.models import Person from person.views import http_rest_json from parser.processor import Processor # load statements from ProPublica API, ignoring any network errors try: statements = http_rest_json( "https://api.propublica.org/congress/v1/{congress}/bills/{bill}/statements.json" .format( congress=bill.congress, bill=bill.bill_type_slug + str(bill.number), ), headers={ 'X-API-Key': settings.PROPUBLICA_CONGRESS_API_KEY, 'Content-Type': 'application/json', 'Accept': 'application/json' }) if statements["status"] != "OK": raise Exception() statements = statements["results"] except: return [] # bulk-fetch all legislators mentioned and make a mapping from bioguide ID to Person object legislators = { p.bioguideid: p for p in Person.objects.filter( bioguideid__in=[s['member_id'] for s in statements]) } # make simplified statements records statements = [{ "date": Processor.parse_datetime(s["date"]).date(), "person": legislators.get(s["member_id"]), "type": s["statement_type"], "title": s["title"], "url": s["url"], } for s in statements if s["date"]] # downcase all-caps titles for s in statements: if s["title"] != s["title"].upper(): continue s["title"] = s["title"].lower() if s["person"]: s["title"] = s["title"].replace( s["person"].lastname.lower(), s["person"].lastname) # common easy case fix # sort by date because we want to give a diversity of viewpoints by showing only one # statement per legislator statements.sort(key=lambda s: s["date"], reverse=True) # Put all of the statements that are the most recent for the legislator first, then all of the second-most-recent, and so on. # Since sort is stable, it will remain in reverse-date order within each group. seen = {} for s in statements: seen[s["person"]] = seen.get(s["person"], 0) + 1 s["legislator_ordinal"] = seen[s["person"]] statements.sort(key=lambda s: s["legislator_ordinal"]) # bulk-fetch ideology & leadership scores from person.models import RoleType from person.analysis import load_sponsorship_analysis2 scores = load_sponsorship_analysis2( bill.congress, RoleType.representative, None)["all"] + load_sponsorship_analysis2( bill.congress, RoleType.senator, None)["all"] leadership_scores = {s["id"]: float(s["leadership"]) for s in scores} ideology_scores = {s["id"]: float(s["ideology"]) for s in scores} # Tag relevance of the person making the press statement. (We'd love to prioritize relevant # committee chairs but we only have current committee info so we couldn't do it for # past bills.) cosponsors = set(bill.cosponsors.all()) for s in statements: if s["person"] == bill.sponsor: s["relevance"] = "Sponsor" s["priority"] = (0, -leadership_scores.get(s["person"].id, 0)) elif s["person"] in cosponsors: s["relevance"] = "Co-sponsor" s["priority"] = (1, -leadership_scores.get( s["person"].id if s["person"] else 0, 0)) # Get the prioritized statements to show. ret = [] # For up to the first two, show the sponsor and the cosponsor with the highest leadership # score (or if there is no sponsor statement, the two top cosponsors), but prioritizing # the most recent statements for each legislator statements.sort( key=lambda s: (s["legislator_ordinal"], s.get("priority", (2, )))) while statements and statements[0].get("priority") and statements[0][ "legislator_ordinal"] == 1 and len(ret) < 2: ret.append(statements.pop(0)) # For the third statement, take the legislator with the ideology score furthest from # the sponsor's score, to hopefully get an opposing viewpoint. if bill.sponsor and bill.sponsor.id in ideology_scores: statements.sort(key=lambda s: ( s["legislator_ordinal" ], # most recent statements by legislators first not (s["person"] and s["person"].id in ideology_scores ), # people with ideology scores first "relevance" in s, # people who aren't a sponsor/cosponsor first -abs( ideology_scores.get(s["person"].id if s["person"] else 0, 0) - ideology_scores[bill.sponsor.id]))) while statements and len(ret) < 3: ret.append(statements.pop(0)) return ret
def vote_details(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = vote.get_voters() # If this is a vote on passage, but it's not the final vote on passage in this # chamber, issue a warning. has_subsequent_vote = False if vote.related_bill and vote.category in (VoteCategory.passage, VoteCategory.passage_suspension) \ and vote.related_bill.votes.filter(chamber=vote.chamber, category__in=(VoteCategory.passage, VoteCategory.passage_suspension)).order_by('-created').first() != vote: has_subsequent_vote = True # Test if we have diagrams for this vote. The only # way to test is to try to make it. has_diagram = {} for image_type in ("map", "diagram"): try: vote_thumbnail_image(request, congress, session, chamber_code, number, image_type) has_diagram[image_type] = True except Http404: has_diagram[image_type] = False # sorting by party actually sorts by party first and by ideology score # second. has_ideology_scores = attach_ideology_scores(voters, vote.congress) # perform an initial sort for display voters.sort( key=lambda x: (x.option.key, x.person_role.party if x.person and x.person_role and x.person_role.party else "", x.person.name_no_details_lastfirst() if x.person else x.get_voter_type_display())) # did any Senate leaders switch their vote for a motion to reconsider? reconsiderers = vote.possible_reconsideration_votes(voters) reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers) # compute statistical outliers (this marks the Voter instances with an is_outlier attribute) get_vote_outliers(voters) # get Explanations from ProPublica for House votes and attach to voter instances propublica_url = None propublica_count = 0 if vote.chamber == CongressChamber.house and vote.congress >= 110: # ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where # we know the session is an integer (the session/legislative year). propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % ( int(vote.session), vote.number) try: explanations = http_rest_json( "https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number)) except: # squash all errors explanations = {} propublica_count = explanations.get("vote", {}).get("total_explanations", 0) expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) } for voter in voters: voter.explanation = expl_map.get(voter.person.bioguideid) return { 'vote': vote, 'voters': voters, 'CongressChamber': CongressChamber, "VoterType": VoterType, "VoteCategory": VoteCategory._items, 'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0, 'has_diagram': has_diagram, 'has_ideology_scores': has_ideology_scores, 'has_subsequent_vote': has_subsequent_vote, 'diagram_key_colors': [(k, "rgb(%d,%d,%d)" % tuple([c * 256 for c in clr])) for k, clr in vote_diagram_colors.items()], 'reconsiderers': (reconsiderers, reconsiderers_titles), 'propublica_url': propublica_url, 'propublica_count': propublica_count, }
def vote_details(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = vote.get_voters() # If this is a vote on passage, but it's not the final vote on passage in this # chamber, issue a warning. has_subsequent_vote = False if vote.related_bill and vote.category in (VoteCategory.passage, VoteCategory.passage_suspension) \ and vote.related_bill.votes.filter(chamber=vote.chamber, category__in=(VoteCategory.passage, VoteCategory.passage_suspension)).order_by('-created').first() != vote: has_subsequent_vote = True # Test if we have diagrams for this vote. The only # way to test is to try to make it. has_diagram = { } for image_type in ("map", "diagram"): try: vote_thumbnail_image(request, congress, session, chamber_code, number, image_type) has_diagram[image_type] = True except Http404: has_diagram[image_type] = False # sorting by party actually sorts by party first and by ideology score # second. has_ideology_scores = attach_ideology_scores(voters, vote.congress) # perform an initial sort for display voters.sort(key = lambda x : (x.option.key, x.person_role.party if x.person and x.person_role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display())) # did any Senate leaders switch their vote for a motion to reconsider? reconsiderers = vote.possible_reconsideration_votes(voters) reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers) # compute statistical outliers (this marks the Voter instances with an is_outlier attribute) get_vote_outliers(voters) # get Explanations from ProPublica for House votes and attach to voter instances propublica_url = None propublica_count = 0 if vote.chamber == CongressChamber.house and vote.congress >= 110: # ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where # we know the session is an integer (the session/legislative year). propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (int(vote.session), vote.number) try: explanations = http_rest_json("https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number)) except: # squash all errors explanations = { } propublica_count = explanations.get("vote", {}).get("total_explanations", 0) expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) } for voter in voters: voter.explanation = expl_map.get(voter.person.bioguideid) return {'vote': vote, 'voters': voters, 'CongressChamber': CongressChamber, "VoterType": VoterType, "VoteCategory": VoteCategory._items, 'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0, 'has_diagram': has_diagram, 'has_ideology_scores': has_ideology_scores, 'has_subsequent_vote': has_subsequent_vote, 'diagram_key_colors': [ (k, "rgb(%d,%d,%d)" % tuple([c*256 for c in clr])) for k, clr in vote_diagram_colors.items() ], 'reconsiderers': (reconsiderers, reconsiderers_titles), 'propublica_url': propublica_url, 'propublica_count': propublica_count, }
def vote_details(request, congress, session, chamber_code, number): vote = load_vote(congress, session, chamber_code, number) voters = vote.get_voters() # Test if we have a diagram for this vote. The only # way to test is to try to make it. try: vote_thumbnail_image(request, congress, session, chamber_code, number, "diagram") has_diagram = True except Http404: has_diagram = False # sorting by party actually sorts by party first and by ideology score # second. has_ideology_scores = False congress = int(congress) global ideology_scores load_ideology_scores(congress) if ideology_scores[congress]: for voter in voters: if voter.person and voter.person.id in ideology_scores[congress]: voter.ideolog_score = ideology_scores[congress][voter.person.id] has_ideology_scores = True else: voter.ideolog_score = \ ideology_scores[congress].get("MEDIAN:" + (voter.person_role.party if voter.person and voter.person_role else ""), ideology_scores[congress]["MEDIAN"]) # perform an initial sort for display voters.sort(key = lambda x : (x.option.key, x.person_role.party if x.person and x.person_role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display())) # did any Senate leaders switch their vote for a motion to reconsider? reconsiderers = vote.possible_reconsideration_votes(voters) reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers) # compute statistical outliers (this marks the Voter instances with an is_outlier attribute) get_vote_outliers(voters) # get Explanations from ProPublica for House votes and attach to voter instances propublica_url = None propublica_count = 0 if vote.chamber == CongressChamber.house and vote.congress >= 110: # ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where # we know the session is an integer (the session/legislative year). propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (int(vote.session), vote.number) try: explanations = http_rest_json("https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number)) except: # squash all errors explanations = { } propublica_count = explanations.get("vote", {}).get("total_explanations", 0) expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) } for voter in voters: voter.explanation = expl_map.get(voter.person.bioguideid) return {'vote': vote, 'voters': voters, 'CongressChamber': CongressChamber, "VoterType": VoterType, "VoteCategory": VoteCategory._items, 'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0, 'has_diagram': has_diagram, 'has_ideology_scores': has_ideology_scores, 'reconsiderers': (reconsiderers, reconsiderers_titles), 'propublica_url': propublica_url, 'propublica_count': propublica_count, }