def fetch_statements(committee):
from person.views import http_rest_json
from parser.processor import Processor
# only full committee statements are available
if committee.committee: # has a parent committee
return []
# load statements from ProPublica API, ignoring any network errors
try:
statements = http_rest_json(
"https://api.propublica.org/congress/v1/statements/committees/{committee_id}.json".format(
committee_id=committee.code,
),
headers={
'X-API-Key': settings.PROPUBLICA_CONGRESS_API_KEY,
'Content-Type': 'application/json',
'Accept': 'application/json'
})
if statements["status"] != "OK": raise Exception()
statements = statements["results"]
except:
return []
# make simplified statements records
statements = [{
"date": Processor.parse_datetime(s["date"]).date(),
"type": s["statement_type"],
"title": s["title"],
"url": s["url"],
} for s in statements
if s["date"]]
# # downcase all-caps titles
# for s in statements:
# if s["title"] != s["title"].upper(): continue
# s["title"] = s["title"].lower()
# if s["person"]: s["title"] = s["title"].replace(s["person"].lastname.lower(), s["person"].lastname) # common easy case fix
return statements
def fetch_statements(bill):
from person.models import Person
from person.views import http_rest_json
from parser.processor import Processor
# load statements from ProPublica API, ignoring any network errors
try:
statements = http_rest_json(
"https://api.propublica.org/congress/v1/{congress}/bills/{bill}/statements.json"
.format(
congress=bill.congress,
bill=bill.bill_type_slug + str(bill.number),
),
headers={
'X-API-Key': settings.PROPUBLICA_CONGRESS_API_KEY,
'Content-Type': 'application/json',
'Accept': 'application/json'
})
if statements["status"] != "OK": raise Exception()
statements = statements["results"]
except:
return []
# bulk-fetch all legislators mentioned and make a mapping from bioguide ID to Person object
legislators = {
p.bioguideid: p
for p in Person.objects.filter(
bioguideid__in=[s['member_id'] for s in statements])
}
# make simplified statements records
statements = [{
"date": Processor.parse_datetime(s["date"]).date(),
"person": legislators.get(s["member_id"]),
"type": s["statement_type"],
"title": s["title"],
"url": s["url"],
} for s in statements if s["date"]]
# downcase all-caps titles
for s in statements:
if s["title"] != s["title"].upper(): continue
s["title"] = s["title"].lower()
if s["person"]:
s["title"] = s["title"].replace(
s["person"].lastname.lower(),
s["person"].lastname) # common easy case fix
# sort by date because we want to give a diversity of viewpoints by showing only one
# statement per legislator
statements.sort(key=lambda s: s["date"], reverse=True)
# Put all of the statements that are the most recent for the legislator first, then all of the second-most-recent, and so on.
# Since sort is stable, it will remain in reverse-date order within each group.
seen = {}
for s in statements:
seen[s["person"]] = seen.get(s["person"], 0) + 1
s["legislator_ordinal"] = seen[s["person"]]
statements.sort(key=lambda s: s["legislator_ordinal"])
# bulk-fetch ideology & leadership scores
from person.models import RoleType
from person.analysis import load_sponsorship_analysis2
scores = load_sponsorship_analysis2(
bill.congress,
RoleType.representative, None)["all"] + load_sponsorship_analysis2(
bill.congress, RoleType.senator, None)["all"]
leadership_scores = {s["id"]: float(s["leadership"]) for s in scores}
ideology_scores = {s["id"]: float(s["ideology"]) for s in scores}
# Tag relevance of the person making the press statement. (We'd love to prioritize relevant
# committee chairs but we only have current committee info so we couldn't do it for
# past bills.)
cosponsors = set(bill.cosponsors.all())
for s in statements:
if s["person"] == bill.sponsor:
s["relevance"] = "Sponsor"
s["priority"] = (0, -leadership_scores.get(s["person"].id, 0))
elif s["person"] in cosponsors:
s["relevance"] = "Co-sponsor"
s["priority"] = (1, -leadership_scores.get(
s["person"].id if s["person"] else 0, 0))
# Get the prioritized statements to show.
ret = []
# For up to the first two, show the sponsor and the cosponsor with the highest leadership
# score (or if there is no sponsor statement, the two top cosponsors), but prioritizing
# the most recent statements for each legislator
statements.sort(
key=lambda s: (s["legislator_ordinal"], s.get("priority", (2, ))))
while statements and statements[0].get("priority") and statements[0][
"legislator_ordinal"] == 1 and len(ret) < 2:
ret.append(statements.pop(0))
# For the third statement, take the legislator with the ideology score furthest from
# the sponsor's score, to hopefully get an opposing viewpoint.
if bill.sponsor and bill.sponsor.id in ideology_scores:
statements.sort(key=lambda s: (
s["legislator_ordinal"
], # most recent statements by legislators first
not (s["person"] and s["person"].id in ideology_scores
), # people with ideology scores first
"relevance" in s, # people who aren't a sponsor/cosponsor first
-abs(
ideology_scores.get(s["person"].id if s["person"] else 0, 0) -
ideology_scores[bill.sponsor.id])))
while statements and len(ret) < 3:
ret.append(statements.pop(0))
return ret
def vote_details(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = vote.get_voters()
# If this is a vote on passage, but it's not the final vote on passage in this
# chamber, issue a warning.
has_subsequent_vote = False
if vote.related_bill and vote.category in (VoteCategory.passage, VoteCategory.passage_suspension) \
and vote.related_bill.votes.filter(chamber=vote.chamber, category__in=(VoteCategory.passage, VoteCategory.passage_suspension)).order_by('-created').first() != vote:
has_subsequent_vote = True
# Test if we have diagrams for this vote. The only
# way to test is to try to make it.
has_diagram = {}
for image_type in ("map", "diagram"):
try:
vote_thumbnail_image(request, congress, session, chamber_code,
number, image_type)
has_diagram[image_type] = True
except Http404:
has_diagram[image_type] = False
# sorting by party actually sorts by party first and by ideology score
# second.
has_ideology_scores = attach_ideology_scores(voters, vote.congress)
# perform an initial sort for display
voters.sort(
key=lambda x: (x.option.key, x.person_role.party
if x.person and x.person_role and x.person_role.party
else "", x.person.name_no_details_lastfirst()
if x.person else x.get_voter_type_display()))
# did any Senate leaders switch their vote for a motion to reconsider?
reconsiderers = vote.possible_reconsideration_votes(voters)
reconsiderers_titles = "/".join(v.person_role.leadership_title
for v in reconsiderers)
# compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
get_vote_outliers(voters)
# get Explanations from ProPublica for House votes and attach to voter instances
propublica_url = None
propublica_count = 0
if vote.chamber == CongressChamber.house and vote.congress >= 110:
# ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where
# we know the session is an integer (the session/legislative year).
propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (
int(vote.session), vote.number)
try:
explanations = http_rest_json(
"https://projects.propublica.org/explanations/api/votes/%d/%d.json"
% (int(vote.session), vote.number))
except:
# squash all errors
explanations = {}
propublica_count = explanations.get("vote",
{}).get("total_explanations", 0)
expl_map = {
e["bioguide_id"]: e
for e in explanations.get("vote", {}).get("explanations", [])
}
for voter in voters:
voter.explanation = expl_map.get(voter.person.bioguideid)
return {
'vote':
vote,
'voters':
voters,
'CongressChamber':
CongressChamber,
"VoterType":
VoterType,
"VoteCategory":
VoteCategory._items,
'has_vp_vote':
len([v
for v in voters if v.voter_type == VoterType.vice_president]) > 0,
'has_diagram':
has_diagram,
'has_ideology_scores':
has_ideology_scores,
'has_subsequent_vote':
has_subsequent_vote,
'diagram_key_colors':
[(k, "rgb(%d,%d,%d)" % tuple([c * 256 for c in clr]))
for k, clr in vote_diagram_colors.items()],
'reconsiderers': (reconsiderers, reconsiderers_titles),
'propublica_url':
propublica_url,
'propublica_count':
propublica_count,
}
def vote_details(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = vote.get_voters()
# If this is a vote on passage, but it's not the final vote on passage in this
# chamber, issue a warning.
has_subsequent_vote = False
if vote.related_bill and vote.category in (VoteCategory.passage, VoteCategory.passage_suspension) \
and vote.related_bill.votes.filter(chamber=vote.chamber, category__in=(VoteCategory.passage, VoteCategory.passage_suspension)).order_by('-created').first() != vote:
has_subsequent_vote = True
# Test if we have diagrams for this vote. The only
# way to test is to try to make it.
has_diagram = { }
for image_type in ("map", "diagram"):
try:
vote_thumbnail_image(request, congress, session, chamber_code, number, image_type)
has_diagram[image_type] = True
except Http404:
has_diagram[image_type] = False
# sorting by party actually sorts by party first and by ideology score
# second.
has_ideology_scores = attach_ideology_scores(voters, vote.congress)
# perform an initial sort for display
voters.sort(key = lambda x : (x.option.key, x.person_role.party if x.person and x.person_role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display()))
# did any Senate leaders switch their vote for a motion to reconsider?
reconsiderers = vote.possible_reconsideration_votes(voters)
reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers)
# compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
get_vote_outliers(voters)
# get Explanations from ProPublica for House votes and attach to voter instances
propublica_url = None
propublica_count = 0
if vote.chamber == CongressChamber.house and vote.congress >= 110:
# ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where
# we know the session is an integer (the session/legislative year).
propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (int(vote.session), vote.number)
try:
explanations = http_rest_json("https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number))
except:
# squash all errors
explanations = { }
propublica_count = explanations.get("vote", {}).get("total_explanations", 0)
expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) }
for voter in voters:
voter.explanation = expl_map.get(voter.person.bioguideid)
return {'vote': vote,
'voters': voters,
'CongressChamber': CongressChamber,
"VoterType": VoterType,
"VoteCategory": VoteCategory._items,
'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0,
'has_diagram': has_diagram,
'has_ideology_scores': has_ideology_scores,
'has_subsequent_vote': has_subsequent_vote,
'diagram_key_colors': [ (k, "rgb(%d,%d,%d)" % tuple([c*256 for c in clr])) for k, clr in vote_diagram_colors.items() ],
'reconsiderers': (reconsiderers, reconsiderers_titles),
'propublica_url': propublica_url,
'propublica_count': propublica_count,
}
def vote_details(request, congress, session, chamber_code, number):
vote = load_vote(congress, session, chamber_code, number)
voters = vote.get_voters()
# Test if we have a diagram for this vote. The only
# way to test is to try to make it.
try:
vote_thumbnail_image(request, congress, session, chamber_code, number, "diagram")
has_diagram = True
except Http404:
has_diagram = False
# sorting by party actually sorts by party first and by ideology score
# second.
has_ideology_scores = False
congress = int(congress)
global ideology_scores
load_ideology_scores(congress)
if ideology_scores[congress]:
for voter in voters:
if voter.person and voter.person.id in ideology_scores[congress]:
voter.ideolog_score = ideology_scores[congress][voter.person.id]
has_ideology_scores = True
else:
voter.ideolog_score = \
ideology_scores[congress].get("MEDIAN:" + (voter.person_role.party if voter.person and voter.person_role else ""),
ideology_scores[congress]["MEDIAN"])
# perform an initial sort for display
voters.sort(key = lambda x : (x.option.key, x.person_role.party if x.person and x.person_role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display()))
# did any Senate leaders switch their vote for a motion to reconsider?
reconsiderers = vote.possible_reconsideration_votes(voters)
reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers)
# compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
get_vote_outliers(voters)
# get Explanations from ProPublica for House votes and attach to voter instances
propublica_url = None
propublica_count = 0
if vote.chamber == CongressChamber.house and vote.congress >= 110:
# ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where
# we know the session is an integer (the session/legislative year).
propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (int(vote.session), vote.number)
try:
explanations = http_rest_json("https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number))
except:
# squash all errors
explanations = { }
propublica_count = explanations.get("vote", {}).get("total_explanations", 0)
expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) }
for voter in voters:
voter.explanation = expl_map.get(voter.person.bioguideid)
return {'vote': vote,
'voters': voters,
'CongressChamber': CongressChamber,
"VoterType": VoterType,
"VoteCategory": VoteCategory._items,
'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0,
'has_diagram': has_diagram,
'has_ideology_scores': has_ideology_scores,
'reconsiderers': (reconsiderers, reconsiderers_titles),
'propublica_url': propublica_url,
'propublica_count': propublica_count,
}