Ejemplo n.º 1
0
def fetch_statements(committee):
    from person.views import http_rest_json
    from parser.processor import Processor

    # only full committee statements are available
    if committee.committee: # has a parent committee
        return []

    # load statements from ProPublica API, ignoring any network errors
    try:
        statements = http_rest_json(
          "https://api.propublica.org/congress/v1/statements/committees/{committee_id}.json".format(
            committee_id=committee.code,
          ),
          headers={
            'X-API-Key': settings.PROPUBLICA_CONGRESS_API_KEY,
            'Content-Type': 'application/json',
            'Accept': 'application/json'
          })
        if statements["status"] != "OK": raise Exception()
        statements = statements["results"]
    except:
        return []

    # make simplified statements records
    statements = [{
        "date": Processor.parse_datetime(s["date"]).date(),
        "type": s["statement_type"],
        "title": s["title"],
        "url": s["url"],
    } for s in statements
      if s["date"]]

    # # downcase all-caps titles
    # for s in statements:
    #   if s["title"] != s["title"].upper(): continue
    #   s["title"] = s["title"].lower()
    #   if s["person"]: s["title"] = s["title"].replace(s["person"].lastname.lower(), s["person"].lastname) # common easy case fix

    return statements
Ejemplo n.º 2
0
def fetch_statements(bill):
    from person.models import Person
    from person.views import http_rest_json
    from parser.processor import Processor

    # load statements from ProPublica API, ignoring any network errors
    try:
        statements = http_rest_json(
            "https://api.propublica.org/congress/v1/{congress}/bills/{bill}/statements.json"
            .format(
                congress=bill.congress,
                bill=bill.bill_type_slug + str(bill.number),
            ),
            headers={
                'X-API-Key': settings.PROPUBLICA_CONGRESS_API_KEY,
                'Content-Type': 'application/json',
                'Accept': 'application/json'
            })
        if statements["status"] != "OK": raise Exception()
        statements = statements["results"]
    except:
        return []

    # bulk-fetch all legislators mentioned and make a mapping from bioguide ID to Person object
    legislators = {
        p.bioguideid: p
        for p in Person.objects.filter(
            bioguideid__in=[s['member_id'] for s in statements])
    }

    # make simplified statements records
    statements = [{
        "date": Processor.parse_datetime(s["date"]).date(),
        "person": legislators.get(s["member_id"]),
        "type": s["statement_type"],
        "title": s["title"],
        "url": s["url"],
    } for s in statements if s["date"]]

    # downcase all-caps titles
    for s in statements:
        if s["title"] != s["title"].upper(): continue
        s["title"] = s["title"].lower()
        if s["person"]:
            s["title"] = s["title"].replace(
                s["person"].lastname.lower(),
                s["person"].lastname)  # common easy case fix

    # sort by date because we want to give a diversity of viewpoints by showing only one
    # statement per legislator
    statements.sort(key=lambda s: s["date"], reverse=True)

    # Put all of the statements that are the most recent for the legislator first, then all of the second-most-recent, and so on.
    # Since sort is stable, it will remain in reverse-date order within each group.
    seen = {}
    for s in statements:
        seen[s["person"]] = seen.get(s["person"], 0) + 1
        s["legislator_ordinal"] = seen[s["person"]]
    statements.sort(key=lambda s: s["legislator_ordinal"])

    # bulk-fetch ideology & leadership scores
    from person.models import RoleType
    from person.analysis import load_sponsorship_analysis2
    scores = load_sponsorship_analysis2(
        bill.congress,
        RoleType.representative, None)["all"] + load_sponsorship_analysis2(
            bill.congress, RoleType.senator, None)["all"]
    leadership_scores = {s["id"]: float(s["leadership"]) for s in scores}
    ideology_scores = {s["id"]: float(s["ideology"]) for s in scores}

    # Tag relevance of the person making the press statement. (We'd love to prioritize relevant
    # committee chairs but we only have current committee info so we couldn't do it for
    # past bills.)
    cosponsors = set(bill.cosponsors.all())
    for s in statements:
        if s["person"] == bill.sponsor:
            s["relevance"] = "Sponsor"
            s["priority"] = (0, -leadership_scores.get(s["person"].id, 0))
        elif s["person"] in cosponsors:
            s["relevance"] = "Co-sponsor"
            s["priority"] = (1, -leadership_scores.get(
                s["person"].id if s["person"] else 0, 0))

    # Get the prioritized statements to show.
    ret = []

    # For up to the first two, show the sponsor and the cosponsor with the highest leadership
    # score (or if there is no sponsor statement, the two top cosponsors), but prioritizing
    # the most recent statements for each legislator
    statements.sort(
        key=lambda s: (s["legislator_ordinal"], s.get("priority", (2, ))))
    while statements and statements[0].get("priority") and statements[0][
            "legislator_ordinal"] == 1 and len(ret) < 2:
        ret.append(statements.pop(0))

    # For the third statement, take the legislator with the ideology score furthest from
    # the sponsor's score, to hopefully get an opposing viewpoint.
    if bill.sponsor and bill.sponsor.id in ideology_scores:
        statements.sort(key=lambda s: (
            s["legislator_ordinal"
              ],  # most recent statements by legislators first
            not (s["person"] and s["person"].id in ideology_scores
                 ),  # people with ideology scores first
            "relevance" in s,  # people who aren't a sponsor/cosponsor first
            -abs(
                ideology_scores.get(s["person"].id if s["person"] else 0, 0) -
                ideology_scores[bill.sponsor.id])))
        while statements and len(ret) < 3:
            ret.append(statements.pop(0))

    return ret
Ejemplo n.º 3
0
def vote_details(request, congress, session, chamber_code, number):
    vote = load_vote(congress, session, chamber_code, number)
    voters = vote.get_voters()

    # If this is a vote on passage, but it's not the final vote on passage in this
    # chamber, issue a warning.
    has_subsequent_vote = False
    if vote.related_bill and vote.category in (VoteCategory.passage, VoteCategory.passage_suspension) \
      and vote.related_bill.votes.filter(chamber=vote.chamber, category__in=(VoteCategory.passage, VoteCategory.passage_suspension)).order_by('-created').first() != vote:
        has_subsequent_vote = True

    # Test if we have diagrams for this vote. The only
    # way to test is to try to make it.
    has_diagram = {}
    for image_type in ("map", "diagram"):
        try:
            vote_thumbnail_image(request, congress, session, chamber_code,
                                 number, image_type)
            has_diagram[image_type] = True
        except Http404:
            has_diagram[image_type] = False

    # sorting by party actually sorts by party first and by ideology score
    # second.
    has_ideology_scores = attach_ideology_scores(voters, vote.congress)

    # perform an initial sort for display
    voters.sort(
        key=lambda x: (x.option.key, x.person_role.party
                       if x.person and x.person_role and x.person_role.party
                       else "", x.person.name_no_details_lastfirst()
                       if x.person else x.get_voter_type_display()))

    # did any Senate leaders switch their vote for a motion to reconsider?
    reconsiderers = vote.possible_reconsideration_votes(voters)
    reconsiderers_titles = "/".join(v.person_role.leadership_title
                                    for v in reconsiderers)

    # compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
    get_vote_outliers(voters)

    # get Explanations from ProPublica for House votes and attach to voter instances
    propublica_url = None
    propublica_count = 0
    if vote.chamber == CongressChamber.house and vote.congress >= 110:
        # ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where
        # we know the session is an integer (the session/legislative year).
        propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (
            int(vote.session), vote.number)
        try:
            explanations = http_rest_json(
                "https://projects.propublica.org/explanations/api/votes/%d/%d.json"
                % (int(vote.session), vote.number))
        except:
            # squash all errors
            explanations = {}
        propublica_count = explanations.get("vote",
                                            {}).get("total_explanations", 0)
        expl_map = {
            e["bioguide_id"]: e
            for e in explanations.get("vote", {}).get("explanations", [])
        }
        for voter in voters:
            voter.explanation = expl_map.get(voter.person.bioguideid)

    return {
        'vote':
        vote,
        'voters':
        voters,
        'CongressChamber':
        CongressChamber,
        "VoterType":
        VoterType,
        "VoteCategory":
        VoteCategory._items,
        'has_vp_vote':
        len([v
             for v in voters if v.voter_type == VoterType.vice_president]) > 0,
        'has_diagram':
        has_diagram,
        'has_ideology_scores':
        has_ideology_scores,
        'has_subsequent_vote':
        has_subsequent_vote,
        'diagram_key_colors':
        [(k, "rgb(%d,%d,%d)" % tuple([c * 256 for c in clr]))
         for k, clr in vote_diagram_colors.items()],
        'reconsiderers': (reconsiderers, reconsiderers_titles),
        'propublica_url':
        propublica_url,
        'propublica_count':
        propublica_count,
    }
Ejemplo n.º 4
0
def vote_details(request, congress, session, chamber_code, number):
    vote = load_vote(congress, session, chamber_code, number)
    voters = vote.get_voters()

    # If this is a vote on passage, but it's not the final vote on passage in this
    # chamber, issue a warning.
    has_subsequent_vote = False
    if vote.related_bill and vote.category in (VoteCategory.passage, VoteCategory.passage_suspension) \
      and vote.related_bill.votes.filter(chamber=vote.chamber, category__in=(VoteCategory.passage, VoteCategory.passage_suspension)).order_by('-created').first() != vote:
      has_subsequent_vote = True
    
    # Test if we have diagrams for this vote. The only
    # way to test is to try to make it.
    has_diagram = { }
    for image_type in ("map", "diagram"):
        try:
            vote_thumbnail_image(request, congress, session, chamber_code, number, image_type)
            has_diagram[image_type] = True
        except Http404:
            has_diagram[image_type] = False
    
    # sorting by party actually sorts by party first and by ideology score
    # second.
    has_ideology_scores = attach_ideology_scores(voters, vote.congress)
        
    # perform an initial sort for display
    voters.sort(key = lambda x : (x.option.key, x.person_role.party if x.person and x.person_role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display()))

    # did any Senate leaders switch their vote for a motion to reconsider?
    reconsiderers = vote.possible_reconsideration_votes(voters)
    reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers)

    # compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
    get_vote_outliers(voters)

    # get Explanations from ProPublica for House votes and attach to voter instances
    propublica_url = None
    propublica_count = 0
    if vote.chamber == CongressChamber.house and vote.congress >= 110:
        # ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where
        # we know the session is an integer (the session/legislative year).
        propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (int(vote.session), vote.number)
        try:
            explanations = http_rest_json("https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number))
        except:
            # squash all errors
            explanations = { }
        propublica_count = explanations.get("vote", {}).get("total_explanations", 0)
        expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) }
        for voter in voters:
            voter.explanation = expl_map.get(voter.person.bioguideid)
    
    return {'vote': vote,
            'voters': voters,
            'CongressChamber': CongressChamber,
            "VoterType": VoterType,
            "VoteCategory": VoteCategory._items,
            'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0,
            'has_diagram': has_diagram,
            'has_ideology_scores': has_ideology_scores,
            'has_subsequent_vote': has_subsequent_vote,
            'diagram_key_colors': [ (k, "rgb(%d,%d,%d)" % tuple([c*256 for c in clr])) for k, clr in vote_diagram_colors.items() ],
            'reconsiderers': (reconsiderers, reconsiderers_titles),
            'propublica_url': propublica_url,
            'propublica_count': propublica_count,
            }
Ejemplo n.º 5
0
def vote_details(request, congress, session, chamber_code, number):
    vote = load_vote(congress, session, chamber_code, number)
    voters = vote.get_voters()
    
    # Test if we have a diagram for this vote. The only
    # way to test is to try to make it.
    try:
        vote_thumbnail_image(request, congress, session, chamber_code, number, "diagram")
        has_diagram = True
    except Http404:
        has_diagram = False
    
    # sorting by party actually sorts by party first and by ideology score
    # second.
    has_ideology_scores = False
    congress = int(congress)
    global ideology_scores
    load_ideology_scores(congress)
    if ideology_scores[congress]:
        for voter in voters:
            if voter.person and voter.person.id in ideology_scores[congress]:
                voter.ideolog_score = ideology_scores[congress][voter.person.id]
                has_ideology_scores = True
            else:
                voter.ideolog_score = \
            	ideology_scores[congress].get("MEDIAN:" + (voter.person_role.party if voter.person and voter.person_role else ""),
            		ideology_scores[congress]["MEDIAN"])
        
    # perform an initial sort for display
    voters.sort(key = lambda x : (x.option.key, x.person_role.party if x.person and x.person_role else "", x.person.name_no_details_lastfirst if x.person else x.get_voter_type_display()))

    # did any Senate leaders switch their vote for a motion to reconsider?
    reconsiderers = vote.possible_reconsideration_votes(voters)
    reconsiderers_titles = "/".join(v.person_role.leadership_title for v in reconsiderers)

    # compute statistical outliers (this marks the Voter instances with an is_outlier attribute)
    get_vote_outliers(voters)

    # get Explanations from ProPublica for House votes and attach to voter instances
    propublica_url = None
    propublica_count = 0
    if vote.chamber == CongressChamber.house and vote.congress >= 110:
        # ProPublica data starts at the 110th Congress, and also be sure to only do 77th forward where
        # we know the session is an integer (the session/legislative year).
        propublica_url = "https://projects.propublica.org/explanations/votes/%d/%d" % (int(vote.session), vote.number)
        try:
            explanations = http_rest_json("https://projects.propublica.org/explanations/api/votes/%d/%d.json" % (int(vote.session), vote.number))
        except:
            # squash all errors
            explanations = { }
        propublica_count = explanations.get("vote", {}).get("total_explanations", 0)
        expl_map = { e["bioguide_id"]: e for e in explanations.get("vote", {}).get("explanations", []) }
        for voter in voters:
            voter.explanation = expl_map.get(voter.person.bioguideid)
    
    return {'vote': vote,
            'voters': voters,
            'CongressChamber': CongressChamber,
            "VoterType": VoterType,
            "VoteCategory": VoteCategory._items,
            'has_vp_vote': len([v for v in voters if v.voter_type == VoterType.vice_president]) > 0,
            'has_diagram': has_diagram,
            'has_ideology_scores': has_ideology_scores,
            'reconsiderers': (reconsiderers, reconsiderers_titles),
            'propublica_url': propublica_url,
            'propublica_count': propublica_count,
            }