def vote_search_manager(): sm = SearchManager(Vote, qs=Vote.objects.select_related('oursummary')) # show sessions as year+session for non-year-based sessions, # and then just the session number (the year) for year-based # sessions. def format_session(s): if s[0] >= 77: # year and congress number in parens return s[1] + " (" + ordinal(s[0]) + " Congress)" else: # date range and congress number in parens if s[2].year == s[3].year: # strftime requires year>=1900, so fool it for generating # month names by replacing old years with 1900 if s[2].month == s[3].month: return str(s[2].year) + " " + s[2].replace(1900).strftime("%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + " " + s[2].replace(1900).strftime("%b-") + s[3].replace(1900).strftime("%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + "-" + str(s[3].year) + " (" + ordinal(s[0]) + " Congress)" session_choices = reversed([(i, format_session(cs)) for (i,cs) in enumerate(get_all_sessions()) if cs[2] <= datetime.now().date()]) sm.add_option('session', type="select", choices=session_choices, filter=session_filter, help="Note: Even-year sessions extend a few days into the next year.") sm.add_option('chamber') sm.add_option('category') sm.add_sort('Date (Latest First)','-created', default=True) sm.add_sort('Most Supported','-percent_plus', func=lambda qs : qs.order_by('-percent_plus', '-total_plus').exclude(percent_plus=None)) sm.add_sort('Most Opposed','percent_plus', func=lambda qs : qs.order_by('percent_plus', '-total_minus').exclude(percent_plus=None)) sm.add_sort('Widest Margin','-margin', func=lambda qs : qs.order_by('-margin', '-total_plus').exclude(margin=None)) sm.add_sort('Narrowest Margin','margin', func=lambda qs : qs.order_by('margin', 'total_plus').exclude(margin=None)) #def safe_strftime(date, format): # return date.replace(year=3456).strftime(format).replace("3456", str(date.year)).replace(" 12:00AM", "") sm.set_template(""" <div class="row"> <div class="col-xs-12"> <div style="margin-bottom: .2em"><a href="{{object.get_absolute_url}}">{{object.question|truncatewords_html:50}}</a></div> </div> <div style="font-size: 93%"> <div class="col-sm-6 col-md-4"> <div><span class="fa fa-barcode fa-fw" aria-hidden="true" style="margin-left: 4px; color: #888"></span> {{object.name}}</div> <div><span class="fa fa-calendar fa-fw" aria-hidden="true" style="margin-left: 4px; color: #888"></span> {{object.created|date}} {{object.created|time|cut:"midnight"}}</div> </div> <div class="col-sm-6 col-md-8"> <div><span class="fa fa-info fa-fw" aria-hidden="true" style="color: #888"></span> {{object.summary}}</div> </div> <div class="col-xs-12" style="padding-top: .25em"> {% if object.question_details and not object.oursummary %}<div style="margin-left: 5px">{{object.question_details}}</div>{% endif %} {% if object.oursummary %}<div style="font-style: italic">{{object.oursummary.plain_text|truncatewords:50}}</div>{% endif %} </div> </div> </div> """) return sm
def collect_stats(session): # Get the congress and start/end dates of the session that this corresponds to. for congress, s, startdate, enddate in us.get_all_sessions(): if s == session: break else: raise ValueError("Invalid session: " + session) # Who was serving on the last day of the session? people = [(r.person, r) for r in PersonRole.objects .filter( role_type__in=(RoleType.representative, RoleType.senator), startdate__lt=enddate, # use __lt and not __lte in case of multiple roles on the same day enddate__gte=enddate, # use __lte in case anyone's term ended exactly on this day ) .select_related("person")] # Do a sponsorship analysis for bills in this session only. run_sponsorship_analysis(people, congress, startdate, enddate) # Get the committee members. from bill.prognosis import load_committee_membership committee_membership = load_committee_membership(congress) # Pre-fetch all of the votes in this session. votes_this_year = Vote.objects.filter(congress=congress, session=session) votes_this_year = { RoleType.representative: set(votes_this_year.filter(chamber=CongressChamber.house).values_list("id", flat=True)), RoleType.senator: set(votes_this_year.filter(chamber=CongressChamber.senate).values_list("id", flat=True)), } # Generate raw statistics. AllStats = { } for person, role in people: AllStats[person.id] = { "id": person.id, "role_id": role.id, "role_type": role.role_type, "role_start": role.startdate.isoformat(), "role_end": role.enddate.isoformat(), "stats": { }, "cohorts": get_cohorts(person, role, congress, session, committee_membership), } stats = AllStats[person.id]["stats"] get_vote_stats(person, role, stats, votes_this_year) get_sponsor_stats(person, role, stats, congress, startdate, enddate, committee_membership) get_cosponsor_stats(person, role, stats, congress, startdate, enddate) get_cosponsored_stats(person, role, stats, congress, startdate, enddate) get_sponsorship_analysis_stats(person, role, stats) get_committee_stats(person, role, stats, committee_membership) get_transparency_stats(person, role, stats, congress, startdate, enddate) return AllStats
def get_most_recent_session_stats(self): # Which Congress and session's end date is the most recently covered by this role? errs = [] congresses = self.congress_numbers() for congress, session, sd, ed in reversed(get_all_sessions()): if congress not in congresses: continue if self.startdate < ed <= self.enddate: try: return self.person.get_session_stats(session) except ValueError as e: errs.append(unicode(e)) raise ValueError("No statistics are available for this role: %s" % "; ".join(errs))
def vote_list(request): # Get the default session to show. We may have sessions listed that are # in the future, during a transition, so take the most recent that at # least has started. default_session = None for i, (cn, sn, sd, ed) in enumerate(get_all_sessions()): if sd > datetime.now().date(): break default_session = i return vote_search_manager().view(request, "vote/vote_list.html", defaults = { "session": default_session }, paginate = lambda form : "session" not in form ) # people like to see all votes for a year on one page
def vote_search_manager(): sm = SearchManager(Vote, qs=Vote.objects.order_by('-created')) # show sessions as year+session for non-year-based sessions, # and then just the session number (the year) for year-based # sessions. def format_session(s): if s[0] >= 77: # year and congress number in parens return s[1] + " (" + ordinal(s[0]) + " Congress)" else: # date range and congress number in parens if s[2].year == s[3].year: # strftime requires year>=1900, so fool it for generating # month names by replacing old years with 1900 if s[2].month == s[3].month: return str(s[2].year) + " " + s[2].replace(1900).strftime( "%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + " " + s[2].replace(1900).strftime( "%b-") + s[3].replace(1900).strftime( "%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + "-" + str(s[3].year) + " (" + ordinal( s[0]) + " Congress)" session_choices = reversed([(i, format_session(cs)) for (i, cs) in enumerate(get_all_sessions()) if cs[2] <= datetime.now().date()]) sm.add_option( 'session', type="select", choices=session_choices, filter=session_filter, help="Note: Even-year sessions extend a few days into the next year.") sm.add_option('chamber') sm.add_option('category') #def safe_strftime(date, format): # return date.replace(year=3456).strftime(format).replace("3456", str(date.year)).replace(" 12:00AM", "") sm.set_template(""" <div><a href="{{object.get_absolute_url}}">{{object.question|truncatewords_html:50}}</a></div> {% if object.question_details %}<div>{{object.question_details}}</div>{% endif %} <div>{{object.name}}</div> <div>{{object.created|date}} {{object.created|time|cut:"midnight"}}</div> <div>{{object.summary}}</div> """) return sm
def vote_redirect(request): if not "-" in request.GET.get("vote", ""): return HttpResponseRedirect("/congress/votes") try: a, roll = request.GET["vote"].split("-") except: raise Http404() chamber = a[0] session = a[1:] from us import get_all_sessions for cong, sess, start, end in get_all_sessions(): if sess == session or str(cong) + "_" + sess == session: return HttpResponseRedirect("/congress/votes/%s-%s/%s%s" % (cong, sess, chamber, roll)) raise Http404()
def vote_list(request): # Get the default session to show. We may have sessions listed that are # in the future, during a transition, so take the most recent that at # least has started. default_session = None for i, (cn, sn, sd, ed) in reversed(list(enumerate(get_all_sessions()))): if sd > datetime.now().date(): continue if not Vote.objects.filter(congress=cn, session=sn).exists(): continue default_session = i break return vote_search_manager().view(request, "vote/vote_list.html", defaults = { "session": default_session }, paginate = lambda form : "session" not in form, # people like to see all votes for a year on one page context = { "feed": Feed(feedname="misc:allvotes") })
def load_session_stats(session): # Which Congress is it? for congress, s, sd, ed in get_all_sessions(): if s == session: break # leaves "congress" variable set else: raise ValueError("Invalid session: %s" % session) fn = "data/us/%d/stats/session-%s.json" % (congress, session) try: datafile = json.load(open(fn)) datafile["meta"]["congress"] = congress # save where we got this from datafile["meta"]["session"] = session # save where we got this from datafile["meta"]["startdate"] = sd datafile["meta"]["enddate"] = ed except IOError: raise ValueError("No statistics are available for session %s." % session) return datafile
def load_session_stats(session): # Which Congress is it? for congress, s, sd, ed in get_all_sessions(): if s == session: break # leaves "congress" variable set else: raise ValueError("Invalid session: %s" % session) fn = "data/us/%d/stats/session-%s.json" % (congress, session) try: datafile = json.load(open(fn)) datafile["meta"]["pub_year"] = session if datafile["meta"]["is_full_congress_stats"]: datafile["meta"]["startdate"] = get_congress_dates(congress)[0] datafile["meta"]["enddate"] = get_congress_dates(congress)[1] else: datafile["meta"]["startdate"] = sd datafile["meta"]["enddate"] = ed except IOError: raise ValueError("No statistics are available for session %s." % session) return datafile
def vote_search_manager(): sm = SearchManager(Vote, qs=Vote.objects.order_by('-created')) # show sessions as year+session for non-year-based sessions, # and then just the session number (the year) for year-based # sessions. def format_session(s): if s[0] >= 77: # year and congress number in parens return s[1] + " (" + ordinal(s[0]) + " Congress)" else: # date range and congress number in parens if s[2].year == s[3].year: # strftime requires year>=1900, so fool it for generating # month names by replacing old years with 1900 if s[2].month == s[3].month: return str(s[2].year) + " " + s[2].replace(1900).strftime("%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + " " + s[2].replace(1900).strftime("%b-") + s[3].replace(1900).strftime("%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + "-" + str(s[3].year) + " (" + ordinal(s[0]) + " Congress)" session_choices = reversed([(i, format_session(cs)) for (i,cs) in enumerate(get_all_sessions()) if cs[2] <= datetime.now().date()]) sm.add_option('session', type="select", choices=session_choices, filter=session_filter, help="Note: Even-year sessions extend a few days into the next year.") sm.add_option('chamber') sm.add_option('category') #def safe_strftime(date, format): # return date.replace(year=3456).strftime(format).replace("3456", str(date.year)).replace(" 12:00AM", "") sm.set_template(""" <div><a href="{{object.get_absolute_url}}">{{object.question|truncatewords_html:50}}</a></div> {% if object.question_details %}<div>{{object.question_details}}</div>{% endif %} <div>{{object.name}}</div> <div>{{object.created|date}} {{object.created|time|cut:"midnight"}}</div> <div>{{object.summary}}</div> """) return sm
people_sort_order.append(p) # build the output matrix for bc in bill_counts: data_matrix.setdefault(people_map[bc['sponsor_role']].person, {})[(congress, session)] = bc["count"] return len(bill_counts) > 0 for role_type in (RoleType.representative, RoleType.senator): people_sort_order = [] data_matrix = {} sessions = [] for congress, session, startdate, enddate in get_all_sessions(): if congress < 109: continue # make a smaller table if build(role_type, congress, session, startdate, enddate, people_sort_order, data_matrix): print(role_type.congress_chamber, congress, session) sessions.append((congress, session)) writer = csv.writer( open( "sponsorship_counts_%s.csv" % role_type.congress_chamber.lower()[0], "w")) writer.writerow(["id", "name"] + [cs[1] for cs in sessions]) def zero(value): if value is None: return 0
def collect_stats(session): # Get the congress and start/end dates of the session that this corresponds to. if int(session) < 1000: # Specifies a Congress. congress = int(session) session = None is_full_congress_stats = True # Get the last session in the Congress. for c, s, x, y in us.get_all_sessions(): if c == congress: session2 = s last_day_of_session = y # Dummy dates. Don't want to use the congress dates because a bill can be # enacted after the end of the Congress. startdate = datetime.date.min enddate = datetime.date.max else: is_full_congress_stats = False session2 = session for congress, s, startdate, enddate in us.get_all_sessions(): if s == session: break else: raise ValueError("Invalid session: " + session) last_day_of_session = enddate # Who was serving on the last day of the session? people = [(r.person, r) for r in PersonRole.objects .filter( role_type__in=(RoleType.representative, RoleType.senator), startdate__lt=last_day_of_session, # use __lt and not __lte in case of multiple roles on the same day enddate__gte=last_day_of_session, # use __lte in case anyone's term ended exactly on this day ) .select_related("person")] # Do a sponsorship analysis for bills in this session only. run_sponsorship_analysis(people, congress, startdate, enddate) # Get the committee members. from bill.prognosis import load_committee_membership committee_membership = load_committee_membership(congress) # Pre-fetch all of the votes in this session. votes_this_year = Vote.objects.filter(congress=congress) if session: votes_this_year = votes_this_year.filter(session=session) votes_this_year = { RoleType.representative: set(votes_this_year.filter(chamber=CongressChamber.house).values_list("id", flat=True)), RoleType.senator: set(votes_this_year.filter(chamber=CongressChamber.senate).values_list("id", flat=True)), } # Generate raw statistics. AllStats = { } for person, role in people: AllStats[person.id] = { "id": person.id, "role_id": role.id, "role_type": role.role_type, "role_start": role.startdate.isoformat(), "role_end": role.enddate.isoformat(), "stats": { }, "cohorts": get_cohorts(person, role, congress, session2, committee_membership), } stats = AllStats[person.id]["stats"] get_vote_stats(person, role, stats, votes_this_year) get_sponsor_stats(person, role, stats, congress, startdate, enddate, committee_membership) get_cosponsor_stats(person, role, stats, congress, startdate, enddate) get_cosponsored_stats(person, role, stats, congress, startdate, enddate) get_sponsorship_analysis_stats(person, role, stats) get_committee_stats(person, role, stats, committee_membership) get_transparency_stats(person, role, stats, congress, startdate, enddate) return AllStats, congress, is_full_congress_stats
for p in people: if p not in data_matrix: people_sort_order.append(p) # build the output matrix for bc in bill_counts: data_matrix.setdefault(people_map[bc['sponsor_role']].person, {})[(congress, session)] = bc["count"] return len(bill_counts) > 0 for role_type in (RoleType.representative, RoleType.senator): people_sort_order = [] data_matrix = { } sessions = [] for congress, session, startdate, enddate in get_all_sessions(): if congress < 109: continue # make a smaller table if build(role_type, congress, session, startdate, enddate, people_sort_order, data_matrix): print(role_type.congress_chamber, congress, session) sessions.append((congress, session)) writer = csv.writer(open("sponsorship_counts_%s.csv" % role_type.congress_chamber.lower()[0], "w")) writer.writerow(["id", "name"] + [cs[1] for cs in sessions]) def zero(value): if value is None: return 0 return value for p in people_sort_order: writer.writerow(
def session_filter(qs, form): session_index = form["session"] if session_index != None: s = get_all_sessions()[int(session_index)] qs = qs.filter(congress=s[0], session=s[1]) return qs
def vote_search_manager(): sm = SearchManager(Vote, qs=Vote.objects.select_related('oursummary')) # show sessions as year+session for non-year-based sessions, # and then just the session number (the year) for year-based # sessions. def format_session(s): if s[0] >= 77: # year and congress number in parens return s[1] + " (" + ordinal(s[0]) + " Congress)" else: # date range and congress number in parens if s[2].year == s[3].year: # strftime requires year>=1900, so fool it for generating # month names by replacing old years with 1900 if s[2].month == s[3].month: return str(s[2].year) + " " + s[2].replace(1900).strftime( "%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + " " + s[2].replace(1900).strftime( "%b-") + s[3].replace(1900).strftime( "%b") + " (" + ordinal(s[0]) + " Congress)" else: return str(s[2].year) + "-" + str(s[3].year) + " (" + ordinal( s[0]) + " Congress)" session_choices = reversed([(i, format_session(cs)) for (i, cs) in enumerate(get_all_sessions()) if cs[2] <= datetime.now().date()]) sm.add_option( 'session', type="select", choices=session_choices, filter=session_filter, help="Note: Even-year sessions extend a few days into the next year.") sm.add_option('chamber') sm.add_option('category') sm.add_sort('Date (Latest First)', '-created', default=True) sm.add_sort('Most Supported', '-percent_plus', func=lambda qs: qs.order_by('-percent_plus', '-total_plus'). exclude(percent_plus=None)) sm.add_sort('Most Opposed', 'percent_plus', func=lambda qs: qs.order_by('percent_plus', '-total_minus'). exclude(percent_plus=None)) sm.add_sort('Widest Margin', '-margin', func=lambda qs: qs.order_by('-margin', '-total_plus').exclude( margin=None)) sm.add_sort('Narrowest Margin', 'margin', func=lambda qs: qs.order_by('margin', 'total_plus').exclude( margin=None)) #def safe_strftime(date, format): # return date.replace(year=3456).strftime(format).replace("3456", str(date.year)).replace(" 12:00AM", "") sm.set_template(""" <div class="row"> <div class="col-xs-12"> <div style="margin-bottom: .2em"><a href="{{object.get_absolute_url}}">{{object.question|truncatewords_html:50}}</a></div> </div> <div style="font-size: 93%"> <div class="col-xs-12 col-sm-6 col-md-4"> <div><span class="fa fa-barcode fa-fw" aria-hidden="true" style="margin-left: 4px; color: #888"></span> {{object.name}}</div> <div><span class="fa fa-calendar fa-fw" aria-hidden="true" style="margin-left: 4px; color: #888"></span> {{object.created|date}} {{object.created|time|cut:"midnight"}}</div> </div> <div class="col-xs-12 col-sm-6 col-md-8"> <div><span class="fa fa-info fa-fw" aria-hidden="true" style="color: #888"></span> {{object.summary}}</div> </div> <div class="col-xs-12"> <div style="margin-left: 5px"> {% if object.question_details and not object.oursummary %}<div style="margin-top: .25em">{{object.question_details}}</div>{% endif %} {% if object.get_summary %}<div style="margin-top: .25em; font-style: italic; line-height: 126%;">{{object.get_summary.plain_text|truncatewords:50}}</div>{% endif %} </div> </div> </div> </div> """) return sm
def collect_stats(session): # Get the congress and start/end dates of the session that this corresponds to. for congress, s, startdate, enddate in us.get_all_sessions(): if s == session: break else: raise ValueError("Invalid session: " + session) # Who was serving on the last day of the session? people = [ (r.person, r) for r in PersonRole.objects.filter( role_type__in=(RoleType.representative, RoleType.senator), startdate__lt= enddate, # use __lt and not __lte in case of multiple roles on the same day enddate__gte= enddate, # use __lte in case anyone's term ended exactly on this day ).select_related("person") ] # Do a sponsorship analysis for bills in this session only. run_sponsorship_analysis(people, congress, startdate, enddate) # Get the committee members. from bill.prognosis import load_committee_membership committee_membership = load_committee_membership(congress) # Pre-fetch all of the votes in this session. votes_this_year = Vote.objects.filter(congress=congress, session=session) votes_this_year = { RoleType.representative: set( votes_this_year.filter(chamber=CongressChamber.house).values_list( "id", flat=True)), RoleType.senator: set( votes_this_year.filter(chamber=CongressChamber.senate).values_list( "id", flat=True)), } # Generate raw statistics. AllStats = {} for person, role in people: AllStats[person.id] = { "id": person.id, "role_id": role.id, "role_type": role.role_type, "role_start": role.startdate.isoformat(), "role_end": role.enddate.isoformat(), "stats": {}, "cohorts": get_cohorts(person, role, congress, session, committee_membership), } stats = AllStats[person.id]["stats"] get_vote_stats(person, role, stats, votes_this_year) get_sponsor_stats(person, role, stats, congress, startdate, enddate, committee_membership) get_cosponsor_stats(person, role, stats, congress, startdate, enddate) get_cosponsored_stats(person, role, stats, congress, startdate, enddate) get_sponsorship_analysis_stats(person, role, stats) get_committee_stats(person, role, stats, committee_membership) get_transparency_stats(person, role, stats, congress, startdate, enddate) return AllStats
# # Forgive me for generating SVG without a proper library. from datetime import date from us import get_all_sessions bar_width = 3.5 height = 20.0 print("""<svg xmlns="http://www.w3.org/2000/svg" version="1.1">""") def break_session(s1, s2): for y in range(s1.year, s2.year+1): yield (s1 if y == s1.year else date(y, 1, 1)), (s2 if y == s2.year else date(y, 12, 31)) for cong, sess, s1, s2 in get_all_sessions(): clr = "rgb(200,200,200)" if (cong % 2) == 0: clr = "rgb(100,100,100)" for startdate, enddate in break_session(s1, s2): print("""<rect x="%f" y="%f" width="%f" height="%f" style="fill:%s; stroke-width:%f; stroke:rgb(0,0,0)" />""" % ( (startdate.year - 1789) * bar_width, (startdate - date(startdate.year, 1, 1)).days/365.0 * height, .5 * bar_width, (enddate - startdate).days/365.0*5.0 * height, clr, bar_width * .05, )) print("""</svg>""")
from datetime import date from us import get_all_sessions bar_width = 3.5 height = 20.0 print("""<svg xmlns="http://www.w3.org/2000/svg" version="1.1">""") def break_session(s1, s2): for y in range(s1.year, s2.year + 1): yield (s1 if y == s1.year else date(y, 1, 1)), (s2 if y == s2.year else date(y, 12, 31)) for cong, sess, s1, s2 in get_all_sessions(): clr = "rgb(200,200,200)" if (cong % 2) == 0: clr = "rgb(100,100,100)" for startdate, enddate in break_session(s1, s2): print( """<rect x="%f" y="%f" width="%f" height="%f" style="fill:%s; stroke-width:%f; stroke:rgb(0,0,0)" />""" % ( (startdate.year - 1789) * bar_width, (startdate - date(startdate.year, 1, 1)).days / 365.0 * height, .5 * bar_width, (enddate - startdate).days / 365.0 * 5.0 * height, clr, bar_width * .05, ))