def membersoverview(request): def get_current_members(role_type, delegates, by_party): qs = PersonRole.objects.filter( role_type=role_type, current=True, state__in=set(s for s, t in stateapportionment.items() if (t != "T") ^ delegates) ) if by_party: return qs.values('party').annotate(count=Count('party')).order_by('-count') else: return qs.count() congress_current = (CURRENT_CONGRESS, get_congress_dates(CURRENT_CONGRESS)[0]) congress_previous = (CURRENT_CONGRESS-1, get_congress_dates(CURRENT_CONGRESS-1)[1]) return { "statelist": statelist, "senate_by_party": get_current_members(RoleType.senator, False, True), "senate_vacancies": 100-get_current_members(RoleType.senator, False, False), "house_by_party": get_current_members(RoleType.representative, False, True), "house_vacancies": 435-get_current_members(RoleType.representative, False, False), "house_delegate_vacancies": 6-get_current_members(RoleType.representative, True, False), "congress_current": congress_current, "congress_previous": congress_previous, }
def compute_productivity(congress, date_range): # laws enacted_bills = Bill.objects.filter( congress=congress, # if we're measuring presidential activity, the date of signing could be outside of the Congress, so change this current_status__in=BillStatus.final_status_passed_bill, #current_status_date__gte=date_range[0], #current_status_date__lte=date_range[1] ) #enacted_bills = (enacted_bills.filter(title__contains="Appropriations") | enacted_bills.filter(title__contains="Authorization")).distinct() enacted_bills = list(enacted_bills) enacted_bills_count = len(enacted_bills) enacted_bill_pages = 0 enacted_bill_words = 0 enacted_bill_pages_missing = 0 for b in enacted_bills: try: pp = load_bill_text(b, None, mods_only=True).get("numpages") except IOError: pp = None if pp is None: enacted_bill_pages_missing += 1 continue pp = int(pp.replace(" pages", "")) enacted_bill_pages += pp wds = len(load_bill_text(b, None, plain_text=True).split(" ")) enacted_bill_words += wds if congress < 103: enacted_bill_pages = "(no data)" if congress < 103: enacted_bill_words = "(no data)" # votes house_votes = Vote.objects.filter( congress=congress, created__gte=date_range[0], created__lte=date_range[1], chamber=CongressChamber.house).count() senate_votes = Vote.objects.filter( congress=congress, created__gte=date_range[0], created__lte=date_range[1], chamber=CongressChamber.senate).count() ## power #congress_same_party = party_control[congress][0] == party_control[congress][1] #branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2]) # timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year-1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year)) row = [congress, timespan, date_range[0].isoformat(), date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes] W.writerow(row)
def compute_productivity(congress, days_in): corresponding_day = get_congress_dates(congress)[0] + days_in # laws enacted_bills = Bill.objects.filter( congress=congress, current_status__in=BillStatus.final_status_passed_bill, current_status_date__lte=corresponding_day) #enacted_bills = (enacted_bills.filter(title__contains="Appropriations") | enacted_bills.filter(title__contains="Authorization")).distinct() enacted_bills = list(enacted_bills) enacted_bills_count = len(enacted_bills) enacted_bill_pages = 0 enacted_bill_words = 0 enacted_bill_pages_missing = 0 for b in enacted_bills: try: pp = load_bill_text(b, None, mods_only=True).get("numpages") except IOError: pp = None if pp is None: enacted_bill_pages_missing += 1 continue pp = int(pp.replace(" pages", "")) enacted_bill_pages += pp wds = len(load_bill_text(b, None, plain_text=True).split(" ")) enacted_bill_words += wds if congress < 103: enacted_bill_pages = "(no data)" if congress < 103: enacted_bill_words = "(no data)" # votes house_votes = Vote.objects.filter( congress=congress, created__lte=corresponding_day, chamber=CongressChamber.house).count() senate_votes = Vote.objects.filter( congress=congress, created__lte=corresponding_day, chamber=CongressChamber.senate).count() # power congress_same_party = party_control[congress][0] == party_control[congress][1] branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2]) # timespan = "%d (%d-%d)" % (congress, get_congress_dates(congress)[0].year, get_congress_dates(congress)[1].year-1) row = [timespan, enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes, "Yes" if congress_same_party else "No", "Yes" if branches_same_party else "No"] W.writerow(row)
def bill_summaries(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) return { "bill": bill, "congressdates": get_congress_dates(bill.congress), "text_info": get_text_info(bill), # for the header tabs }
def bill_widget(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) from person.name import get_person_name if bill.sponsor: bill.sponsor.role = bill.sponsor_role # for rending name sponsor_name = None if not bill.sponsor else \ get_person_name(bill.sponsor, firstname_position='before', show_suffix=True) def get_text_info(): from billtext import load_bill_text try: return load_bill_text(bill, None, mods_only=True) except IOError: return None return { "SITE_ROOT_URL": settings.SITE_ROOT_URL, "bill": bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "sponsor_name": sponsor_name, "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "text": get_text_info, }
def bill_key_questions(request, congress, type_slug, number): # load bill and text bill = load_bill_from_url(congress, type_slug, number) text_info = bill.get_text_info(with_citations=True) # load stakeholder positions from stakeholder.models import BillPosition stakeholder_posts = bill.stakeholder_positions\ .filter(post__stakeholder__verified=True)\ .select_related("post", "post__stakeholder")\ .order_by('-created') def add_position_return_post(bp): bp.post.position = bp.position; return bp.post stakeholder_posts = [add_position_return_post(bp) for bp in stakeholder_posts] # context return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "prognosis": bill.get_prognosis_with_details(), "text_info": text_info, "text_incorporation": fixup_text_incorporation(bill.text_incorporation), "stakeholder_posts": stakeholder_posts, "legislator_statements": fetch_statements(bill), }
def build_info(): feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"] groups = [ ( g[0], # title g[1], # text 1 g[2], # text 2 "/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]), # link load_bill_status_qs(g[4]).count(), # count in category load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category ) for g in bill_status_groups ] dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date')) sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date')) coming_up = list(dhg_bills | sfs_bills) coming_up.sort(key = lambda b : b.docs_house_gov_postdate if (b.docs_house_gov_postdate and (not b.senate_floor_schedule_postdate or b.senate_floor_schedule_postdate < b.docs_house_gov_postdate)) else b.senate_floor_schedule_postdate, reverse=True) start, end = get_congress_dates(CURRENT_CONGRESS) end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year current_congress_years = '%d-%d' % (start.year, end.year) current_congress = ordinal(CURRENT_CONGRESS) return { "feeds": feeds, "total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(), "current_congress_years": current_congress_years, "current_congress": current_congress, "groups": groups, "coming_up": coming_up, "subjects": subject_choices(), "BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.referred, BillStatus.reported), }
def load_roles_at_date(persons, when, congress): """ Find out representative/senator role of each person at given date. This method is optimized for bulk operation. """ if when: when = (when, when) else: from us import get_congress_dates when = get_congress_dates(congress) roles = PersonRole.objects.filter(startdate__lte=when[1], enddate__gte=when[0], role_type__in=(RoleType.representative, RoleType.senator), person__in=persons) roles_by_person = {} for role in roles: if role.congress_numbers( ) is not None and congress not in role.congress_numbers(): continue roles_by_person[role.person_id] = role for person in persons: person.role = roles_by_person.get(person.id) return None
def bill_details(request, congress, type_slug, number): # pre-load info bill = load_bill_from_url(congress, type_slug, number) text_info = bill.get_text_info(with_citations=True) from stakeholder.models import BillPosition stakeholder_posts = bill.stakeholder_positions\ .filter(post__stakeholder__verified=True)\ .select_related("post", "post__stakeholder")\ .order_by('-created') def add_position_return_post(bp): bp.post.position = bp.position; return bp.post stakeholder_posts = [add_position_return_post(bp) for bp in stakeholder_posts] # context return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "feed": bill.get_feed(), "prognosis": bill.get_prognosis_with_details(), "text_info": text_info, "text_incorporation": fixup_text_incorporation(bill.text_incorporation), "show_media_bar": not bill.original_intent_replaced and bill.sponsor and bill.sponsor.has_photo() and text_info and text_info.get("has_thumbnail"), "stakeholder_posts": stakeholder_posts, }
def bill_summaries(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) return { "bill_subpage": "Summary", "bill": bill, "congressdates": get_congress_dates(bill.congress), "text_info": bill.get_text_info(with_citations=True), # for the header tabs }
def get_last_role_at_congress(self, congress): start, end = get_congress_dates(congress) try: return self.roles.filter( startdate__lte=end, enddate__gte=start).order_by('-startdate')[0] except IndexError: return None
def get_roles_of_people(congressnumber): congress_dates = get_congress_dates(congressnumber) return PersonRole.objects.filter( role_type__in=(RoleType.senator, RoleType.representative), startdate__lt=congress_dates[1], # start dates of next congress are on this day too enddate__gt=congress_dates[0] # end dates from previous congress are on this day too ).select_related("person")\ .order_by('-startdate') # so we put people in the right list by their most recent term
def build_info(): # feeds about all legislation that we offer the user to subscribe to feeds = [ f for f in Feed.get_simple_feeds() if f.category == "federal-bills" ] # info about bills by status groups = [ ( g[0], # title g[1], # text 1 g[2], # text 2 "/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]) + "&sort=-current_status_date", # link load_bill_status_qs(g[4]).count(), # count in category load_bill_status_qs(g[4]).order_by( '-current_status_date')[0:6], # top 6 in this category ) for g in bill_status_groups ] # legislation coming up dhg_bills = Bill.objects.filter( congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter( docs_house_gov_postdate__gt=F('current_status_date')) sfs_bills = Bill.objects.filter( congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter( senate_floor_schedule_postdate__gt=F('current_status_date')) coming_up = list( (dhg_bills | sfs_bills).order_by('scheduled_consideration_date')) # top tracked bills top_bills = Feed.objects\ .filter(feedname__startswith='bill:')\ .filter(feedname__regex='^bill:[hs][jcr]?%d-' % CURRENT_CONGRESS) top_bills = top_bills\ .annotate(count=Count('tracked_in_lists'))\ .order_by('-count')\ .values('feedname', 'count')\ [0:25] top_bills = [(Bill.from_feed(Feed.from_name(bf["feedname"])), bf["count"]) for bf in top_bills] return { "feeds": feeds, "total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(), "current_congress": CURRENT_CONGRESS, "current_congress_dates": get_congress_dates(CURRENT_CONGRESS), "groups": groups, "coming_up": coming_up, "top_tracked_bills": top_bills, "subjects": subject_choices(), "BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.reported), }
def bill_text(request, congress, type_slug, number, version=None): if version == "": version = None try: bill_type = BillType.by_slug(type_slug) except BillType.NotFound: raise Http404("Invalid bill type: " + type_slug) bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number) from .billtext import load_bill_text, get_bill_text_versions try: textdata = load_bill_text(bill, version) except IOError: textdata = None # Get a list of the alternate versions of this bill. alternates = None is_latest = True if textdata: alternates = [] for v in get_bill_text_versions(bill): try: alternates.append(load_bill_text(bill, v, mods_only=True)) except IOError: pass alternates.sort(key = lambda mods : mods["docdate"]) if len(alternates) > 0: is_latest = False if textdata["doc_version"] == alternates[-1]["doc_version"]: is_latest = True # Get a list of related bills. from .billtext import get_current_version related_bills = [] for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]: try: rbv = get_current_version(rb) if not (rb, rbv) in related_bills: related_bills.append((rb, rbv)) except IOError: pass # text not available for btc in BillTextComparison.objects.filter(bill1=bill).exclude(bill2=bill): if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2)) for btc in BillTextComparison.objects.filter(bill2=bill).exclude(bill1=bill): if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1)) return { "bill_subpage": "Text", 'bill': bill, "congressdates": get_congress_dates(bill.congress), "textdata": textdata, "version": version, "is_latest": is_latest, "alternates": alternates, "related_bills": related_bills, "days_old": (datetime.datetime.now().date() - bill.current_status_date).days, "is_on_bill_text_page": True, # for the header tabs }
def analysis_methodology(request): from settings import CURRENT_CONGRESS from person.models import RoleType from bill.models import BillType from us import get_congress_dates import json from person.analysis import load_sponsorship_analysis2 def make_chart_series(role_type): data = load_sponsorship_analysis2(CURRENT_CONGRESS, role_type, None) if not data: return None ret = { } for p in data["all"]: ret.setdefault(p["party"], { "type": "party", "party": p["party"], "data": [], })["data"].append({ "x": float(p["ideology"]), "y": float(p["leadership"]), "name": p["name"], }) ret = list(ret.values()) ret.sort(key = lambda s : len(s["data"]), reverse=True) data = dict(data) # clone before modifying, just in case data["series"] = json.dumps(ret) return data import bill.prognosis import bill.prognosis_model import bill.prognosis_model_test prognosis_factors = sorted([dict(v) for v in bill.prognosis_model.factors.values()], key = lambda m : m["count"], reverse=True) for v in prognosis_factors: v["factors"] = sorted(v["factors"].values(), key = lambda f : f["regression_beta"], reverse=True) prognosis_test = sorted(bill.prognosis_model_test.model_test_results.values(), key = lambda v : v["count"], reverse=True) return { "ideology": lambda : { # defer until cache miss "house": make_chart_series(RoleType.representative), "senate": make_chart_series(RoleType.senator), }, "current_congress": CURRENT_CONGRESS, "prognosis_training_congress": bill.prognosis_model.congress, "prognosis_training_congress_dates": get_congress_dates(bill.prognosis_model.congress), "prognosis_factors": prognosis_factors, "prognosis_test": prognosis_test, "prognosis_testing_traincongress": bill.prognosis_model_test.train_congress, "prognosis_testing_testcongress": bill.prognosis_model_test.test_congress, }
def bill_details(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "feed": bill.get_feed(), "text_info": get_text_info(bill), }
def load_session_stats(session): # Which Congress is it? for congress, s, sd, ed in get_all_sessions(): if s == session: break # leaves "congress" variable set else: raise ValueError("Invalid session: %s" % session) fn = "data/us/%d/stats/session-%s.json" % (congress, session) try: datafile = json.load(open(fn)) datafile["meta"]["pub_year"] = session if datafile["meta"]["is_full_congress_stats"]: datafile["meta"]["startdate"] = get_congress_dates(congress)[0] datafile["meta"]["enddate"] = get_congress_dates(congress)[1] else: datafile["meta"]["startdate"] = sd datafile["meta"]["enddate"] = ed except IOError: raise ValueError("No statistics are available for session %s." % session) return datafile
def load_majority_party(congress): majority_party = { } start, end = get_congress_dates(congress) for rt, bts in ( (RoleType.senator, (BillType.senate_bill, BillType.senate_resolution, BillType.senate_concurrent_resolution, BillType.senate_joint_resolution)), (RoleType.representative, (BillType.house_bill, BillType.house_resolution, BillType.house_concurrent_resolution, BillType.house_joint_resolution)) ): p = PersonRole.objects.filter(startdate__lte=end, enddate__gte=start, role_type=rt).values("party").annotate(count=Count("id")).order_by("-count")[0]["party"] for bt in bts: majority_party[bt] = p return majority_party
def browse_state(request, state): state = normalize_state_arg(state) if state not in stateapportionment: raise Http404() return { "state": state, "stateapp": stateapportionment[state], "statename": statenames[state], "senators": get_senators(state), "representatives": get_representatives(state), "end_of_congress_date": get_congress_dates(CURRENT_CONGRESS)[1], }
def build_info(): # feeds about all legislation that we offer the user to subscribe to feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"] # info about bills by status groups = [ ( g[0], # title g[1], # text 1 g[2], # text 2 "/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]) + "&sort=-current_status_date", # link load_bill_status_qs(g[4]).count(), # count in category load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category ) for g in bill_status_groups ] # legislation coming up dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date')) sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date')) coming_up = list(dhg_bills | sfs_bills) coming_up.sort(key = lambda b : b.docs_house_gov_postdate if (b.docs_house_gov_postdate and (not b.senate_floor_schedule_postdate or b.senate_floor_schedule_postdate < b.docs_house_gov_postdate)) else b.senate_floor_schedule_postdate, reverse=True) # top tracked bills top_bills = Feed.objects\ .filter(feedname__startswith='bill:')\ .filter(feedname__regex='^bill:[hs][jcr]?%d-' % CURRENT_CONGRESS) top_bills = top_bills\ .annotate(count=Count('tracked_in_lists'))\ .order_by('-count')\ .values('feedname', 'count')\ [0:25] top_bills = [(Bill.from_feed(Feed.from_name(bf["feedname"])), bf["count"]) for bf in top_bills] # current congrss years start, end = get_congress_dates(CURRENT_CONGRESS) end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year current_congress_years = '%d-%d' % (start.year, end.year) current_congress = ordinal(CURRENT_CONGRESS) return { "feeds": feeds, "total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(), "current_congress_years": current_congress_years, "current_congress": current_congress, "groups": groups, "coming_up": coming_up, "top_tracked_bills": top_bills, "subjects": subject_choices(), "BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.referred, BillStatus.reported), }
def build_info(): # feeds about all legislation that we offer the user to subscribe to feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"] # info about bills by status groups = [ ( g[0], # title g[1], # text 1 g[2], # text 2 "/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]) + "&sort=-current_status_date", # link load_bill_status_qs(g[4]).count(), # count in category load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category ) for g in bill_status_groups ] # legislation coming up dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date')) sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date')) coming_up = list((dhg_bills | sfs_bills).order_by('scheduled_consideration_date')) # top tracked bills top_bills = Feed.objects\ .filter(feedname__startswith='bill:')\ .filter(feedname__regex='^bill:[hs][jcr]?%d-' % CURRENT_CONGRESS) top_bills = top_bills\ .annotate(count=Count('tracked_in_lists'))\ .order_by('-count')\ .values('feedname', 'count')\ [0:25] top_bills = [(Bill.from_feed(Feed.from_name(bf["feedname"])), bf["count"]) for bf in top_bills] # trending bills trf = Feed.get_trending_feeds() trf = [Feed.objects.get(id=f) for f in trf] trending_bill_feeds = [f for f in trf if f.feedname.startswith("bill:")] return { "feeds": feeds, "total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(), "current_congress": CURRENT_CONGRESS, "current_congress_dates": get_congress_dates(CURRENT_CONGRESS), "groups": groups, "coming_up": coming_up, "top_tracked_bills": top_bills, "trending_bill_feeds": trending_bill_feeds, "subjects": subject_choices(), "BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.reported), }
def person_session_stats(request, pk, session): # get the person and the statistics person = get_object_or_404(Person, pk=pk) try: stats = person.get_session_stats(session) except ValueError: # no stats raise Http404() # get the role as stored in the file role = PersonRole.objects.get(id=stats["role_id"]) # mark the role as current if the logical end date is in the future, to fix the display of Served/Serving role.current = (role.logical_dates()[1] > datetime.now().date()) # clean and sort the stats for this person so they're ready for display from person.views_sessionstats import clean_person_stats clean_person_stats(stats) # group into an order for navigation nav_groups = [] for stat in stats["stats"]: for group in nav_groups: if group["icon"] == stat["icon"]: group["stats"].append(stat) break else: nav_groups.append({ "icon": stat["icon"], "stats": [stat] }) import dateutil.parser from person.types import Gender, RoleType # what dates specifically for the congress? (period_min, period_max) = get_congress_dates(stats["meta"]["congress"]) period_min = max(period_min, role.logical_dates()[0]) period_max = min(period_max, role.logical_dates()[1]) return { "publishdate": dateutil.parser.parse(stats["meta"]["as-of"]), "period": session_stats_period(session, stats), "congress_dates": (period_min, period_max), "person": person, "photo": person.get_photo()[0], "himher": Gender.by_value(person.gender).pronoun_object, "role": role, "class": RoleType.by_value(role.role_type).label.lower() + "s", "session": session, "meta": stats["meta"], "stats": stats["stats"], "nav_groups": nav_groups, }
def person_session_stats(request, pk, session): # get the person and the statistics person = get_object_or_404(Person, pk=pk) try: stats = person.get_session_stats(session) except ValueError: # no stats raise Http404() # get the role as stored in the file role = PersonRole.objects.get(id=stats["role_id"]) # mark the role as current if the logical end date is in the future, to fix the display of Served/Serving role.current = (role.logical_dates()[1] > datetime.now().date()) # clean and sort the stats for this person so they're ready for display from person.views_sessionstats import clean_person_stats clean_person_stats(stats) # group into an order for navigation nav_groups = [] for stat in stats["stats"]: for group in nav_groups: if group["icon"] == stat["icon"]: group["stats"].append(stat) break else: nav_groups.append({"icon": stat["icon"], "stats": [stat]}) import dateutil.parser from person.types import Gender, RoleType # what dates specifically for the congress? (period_min, period_max) = get_congress_dates(stats["meta"]["congress"]) period_min = max(period_min, role.logical_dates()[0]) period_max = min(period_max, role.logical_dates()[1]) return { "publishdate": dateutil.parser.parse(stats["meta"]["as-of"]), "period": session_stats_period(session, stats), "congress_dates": (period_min, period_max), "person": person, "photo": person.get_photo()[0], "himher": Gender.by_value(person.gender).pronoun_object, "role": role, "class": RoleType.by_value(role.role_type).label.lower() + "s", "session": session, "meta": stats["meta"], "stats": stats["stats"], "nav_groups": nav_groups, }
def bill_text(request, congress, type_slug, number, version=None): if version == "": version = None try: bill_type = BillType.by_slug(type_slug) except BillType.NotFound: raise Http404("Invalid bill type: " + type_slug) bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number) from billtext import load_bill_text, bill_gpo_status_codes try: textdata = load_bill_text(bill, version) except IOError: textdata = None # Get a list of the alternate versions of this bill. alternates = None if textdata: alternates = [] for v in bill_gpo_status_codes: try: alternates.append(load_bill_text(bill, v, mods_only=True)) except IOError: pass alternates.sort(key = lambda mods : mods["docdate"]) # Get a list of related bills. from billtext import get_current_version related_bills = [] for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]: try: rbv = get_current_version(rb) if not (rb, rbv) in related_bills: related_bills.append((rb, rbv)) except IOError: pass # text not available for btc in BillTextComparison.objects.filter(bill1=bill).exclude(bill2=bill): if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2)) for btc in BillTextComparison.objects.filter(bill2=bill).exclude(bill1=bill): if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1)) return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "textdata": textdata, "version": version, "alternates": alternates, "related_bills": related_bills, }
def bill_text(request, congress, type_slug, number, version=None): if int(congress) < 103: raise Http404("Bill text is not available before the 103rd congress.") if version == "": version = None try: bill_type = BillType.by_slug(type_slug) except BillType.NotFound: raise Http404("Invalid bill type: " + type_slug) bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number) from billtext import load_bill_text, bill_gpo_status_codes try: textdata = load_bill_text(bill, version) except IOError: textdata = None # Get a list of the alternate versions of this bill. alternates = None if textdata: alternates = [] for v in bill_gpo_status_codes: fn = "data/us/bills.text/%s/%s/%s%d%s.mods.xml" % (bill.congress, BillType.by_value(bill.bill_type).xml_code, BillType.by_value(bill.bill_type).xml_code, bill.number, v) if os.path.exists(fn): alternates.append(load_bill_text(bill, v, mods_only=True)) alternates.sort(key = lambda mods : mods["docdate"]) # Get a list of related bills. related_bills = [] for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]: if not (rb, "") in related_bills: related_bills.append((rb, "")) for btc in BillTextComparison.objects.filter(bill1=bill): if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2)) for btc in BillTextComparison.objects.filter(bill2=bill): if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1)) return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "textdata": textdata, "version": version, "alternates": alternates, "related_bills": related_bills, }
def bill_statistics(request): # Get the count of bills by status and by Congress. counts_by_congress = [] for c in xrange(93, CURRENT_CONGRESS+1): total = Bill.objects.filter(congress=c).count() if total == 0: continue # during transitions between Congresses counts_by_congress.append({ "congress": c, "dates": get_congress_dates(c), "counts": [ ], "total": total, }) for g in bill_status_groups: t = load_bill_status_qs(g[4], congress=c).count() counts_by_congress[-1]["counts"].append( { "count": t, "percent": "%0.0f" % float(100.0*t/total), "link": "/congress/bills/browse?congress=%s&status=%s" % (c, ",".join(str(s) for s in g[4])), } ) counts_by_congress.reverse() # When does activity occur within the session cycle? if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3': from django.db import connection def pull_time_stat(field, where, cursor): historical = False cursor.execute("SELECT YEAR(%s) - congress*2 - 1787, MONTH(%s), COUNT(*) FROM bill_bill WHERE congress>=93 AND congress%s%d AND %s GROUP BY YEAR(%s) - congress*2, MONTH(%s)" % (field, field, "<" if historical else "=", CURRENT_CONGRESS, where, field, field)) activity = [{ "x": r[0]*12 + (r[1]-1), "count": r[2], "year": r[0] } for r in cursor.fetchall()] total = sum(m["count"] for m in activity) for i, m in enumerate(activity): m["cumulative_count"] = m["count"]/float(total) + (0.0 if i==0 else activity[i-1]["cumulative_count"]) for m in activity: m["count"] = round(m["count"] / (CURRENT_CONGRESS-96), 1) for m in activity: m["cumulative_count"] = round(m["cumulative_count"] * 100.0) return activity with connection.cursor() as cursor: activity_introduced_by_month = pull_time_stat('introduced_date', "1", cursor) activity_enacted_by_month = pull_time_stat('current_status_date', "current_status IN (%d,%d,%d)" % (int(BillStatus.enacted_signed), int(BillStatus.enacted_veto_override), int(BillStatus.enacted_tendayrule)), cursor) else: activity_introduced_by_month = [] activity_enacted_by_month = [] return { "groups2": bill_status_groups, "counts_by_congress": counts_by_congress, "activity": (("Bills and Resolutions Introduced", activity_introduced_by_month), ("Bills and Joint Resolutions Enacted", activity_enacted_by_month) ) }
def bill_statistics(request): # Get the count of bills by status and by Congress. counts_by_congress = [] for c in range(93, CURRENT_CONGRESS+1): total = Bill.objects.filter(congress=c).count() if total == 0: continue # during transitions between Congresses counts_by_congress.append({ "congress": c, "dates": get_congress_dates(c), "counts": [ ], "total": total, }) for g in bill_status_groups: t = load_bill_status_qs(g[4], congress=c).count() counts_by_congress[-1]["counts"].append( { "count": t, "percent": "%0.0f" % float(100.0*t/total), "link": "/congress/bills/browse?congress=%s&status=%s" % (c, ",".join(str(s) for s in g[4])), } ) counts_by_congress.reverse() # When does activity occur within the session cycle? if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3': from django.db import connection def pull_time_stat(field, where, cursor): cursor.execute("SELECT YEAR(%s) - congress*2 - 1787, MONTH(%s), COUNT(*) FROM bill_bill WHERE congress >= 93 AND %s GROUP BY YEAR(%s) - congress*2, MONTH(%s)" % (field, field, where, field, field)) activity = [{ "x": r[0]*12 + (r[1]-1), "count": r[2], "year": r[0] } for r in cursor.fetchall()] total = sum(m["count"] for m in activity) for i, m in enumerate(activity): m["cumulative_count"] = m["count"]/float(total) + (0.0 if i==0 else activity[i-1]["cumulative_count"]) for m in activity: m["count"] = round(m["count"] / float(total) * 100.0, 1) for m in activity: m["cumulative_count"] = round(m["cumulative_count"] * 100.0) return activity with connection.cursor() as cursor: activity_introduced_by_month = pull_time_stat('introduced_date', "1", cursor) activity_enacted_by_month = pull_time_stat('current_status_date', "current_status IN (%d,%d,%d)" % (int(BillStatus.enacted_signed), int(BillStatus.enacted_veto_override), int(BillStatus.enacted_tendayrule)), cursor) else: activity_introduced_by_month = [] activity_enacted_by_month = [] return { "groups2": bill_status_groups, "counts_by_congress": counts_by_congress, "activity": (("Bills Introduced", activity_introduced_by_month), ("Laws Enacted", activity_enacted_by_month) ) }
def proscore(self): """A modified prognosis score that omits factors associated with uninteresting bills, such as naming post offices. Only truly valid for current bills, and useless to compare across Congresses, but returns a value for all bills.""" # To aid search, especially for non-current bills, add in something to give most recently active bills a boost. type_boost = { BillType.senate_bill: 1.0, BillType.house_bill: 1.0, BillType.senate_resolution: 0.2, BillType.house_resolution: 0.2, BillType.senate_concurrent_resolution: 0.3, BillType.house_concurrent_resolution: 0.3, BillType.senate_joint_resolution: 0.75, BillType.house_joint_resolution: 0.75, } cstart, cend = get_congress_dates(self.congress) csd = self.current_status_date if hasattr(csd, 'date'): csd = csd.date() r = (csd - cstart).days / 365.0 # ranges from 0.0 to about 2.0. if self.is_current: from prognosis import compute_prognosis r += compute_prognosis(self, proscore=True)["prediction"] r *= type_boost[self.bill_type] return r
def bill_details(request, congress, type_slug, number): if type_slug.isdigit(): bill_type = type_slug else: try: bill_type = BillType.by_slug(type_slug) except BillType.NotFound: raise Http404("Invalid bill type: " + type_slug) bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number) from person.name import get_person_name sponsor_name = None if not bill.sponsor else \ get_person_name(bill.sponsor, role_date=bill.introduced_date, firstname_position='before', show_suffix=True) def get_reintroductions(): reintro_prev = None reintro_next = None for reintro in bill.find_reintroductions(): if reintro.congress < bill.congress: reintro_prev = reintro if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro return reintro_prev, reintro_next def get_text_info(): from billtext import load_bill_text try: return load_bill_text(bill, None, mods_only=True) except IOError: return None return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "sponsor_name": sponsor_name, "reintros": get_reintroductions, # defer so we can use template caching "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "feed": Feed.BillFeed(bill), "text": get_text_info, }
def bill_details(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) # get related bills related_bills = [] reintro_prev = None reintro_next = None for reintro in bill.find_reintroductions(): if reintro.congress < bill.congress: reintro_prev = reintro if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro if reintro_prev: related_bills.append({ "bill": reintro_prev, "note": "was a previous version of this bill.", "show_title": False }) if reintro_next: related_bills.append({ "bill": reintro_next, "note": "was a re-introduction of this bill in a later Congress.", "show_title": False }) for rb in bill.get_related_bills(): if rb.relation in ("identical", "rule"): related_bills.append({ "bill": rb.related_bill, "note": "(%s)" % rb.relation, "show_title": False }) elif rb.relation == "ruled-by": related_bills.append({ "bill": rb.related_bill, "prenote": "Debate on", "note": " is governed by these rules.", "show_title": False }) else: related_bills.append({ "bill": rb.related_bill, "note": ("(%s)" % (rb.relation.title() if rb.relation != "unknown" else "Related")), "show_title": True }) # bill text info and areas of law affected from billtext import load_bill_text try: text_info = load_bill_text(bill, None, mods_only=True, with_citations=True) except IOError: text_info = None return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "feed": bill.get_feed(), "text_info": text_info, "related": related_bills, }
def bill_details(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) from person.name import get_person_name sponsor_name = None if not bill.sponsor else \ get_person_name(bill.sponsor, role_date=bill.introduced_date, firstname_position='before', show_suffix=True) def get_reintroductions(): reintro_prev = None reintro_next = None for reintro in bill.find_reintroductions(): if reintro.congress < bill.congress: reintro_prev = reintro if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro return reintro_prev, reintro_next def get_text_info(): from models import USCSection from billtext import load_bill_text from search import parse_slip_law_number import re try: metadata = load_bill_text(bill, None, mods_only=True) # do interesting stuff with citations if "citations" in metadata and not settings.DEBUG: slip_laws = [] statutes = [] usc = { } other = [] usc_other = USCSection(name="Other Citations", ordering=99999) for cite in metadata["citations"]: if cite["type"] == "slip_law": slip_laws.append(cite) cite["bill"] = parse_slip_law_number(cite["text"]) elif cite["type"] == "statutes_at_large": statutes.append(cite) elif cite["type"] in ("usc-section", "usc-chapter"): # Build a tree of title-chapter-...-section nodes so we can # display the citations in context. try: sec_obj = USCSection.objects.get(citation=cite["key"]) except: # USCSection.DoesNotExist and MultipleObjectsReturned both possible # create a fake entry for the sake of output # the 'id' field is set to make these objects properly hashable sec_obj = USCSection(id=cite["text"], name=cite["text"], parent_section=usc_other) if "range_to_section" in cite: sec_obj.range_to_section = cite["range_to_section"] # recursively go up to the title path = [sec_obj] so = sec_obj while so.parent_section: so = so.parent_section path.append(so) # build a link to LII if cite["type"] == "usc-section": cite_link = "http://www.law.cornell.edu/uscode/text/" + cite["title"] if cite["section"]: cite_link += "/" + cite["section"] if cite["paragraph"]: cite_link += "#" + "_".join(re.findall(r"\(([^)]+)\)", cite["paragraph"])) elif cite["type"] == "usc-chapter": cite_link = "http://www.law.cornell.edu/uscode/text/" + cite["title"] + "/" + "/".join( (so.level_type + "-" + so.number) for so in reversed(path[:-1]) ) sec_obj.link = cite_link # now pop off from the path to put the node at the right point in a tree container = usc while path: p = path.pop(-1) if p not in container: container[p] = { } container = container[p] else: other.append(cite) slip_laws.sort(key = lambda x : (x["congress"], x["number"])) # restructure data format def ucfirst(s): return s[0].upper() + s[1:] def rebuild_usc_sec(seclist, indent=0): ret = [] seclist = sorted(seclist.items(), key=lambda x : x[0].ordering) for sec, subparts in seclist: ret.append({ "text": (ucfirst(sec.level_type + ((" " + sec.number) if sec.number else "") + (": " if sec.name else "")) if sec.level_type else "") + (sec.name_recased if sec.name else ""), "link": getattr(sec, "link", None), "range_to_section": getattr(sec, "range_to_section", None), "indent": indent, }) ret.extend(rebuild_usc_sec(subparts, indent=indent+1)) return ret usc = rebuild_usc_sec(usc) metadata["citations"] = { "slip_laws": slip_laws, "statutes": statutes, "usc": usc, "other": other, "count": len(slip_laws)+len(statutes)+len(usc)+len(other) } return metadata except IOError: return None return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "sponsor_name": sponsor_name, "reintros": get_reintroductions, # defer so we can use template caching "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "feed": Feed.BillFeed(bill), "text": get_text_info, "care2_category_id": { 5816: '793', # Agriculture and Food=>Health 5840: '789', # Animals=>Animal Welfare 5996: '794', # Civil Rights=>Human Rights 5991: '791', # Education=>Education 6021: '792', # Energy=>Environment & Wildlife 6038: '792', # Environmental Protection=>Environment & Wildlife 6053: '793', # Families=>Health 6130: '793', # Health=>Health 6206: '794', # Immigration=>Human Rights 6279: '792', # Public Lands/Natural Resources=>Environment & Wildlife 6321: '791', # Social Sciences=>Education 6328: '793', # Social Welfare => Health }.get(bill.get_top_term_id(), '795') # fall back to Politics category }
def format_congress_number(value): start, end = get_congress_dates(value) end_year = end.year if end.month > 1 else end.year - 1 # count January finishes as the prev year return '%s Congress: %d-%d' % (ordinal(value), start.year, end.year)
def main(options): """ Update Person and PersonRole models. Do safe update: touch only those records which have been changed. """ BASE_PATH = CONGRESS_LEGISLATORS_PATH SRC_FILES = ['legislators-current', 'legislators-historical', 'legislators-social-media', 'executive'] # order matters for p in SRC_FILES: f = BASE_PATH + p + ".yaml" if not File.objects.is_changed(f) and not options.force: log.info('File %s was not changed' % f) else: # file modified... break else: # no 'break' ==> no files modified return # Start parsing. had_error = False # Get combined data. legislator_data = { } leg_id_map = { } for p in SRC_FILES: log.info('Opening %s...' % p) f = BASE_PATH + p + ".yaml" y = yaml_load(f) for m in y: if p != 'legislators-social-media': govtrack_id = m["id"].get("govtrack") # For the benefit of the social media file, make a mapping of IDs. for k, v in m["id"].items(): if type(v) != list: leg_id_map[(k,v)] = govtrack_id else: # GovTrack IDs are not always listed in this file. govtrack_id = None for k, v in m["id"].items(): if type(v) != list and (k, v) in leg_id_map: govtrack_id = leg_id_map[(k,v)] break if not govtrack_id: print "No GovTrack ID:" pprint.pprint(m) had_error = True continue if govtrack_id not in legislator_data: legislator_data[govtrack_id] = m elif p == "legislators-social-media": legislator_data[govtrack_id]["social"] = m["social"] elif p == "executive": legislator_data[govtrack_id]["terms"].extend( m["terms"] ) else: raise ValueError("Duplication in an unexpected way (%d, %s)." % (govtrack_id, p)) person_processor = PersonProcessor() role_processor = PersonRoleProcessor() existing_persons = set(Person.objects.values_list('pk', flat=True)) processed_persons = set() created_persons = set() progress = Progress(total=len(legislator_data)) log.info('Processing persons') for node in legislator_data.values(): # Wrap each iteration in try/except # so that if some node breaks the parsing process # then other nodes could be parsed try: person = person_processor.process(Person(), node) # Create cached name strings. This is done again later # after the roles are updated. person.set_names() # Now try to load the person with such ID from # database. If found it then just update it # else create new Person object try: ex_person = Person.objects.get(pk=person.pk) if person_processor.changed(ex_person, person) or options.force: # If the person has PK of existing record, # coming in via the YAML-specified GovTrack ID, # then Django ORM will update existing record if not options.force: log.warn("Updated %s" % person) person.save() except Person.DoesNotExist: created_persons.add(person.pk) person.save() log.warn("Created %s" % person) processed_persons.add(person.pk) # Process roles of the person roles = list(PersonRole.objects.filter(person=person)) existing_roles = set(PersonRole.objects.filter(person=person).values_list('pk', flat=True)) processed_roles = set() role_list = [] for termnode in node['terms']: role = role_processor.process(PersonRole(), termnode) role.person = person role.extra = filter_yaml_term_structure(termnode) # copy in the whole YAML structure now = datetime.now().date() if now < get_congress_dates(CURRENT_CONGRESS)[0]: # in the transition time before the next Congress, just count as that Congress now = get_congress_dates(CURRENT_CONGRESS)[0] role.current = role.startdate <= now and role.enddate >= now # Scan for most recent leadership role within the time period of this term, # which isn't great for Senators because it's likely it changed a few times # within a term, especially if there was a party switch. role.leadership_title = None for leadership_node in node.get("leadership_roles", []): # must match on date and chamber if leadership_node["start"] >= role.enddate.isoformat(): continue # might start on the same day but is for the next Congress if "end" in leadership_node and leadership_node["end"] <= role.startdate.isoformat(): continue # might start on the same day but is for the previous Congress if leadership_node["chamber"] != RoleType.by_value(role.role_type).congress_chamber.lower(): continue role.leadership_title = leadership_node["title"] # Try to match this role with one already in the database. # First search for an exact match on type/start/end. ex_role = None for r in roles: if role.role_type == r.role_type and r.startdate == role.startdate and r.enddate == role.enddate: ex_role = r break # Otherwise match on type & either start or end only. if not ex_role: for r in roles: if role.role_type == r.role_type and (r.startdate == role.startdate or r.enddate == role.enddate): ex_role = r break if ex_role: # These roles correspond. processed_roles.add(ex_role.id) role.id = ex_role.id if role_processor.changed(ex_role, role) or options.force: role.save() role_list.append(role) if not options.force: log.warn("Updated %s" % role) roles.remove(ex_role) # don't need to try matching this to any other node else: # Didn't find a matching role. if len([r for r in roles if r.role_type == role.role_type]) > 0: print role, "is one of these?" for ex_role in roles: print "\t", ex_role raise Exception("There is an unmatched role.") log.warn("Created %s" % role) role.save() role_list.append(role) # create the events for the roles after all have been loaded # because we don't create events for ends of terms and # starts of terms that are adjacent. if not options.disable_events: for i in xrange(len(role_list)): role_list[i].create_events( role_list[i-1] if i > 0 else None, role_list[i+1] if i < len(role_list)-1 else None ) removed_roles = existing_roles - processed_roles for pk in removed_roles: pr = PersonRole.objects.get(pk=pk) print pr.person.id, pr raise ValueError("Deleted role??") log.warn("Deleted %s" % pr) pr.delete() # The name can't be determined until all of the roles are set. If # it changes, re-save. Unfortunately roles are cached so this actually # doesn't work yet. Re-run the parser to fix names. nn = (person.name, person.sortname) if hasattr(person, "role"): delattr(person, "role") # clear the cached info person.set_names() if nn != (person.name, person.sortname): log.warn("%s is now %s." % (nn[0], person.name)) person.save() except Exception, ex: # Catch unexpected exceptions and log them pprint.pprint(node) log.error('', exc_info=ex) had_error = True progress.tick()
def get_prognosis(self): if self.congress != settings.CURRENT_CONGRESS: return None import prognosis prog = prognosis.compute_prognosis(self) prog["congressdates"] = get_congress_dates(prog["congress"]) return prog
def bill_details(request, congress, type_slug, number): if type_slug.isdigit(): bill_type = type_slug else: try: bill_type = BillType.by_slug(type_slug) except BillType.NotFound: raise Http404("Invalid bill type: " + type_slug) bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number) from person.name import get_person_name sponsor_name = None if not bill.sponsor else \ get_person_name(bill.sponsor, role_date=bill.introduced_date, firstname_position='before', show_suffix=True) def get_reintroductions(): reintro_prev = None reintro_next = None for reintro in bill.find_reintroductions(): if reintro.congress < bill.congress: reintro_prev = reintro if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro return reintro_prev, reintro_next def get_text_info(): from models import USCSection from billtext import load_bill_text from search import parse_slip_law_number import re try: metadata = load_bill_text(bill, None, mods_only=True) # do interesting stuff with citations if "citations" in metadata: slip_laws = [] statutes = [] usc = { } other = [] usc_other = USCSection(name="Other Citations", ordering=99999) for cite in metadata["citations"]: if cite["type"] == "slip_law": slip_laws.append(cite) cite["bill"] = parse_slip_law_number(cite["text"]) elif cite["type"] == "statutes_at_large": statutes.append(cite) elif cite["type"] == "usc": # build a normalized citation and a link to LII cite_norm = "usc/" + cite["title"] cite_link = "http://www.law.cornell.edu/uscode/text/" + cite["title"] if cite["section"]: cite_link += "/" + cite["section"] cite_norm += "/" + cite["section"] if cite["paragraph"]: cite_link += "#" + "_".join(re.findall(r"\(([^)]+)\)", cite["paragraph"])) # Build a tree of title-chapter-...-section nodes so we can # display the citations in context. try: sec_obj = USCSection.objects.get(citation=cite_norm) except: # USCSection.DoesNotExist and MultipleObjectsReturned both possible # the 'id' field is set to make these objects properly hashable sec_obj = USCSection(id=cite["text"], name=cite["text"], parent_section=usc_other) sec_obj.link = cite_link if "range_to_section" in cite: sec_obj.range_to_section = cite["range_to_section"] # recursively go up to the title path = [sec_obj] while sec_obj.parent_section: sec_obj = sec_obj.parent_section path.append(sec_obj) # now pop off from the path to put the node at the right point in a tree container = usc while path: p = path.pop(-1) if p not in container: container[p] = { } container = container[p] else: other.append(cite) slip_laws.sort(key = lambda x : (x["congress"], x["number"])) # restructure data format def ucfirst(s): return s[0].upper() + s[1:] def rebuild_usc_sec(seclist, indent=0): ret = [] seclist = sorted(seclist.items(), key=lambda x : x[0].ordering) for sec, subparts in seclist: ret.append({ "text": (ucfirst(sec.level_type + ((" " + sec.number) if sec.number else "") + (": " if sec.name else "")) if sec.level_type else "") + (sec.name if sec.name else ""), "link": getattr(sec, "link", None), "range_to_section": getattr(sec, "range_to_section", None), "indent": indent, }) ret.extend(rebuild_usc_sec(subparts, indent=indent+1)) return ret usc = rebuild_usc_sec(usc) metadata["citations"] = { "slip_laws": slip_laws, "statutes": statutes, "usc": usc, "other": other, "count": len(slip_laws)+len(statutes)+len(usc)+len(other) } return metadata except IOError: return None return { 'bill': bill, "congressdates": get_congress_dates(bill.congress), "subtitle": get_secondary_bill_title(bill, bill.titles), "sponsor_name": sponsor_name, "reintros": get_reintroductions, # defer so we can use template caching "current": bill.congress == CURRENT_CONGRESS, "dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious, "feed": Feed.BillFeed(bill), "text": get_text_info, }
def format_congress_number(value): start, end = get_congress_dates(value) end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year return '%s Congress: %d-%d' % (ordinal(value), start.year, end.year)
def run_analysis(label, congress): global stats global columns congress_dates = get_congress_dates(congress) start_date = datetime.date(congress_dates[0].year, 1, 1) # limit to now, for the current congress end_date = datetime.date(congress_dates[0].year, 9, 11) end_date = min(end_date, datetime.datetime.now().date()) bills = Bill.objects.filter(congress=congress, bill_type__in=(BillType.house_bill, BillType.senate_bill, BillType.house_joint_resolution, BillType.senate_joint_resolution), introduced_date__lte=end_date)\ .order_by('current_status_date') by_day = { } for b in tqdm.tqdm(bills, desc=label): match_date = None for datestr, state, text, srcxml in b.major_actions: action_date = eval(datestr) #if state in (BillStatus.pass_over_house, BillStatus.pass_back_house, BillStatus.passed_bill): if state in BillStatus.final_status_enacted_bill: match_date = action_date.date() break else: # No event matched. continue # Load plain text. text = load_bill_text(b, None, plain_text=True) # Bills since 1993 have GPO MODS XML metadata with page counts. try: mods = load_bill_text(b, None, mods_only=True) pages = mods.get("numpages") except (IOError, AttributeError) as e: # For historical statutes we only have plain text from the # Statutes at Large, extracted from PDFs. We can get page # counts by looking for our replacement of the form feed # character put in by pdftotext. We only have that when # we extracted text from PDFs, which we only did for # the Statutes at Large. We can't do this on modern bills # where the text came from GPO plain text format. if b.congress < 103: pages = len([pgtext for pgtext in text.split("\n=============================================\n") if pgtext.strip() != ""]) else: print b.id, b, e raise ValueError("page date missing") #words = len(re.split(r"\s+", text)) # not very good for pre-GPO bills because Statutes at Large pages may have multiple statutes on them ################ EEK #if pages == 1: continue ################ EEK rel_date = (match_date - start_date).days rec = by_day.setdefault(rel_date, { "bills": 0, "pages": 0 } ) rec["bills"] += 1 rec["pages"] += pages #rec["words"] += words # Compute cumulative counts starting on day 0 and for every day till the # last day a bill was signed. columns.append(label) bills = 0 pages = 0 #words = 0 for rel_date in range((end_date-start_date).days+1): if rel_date in by_day: bills += by_day[rel_date]["bills"] pages += by_day[rel_date]["pages"] #words += by_day[rel_date]["words"] stats.setdefault(rel_date, {})[label] = (bills, pages)
def get_last_role_at_congress(self, congress): start, end = get_congress_dates(congress) try: return self.roles.filter(startdate__lte=end, enddate__gte=start).order_by('-startdate')[0] except IndexError: return None
def get_bill_factors(bill, pop_title_prefixes, committee_membership, majority_party, lobbying_data, include_related_bills=True): factors = list() # introduced date (idea from Yano, Smith and Wilkerson 2012 paper) idate = bill.introduced_date if hasattr(idate, 'date'): idate = idate.date() # not sure how this is possible if (idate - get_congress_dates(bill.congress)[0]).days < 90: factors.append(("introduced_first90days", "The %s was introduced in the first 90 days of the Congress." % bill.noun, "Introduced in the first 90 days of the Congress (incl. companion bills).")) if (idate - get_congress_dates(bill.congress)[0]).days < 365: factors.append(("introduced_firstyear", "The %s was introduced in the first year of the Congress." % bill.noun, "Introduced in the first year of the Congress (incl. companion bills).")) if (get_congress_dates(bill.congress)[1] - idate).days < 90: factors.append(("introduced_last90days", "The %s was introduced in the last 90 days of the Congress." % bill.noun, "Introduced in the last 90 days of the Congress (incl. companion bills).")) # does the bill's title start with a common prefix? for prefix in pop_title_prefixes: if bill.title_no_number.startswith(prefix + " "): factors.append(("startswith:" + prefix, "The %s's title starts with \"%s.\"" % (bill.noun, prefix), "Title starts with \"%s\"." % prefix)) cosponsors = list(Cosponsor.objects.filter(bill=bill, withdrawn=None).select_related("person")) committees = list(bill.committees.all()) maj_party = majority_party[bill.bill_type] if bill.sponsor_role: # party of the sponsor sponsor_party = bill.sponsor_role.party if sponsor_party != maj_party: factors.append( ("sponsor_minority", "The sponsor is a member of the minority party.", "Sponsor is a member of the minority party.") ) # is the sponsor a member/chair of a committee to which the bill has # been referred? for rname, rvalue in (("member", CommitteeMemberRole.member), ("rankingmember", CommitteeMemberRole.ranking_member), ("vicechair", CommitteeMemberRole.vice_chairman), ("chair", CommitteeMemberRole.chairman)): for committee in committees: if committee_membership.get(bill.sponsor_id, {}).get(committee.code) == rvalue: if rvalue != CommitteeMemberRole.member: factors.append(("sponsor_committee_%s" % rname, "The sponsor is the %s of a committee to which the %s has been referred." % (CommitteeMemberRole.by_value(rvalue).label.lower(), bill.noun), "Sponsor is a relevant committee %s." % CommitteeMemberRole.by_value(rvalue).label.lower())) if sponsor_party == maj_party: factors.append(("sponsor_committee_member_majority", "The sponsor is on a committee to which the %s has been referred, and the sponsor is a member of the majority party." % bill.noun, "Sponsor is on a relevant committee & in majority party.")) # leadership score of the sponsor, doesn't actually seem to be helpful, # even though leadership score of cosponsors is. if get_leadership_score(bill.sponsor) > .8: if sponsor_party == maj_party: factors.append(("sponsor_leader_majority", "The sponsor is in the majority party and has a high leadership score.", "Sponsor has a high leadership score (majority party).")) else: factors.append(("sponsor_leader_minority", "The sponsor has a high leadership score but is not in the majority party.", "Sponsor has a high leadership score (minority party).")) # count cosponsor assignments to committees by committee role and Member party for rname, rvalue in (("committeemember", CommitteeMemberRole.member), ("rankingmember", CommitteeMemberRole.ranking_member), ("vicechair", CommitteeMemberRole.vice_chairman), ("chair", CommitteeMemberRole.chairman)): num_cosp = 0 for cosponsor in cosponsors: for committee in committees: cvalue = committee_membership.get(cosponsor.person.id, {}).get(committee.code) if cvalue == rvalue or (rvalue==CommitteeMemberRole.member and cvalue != None): num_cosp += 1 break if rvalue == CommitteeMemberRole.member: if num_cosp <= 2: # ranges are tweakable... num_cosp = str(num_cosp) if num_cosp <= 5: num_cosp = "3-5" else: num_cosp = "6+" factors.append( ("cosponsor_%s_%s" % (rname, num_cosp), "%s cosponsors serve on a committee to which the %s has been referred." % (num_cosp, bill.noun), "%s cosponsors are on a relevant committee." % num_cosp) ) elif num_cosp > 0: rname2 = CommitteeMemberRole.by_value(rvalue).label.lower() factors.append( ("cosponsor_%s" % rname, "A cosponsor is the %s of a committee to which the %s has been referred." % (rname2, bill.noun), "A cosponsor is a relevant committee %s." % rname2)) # what committees is the bill assigned to? only look at committees # in the originating chamber, since assignments in the other chamber # indicate the bill had at least one successful vote. for cm in committees: if cm.committee != None: continue # skip subcommittees if CommitteeType.by_value(cm.committee_type).label != bill.originating_chamber: continue factors.append( ("committee_%s" % cm.code, "The bill was referred to %s." % cm.shortname, "Referred to %s." % cm.shortname)) # do we have cosponsors on both parties? num_cosp_majority = 0 for cosponsor in cosponsors: if cosponsor.role.party == maj_party: num_cosp_majority += 1 if bill.sponsor and sponsor_party == maj_party and len(cosponsors) >= 6 and num_cosp_majority < 2.0*len(cosponsors)/3: factors.append(("cosponsors_bipartisan", "The sponsor is in the majority party and at least one third of the %s's cosponsors are from the minority party." % bill.noun, "Sponsor is in majority party and 1/3rd+ of cosponsors are in minority party.")) if num_cosp_majority > 0 and num_cosp_majority < len(cosponsors): factors.append(("cosponsors_crosspartisan", "There is at least one cosponsor from the majority party and one cosponsor outside of the majority party.", "Has cosponsors from both parties.")) for is_majority in (False, True): for cosponsor in cosponsors: if (cosponsor.role.party == maj_party) != is_majority: continue if get_leadership_score(cosponsor.person) > .85: if is_majority: factors.append(("cosponsor_leader_majority", "A cosponsor in the majority party has a high leadership score.", "Cosponsor has high leadership score (majority party).")) else: factors.append(("cosponsor_leader_minority", "A cosponsor in the minority party has a high leadership score.", "Cosponsor has high leadership score (minority party).")) break # Is this bill a re-intro from last Congress, and if so was that bill reported by committee? if bill.sponsor: def normalize_title(title): # remove anything that looks like a year return re.sub(r"of \d\d\d\d$", "", title) for reintro in Bill.objects.filter(congress=bill.congress-1, sponsor=bill.sponsor): if normalize_title(bill.title_no_number) == normalize_title(reintro.title_no_number): if reintro.current_status not in (BillStatus.introduced, BillStatus.referred): factors.append(("reintroduced_of_reported", "This %s was reported by committee as %s in the previous session of Congress." % (bill.noun, reintro.display_number), "Got past committee in a previous Congress.")) else: factors.append(("reintroduced", "This %s was a re-introduction of %s from the previous session of Congress." % (bill.noun, reintro.display_number), "Is a bill reintroduced from a previous Congress.")) break if include_related_bills: # prevent infinite recursion # Add factors from any CRS-identified identical bill, changing most factors' # key into companion_KEY so that they become separate factors to consider. # For some specific factors, lump them in with the factor for the bill itself. for rb in RelatedBill.objects.filter(bill=bill, relation="identical").select_related("related_bill", "related_bill__sponsor_role"): # has a companion factors.append(("companion", "The %s has been introduced in both chambers (the other is %s)." % (bill.noun, rb.related_bill.display_number), "Has a companion bill in the other chamber.")) # companion sponsor's party if bill.sponsor_role and rb.related_bill.sponsor_role: if bill.sponsor_role.party != rb.related_bill.sponsor_role.party: factors.append(("companion_bipartisan", "The %s's companion %s was sponsored by a member of the other party." % (bill.noun, rb.related_bill.display_number), "Has a companion bill sponsored by the other party.")) for f in get_bill_factors(rb.related_bill, pop_title_prefixes, committee_membership, majority_party, lobbying_data, include_related_bills=False): if "startswith" in f[0]: continue # don't include title factors because the title is probs the same if f[0] in ("introduced_first90days", "introduced_last90days", "introduced_firstyear", "reintroduced_of_reported", "reintroduced"): f = (f[0], "%s (on companion bill %s)" % (f[1], rb.related_bill.display_number), f[2]) else: f = ("companion__" + f[0], "Companion bill " + rb.related_bill.display_number + ": " + f[1], "On a companion bill: " + f[2]) # Make sure not to duplicate any factors, especially if we are promoting the companion # bill factor to a main factor, we don't want to double count or override the description # on the main bill. if f[0] in set(k[0] for k in factors): continue factors.append(f) # Are lobbyists registering that they are lobbying on this bill? Does this bill # have more registered lobbying than the median bill? Overall this picks out # bills NOT likely to be enacted. # # Two possible explanations: First, lobbying can be to defeat a bill not just # pass it. So this would indicate that on balance lobbying is having that effect. # # Second it could be because lobbyists don't bother with # the easy bills that don't need their help. Meaning, they pick out a pool of # improbable bllls, and presumably make those bills more likely to be enacted # but still not as likely as the easy bills. (If they truly hurt a bill's future, they # would presumably know and stop lobbying!) # # Looking at lobbying might be more useful if we combined it with another # factor that could pick out the hard bills, and then this might show that for # hard bills, lobbying made the bills more successful. But it's a bit impossible # because surely the lobbyists know better than we do which bills are hard, # so it would be impossible to factor out "hard bills" entirely. if False: if lobbying_data["counts"].get( (bill.bill_type, bill.number), 0 ) > lobbying_data["median"]: factors.append( ("crp-lobby-many", "The Center for Responsive Politics reports that a large number of organizations are lobbying on this %s." % bill.noun, "Has many lobbyists.") ) elif lobbying_data["counts"].get( (bill.bill_type, bill.number), 0 ) > 0: factors.append( ("crp-lobby", "The Center for Responsive Politics reports that organizations are lobbying on this %s." % bill.noun, "Has lobbyists.") ) return factors
## power #congress_same_party = party_control[congress][0] == party_control[congress][1] #branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2]) # timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year-1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year)) row = [congress, timespan, date_range[0].isoformat(), date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes] W.writerow(row) #print("<tr>%s</tr>" % "".join( "<td>%s</td> " % td for td in row) ) if 0: # Look at corresponding time periods from past Congresses. # Go back a few days because our data isn't real time! days_in = (datetime.now().date() - get_congress_dates(CURRENT_CONGRESS)[0]) \ - timedelta(days=4) print("We are about %d days into the %d Congress" % (days_in.days, CURRENT_CONGRESS)) for c in range(93, CURRENT_CONGRESS+1): date_range = get_congress_dates(c) compute_productivity(c, (date_range[0], date_range[1] + days_in)) elif 0: for c in range(93, CURRENT_CONGRESS+1): # January 1 of the second year of the Congress # through March 31 of that year. date_range = get_congress_dates(c) date_range = (datetime(date_range[0].year+1, 1, 1).date(), datetime(date_range[0].year+1, 1, 1).date()+timedelta(days=105)) compute_productivity(c, date_range) elif 0:
((get_congress_dates(congress)[1].year - 1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year)) row = [ congress, timespan, date_range[0].isoformat(), date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes ] W.writerow(row) #print("<tr>%s</tr>" % "".join( "<td>%s</td> " % td for td in row) ) if 0: # Look at corresponding time periods from past Congresses. # Go back a few days because our data isn't real time! days_in = (datetime.now().date() - get_congress_dates(CURRENT_CONGRESS)[0]) \ - timedelta(days=4) print("We are about %d days into the %d Congress" % (days_in.days, CURRENT_CONGRESS)) for c in range(93, 114 + 1): date_range = get_congress_dates(c) compute_productivity(c, (date_range[0], date_range[1] + days_in)) elif 0: for c in range(93, 114 + 1): # January 1 of the second year of the Congress # through March 31 of that year. date_range = get_congress_dates(c) date_range = (datetime(date_range[0].year + 1, 1, 1).date(), datetime(date_range[0].year + 1, 1, 1).date() + timedelta(days=105))
def compute_productivity(congress, date_range): # laws enacted_bills = Bill.objects.filter( #congress=congress, --- because we're mostly measuring presidential activity, the date of signing could be outside of the Congress current_status__in=BillStatus.final_status_passed_bill, current_status_date__gte=date_range[0], current_status_date__lte=date_range[1]) #enacted_bills = (enacted_bills.filter(title__contains="Appropriations") | enacted_bills.filter(title__contains="Authorization")).distinct() enacted_bills = list(enacted_bills) enacted_bills_count = len(enacted_bills) enacted_bill_pages = 0 enacted_bill_words = 0 enacted_bill_pages_missing = 0 for b in enacted_bills: try: pp = load_bill_text(b, None, mods_only=True).get("numpages") except IOError: pp = None if pp is None: enacted_bill_pages_missing += 1 continue pp = int(pp.replace(" pages", "")) enacted_bill_pages += pp wds = len(load_bill_text(b, None, plain_text=True).split(" ")) enacted_bill_words += wds if congress < 103: enacted_bill_pages = "(no data)" if congress < 103: enacted_bill_words = "(no data)" # votes house_votes = Vote.objects.filter(congress=congress, created__gte=date_range[0], created__lte=date_range[1], chamber=CongressChamber.house).count() senate_votes = Vote.objects.filter(congress=congress, created__gte=date_range[0], created__lte=date_range[1], chamber=CongressChamber.senate).count() ## power #congress_same_party = party_control[congress][0] == party_control[congress][1] #branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2]) # timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year - 1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year)) row = [ congress, timespan, date_range[0].isoformat(), date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes ] W.writerow(row)
def bill_summaries(request, congress, type_slug, number): bill = load_bill_from_url(congress, type_slug, number) return { "bill": bill, "congressdates": get_congress_dates(bill.congress), }
# senate_votes = Vote.objects.filter( # congress=congress, # created__gte=date_range[0], # created__lte=date_range[1], # chamber=CongressChamber.senate).count() #timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year-1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year)) row = [label or congress, date_range[0].isoformat(), date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages, enacted_bill_words] #, house_votes, senate_votes] W.writerow(row) #print("<tr>%s</tr>" % "".join( "<td>%s</td> " % td for td in row) ) if 0: # Look at corresponding time periods from past Congresses. # Go back a few days because our data isn't real time! days_in = (datetime.now().date() - get_congress_dates(CURRENT_CONGRESS)[0]) \ - timedelta(days=4) print("We are about %d days into the %d Congress" % (days_in.days, CURRENT_CONGRESS)) for c in range(93, CURRENT_CONGRESS+1): date_range = get_congress_dates(c) compute_productivity(c, (date_range[0], date_range[0] + days_in)) elif 1: # First X days of presidency, minus a few days because of # data delays. days_in = (datetime.now().date() - datetime(get_congress_dates(CURRENT_CONGRESS)[0].year, 1, 20, 0, 0, 0).date()) \ - timedelta(days=0) #days_in = timedelta(days=178) print("We are about %d days into the presidency" % days_in.days) print("Looking at presidents whose first day is Jan 20 of the first year of a Congress.") for label, congress in (("Eisenhower", 83), ("Kennedy", 87), ("Nixon", 91), ("Carter", 95), ("Reagan", 97), ("Bush1", 101), ("Clinton", 103), ("Bush2", 107), ("Obama", 111), ("Trump", 115)):
def get_bill_factors(bill, pop_title_prefixes, committee_membership, majority_party, lobbying_data, include_related_bills=True): factors = list() # introduced date (idea from Yano, Smith and Wilkerson 2012 paper) idate = bill.introduced_date if hasattr(idate, 'date'): idate = idate.date() # not sure how this is possible if (idate - get_congress_dates(bill.congress)[0]).days < 90: factors.append(( "introduced_first90days", "The %s was introduced in the first 90 days of the Congress." % bill.noun, "Introduced in the first 90 days of the Congress (incl. companion bills)." )) if (idate - get_congress_dates(bill.congress)[0]).days < 365: factors.append(( "introduced_firstyear", "The %s was introduced in the first year of the Congress." % bill.noun, "Introduced in the first year of the Congress (incl. companion bills)." )) if (get_congress_dates(bill.congress)[1] - idate).days < 90: factors.append(( "introduced_last90days", "The %s was introduced in the last 90 days of the Congress." % bill.noun, "Introduced in the last 90 days of the Congress (incl. companion bills)." )) # does the bill's title start with a common prefix? for prefix in pop_title_prefixes: if bill.title_no_number.startswith(prefix + " "): factors.append( ("startswith:" + prefix, "The %s's title starts with \"%s.\"" % (bill.noun, prefix), "Title starts with \"%s\"." % prefix)) cosponsors = list( Cosponsor.objects.filter(bill=bill, withdrawn=None).select_related("person")) committees = list(bill.committees.all()) maj_party = majority_party[bill.bill_type] if bill.sponsor_role: # party of the sponsor sponsor_party = bill.sponsor_role.party if sponsor_party != maj_party: factors.append(("sponsor_minority", "The sponsor is a member of the minority party.", "Sponsor is a member of the minority party.")) # is the sponsor a member/chair of a committee to which the bill has # been referred? for rname, rvalue in (("member", CommitteeMemberRole.member), ("rankingmember", CommitteeMemberRole.ranking_member), ("chair", CommitteeMemberRole.vice_chair), ("chair", CommitteeMemberRole.chair)): for committee in committees: if committee_membership.get(bill.sponsor_id, {}).get(committee.code) == rvalue: if rvalue != CommitteeMemberRole.member: factors.append(( "sponsor_committee_%s" % rname, "The sponsor is the %s of a committee to which the %s has been referred." % (CommitteeMemberRole.by_value( rvalue).label.lower(), bill.noun), "Sponsor is a relevant committee %s." % CommitteeMemberRole.by_value(rvalue).label.lower() )) elif sponsor_party == maj_party: factors.append(( "sponsor_committee_member_majority", "The sponsor is on a committee to which the %s has been referred, and the sponsor is a member of the majority party." % bill.noun, "Sponsor is on a relevant committee & in majority party." )) # leadership score of the sponsor, doesn't actually seem to be helpful, # even though leadership score of cosponsors is. if get_leadership_score(bill.sponsor) > .8: if sponsor_party == maj_party: factors.append(( "sponsor_leader_majority", "The sponsor is in the majority party and has a high leadership score.", "Sponsor has a high leadership score (majority party).")) else: factors.append(( "sponsor_leader_minority", "The sponsor has a high leadership score but is not in the majority party.", "Sponsor has a high leadership score (minority party).")) # count cosponsor assignments to committees by committee role and Member party for rname, rvalue in (("committeemember", CommitteeMemberRole.member), ("rankingmember", CommitteeMemberRole.ranking_member), ("chair", CommitteeMemberRole.vice_chair), ("chair", CommitteeMemberRole.chair)): num_cosp = 0 for cosponsor in cosponsors: for committee in committees: cvalue = committee_membership.get(cosponsor.person.id, {}).get(committee.code) if cvalue == rvalue or (rvalue == CommitteeMemberRole.member and cvalue != None): num_cosp += 1 break if rvalue == CommitteeMemberRole.member: if num_cosp >= 2: factors.append(( "cosponsors_%s", "At least two cosponsors serve on a committee to which the %s has been referred." % (bill.noun, ), "2 or more cosponsors are on a relevant committee.")) elif num_cosp > 0: rname2 = CommitteeMemberRole.by_value(rvalue).label.lower() factors.append(( "cosponsor_%s" % rname, "A cosponsor is the %s of a committee to which the %s has been referred." % (rname2, bill.noun), "A cosponsor is a relevant committee %s." % rname2)) # what committees is the bill assigned to? only look at committees # in the originating chamber, since assignments in the other chamber # indicate the bill had at least one successful vote. for cm in committees: if cm.committee != None: continue # skip subcommittees if CommitteeType.by_value( cm.committee_type).label != bill.originating_chamber: continue factors.append(("committee_%s" % cm.code, "The bill was referred to %s." % cm.shortname, "Referred to %s (incl. companion)." % cm.shortname)) # do we have cosponsors on both parties? num_cosp_majority = 0 for cosponsor in cosponsors: if cosponsor.role.party == maj_party: num_cosp_majority += 1 if bill.sponsor and sponsor_party == maj_party and len( cosponsors) >= 6 and num_cosp_majority < 2.0 * len(cosponsors) / 3: factors.append(( "cosponsors_bipartisan", "The sponsor is in the majority party and at least one third of the %s's cosponsors are from the minority party." % bill.noun, "Sponsor is in majority party and 1/3rd+ of cosponsors are in minority party." )) elif num_cosp_majority > 0 and num_cosp_majority < len(cosponsors): factors.append(( "cosponsors_crosspartisan", "There is at least one cosponsor from the majority party and one cosponsor outside of the majority party.", "Has cosponsors from both parties.")) for is_majority in (False, True): for cosponsor in cosponsors: if (cosponsor.role.party == maj_party) != is_majority: continue if get_leadership_score(cosponsor.person) > .85: if is_majority: factors.append(( "cosponsor_leader_majority", "A cosponsor in the majority party has a high leadership score.", "Cosponsor has high leadership score (majority party)." )) else: factors.append(( "cosponsor_leader_minority", "A cosponsor in the minority party has a high leadership score.", "Cosponsor has high leadership score (minority party)." )) break # Is this bill a re-intro from last Congress, and if so was that bill reported by committee? if bill.sponsor: def normalize_title(title): # remove anything that looks like a year return re.sub(r"of \d\d\d\d$", "", title) for reintro in Bill.objects.filter(congress=bill.congress - 1, sponsor=bill.sponsor): if normalize_title(bill.title_no_number) == normalize_title( reintro.title_no_number): if reintro.current_status != BillStatus.introduced: factors.append(( "reintroduced_of_reported", "This %s was reported by committee as %s in the previous session of Congress." % (bill.noun, reintro.display_number), "Got past committee in a previous Congress.")) else: factors.append(( "reintroduced", "This %s was a re-introduction of %s from the previous session of Congress." % (bill.noun, reintro.display_number), "Is a bill reintroduced from a previous Congress.")) break if include_related_bills: # prevent infinite recursion # Add factors from any CRS-identified identical bill, changing most factors' # key into companion_KEY so that they become separate factors to consider. # For some specific factors, lump them in with the factor for the bill itself. for rb in RelatedBill.objects.filter( bill=bill, relation="identical").select_related( "related_bill", "related_bill__sponsor_role"): # has a companion factors.append(( "companion", "The %s has been introduced in both chambers (the other is %s)." % (bill.noun, rb.related_bill.display_number), "Has a companion bill in the other chamber.")) # companion sponsor's party if bill.sponsor_role and rb.related_bill.sponsor_role: if bill.sponsor_role.party != rb.related_bill.sponsor_role.party: factors.append(( "companion_bipartisan", "The %s's companion %s was sponsored by a member of the other party." % (bill.noun, rb.related_bill.display_number), "Has a companion bill sponsored by a member of the other party." )) for f in get_bill_factors(rb.related_bill, pop_title_prefixes, committee_membership, majority_party, lobbying_data, include_related_bills=False): if "startswith" in f[0]: continue # don't include title factors because the title is probs the same if f[0] in ("introduced_first90days", "introduced_last90days", "introduced_firstyear", "reintroduced_of_reported", "reintroduced") or f[0].startswith("committee_"): f = (f[0], "%s (on companion bill %s)" % (f[1], rb.related_bill.display_number), f[2]) else: f = ("companion__" + f[0], "Companion bill " + rb.related_bill.display_number + ": " + f[1], "On a companion bill: " + f[2]) # Make sure not to duplicate any factors, especially if we are promoting the companion # bill factor to a main factor, we don't want to double count or override the description # on the main bill. if f[0] in set(k[0] for k in factors): continue factors.append(f) # Are lobbyists registering that they are lobbying on this bill? Does this bill # have more registered lobbying than the median bill? Overall this picks out # bills NOT likely to be enacted. # # Two possible explanations: First, lobbying can be to defeat a bill not just # pass it. So this would indicate that on balance lobbying is having that effect. # # Second it could be because lobbyists don't bother with # the easy bills that don't need their help. Meaning, they pick out a pool of # improbable bllls, and presumably make those bills more likely to be enacted # but still not as likely as the easy bills. (If they truly hurt a bill's future, they # would presumably know and stop lobbying!) # # Looking at lobbying might be more useful if we combined it with another # factor that could pick out the hard bills, and then this might show that for # hard bills, lobbying made the bills more successful. But it's a bit impossible # because surely the lobbyists know better than we do which bills are hard, # so it would be impossible to factor out "hard bills" entirely. if False: if lobbying_data["counts"].get( (bill.bill_type, bill.number), 0) > lobbying_data["median"]: factors.append(( "crp-lobby-many", "The Center for Responsive Politics reports that a large number of organizations are lobbying on this %s." % bill.noun, "Has many lobbyists.")) elif lobbying_data["counts"].get((bill.bill_type, bill.number), 0) > 0: factors.append(( "crp-lobby", "The Center for Responsive Politics reports that organizations are lobbying on this %s." % bill.noun, "Has lobbyists.")) return factors