def membersoverview(request):
def get_current_members(role_type, delegates, by_party):
qs = PersonRole.objects.filter(
role_type=role_type,
current=True,
state__in=set(s for s, t in stateapportionment.items() if (t != "T") ^ delegates)
)
if by_party:
return qs.values('party').annotate(count=Count('party')).order_by('-count')
else:
return qs.count()
congress_current = (CURRENT_CONGRESS, get_congress_dates(CURRENT_CONGRESS)[0])
congress_previous = (CURRENT_CONGRESS-1, get_congress_dates(CURRENT_CONGRESS-1)[1])
return {
"statelist": statelist,
"senate_by_party": get_current_members(RoleType.senator, False, True),
"senate_vacancies": 100-get_current_members(RoleType.senator, False, False),
"house_by_party": get_current_members(RoleType.representative, False, True),
"house_vacancies": 435-get_current_members(RoleType.representative, False, False),
"house_delegate_vacancies": 6-get_current_members(RoleType.representative, True, False),
"congress_current": congress_current,
"congress_previous": congress_previous,
}
def membersoverview(request):
def get_current_members(role_type, delegates, by_party):
qs = PersonRole.objects.filter(
role_type=role_type,
current=True,
state__in=set(s for s, t in stateapportionment.items() if (t != "T") ^ delegates)
)
if by_party:
return qs.values('party').annotate(count=Count('party')).order_by('-count')
else:
return qs.count()
congress_current = (CURRENT_CONGRESS, get_congress_dates(CURRENT_CONGRESS)[0])
congress_previous = (CURRENT_CONGRESS-1, get_congress_dates(CURRENT_CONGRESS-1)[1])
return {
"statelist": statelist,
"senate_by_party": get_current_members(RoleType.senator, False, True),
"senate_vacancies": 100-get_current_members(RoleType.senator, False, False),
"house_by_party": get_current_members(RoleType.representative, False, True),
"house_vacancies": 435-get_current_members(RoleType.representative, False, False),
"house_delegate_vacancies": 6-get_current_members(RoleType.representative, True, False),
"congress_current": congress_current,
"congress_previous": congress_previous,
}
def compute_productivity(congress, date_range):
# laws
enacted_bills = Bill.objects.filter(
congress=congress, # if we're measuring presidential activity, the date of signing could be outside of the Congress, so change this
current_status__in=BillStatus.final_status_passed_bill,
#current_status_date__gte=date_range[0],
#current_status_date__lte=date_range[1]
)
#enacted_bills = (enacted_bills.filter(title__contains="Appropriations") | enacted_bills.filter(title__contains="Authorization")).distinct()
enacted_bills = list(enacted_bills)
enacted_bills_count = len(enacted_bills)
enacted_bill_pages = 0
enacted_bill_words = 0
enacted_bill_pages_missing = 0
for b in enacted_bills:
try:
pp = load_bill_text(b, None, mods_only=True).get("numpages")
except IOError:
pp = None
if pp is None:
enacted_bill_pages_missing += 1
continue
pp = int(pp.replace(" pages", ""))
enacted_bill_pages += pp
wds = len(load_bill_text(b, None, plain_text=True).split(" "))
enacted_bill_words += wds
if congress < 103: enacted_bill_pages = "(no data)"
if congress < 103: enacted_bill_words = "(no data)"
# votes
house_votes = Vote.objects.filter(
congress=congress,
created__gte=date_range[0],
created__lte=date_range[1],
chamber=CongressChamber.house).count()
senate_votes = Vote.objects.filter(
congress=congress,
created__gte=date_range[0],
created__lte=date_range[1],
chamber=CongressChamber.senate).count()
## power
#congress_same_party = party_control[congress][0] == party_control[congress][1]
#branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2])
#
timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year-1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year))
row = [congress, timespan, date_range[0].isoformat(), date_range[1].isoformat(),
enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes]
W.writerow(row)
def compute_productivity(congress, days_in):
corresponding_day = get_congress_dates(congress)[0] + days_in
# laws
enacted_bills = Bill.objects.filter(
congress=congress,
current_status__in=BillStatus.final_status_passed_bill,
current_status_date__lte=corresponding_day)
#enacted_bills = (enacted_bills.filter(title__contains="Appropriations") | enacted_bills.filter(title__contains="Authorization")).distinct()
enacted_bills = list(enacted_bills)
enacted_bills_count = len(enacted_bills)
enacted_bill_pages = 0
enacted_bill_words = 0
enacted_bill_pages_missing = 0
for b in enacted_bills:
try:
pp = load_bill_text(b, None, mods_only=True).get("numpages")
except IOError:
pp = None
if pp is None:
enacted_bill_pages_missing += 1
continue
pp = int(pp.replace(" pages", ""))
enacted_bill_pages += pp
wds = len(load_bill_text(b, None, plain_text=True).split(" "))
enacted_bill_words += wds
if congress < 103: enacted_bill_pages = "(no data)"
if congress < 103: enacted_bill_words = "(no data)"
# votes
house_votes = Vote.objects.filter(
congress=congress,
created__lte=corresponding_day,
chamber=CongressChamber.house).count()
senate_votes = Vote.objects.filter(
congress=congress,
created__lte=corresponding_day,
chamber=CongressChamber.senate).count()
# power
congress_same_party = party_control[congress][0] == party_control[congress][1]
branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2])
#
timespan = "%d (%d-%d)" % (congress, get_congress_dates(congress)[0].year, get_congress_dates(congress)[1].year-1)
row = [timespan, enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes, "Yes" if congress_same_party else "No", "Yes" if branches_same_party else "No"]
W.writerow(row)
def bill_summaries(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
return {
"bill": bill,
"congressdates": get_congress_dates(bill.congress),
"text_info": get_text_info(bill), # for the header tabs
}
def bill_widget(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
from person.name import get_person_name
if bill.sponsor: bill.sponsor.role = bill.sponsor_role # for rending name
sponsor_name = None if not bill.sponsor else \
get_person_name(bill.sponsor, firstname_position='before', show_suffix=True)
def get_text_info():
from billtext import load_bill_text
try:
return load_bill_text(bill, None, mods_only=True)
except IOError:
return None
return {
"SITE_ROOT_URL": settings.SITE_ROOT_URL,
"bill": bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"sponsor_name": sponsor_name,
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"text": get_text_info,
}
def bill_widget(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
from person.name import get_person_name
if bill.sponsor: bill.sponsor.role = bill.sponsor_role # for rending name
sponsor_name = None if not bill.sponsor else \
get_person_name(bill.sponsor, firstname_position='before', show_suffix=True)
def get_text_info():
from billtext import load_bill_text
try:
return load_bill_text(bill, None, mods_only=True)
except IOError:
return None
return {
"SITE_ROOT_URL":
settings.SITE_ROOT_URL,
"bill":
bill,
"congressdates":
get_congress_dates(bill.congress),
"subtitle":
get_secondary_bill_title(bill, bill.titles),
"sponsor_name":
sponsor_name,
"current":
bill.congress == CURRENT_CONGRESS,
"dead":
bill.congress != CURRENT_CONGRESS
and bill.current_status not in BillStatus.final_status_obvious,
"text":
get_text_info,
}
def bill_key_questions(request, congress, type_slug, number):
# load bill and text
bill = load_bill_from_url(congress, type_slug, number)
text_info = bill.get_text_info(with_citations=True)
# load stakeholder positions
from stakeholder.models import BillPosition
stakeholder_posts = bill.stakeholder_positions\
.filter(post__stakeholder__verified=True)\
.select_related("post", "post__stakeholder")\
.order_by('-created')
def add_position_return_post(bp): bp.post.position = bp.position; return bp.post
stakeholder_posts = [add_position_return_post(bp) for bp in stakeholder_posts]
# context
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"prognosis": bill.get_prognosis_with_details(),
"text_info": text_info,
"text_incorporation": fixup_text_incorporation(bill.text_incorporation),
"stakeholder_posts": stakeholder_posts,
"legislator_statements": fetch_statements(bill),
}
def build_info():
feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"]
groups = [
( g[0], # title
g[1], # text 1
g[2], # text 2
"/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]), # link
load_bill_status_qs(g[4]).count(), # count in category
load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category
)
for g in bill_status_groups ]
dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date'))
sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date'))
coming_up = list(dhg_bills | sfs_bills)
coming_up.sort(key = lambda b : b.docs_house_gov_postdate if (b.docs_house_gov_postdate and (not b.senate_floor_schedule_postdate or b.senate_floor_schedule_postdate < b.docs_house_gov_postdate)) else b.senate_floor_schedule_postdate, reverse=True)
start, end = get_congress_dates(CURRENT_CONGRESS)
end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year
current_congress_years = '%d-%d' % (start.year, end.year)
current_congress = ordinal(CURRENT_CONGRESS)
return {
"feeds": feeds,
"total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(),
"current_congress_years": current_congress_years,
"current_congress": current_congress,
"groups": groups,
"coming_up": coming_up,
"subjects": subject_choices(),
"BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.referred, BillStatus.reported),
}
def bill_summaries(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
return {
"bill": bill,
"congressdates": get_congress_dates(bill.congress),
"text_info": get_text_info(bill), # for the header tabs
}
def load_roles_at_date(persons, when, congress):
"""
Find out representative/senator role of each person at given date.
This method is optimized for bulk operation.
"""
if when:
when = (when, when)
else:
from us import get_congress_dates
when = get_congress_dates(congress)
roles = PersonRole.objects.filter(startdate__lte=when[1],
enddate__gte=when[0],
role_type__in=(RoleType.representative,
RoleType.senator),
person__in=persons)
roles_by_person = {}
for role in roles:
if role.congress_numbers(
) is not None and congress not in role.congress_numbers():
continue
roles_by_person[role.person_id] = role
for person in persons:
person.role = roles_by_person.get(person.id)
return None
def bill_details(request, congress, type_slug, number):
# pre-load info
bill = load_bill_from_url(congress, type_slug, number)
text_info = bill.get_text_info(with_citations=True)
from stakeholder.models import BillPosition
stakeholder_posts = bill.stakeholder_positions\
.filter(post__stakeholder__verified=True)\
.select_related("post", "post__stakeholder")\
.order_by('-created')
def add_position_return_post(bp): bp.post.position = bp.position; return bp.post
stakeholder_posts = [add_position_return_post(bp) for bp in stakeholder_posts]
# context
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": bill.get_feed(),
"prognosis": bill.get_prognosis_with_details(),
"text_info": text_info,
"text_incorporation": fixup_text_incorporation(bill.text_incorporation),
"show_media_bar": not bill.original_intent_replaced and bill.sponsor and bill.sponsor.has_photo() and text_info and text_info.get("has_thumbnail"),
"stakeholder_posts": stakeholder_posts,
}
def build_info():
feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"]
groups = [
( g[0], # title
g[1], # text 1
g[2], # text 2
"/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]), # link
load_bill_status_qs(g[4]).count(), # count in category
load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category
)
for g in bill_status_groups ]
dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date'))
sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date'))
coming_up = list(dhg_bills | sfs_bills)
coming_up.sort(key = lambda b : b.docs_house_gov_postdate if (b.docs_house_gov_postdate and (not b.senate_floor_schedule_postdate or b.senate_floor_schedule_postdate < b.docs_house_gov_postdate)) else b.senate_floor_schedule_postdate, reverse=True)
start, end = get_congress_dates(CURRENT_CONGRESS)
end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year
current_congress_years = '%d-%d' % (start.year, end.year)
current_congress = ordinal(CURRENT_CONGRESS)
return {
"feeds": feeds,
"total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(),
"current_congress_years": current_congress_years,
"current_congress": current_congress,
"groups": groups,
"coming_up": coming_up,
"subjects": subject_choices(),
"BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.referred, BillStatus.reported),
}
def bill_details(request, congress, type_slug, number):
# pre-load info
bill = load_bill_from_url(congress, type_slug, number)
text_info = bill.get_text_info(with_citations=True)
from stakeholder.models import BillPosition
stakeholder_posts = bill.stakeholder_positions\
.filter(post__stakeholder__verified=True)\
.select_related("post", "post__stakeholder")\
.order_by('-created')
def add_position_return_post(bp): bp.post.position = bp.position; return bp.post
stakeholder_posts = [add_position_return_post(bp) for bp in stakeholder_posts]
# context
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": bill.get_feed(),
"prognosis": bill.get_prognosis_with_details(),
"text_info": text_info,
"text_incorporation": fixup_text_incorporation(bill.text_incorporation),
"show_media_bar": not bill.original_intent_replaced and bill.sponsor and bill.sponsor.has_photo() and text_info and text_info.get("has_thumbnail"),
"stakeholder_posts": stakeholder_posts,
}
def bill_summaries(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
return {
"bill_subpage": "Summary",
"bill": bill,
"congressdates": get_congress_dates(bill.congress),
"text_info": bill.get_text_info(with_citations=True), # for the header tabs
}
def get_last_role_at_congress(self, congress):
start, end = get_congress_dates(congress)
try:
return self.roles.filter(
startdate__lte=end,
enddate__gte=start).order_by('-startdate')[0]
except IndexError:
return None
def get_roles_of_people(congressnumber):
congress_dates = get_congress_dates(congressnumber)
return PersonRole.objects.filter(
role_type__in=(RoleType.senator, RoleType.representative),
startdate__lt=congress_dates[1], # start dates of next congress are on this day too
enddate__gt=congress_dates[0] # end dates from previous congress are on this day too
).select_related("person")\
.order_by('-startdate') # so we put people in the right list by their most recent term
def get_roles_of_people(congressnumber):
congress_dates = get_congress_dates(congressnumber)
return PersonRole.objects.filter(
role_type__in=(RoleType.senator, RoleType.representative),
startdate__lt=congress_dates[1], # start dates of next congress are on this day too
enddate__gt=congress_dates[0] # end dates from previous congress are on this day too
).select_related("person")\
.order_by('-startdate') # so we put people in the right list by their most recent term
def bill_summaries(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
return {
"bill_subpage": "Summary",
"bill": bill,
"congressdates": get_congress_dates(bill.congress),
"text_info": bill.get_text_info(with_citations=True), # for the header tabs
}
def build_info():
# feeds about all legislation that we offer the user to subscribe to
feeds = [
f for f in Feed.get_simple_feeds() if f.category == "federal-bills"
]
# info about bills by status
groups = [
(
g[0], # title
g[1], # text 1
g[2], # text 2
"/congress/bills/browse?status=" +
",".join(str(s)
for s in g[4]) + "&sort=-current_status_date", # link
load_bill_status_qs(g[4]).count(), # count in category
load_bill_status_qs(g[4]).order_by(
'-current_status_date')[0:6], # top 6 in this category
) for g in bill_status_groups
]
# legislation coming up
dhg_bills = Bill.objects.filter(
congress=CURRENT_CONGRESS,
docs_house_gov_postdate__gt=datetime.datetime.now() -
datetime.timedelta(days=10)).filter(
docs_house_gov_postdate__gt=F('current_status_date'))
sfs_bills = Bill.objects.filter(
congress=CURRENT_CONGRESS,
senate_floor_schedule_postdate__gt=datetime.datetime.now() -
datetime.timedelta(days=5)).filter(
senate_floor_schedule_postdate__gt=F('current_status_date'))
coming_up = list(
(dhg_bills | sfs_bills).order_by('scheduled_consideration_date'))
# top tracked bills
top_bills = Feed.objects\
.filter(feedname__startswith='bill:')\
.filter(feedname__regex='^bill:[hs][jcr]?%d-' % CURRENT_CONGRESS)
top_bills = top_bills\
.annotate(count=Count('tracked_in_lists'))\
.order_by('-count')\
.values('feedname', 'count')\
[0:25]
top_bills = [(Bill.from_feed(Feed.from_name(bf["feedname"])),
bf["count"]) for bf in top_bills]
return {
"feeds": feeds,
"total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(),
"current_congress": CURRENT_CONGRESS,
"current_congress_dates": get_congress_dates(CURRENT_CONGRESS),
"groups": groups,
"coming_up": coming_up,
"top_tracked_bills": top_bills,
"subjects": subject_choices(),
"BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.reported),
}
def bill_text(request, congress, type_slug, number, version=None):
if version == "":
version = None
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from .billtext import load_bill_text, get_bill_text_versions
try:
textdata = load_bill_text(bill, version)
except IOError:
textdata = None
# Get a list of the alternate versions of this bill.
alternates = None
is_latest = True
if textdata:
alternates = []
for v in get_bill_text_versions(bill):
try:
alternates.append(load_bill_text(bill, v, mods_only=True))
except IOError:
pass
alternates.sort(key = lambda mods : mods["docdate"])
if len(alternates) > 0:
is_latest = False
if textdata["doc_version"] == alternates[-1]["doc_version"]:
is_latest = True
# Get a list of related bills.
from .billtext import get_current_version
related_bills = []
for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]:
try:
rbv = get_current_version(rb)
if not (rb, rbv) in related_bills: related_bills.append((rb, rbv))
except IOError:
pass # text not available
for btc in BillTextComparison.objects.filter(bill1=bill).exclude(bill2=bill):
if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2))
for btc in BillTextComparison.objects.filter(bill2=bill).exclude(bill1=bill):
if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1))
return {
"bill_subpage": "Text",
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"textdata": textdata,
"version": version,
"is_latest": is_latest,
"alternates": alternates,
"related_bills": related_bills,
"days_old": (datetime.datetime.now().date() - bill.current_status_date).days,
"is_on_bill_text_page": True, # for the header tabs
}
def bill_text(request, congress, type_slug, number, version=None):
if version == "":
version = None
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from .billtext import load_bill_text, get_bill_text_versions
try:
textdata = load_bill_text(bill, version)
except IOError:
textdata = None
# Get a list of the alternate versions of this bill.
alternates = None
is_latest = True
if textdata:
alternates = []
for v in get_bill_text_versions(bill):
try:
alternates.append(load_bill_text(bill, v, mods_only=True))
except IOError:
pass
alternates.sort(key = lambda mods : mods["docdate"])
if len(alternates) > 0:
is_latest = False
if textdata["doc_version"] == alternates[-1]["doc_version"]:
is_latest = True
# Get a list of related bills.
from .billtext import get_current_version
related_bills = []
for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]:
try:
rbv = get_current_version(rb)
if not (rb, rbv) in related_bills: related_bills.append((rb, rbv))
except IOError:
pass # text not available
for btc in BillTextComparison.objects.filter(bill1=bill).exclude(bill2=bill):
if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2))
for btc in BillTextComparison.objects.filter(bill2=bill).exclude(bill1=bill):
if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1))
return {
"bill_subpage": "Text",
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"textdata": textdata,
"version": version,
"is_latest": is_latest,
"alternates": alternates,
"related_bills": related_bills,
"days_old": (datetime.datetime.now().date() - bill.current_status_date).days,
"is_on_bill_text_page": True, # for the header tabs
}
def analysis_methodology(request):
from settings import CURRENT_CONGRESS
from person.models import RoleType
from bill.models import BillType
from us import get_congress_dates
import json
from person.analysis import load_sponsorship_analysis2
def make_chart_series(role_type):
data = load_sponsorship_analysis2(CURRENT_CONGRESS, role_type, None)
if not data: return None
ret = { }
for p in data["all"]:
ret.setdefault(p["party"], {
"type": "party",
"party": p["party"],
"data": [],
})["data"].append({
"x": float(p["ideology"]),
"y": float(p["leadership"]),
"name": p["name"],
})
ret = list(ret.values())
ret.sort(key = lambda s : len(s["data"]), reverse=True)
data = dict(data) # clone before modifying, just in case
data["series"] = json.dumps(ret)
return data
import bill.prognosis
import bill.prognosis_model
import bill.prognosis_model_test
prognosis_factors = sorted([dict(v) for v in bill.prognosis_model.factors.values()],
key = lambda m : m["count"], reverse=True)
for v in prognosis_factors:
v["factors"] = sorted(v["factors"].values(), key = lambda f : f["regression_beta"], reverse=True)
prognosis_test = sorted(bill.prognosis_model_test.model_test_results.values(),
key = lambda v : v["count"], reverse=True)
return {
"ideology": lambda : { # defer until cache miss
"house": make_chart_series(RoleType.representative),
"senate": make_chart_series(RoleType.senator),
},
"current_congress": CURRENT_CONGRESS,
"prognosis_training_congress": bill.prognosis_model.congress,
"prognosis_training_congress_dates": get_congress_dates(bill.prognosis_model.congress),
"prognosis_factors": prognosis_factors,
"prognosis_test": prognosis_test,
"prognosis_testing_traincongress": bill.prognosis_model_test.train_congress,
"prognosis_testing_testcongress": bill.prognosis_model_test.test_congress,
}
def bill_details(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": bill.get_feed(),
"text_info": get_text_info(bill),
}
def analysis_methodology(request):
from settings import CURRENT_CONGRESS
from person.models import RoleType
from bill.models import BillType
from us import get_congress_dates
import json
from person.analysis import load_sponsorship_analysis2
def make_chart_series(role_type):
data = load_sponsorship_analysis2(CURRENT_CONGRESS, role_type, None)
if not data: return None
ret = { }
for p in data["all"]:
ret.setdefault(p["party"], {
"type": "party",
"party": p["party"],
"data": [],
})["data"].append({
"x": float(p["ideology"]),
"y": float(p["leadership"]),
"name": p["name"],
})
ret = list(ret.values())
ret.sort(key = lambda s : len(s["data"]), reverse=True)
data = dict(data) # clone before modifying, just in case
data["series"] = json.dumps(ret)
return data
import bill.prognosis
import bill.prognosis_model
import bill.prognosis_model_test
prognosis_factors = sorted([dict(v) for v in bill.prognosis_model.factors.values()],
key = lambda m : m["count"], reverse=True)
for v in prognosis_factors:
v["factors"] = sorted(v["factors"].values(), key = lambda f : f["regression_beta"], reverse=True)
prognosis_test = sorted(bill.prognosis_model_test.model_test_results.values(),
key = lambda v : v["count"], reverse=True)
return {
"ideology": lambda : { # defer until cache miss
"house": make_chart_series(RoleType.representative),
"senate": make_chart_series(RoleType.senator),
},
"current_congress": CURRENT_CONGRESS,
"prognosis_training_congress": bill.prognosis_model.congress,
"prognosis_training_congress_dates": get_congress_dates(bill.prognosis_model.congress),
"prognosis_factors": prognosis_factors,
"prognosis_test": prognosis_test,
"prognosis_testing_traincongress": bill.prognosis_model_test.train_congress,
"prognosis_testing_testcongress": bill.prognosis_model_test.test_congress,
}
def load_session_stats(session):
# Which Congress is it?
for congress, s, sd, ed in get_all_sessions():
if s == session: break # leaves "congress" variable set
else:
raise ValueError("Invalid session: %s" % session)
fn = "data/us/%d/stats/session-%s.json" % (congress, session)
try:
datafile = json.load(open(fn))
datafile["meta"]["pub_year"] = session
if datafile["meta"]["is_full_congress_stats"]:
datafile["meta"]["startdate"] = get_congress_dates(congress)[0]
datafile["meta"]["enddate"] = get_congress_dates(congress)[1]
else:
datafile["meta"]["startdate"] = sd
datafile["meta"]["enddate"] = ed
except IOError:
raise ValueError("No statistics are available for session %s." % session)
return datafile
def load_majority_party(congress):
majority_party = { }
start, end = get_congress_dates(congress)
for rt, bts in (
(RoleType.senator, (BillType.senate_bill, BillType.senate_resolution, BillType.senate_concurrent_resolution, BillType.senate_joint_resolution)),
(RoleType.representative, (BillType.house_bill, BillType.house_resolution, BillType.house_concurrent_resolution, BillType.house_joint_resolution))
):
p = PersonRole.objects.filter(startdate__lte=end, enddate__gte=start, role_type=rt).values("party").annotate(count=Count("id")).order_by("-count")[0]["party"]
for bt in bts:
majority_party[bt] = p
return majority_party
def browse_state(request, state):
state = normalize_state_arg(state)
if state not in stateapportionment: raise Http404()
return {
"state": state,
"stateapp": stateapportionment[state],
"statename": statenames[state],
"senators": get_senators(state),
"representatives": get_representatives(state),
"end_of_congress_date": get_congress_dates(CURRENT_CONGRESS)[1],
}
def load_majority_party(congress):
majority_party = { }
start, end = get_congress_dates(congress)
for rt, bts in (
(RoleType.senator, (BillType.senate_bill, BillType.senate_resolution, BillType.senate_concurrent_resolution, BillType.senate_joint_resolution)),
(RoleType.representative, (BillType.house_bill, BillType.house_resolution, BillType.house_concurrent_resolution, BillType.house_joint_resolution))
):
p = PersonRole.objects.filter(startdate__lte=end, enddate__gte=start, role_type=rt).values("party").annotate(count=Count("id")).order_by("-count")[0]["party"]
for bt in bts:
majority_party[bt] = p
return majority_party
def build_info():
# feeds about all legislation that we offer the user to subscribe to
feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"]
# info about bills by status
groups = [
( g[0], # title
g[1], # text 1
g[2], # text 2
"/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]) + "&sort=-current_status_date", # link
load_bill_status_qs(g[4]).count(), # count in category
load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category
)
for g in bill_status_groups ]
# legislation coming up
dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date'))
sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date'))
coming_up = list(dhg_bills | sfs_bills)
coming_up.sort(key = lambda b : b.docs_house_gov_postdate if (b.docs_house_gov_postdate and (not b.senate_floor_schedule_postdate or b.senate_floor_schedule_postdate < b.docs_house_gov_postdate)) else b.senate_floor_schedule_postdate, reverse=True)
# top tracked bills
top_bills = Feed.objects\
.filter(feedname__startswith='bill:')\
.filter(feedname__regex='^bill:[hs][jcr]?%d-' % CURRENT_CONGRESS)
top_bills = top_bills\
.annotate(count=Count('tracked_in_lists'))\
.order_by('-count')\
.values('feedname', 'count')\
[0:25]
top_bills = [(Bill.from_feed(Feed.from_name(bf["feedname"])), bf["count"]) for bf in top_bills]
# current congrss years
start, end = get_congress_dates(CURRENT_CONGRESS)
end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year
current_congress_years = '%d-%d' % (start.year, end.year)
current_congress = ordinal(CURRENT_CONGRESS)
return {
"feeds": feeds,
"total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(),
"current_congress_years": current_congress_years,
"current_congress": current_congress,
"groups": groups,
"coming_up": coming_up,
"top_tracked_bills": top_bills,
"subjects": subject_choices(),
"BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.referred, BillStatus.reported),
}
def build_info():
# feeds about all legislation that we offer the user to subscribe to
feeds = [f for f in Feed.get_simple_feeds() if f.category == "federal-bills"]
# info about bills by status
groups = [
( g[0], # title
g[1], # text 1
g[2], # text 2
"/congress/bills/browse?status=" + ",".join(str(s) for s in g[4]) + "&sort=-current_status_date", # link
load_bill_status_qs(g[4]).count(), # count in category
load_bill_status_qs(g[4]).order_by('-current_status_date')[0:6], # top 6 in this category
)
for g in bill_status_groups ]
# legislation coming up
dhg_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, docs_house_gov_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=10)).filter(docs_house_gov_postdate__gt=F('current_status_date'))
sfs_bills = Bill.objects.filter(congress=CURRENT_CONGRESS, senate_floor_schedule_postdate__gt=datetime.datetime.now() - datetime.timedelta(days=5)).filter(senate_floor_schedule_postdate__gt=F('current_status_date'))
coming_up = list((dhg_bills | sfs_bills).order_by('scheduled_consideration_date'))
# top tracked bills
top_bills = Feed.objects\
.filter(feedname__startswith='bill:')\
.filter(feedname__regex='^bill:[hs][jcr]?%d-' % CURRENT_CONGRESS)
top_bills = top_bills\
.annotate(count=Count('tracked_in_lists'))\
.order_by('-count')\
.values('feedname', 'count')\
[0:25]
top_bills = [(Bill.from_feed(Feed.from_name(bf["feedname"])), bf["count"]) for bf in top_bills]
# trending bills
trf = Feed.get_trending_feeds()
trf = [Feed.objects.get(id=f) for f in trf]
trending_bill_feeds = [f for f in trf if f.feedname.startswith("bill:")]
return {
"feeds": feeds,
"total": Bill.objects.filter(congress=CURRENT_CONGRESS).count(),
"current_congress": CURRENT_CONGRESS,
"current_congress_dates": get_congress_dates(CURRENT_CONGRESS),
"groups": groups,
"coming_up": coming_up,
"top_tracked_bills": top_bills,
"trending_bill_feeds": trending_bill_feeds,
"subjects": subject_choices(),
"BILL_STATUS_INTRO": (BillStatus.introduced, BillStatus.reported),
}
def person_session_stats(request, pk, session):
# get the person and the statistics
person = get_object_or_404(Person, pk=pk)
try:
stats = person.get_session_stats(session)
except ValueError:
# no stats
raise Http404()
# get the role as stored in the file
role = PersonRole.objects.get(id=stats["role_id"])
# mark the role as current if the logical end date is in the future, to fix the display of Served/Serving
role.current = (role.logical_dates()[1] > datetime.now().date())
# clean and sort the stats for this person so they're ready for display
from person.views_sessionstats import clean_person_stats
clean_person_stats(stats)
# group into an order for navigation
nav_groups = []
for stat in stats["stats"]:
for group in nav_groups:
if group["icon"] == stat["icon"]:
group["stats"].append(stat)
break
else:
nav_groups.append({ "icon": stat["icon"], "stats": [stat] })
import dateutil.parser
from person.types import Gender, RoleType
# what dates specifically for the congress?
(period_min, period_max) = get_congress_dates(stats["meta"]["congress"])
period_min = max(period_min, role.logical_dates()[0])
period_max = min(period_max, role.logical_dates()[1])
return {
"publishdate": dateutil.parser.parse(stats["meta"]["as-of"]),
"period": session_stats_period(session, stats),
"congress_dates": (period_min, period_max),
"person": person,
"photo": person.get_photo()[0],
"himher": Gender.by_value(person.gender).pronoun_object,
"role": role,
"class": RoleType.by_value(role.role_type).label.lower() + "s",
"session": session,
"meta": stats["meta"],
"stats": stats["stats"],
"nav_groups": nav_groups,
}
def person_session_stats(request, pk, session):
# get the person and the statistics
person = get_object_or_404(Person, pk=pk)
try:
stats = person.get_session_stats(session)
except ValueError:
# no stats
raise Http404()
# get the role as stored in the file
role = PersonRole.objects.get(id=stats["role_id"])
# mark the role as current if the logical end date is in the future, to fix the display of Served/Serving
role.current = (role.logical_dates()[1] > datetime.now().date())
# clean and sort the stats for this person so they're ready for display
from person.views_sessionstats import clean_person_stats
clean_person_stats(stats)
# group into an order for navigation
nav_groups = []
for stat in stats["stats"]:
for group in nav_groups:
if group["icon"] == stat["icon"]:
group["stats"].append(stat)
break
else:
nav_groups.append({"icon": stat["icon"], "stats": [stat]})
import dateutil.parser
from person.types import Gender, RoleType
# what dates specifically for the congress?
(period_min, period_max) = get_congress_dates(stats["meta"]["congress"])
period_min = max(period_min, role.logical_dates()[0])
period_max = min(period_max, role.logical_dates()[1])
return {
"publishdate": dateutil.parser.parse(stats["meta"]["as-of"]),
"period": session_stats_period(session, stats),
"congress_dates": (period_min, period_max),
"person": person,
"photo": person.get_photo()[0],
"himher": Gender.by_value(person.gender).pronoun_object,
"role": role,
"class": RoleType.by_value(role.role_type).label.lower() + "s",
"session": session,
"meta": stats["meta"],
"stats": stats["stats"],
"nav_groups": nav_groups,
}
def bill_text(request, congress, type_slug, number, version=None):
if version == "":
version = None
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from billtext import load_bill_text, bill_gpo_status_codes
try:
textdata = load_bill_text(bill, version)
except IOError:
textdata = None
# Get a list of the alternate versions of this bill.
alternates = None
if textdata:
alternates = []
for v in bill_gpo_status_codes:
try:
alternates.append(load_bill_text(bill, v, mods_only=True))
except IOError:
pass
alternates.sort(key = lambda mods : mods["docdate"])
# Get a list of related bills.
from billtext import get_current_version
related_bills = []
for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]:
try:
rbv = get_current_version(rb)
if not (rb, rbv) in related_bills: related_bills.append((rb, rbv))
except IOError:
pass # text not available
for btc in BillTextComparison.objects.filter(bill1=bill).exclude(bill2=bill):
if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2))
for btc in BillTextComparison.objects.filter(bill2=bill).exclude(bill1=bill):
if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1))
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"textdata": textdata,
"version": version,
"alternates": alternates,
"related_bills": related_bills,
}
def bill_text(request, congress, type_slug, number, version=None):
if version == "":
version = None
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from billtext import load_bill_text, bill_gpo_status_codes
try:
textdata = load_bill_text(bill, version)
except IOError:
textdata = None
# Get a list of the alternate versions of this bill.
alternates = None
if textdata:
alternates = []
for v in bill_gpo_status_codes:
try:
alternates.append(load_bill_text(bill, v, mods_only=True))
except IOError:
pass
alternates.sort(key = lambda mods : mods["docdate"])
# Get a list of related bills.
from billtext import get_current_version
related_bills = []
for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]:
try:
rbv = get_current_version(rb)
if not (rb, rbv) in related_bills: related_bills.append((rb, rbv))
except IOError:
pass # text not available
for btc in BillTextComparison.objects.filter(bill1=bill).exclude(bill2=bill):
if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2))
for btc in BillTextComparison.objects.filter(bill2=bill).exclude(bill1=bill):
if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1))
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"textdata": textdata,
"version": version,
"alternates": alternates,
"related_bills": related_bills,
}
def bill_text(request, congress, type_slug, number, version=None):
if int(congress) < 103:
raise Http404("Bill text is not available before the 103rd congress.")
if version == "":
version = None
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from billtext import load_bill_text, bill_gpo_status_codes
try:
textdata = load_bill_text(bill, version)
except IOError:
textdata = None
# Get a list of the alternate versions of this bill.
alternates = None
if textdata:
alternates = []
for v in bill_gpo_status_codes:
fn = "data/us/bills.text/%s/%s/%s%d%s.mods.xml" % (bill.congress, BillType.by_value(bill.bill_type).xml_code, BillType.by_value(bill.bill_type).xml_code, bill.number, v)
if os.path.exists(fn):
alternates.append(load_bill_text(bill, v, mods_only=True))
alternates.sort(key = lambda mods : mods["docdate"])
# Get a list of related bills.
related_bills = []
for rb in list(bill.find_reintroductions()) + [r.related_bill for r in bill.get_related_bills()]:
if not (rb, "") in related_bills: related_bills.append((rb, ""))
for btc in BillTextComparison.objects.filter(bill1=bill):
if not (btc.bill2, btc.ver2) in related_bills: related_bills.append((btc.bill2, btc.ver2))
for btc in BillTextComparison.objects.filter(bill2=bill):
if not (btc.bill1, btc.ver1) in related_bills: related_bills.append((btc.bill1, btc.ver1))
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"textdata": textdata,
"version": version,
"alternates": alternates,
"related_bills": related_bills,
}
def bill_statistics(request):
# Get the count of bills by status and by Congress.
counts_by_congress = []
for c in xrange(93, CURRENT_CONGRESS+1):
total = Bill.objects.filter(congress=c).count()
if total == 0: continue # during transitions between Congresses
counts_by_congress.append({
"congress": c,
"dates": get_congress_dates(c),
"counts": [ ],
"total": total,
})
for g in bill_status_groups:
t = load_bill_status_qs(g[4], congress=c).count()
counts_by_congress[-1]["counts"].append(
{ "count": t,
"percent": "%0.0f" % float(100.0*t/total),
"link": "/congress/bills/browse?congress=%s&status=%s" % (c, ",".join(str(s) for s in g[4])),
} )
counts_by_congress.reverse()
# When does activity occur within the session cycle?
if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
from django.db import connection
def pull_time_stat(field, where, cursor):
historical = False
cursor.execute("SELECT YEAR(%s) - congress*2 - 1787, MONTH(%s), COUNT(*) FROM bill_bill WHERE congress>=93 AND congress%s%d AND %s GROUP BY YEAR(%s) - congress*2, MONTH(%s)" % (field, field, "<" if historical else "=", CURRENT_CONGRESS, where, field, field))
activity = [{ "x": r[0]*12 + (r[1]-1), "count": r[2], "year": r[0] } for r in cursor.fetchall()]
total = sum(m["count"] for m in activity)
for i, m in enumerate(activity): m["cumulative_count"] = m["count"]/float(total) + (0.0 if i==0 else activity[i-1]["cumulative_count"])
for m in activity: m["count"] = round(m["count"] / (CURRENT_CONGRESS-96), 1)
for m in activity: m["cumulative_count"] = round(m["cumulative_count"] * 100.0)
return activity
with connection.cursor() as cursor:
activity_introduced_by_month = pull_time_stat('introduced_date', "1", cursor)
activity_enacted_by_month = pull_time_stat('current_status_date', "current_status IN (%d,%d,%d)" % (int(BillStatus.enacted_signed), int(BillStatus.enacted_veto_override), int(BillStatus.enacted_tendayrule)), cursor)
else:
activity_introduced_by_month = []
activity_enacted_by_month = []
return {
"groups2": bill_status_groups,
"counts_by_congress": counts_by_congress,
"activity": (("Bills and Resolutions Introduced", activity_introduced_by_month),
("Bills and Joint Resolutions Enacted", activity_enacted_by_month) )
}
def bill_statistics(request):
# Get the count of bills by status and by Congress.
counts_by_congress = []
for c in range(93, CURRENT_CONGRESS+1):
total = Bill.objects.filter(congress=c).count()
if total == 0: continue # during transitions between Congresses
counts_by_congress.append({
"congress": c,
"dates": get_congress_dates(c),
"counts": [ ],
"total": total,
})
for g in bill_status_groups:
t = load_bill_status_qs(g[4], congress=c).count()
counts_by_congress[-1]["counts"].append(
{ "count": t,
"percent": "%0.0f" % float(100.0*t/total),
"link": "/congress/bills/browse?congress=%s&status=%s" % (c, ",".join(str(s) for s in g[4])),
} )
counts_by_congress.reverse()
# When does activity occur within the session cycle?
if settings.DATABASES['default']['ENGINE'] != 'django.db.backends.sqlite3':
from django.db import connection
def pull_time_stat(field, where, cursor):
cursor.execute("SELECT YEAR(%s) - congress*2 - 1787, MONTH(%s), COUNT(*) FROM bill_bill WHERE congress >= 93 AND %s GROUP BY YEAR(%s) - congress*2, MONTH(%s)" % (field, field, where, field, field))
activity = [{ "x": r[0]*12 + (r[1]-1), "count": r[2], "year": r[0] } for r in cursor.fetchall()]
total = sum(m["count"] for m in activity)
for i, m in enumerate(activity): m["cumulative_count"] = m["count"]/float(total) + (0.0 if i==0 else activity[i-1]["cumulative_count"])
for m in activity: m["count"] = round(m["count"] / float(total) * 100.0, 1)
for m in activity: m["cumulative_count"] = round(m["cumulative_count"] * 100.0)
return activity
with connection.cursor() as cursor:
activity_introduced_by_month = pull_time_stat('introduced_date', "1", cursor)
activity_enacted_by_month = pull_time_stat('current_status_date', "current_status IN (%d,%d,%d)" % (int(BillStatus.enacted_signed), int(BillStatus.enacted_veto_override), int(BillStatus.enacted_tendayrule)), cursor)
else:
activity_introduced_by_month = []
activity_enacted_by_month = []
return {
"groups2": bill_status_groups,
"counts_by_congress": counts_by_congress,
"activity": (("Bills Introduced", activity_introduced_by_month),
("Laws Enacted", activity_enacted_by_month) )
}
def proscore(self):
"""A modified prognosis score that omits factors associated with uninteresting bills, such as naming post offices. Only truly valid for current bills, and useless to compare across Congresses, but returns a value for all bills."""
# To aid search, especially for non-current bills, add in something to give most recently active bills a boost.
type_boost = {
BillType.senate_bill: 1.0, BillType.house_bill: 1.0,
BillType.senate_resolution: 0.2, BillType.house_resolution: 0.2,
BillType.senate_concurrent_resolution: 0.3, BillType.house_concurrent_resolution: 0.3,
BillType.senate_joint_resolution: 0.75, BillType.house_joint_resolution: 0.75,
}
cstart, cend = get_congress_dates(self.congress)
csd = self.current_status_date
if hasattr(csd, 'date'): csd = csd.date()
r = (csd - cstart).days / 365.0 # ranges from 0.0 to about 2.0.
if self.is_current:
from prognosis import compute_prognosis
r += compute_prognosis(self, proscore=True)["prediction"]
r *= type_boost[self.bill_type]
return r
def bill_details(request, congress, type_slug, number):
if type_slug.isdigit():
bill_type = type_slug
else:
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from person.name import get_person_name
sponsor_name = None if not bill.sponsor else \
get_person_name(bill.sponsor, role_date=bill.introduced_date, firstname_position='before', show_suffix=True)
def get_reintroductions():
reintro_prev = None
reintro_next = None
for reintro in bill.find_reintroductions():
if reintro.congress < bill.congress: reintro_prev = reintro
if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro
return reintro_prev, reintro_next
def get_text_info():
from billtext import load_bill_text
try:
return load_bill_text(bill, None, mods_only=True)
except IOError:
return None
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"sponsor_name": sponsor_name,
"reintros": get_reintroductions, # defer so we can use template caching
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": Feed.BillFeed(bill),
"text": get_text_info,
}
def bill_details(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
# get related bills
related_bills = []
reintro_prev = None
reintro_next = None
for reintro in bill.find_reintroductions():
if reintro.congress < bill.congress: reintro_prev = reintro
if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro
if reintro_prev: related_bills.append({ "bill": reintro_prev, "note": "was a previous version of this bill.", "show_title": False })
if reintro_next: related_bills.append({ "bill": reintro_next, "note": "was a re-introduction of this bill in a later Congress.", "show_title": False })
for rb in bill.get_related_bills():
if rb.relation in ("identical", "rule"):
related_bills.append({ "bill": rb.related_bill, "note": "(%s)" % rb.relation, "show_title": False })
elif rb.relation == "ruled-by":
related_bills.append({ "bill": rb.related_bill, "prenote": "Debate on", "note": " is governed by these rules.", "show_title": False })
else:
related_bills.append({ "bill": rb.related_bill, "note": ("(%s)" % (rb.relation.title() if rb.relation != "unknown" else "Related")), "show_title": True })
# bill text info and areas of law affected
from billtext import load_bill_text
try:
text_info = load_bill_text(bill, None, mods_only=True, with_citations=True)
except IOError:
text_info = None
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": bill.get_feed(),
"text_info": text_info,
"related": related_bills,
}
def bill_details(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
# get related bills
related_bills = []
reintro_prev = None
reintro_next = None
for reintro in bill.find_reintroductions():
if reintro.congress < bill.congress: reintro_prev = reintro
if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro
if reintro_prev: related_bills.append({ "bill": reintro_prev, "note": "was a previous version of this bill.", "show_title": False })
if reintro_next: related_bills.append({ "bill": reintro_next, "note": "was a re-introduction of this bill in a later Congress.", "show_title": False })
for rb in bill.get_related_bills():
if rb.relation in ("identical", "rule"):
related_bills.append({ "bill": rb.related_bill, "note": "(%s)" % rb.relation, "show_title": False })
elif rb.relation == "ruled-by":
related_bills.append({ "bill": rb.related_bill, "prenote": "Debate on", "note": " is governed by these rules.", "show_title": False })
else:
related_bills.append({ "bill": rb.related_bill, "note": ("(%s)" % (rb.relation.title() if rb.relation != "unknown" else "Related")), "show_title": True })
# bill text info and areas of law affected
from billtext import load_bill_text
try:
text_info = load_bill_text(bill, None, mods_only=True, with_citations=True)
except IOError:
text_info = None
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": bill.get_feed(),
"text_info": text_info,
"related": related_bills,
}
def bill_details(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
from person.name import get_person_name
sponsor_name = None if not bill.sponsor else \
get_person_name(bill.sponsor, role_date=bill.introduced_date, firstname_position='before', show_suffix=True)
def get_reintroductions():
reintro_prev = None
reintro_next = None
for reintro in bill.find_reintroductions():
if reintro.congress < bill.congress: reintro_prev = reintro
if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro
return reintro_prev, reintro_next
def get_text_info():
from models import USCSection
from billtext import load_bill_text
from search import parse_slip_law_number
import re
try:
metadata = load_bill_text(bill, None, mods_only=True)
# do interesting stuff with citations
if "citations" in metadata and not settings.DEBUG:
slip_laws = []
statutes = []
usc = { }
other = []
usc_other = USCSection(name="Other Citations", ordering=99999)
for cite in metadata["citations"]:
if cite["type"] == "slip_law":
slip_laws.append(cite)
cite["bill"] = parse_slip_law_number(cite["text"])
elif cite["type"] == "statutes_at_large":
statutes.append(cite)
elif cite["type"] in ("usc-section", "usc-chapter"):
# Build a tree of title-chapter-...-section nodes so we can
# display the citations in context.
try:
sec_obj = USCSection.objects.get(citation=cite["key"])
except: # USCSection.DoesNotExist and MultipleObjectsReturned both possible
# create a fake entry for the sake of output
# the 'id' field is set to make these objects properly hashable
sec_obj = USCSection(id=cite["text"], name=cite["text"], parent_section=usc_other)
if "range_to_section" in cite:
sec_obj.range_to_section = cite["range_to_section"]
# recursively go up to the title
path = [sec_obj]
so = sec_obj
while so.parent_section:
so = so.parent_section
path.append(so)
# build a link to LII
if cite["type"] == "usc-section":
cite_link = "http://www.law.cornell.edu/uscode/text/" + cite["title"]
if cite["section"]:
cite_link += "/" + cite["section"]
if cite["paragraph"]: cite_link += "#" + "_".join(re.findall(r"\(([^)]+)\)", cite["paragraph"]))
elif cite["type"] == "usc-chapter":
cite_link = "http://www.law.cornell.edu/uscode/text/" + cite["title"] + "/" + "/".join(
(so.level_type + "-" + so.number) for so in reversed(path[:-1])
)
sec_obj.link = cite_link
# now pop off from the path to put the node at the right point in a tree
container = usc
while path:
p = path.pop(-1)
if p not in container: container[p] = { }
container = container[p]
else:
other.append(cite)
slip_laws.sort(key = lambda x : (x["congress"], x["number"]))
# restructure data format
def ucfirst(s): return s[0].upper() + s[1:]
def rebuild_usc_sec(seclist, indent=0):
ret = []
seclist = sorted(seclist.items(), key=lambda x : x[0].ordering)
for sec, subparts in seclist:
ret.append({
"text": (ucfirst(sec.level_type + ((" " + sec.number) if sec.number else "") + (": " if sec.name else "")) if sec.level_type else "") + (sec.name_recased if sec.name else ""),
"link": getattr(sec, "link", None),
"range_to_section": getattr(sec, "range_to_section", None),
"indent": indent,
})
ret.extend(rebuild_usc_sec(subparts, indent=indent+1))
return ret
usc = rebuild_usc_sec(usc)
metadata["citations"] = {
"slip_laws": slip_laws, "statutes": statutes, "usc": usc, "other": other,
"count": len(slip_laws)+len(statutes)+len(usc)+len(other) }
return metadata
except IOError:
return None
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"sponsor_name": sponsor_name,
"reintros": get_reintroductions, # defer so we can use template caching
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": Feed.BillFeed(bill),
"text": get_text_info,
"care2_category_id": {
5816: '793', # Agriculture and Food=>Health
5840: '789', # Animals=>Animal Welfare
5996: '794', # Civil Rights=>Human Rights
5991: '791', # Education=>Education
6021: '792', # Energy=>Environment & Wildlife
6038: '792', # Environmental Protection=>Environment & Wildlife
6053: '793', # Families=>Health
6130: '793', # Health=>Health
6206: '794', # Immigration=>Human Rights
6279: '792', # Public Lands/Natural Resources=>Environment & Wildlife
6321: '791', # Social Sciences=>Education
6328: '793', # Social Welfare => Health
}.get(bill.get_top_term_id(), '795') # fall back to Politics category
}
def format_congress_number(value):
start, end = get_congress_dates(value)
end_year = end.year if end.month > 1 else end.year - 1 # count January finishes as the prev year
return '%s Congress: %d-%d' % (ordinal(value), start.year, end.year)
def main(options):
"""
Update Person and PersonRole models.
Do safe update: touch only those records
which have been changed.
"""
BASE_PATH = CONGRESS_LEGISLATORS_PATH
SRC_FILES = ['legislators-current', 'legislators-historical', 'legislators-social-media', 'executive'] # order matters
for p in SRC_FILES:
f = BASE_PATH + p + ".yaml"
if not File.objects.is_changed(f) and not options.force:
log.info('File %s was not changed' % f)
else:
# file modified...
break
else:
# no 'break' ==> no files modified
return
# Start parsing.
had_error = False
# Get combined data.
legislator_data = { }
leg_id_map = { }
for p in SRC_FILES:
log.info('Opening %s...' % p)
f = BASE_PATH + p + ".yaml"
y = yaml_load(f)
for m in y:
if p != 'legislators-social-media':
govtrack_id = m["id"].get("govtrack")
# For the benefit of the social media file, make a mapping of IDs.
for k, v in m["id"].items():
if type(v) != list:
leg_id_map[(k,v)] = govtrack_id
else:
# GovTrack IDs are not always listed in this file.
govtrack_id = None
for k, v in m["id"].items():
if type(v) != list and (k, v) in leg_id_map:
govtrack_id = leg_id_map[(k,v)]
break
if not govtrack_id:
print "No GovTrack ID:"
pprint.pprint(m)
had_error = True
continue
if govtrack_id not in legislator_data:
legislator_data[govtrack_id] = m
elif p == "legislators-social-media":
legislator_data[govtrack_id]["social"] = m["social"]
elif p == "executive":
legislator_data[govtrack_id]["terms"].extend( m["terms"] )
else:
raise ValueError("Duplication in an unexpected way (%d, %s)." % (govtrack_id, p))
person_processor = PersonProcessor()
role_processor = PersonRoleProcessor()
existing_persons = set(Person.objects.values_list('pk', flat=True))
processed_persons = set()
created_persons = set()
progress = Progress(total=len(legislator_data))
log.info('Processing persons')
for node in legislator_data.values():
# Wrap each iteration in try/except
# so that if some node breaks the parsing process
# then other nodes could be parsed
try:
person = person_processor.process(Person(), node)
# Create cached name strings. This is done again later
# after the roles are updated.
person.set_names()
# Now try to load the person with such ID from
# database. If found it then just update it
# else create new Person object
try:
ex_person = Person.objects.get(pk=person.pk)
if person_processor.changed(ex_person, person) or options.force:
# If the person has PK of existing record,
# coming in via the YAML-specified GovTrack ID,
# then Django ORM will update existing record
if not options.force:
log.warn("Updated %s" % person)
person.save()
except Person.DoesNotExist:
created_persons.add(person.pk)
person.save()
log.warn("Created %s" % person)
processed_persons.add(person.pk)
# Process roles of the person
roles = list(PersonRole.objects.filter(person=person))
existing_roles = set(PersonRole.objects.filter(person=person).values_list('pk', flat=True))
processed_roles = set()
role_list = []
for termnode in node['terms']:
role = role_processor.process(PersonRole(), termnode)
role.person = person
role.extra = filter_yaml_term_structure(termnode) # copy in the whole YAML structure
now = datetime.now().date()
if now < get_congress_dates(CURRENT_CONGRESS)[0]:
# in the transition time before the next Congress, just count as that Congress
now = get_congress_dates(CURRENT_CONGRESS)[0]
role.current = role.startdate <= now and role.enddate >= now
# Scan for most recent leadership role within the time period of this term,
# which isn't great for Senators because it's likely it changed a few times
# within a term, especially if there was a party switch.
role.leadership_title = None
for leadership_node in node.get("leadership_roles", []):
# must match on date and chamber
if leadership_node["start"] >= role.enddate.isoformat(): continue # might start on the same day but is for the next Congress
if "end" in leadership_node and leadership_node["end"] <= role.startdate.isoformat(): continue # might start on the same day but is for the previous Congress
if leadership_node["chamber"] != RoleType.by_value(role.role_type).congress_chamber.lower(): continue
role.leadership_title = leadership_node["title"]
# Try to match this role with one already in the database.
# First search for an exact match on type/start/end.
ex_role = None
for r in roles:
if role.role_type == r.role_type and r.startdate == role.startdate and r.enddate == role.enddate:
ex_role = r
break
# Otherwise match on type & either start or end only.
if not ex_role:
for r in roles:
if role.role_type == r.role_type and (r.startdate == role.startdate or r.enddate == role.enddate):
ex_role = r
break
if ex_role:
# These roles correspond.
processed_roles.add(ex_role.id)
role.id = ex_role.id
if role_processor.changed(ex_role, role) or options.force:
role.save()
role_list.append(role)
if not options.force:
log.warn("Updated %s" % role)
roles.remove(ex_role) # don't need to try matching this to any other node
else:
# Didn't find a matching role.
if len([r for r in roles if r.role_type == role.role_type]) > 0:
print role, "is one of these?"
for ex_role in roles:
print "\t", ex_role
raise Exception("There is an unmatched role.")
log.warn("Created %s" % role)
role.save()
role_list.append(role)
# create the events for the roles after all have been loaded
# because we don't create events for ends of terms and
# starts of terms that are adjacent.
if not options.disable_events:
for i in xrange(len(role_list)):
role_list[i].create_events(
role_list[i-1] if i > 0 else None,
role_list[i+1] if i < len(role_list)-1 else None
)
removed_roles = existing_roles - processed_roles
for pk in removed_roles:
pr = PersonRole.objects.get(pk=pk)
print pr.person.id, pr
raise ValueError("Deleted role??")
log.warn("Deleted %s" % pr)
pr.delete()
# The name can't be determined until all of the roles are set. If
# it changes, re-save. Unfortunately roles are cached so this actually
# doesn't work yet. Re-run the parser to fix names.
nn = (person.name, person.sortname)
if hasattr(person, "role"): delattr(person, "role") # clear the cached info
person.set_names()
if nn != (person.name, person.sortname):
log.warn("%s is now %s." % (nn[0], person.name))
person.save()
except Exception, ex:
# Catch unexpected exceptions and log them
pprint.pprint(node)
log.error('', exc_info=ex)
had_error = True
progress.tick()
def get_prognosis(self):
if self.congress != settings.CURRENT_CONGRESS: return None
import prognosis
prog = prognosis.compute_prognosis(self)
prog["congressdates"] = get_congress_dates(prog["congress"])
return prog
def bill_details(request, congress, type_slug, number):
if type_slug.isdigit():
bill_type = type_slug
else:
try:
bill_type = BillType.by_slug(type_slug)
except BillType.NotFound:
raise Http404("Invalid bill type: " + type_slug)
bill = get_object_or_404(Bill, congress=congress, bill_type=bill_type, number=number)
from person.name import get_person_name
sponsor_name = None if not bill.sponsor else \
get_person_name(bill.sponsor, role_date=bill.introduced_date, firstname_position='before', show_suffix=True)
def get_reintroductions():
reintro_prev = None
reintro_next = None
for reintro in bill.find_reintroductions():
if reintro.congress < bill.congress: reintro_prev = reintro
if reintro.congress > bill.congress and not reintro_next: reintro_next = reintro
return reintro_prev, reintro_next
def get_text_info():
from models import USCSection
from billtext import load_bill_text
from search import parse_slip_law_number
import re
try:
metadata = load_bill_text(bill, None, mods_only=True)
# do interesting stuff with citations
if "citations" in metadata:
slip_laws = []
statutes = []
usc = { }
other = []
usc_other = USCSection(name="Other Citations", ordering=99999)
for cite in metadata["citations"]:
if cite["type"] == "slip_law":
slip_laws.append(cite)
cite["bill"] = parse_slip_law_number(cite["text"])
elif cite["type"] == "statutes_at_large":
statutes.append(cite)
elif cite["type"] == "usc":
# build a normalized citation and a link to LII
cite_norm = "usc/" + cite["title"]
cite_link = "http://www.law.cornell.edu/uscode/text/" + cite["title"]
if cite["section"]:
cite_link += "/" + cite["section"]
cite_norm += "/" + cite["section"]
if cite["paragraph"]: cite_link += "#" + "_".join(re.findall(r"\(([^)]+)\)", cite["paragraph"]))
# Build a tree of title-chapter-...-section nodes so we can
# display the citations in context.
try:
sec_obj = USCSection.objects.get(citation=cite_norm)
except: # USCSection.DoesNotExist and MultipleObjectsReturned both possible
# the 'id' field is set to make these objects properly hashable
sec_obj = USCSection(id=cite["text"], name=cite["text"], parent_section=usc_other)
sec_obj.link = cite_link
if "range_to_section" in cite:
sec_obj.range_to_section = cite["range_to_section"]
# recursively go up to the title
path = [sec_obj]
while sec_obj.parent_section:
sec_obj = sec_obj.parent_section
path.append(sec_obj)
# now pop off from the path to put the node at the right point in a tree
container = usc
while path:
p = path.pop(-1)
if p not in container: container[p] = { }
container = container[p]
else:
other.append(cite)
slip_laws.sort(key = lambda x : (x["congress"], x["number"]))
# restructure data format
def ucfirst(s): return s[0].upper() + s[1:]
def rebuild_usc_sec(seclist, indent=0):
ret = []
seclist = sorted(seclist.items(), key=lambda x : x[0].ordering)
for sec, subparts in seclist:
ret.append({
"text": (ucfirst(sec.level_type + ((" " + sec.number) if sec.number else "") + (": " if sec.name else "")) if sec.level_type else "") + (sec.name if sec.name else ""),
"link": getattr(sec, "link", None),
"range_to_section": getattr(sec, "range_to_section", None),
"indent": indent,
})
ret.extend(rebuild_usc_sec(subparts, indent=indent+1))
return ret
usc = rebuild_usc_sec(usc)
metadata["citations"] = {
"slip_laws": slip_laws, "statutes": statutes, "usc": usc, "other": other,
"count": len(slip_laws)+len(statutes)+len(usc)+len(other) }
return metadata
except IOError:
return None
return {
'bill': bill,
"congressdates": get_congress_dates(bill.congress),
"subtitle": get_secondary_bill_title(bill, bill.titles),
"sponsor_name": sponsor_name,
"reintros": get_reintroductions, # defer so we can use template caching
"current": bill.congress == CURRENT_CONGRESS,
"dead": bill.congress != CURRENT_CONGRESS and bill.current_status not in BillStatus.final_status_obvious,
"feed": Feed.BillFeed(bill),
"text": get_text_info,
}
def format_congress_number(value):
start, end = get_congress_dates(value)
end_year = end.year if end.month > 1 else end.year-1 # count January finishes as the prev year
return '%s Congress: %d-%d' % (ordinal(value), start.year, end.year)
def run_analysis(label, congress):
global stats
global columns
congress_dates = get_congress_dates(congress)
start_date = datetime.date(congress_dates[0].year, 1, 1)
# limit to now, for the current congress
end_date = datetime.date(congress_dates[0].year, 9, 11)
end_date = min(end_date, datetime.datetime.now().date())
bills = Bill.objects.filter(congress=congress, bill_type__in=(BillType.house_bill, BillType.senate_bill, BillType.house_joint_resolution, BillType.senate_joint_resolution), introduced_date__lte=end_date)\
.order_by('current_status_date')
by_day = { }
for b in tqdm.tqdm(bills, desc=label):
match_date = None
for datestr, state, text, srcxml in b.major_actions:
action_date = eval(datestr)
#if state in (BillStatus.pass_over_house, BillStatus.pass_back_house, BillStatus.passed_bill):
if state in BillStatus.final_status_enacted_bill:
match_date = action_date.date()
break
else:
# No event matched.
continue
# Load plain text.
text = load_bill_text(b, None, plain_text=True)
# Bills since 1993 have GPO MODS XML metadata with page counts.
try:
mods = load_bill_text(b, None, mods_only=True)
pages = mods.get("numpages")
except (IOError, AttributeError) as e:
# For historical statutes we only have plain text from the
# Statutes at Large, extracted from PDFs. We can get page
# counts by looking for our replacement of the form feed
# character put in by pdftotext. We only have that when
# we extracted text from PDFs, which we only did for
# the Statutes at Large. We can't do this on modern bills
# where the text came from GPO plain text format.
if b.congress < 103:
pages = len([pgtext for pgtext in text.split("\n=============================================\n") if pgtext.strip() != ""])
else:
print b.id, b, e
raise ValueError("page date missing")
#words = len(re.split(r"\s+", text)) # not very good for pre-GPO bills because Statutes at Large pages may have multiple statutes on them
################ EEK
#if pages == 1: continue
################ EEK
rel_date = (match_date - start_date).days
rec = by_day.setdefault(rel_date, { "bills": 0, "pages": 0 } )
rec["bills"] += 1
rec["pages"] += pages
#rec["words"] += words
# Compute cumulative counts starting on day 0 and for every day till the
# last day a bill was signed.
columns.append(label)
bills = 0
pages = 0
#words = 0
for rel_date in range((end_date-start_date).days+1):
if rel_date in by_day:
bills += by_day[rel_date]["bills"]
pages += by_day[rel_date]["pages"]
#words += by_day[rel_date]["words"]
stats.setdefault(rel_date, {})[label] = (bills, pages)
def get_last_role_at_congress(self, congress):
start, end = get_congress_dates(congress)
try:
return self.roles.filter(startdate__lte=end, enddate__gte=start).order_by('-startdate')[0]
except IndexError:
return None
def get_bill_factors(bill, pop_title_prefixes, committee_membership, majority_party, lobbying_data, include_related_bills=True):
factors = list()
# introduced date (idea from Yano, Smith and Wilkerson 2012 paper)
idate = bill.introduced_date
if hasattr(idate, 'date'): idate = idate.date() # not sure how this is possible
if (idate - get_congress_dates(bill.congress)[0]).days < 90:
factors.append(("introduced_first90days", "The %s was introduced in the first 90 days of the Congress." % bill.noun, "Introduced in the first 90 days of the Congress (incl. companion bills)."))
if (idate - get_congress_dates(bill.congress)[0]).days < 365:
factors.append(("introduced_firstyear", "The %s was introduced in the first year of the Congress." % bill.noun, "Introduced in the first year of the Congress (incl. companion bills)."))
if (get_congress_dates(bill.congress)[1] - idate).days < 90:
factors.append(("introduced_last90days", "The %s was introduced in the last 90 days of the Congress." % bill.noun, "Introduced in the last 90 days of the Congress (incl. companion bills)."))
# does the bill's title start with a common prefix?
for prefix in pop_title_prefixes:
if bill.title_no_number.startswith(prefix + " "):
factors.append(("startswith:" + prefix, "The %s's title starts with \"%s.\"" % (bill.noun, prefix), "Title starts with \"%s\"." % prefix))
cosponsors = list(Cosponsor.objects.filter(bill=bill, withdrawn=None).select_related("person"))
committees = list(bill.committees.all())
maj_party = majority_party[bill.bill_type]
if bill.sponsor_role:
# party of the sponsor
sponsor_party = bill.sponsor_role.party
if sponsor_party != maj_party:
factors.append( ("sponsor_minority", "The sponsor is a member of the minority party.", "Sponsor is a member of the minority party.") )
# is the sponsor a member/chair of a committee to which the bill has
# been referred?
for rname, rvalue in (("member", CommitteeMemberRole.member), ("rankingmember", CommitteeMemberRole.ranking_member), ("vicechair", CommitteeMemberRole.vice_chairman), ("chair", CommitteeMemberRole.chairman)):
for committee in committees:
if committee_membership.get(bill.sponsor_id, {}).get(committee.code) == rvalue:
if rvalue != CommitteeMemberRole.member:
factors.append(("sponsor_committee_%s" % rname, "The sponsor is the %s of a committee to which the %s has been referred." % (CommitteeMemberRole.by_value(rvalue).label.lower(), bill.noun), "Sponsor is a relevant committee %s." % CommitteeMemberRole.by_value(rvalue).label.lower()))
if sponsor_party == maj_party:
factors.append(("sponsor_committee_member_majority", "The sponsor is on a committee to which the %s has been referred, and the sponsor is a member of the majority party." % bill.noun, "Sponsor is on a relevant committee & in majority party."))
# leadership score of the sponsor, doesn't actually seem to be helpful,
# even though leadership score of cosponsors is.
if get_leadership_score(bill.sponsor) > .8:
if sponsor_party == maj_party:
factors.append(("sponsor_leader_majority", "The sponsor is in the majority party and has a high leadership score.", "Sponsor has a high leadership score (majority party)."))
else:
factors.append(("sponsor_leader_minority", "The sponsor has a high leadership score but is not in the majority party.", "Sponsor has a high leadership score (minority party)."))
# count cosponsor assignments to committees by committee role and Member party
for rname, rvalue in (("committeemember", CommitteeMemberRole.member), ("rankingmember", CommitteeMemberRole.ranking_member), ("vicechair", CommitteeMemberRole.vice_chairman), ("chair", CommitteeMemberRole.chairman)):
num_cosp = 0
for cosponsor in cosponsors:
for committee in committees:
cvalue = committee_membership.get(cosponsor.person.id, {}).get(committee.code)
if cvalue == rvalue or (rvalue==CommitteeMemberRole.member and cvalue != None):
num_cosp += 1
break
if rvalue == CommitteeMemberRole.member:
if num_cosp <= 2: # ranges are tweakable...
num_cosp = str(num_cosp)
if num_cosp <= 5:
num_cosp = "3-5"
else:
num_cosp = "6+"
factors.append( ("cosponsor_%s_%s" % (rname, num_cosp), "%s cosponsors serve on a committee to which the %s has been referred." % (num_cosp, bill.noun), "%s cosponsors are on a relevant committee." % num_cosp) )
elif num_cosp > 0:
rname2 = CommitteeMemberRole.by_value(rvalue).label.lower()
factors.append( ("cosponsor_%s" % rname, "A cosponsor is the %s of a committee to which the %s has been referred." % (rname2, bill.noun), "A cosponsor is a relevant committee %s." % rname2))
# what committees is the bill assigned to? only look at committees
# in the originating chamber, since assignments in the other chamber
# indicate the bill had at least one successful vote.
for cm in committees:
if cm.committee != None: continue # skip subcommittees
if CommitteeType.by_value(cm.committee_type).label != bill.originating_chamber: continue
factors.append( ("committee_%s" % cm.code, "The bill was referred to %s." % cm.shortname, "Referred to %s." % cm.shortname))
# do we have cosponsors on both parties?
num_cosp_majority = 0
for cosponsor in cosponsors:
if cosponsor.role.party == maj_party:
num_cosp_majority += 1
if bill.sponsor and sponsor_party == maj_party and len(cosponsors) >= 6 and num_cosp_majority < 2.0*len(cosponsors)/3:
factors.append(("cosponsors_bipartisan", "The sponsor is in the majority party and at least one third of the %s's cosponsors are from the minority party." % bill.noun, "Sponsor is in majority party and 1/3rd+ of cosponsors are in minority party."))
if num_cosp_majority > 0 and num_cosp_majority < len(cosponsors):
factors.append(("cosponsors_crosspartisan", "There is at least one cosponsor from the majority party and one cosponsor outside of the majority party.", "Has cosponsors from both parties."))
for is_majority in (False, True):
for cosponsor in cosponsors:
if (cosponsor.role.party == maj_party) != is_majority: continue
if get_leadership_score(cosponsor.person) > .85:
if is_majority:
factors.append(("cosponsor_leader_majority", "A cosponsor in the majority party has a high leadership score.", "Cosponsor has high leadership score (majority party)."))
else:
factors.append(("cosponsor_leader_minority", "A cosponsor in the minority party has a high leadership score.", "Cosponsor has high leadership score (minority party)."))
break
# Is this bill a re-intro from last Congress, and if so was that bill reported by committee?
if bill.sponsor:
def normalize_title(title):
# remove anything that looks like a year
return re.sub(r"of \d\d\d\d$", "", title)
for reintro in Bill.objects.filter(congress=bill.congress-1, sponsor=bill.sponsor):
if normalize_title(bill.title_no_number) == normalize_title(reintro.title_no_number):
if reintro.current_status not in (BillStatus.introduced, BillStatus.referred):
factors.append(("reintroduced_of_reported", "This %s was reported by committee as %s in the previous session of Congress." % (bill.noun, reintro.display_number), "Got past committee in a previous Congress."))
else:
factors.append(("reintroduced", "This %s was a re-introduction of %s from the previous session of Congress." % (bill.noun, reintro.display_number), "Is a bill reintroduced from a previous Congress."))
break
if include_related_bills: # prevent infinite recursion
# Add factors from any CRS-identified identical bill, changing most factors'
# key into companion_KEY so that they become separate factors to consider.
# For some specific factors, lump them in with the factor for the bill itself.
for rb in RelatedBill.objects.filter(bill=bill, relation="identical").select_related("related_bill", "related_bill__sponsor_role"):
# has a companion
factors.append(("companion", "The %s has been introduced in both chambers (the other is %s)." % (bill.noun, rb.related_bill.display_number), "Has a companion bill in the other chamber."))
# companion sponsor's party
if bill.sponsor_role and rb.related_bill.sponsor_role:
if bill.sponsor_role.party != rb.related_bill.sponsor_role.party:
factors.append(("companion_bipartisan", "The %s's companion %s was sponsored by a member of the other party." % (bill.noun, rb.related_bill.display_number), "Has a companion bill sponsored by the other party."))
for f in get_bill_factors(rb.related_bill, pop_title_prefixes, committee_membership, majority_party, lobbying_data, include_related_bills=False):
if "startswith" in f[0]: continue # don't include title factors because the title is probs the same
if f[0] in ("introduced_first90days", "introduced_last90days", "introduced_firstyear", "reintroduced_of_reported", "reintroduced"):
f = (f[0], "%s (on companion bill %s)" % (f[1], rb.related_bill.display_number), f[2])
else:
f = ("companion__" + f[0], "Companion bill " + rb.related_bill.display_number + ": " + f[1], "On a companion bill: " + f[2])
# Make sure not to duplicate any factors, especially if we are promoting the companion
# bill factor to a main factor, we don't want to double count or override the description
# on the main bill.
if f[0] in set(k[0] for k in factors): continue
factors.append(f)
# Are lobbyists registering that they are lobbying on this bill? Does this bill
# have more registered lobbying than the median bill? Overall this picks out
# bills NOT likely to be enacted.
#
# Two possible explanations: First, lobbying can be to defeat a bill not just
# pass it. So this would indicate that on balance lobbying is having that effect.
#
# Second it could be because lobbyists don't bother with
# the easy bills that don't need their help. Meaning, they pick out a pool of
# improbable bllls, and presumably make those bills more likely to be enacted
# but still not as likely as the easy bills. (If they truly hurt a bill's future, they
# would presumably know and stop lobbying!)
#
# Looking at lobbying might be more useful if we combined it with another
# factor that could pick out the hard bills, and then this might show that for
# hard bills, lobbying made the bills more successful. But it's a bit impossible
# because surely the lobbyists know better than we do which bills are hard,
# so it would be impossible to factor out "hard bills" entirely.
if False:
if lobbying_data["counts"].get( (bill.bill_type, bill.number), 0 ) > lobbying_data["median"]:
factors.append( ("crp-lobby-many", "The Center for Responsive Politics reports that a large number of organizations are lobbying on this %s." % bill.noun, "Has many lobbyists.") )
elif lobbying_data["counts"].get( (bill.bill_type, bill.number), 0 ) > 0:
factors.append( ("crp-lobby", "The Center for Responsive Politics reports that organizations are lobbying on this %s." % bill.noun, "Has lobbyists.") )
return factors
## power
#congress_same_party = party_control[congress][0] == party_control[congress][1]
#branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2])
#
timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year-1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year))
row = [congress, timespan, date_range[0].isoformat(), date_range[1].isoformat(),
enacted_bills_count, enacted_bill_pages, enacted_bill_words, house_votes, senate_votes]
W.writerow(row)
#print("<tr>%s</tr>" % "".join( "<td>%s</td> " % td for td in row) )
if 0:
# Look at corresponding time periods from past Congresses.
# Go back a few days because our data isn't real time!
days_in = (datetime.now().date() - get_congress_dates(CURRENT_CONGRESS)[0]) \
- timedelta(days=4)
print("We are about %d days into the %d Congress" % (days_in.days, CURRENT_CONGRESS))
for c in range(93, CURRENT_CONGRESS+1):
date_range = get_congress_dates(c)
compute_productivity(c, (date_range[0], date_range[1] + days_in))
elif 0:
for c in range(93, CURRENT_CONGRESS+1):
# January 1 of the second year of the Congress
# through March 31 of that year.
date_range = get_congress_dates(c)
date_range = (datetime(date_range[0].year+1, 1, 1).date(), datetime(date_range[0].year+1, 1, 1).date()+timedelta(days=105))
compute_productivity(c, date_range)
elif 0:
((get_congress_dates(congress)[1].year -
1) if get_congress_dates(congress)[1].month == 1
else get_congress_dates(congress)[1].year))
row = [
congress, timespan, date_range[0].isoformat(),
date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages,
enacted_bill_words, house_votes, senate_votes
]
W.writerow(row)
#print("<tr>%s</tr>" % "".join( "<td>%s</td> " % td for td in row) )
if 0:
# Look at corresponding time periods from past Congresses.
# Go back a few days because our data isn't real time!
days_in = (datetime.now().date() - get_congress_dates(CURRENT_CONGRESS)[0]) \
- timedelta(days=4)
print("We are about %d days into the %d Congress" %
(days_in.days, CURRENT_CONGRESS))
for c in range(93, 114 + 1):
date_range = get_congress_dates(c)
compute_productivity(c, (date_range[0], date_range[1] + days_in))
elif 0:
for c in range(93, 114 + 1):
# January 1 of the second year of the Congress
# through March 31 of that year.
date_range = get_congress_dates(c)
date_range = (datetime(date_range[0].year + 1, 1, 1).date(),
datetime(date_range[0].year + 1, 1, 1).date() +
timedelta(days=105))
def compute_productivity(congress, date_range):
# laws
enacted_bills = Bill.objects.filter(
#congress=congress, --- because we're mostly measuring presidential activity, the date of signing could be outside of the Congress
current_status__in=BillStatus.final_status_passed_bill,
current_status_date__gte=date_range[0],
current_status_date__lte=date_range[1])
#enacted_bills = (enacted_bills.filter(title__contains="Appropriations") | enacted_bills.filter(title__contains="Authorization")).distinct()
enacted_bills = list(enacted_bills)
enacted_bills_count = len(enacted_bills)
enacted_bill_pages = 0
enacted_bill_words = 0
enacted_bill_pages_missing = 0
for b in enacted_bills:
try:
pp = load_bill_text(b, None, mods_only=True).get("numpages")
except IOError:
pp = None
if pp is None:
enacted_bill_pages_missing += 1
continue
pp = int(pp.replace(" pages", ""))
enacted_bill_pages += pp
wds = len(load_bill_text(b, None, plain_text=True).split(" "))
enacted_bill_words += wds
if congress < 103: enacted_bill_pages = "(no data)"
if congress < 103: enacted_bill_words = "(no data)"
# votes
house_votes = Vote.objects.filter(congress=congress,
created__gte=date_range[0],
created__lte=date_range[1],
chamber=CongressChamber.house).count()
senate_votes = Vote.objects.filter(congress=congress,
created__gte=date_range[0],
created__lte=date_range[1],
chamber=CongressChamber.senate).count()
## power
#congress_same_party = party_control[congress][0] == party_control[congress][1]
#branches_same_party = (party_control[congress][0] == party_control[congress][1]) and (party_control[congress][0] == party_control[congress][2])
#
timespan = "%d-%d" % (get_congress_dates(congress)[0].year,
((get_congress_dates(congress)[1].year -
1) if get_congress_dates(congress)[1].month == 1
else get_congress_dates(congress)[1].year))
row = [
congress, timespan, date_range[0].isoformat(),
date_range[1].isoformat(), enacted_bills_count, enacted_bill_pages,
enacted_bill_words, house_votes, senate_votes
]
W.writerow(row)
def bill_summaries(request, congress, type_slug, number):
bill = load_bill_from_url(congress, type_slug, number)
return {
"bill": bill,
"congressdates": get_congress_dates(bill.congress),
}
# senate_votes = Vote.objects.filter(
# congress=congress,
# created__gte=date_range[0],
# created__lte=date_range[1],
# chamber=CongressChamber.senate).count()
#timespan = "%d-%d" % (get_congress_dates(congress)[0].year, ((get_congress_dates(congress)[1].year-1) if get_congress_dates(congress)[1].month == 1 else get_congress_dates(congress)[1].year))
row = [label or congress, date_range[0].isoformat(), date_range[1].isoformat(),
enacted_bills_count, enacted_bill_pages, enacted_bill_words] #, house_votes, senate_votes]
W.writerow(row)
#print("<tr>%s</tr>" % "".join( "<td>%s</td> " % td for td in row) )
if 0:
# Look at corresponding time periods from past Congresses.
# Go back a few days because our data isn't real time!
days_in = (datetime.now().date() - get_congress_dates(CURRENT_CONGRESS)[0]) \
- timedelta(days=4)
print("We are about %d days into the %d Congress" % (days_in.days, CURRENT_CONGRESS))
for c in range(93, CURRENT_CONGRESS+1):
date_range = get_congress_dates(c)
compute_productivity(c, (date_range[0], date_range[0] + days_in))
elif 1:
# First X days of presidency, minus a few days because of
# data delays.
days_in = (datetime.now().date() - datetime(get_congress_dates(CURRENT_CONGRESS)[0].year, 1, 20, 0, 0, 0).date()) \
- timedelta(days=0)
#days_in = timedelta(days=178)
print("We are about %d days into the presidency" % days_in.days)
print("Looking at presidents whose first day is Jan 20 of the first year of a Congress.")
for label, congress in (("Eisenhower", 83), ("Kennedy", 87), ("Nixon", 91), ("Carter", 95), ("Reagan", 97), ("Bush1", 101), ("Clinton", 103), ("Bush2", 107), ("Obama", 111), ("Trump", 115)):
def get_bill_factors(bill,
pop_title_prefixes,
committee_membership,
majority_party,
lobbying_data,
include_related_bills=True):
factors = list()
# introduced date (idea from Yano, Smith and Wilkerson 2012 paper)
idate = bill.introduced_date
if hasattr(idate, 'date'):
idate = idate.date() # not sure how this is possible
if (idate - get_congress_dates(bill.congress)[0]).days < 90:
factors.append((
"introduced_first90days",
"The %s was introduced in the first 90 days of the Congress." %
bill.noun,
"Introduced in the first 90 days of the Congress (incl. companion bills)."
))
if (idate - get_congress_dates(bill.congress)[0]).days < 365:
factors.append((
"introduced_firstyear",
"The %s was introduced in the first year of the Congress." %
bill.noun,
"Introduced in the first year of the Congress (incl. companion bills)."
))
if (get_congress_dates(bill.congress)[1] - idate).days < 90:
factors.append((
"introduced_last90days",
"The %s was introduced in the last 90 days of the Congress." %
bill.noun,
"Introduced in the last 90 days of the Congress (incl. companion bills)."
))
# does the bill's title start with a common prefix?
for prefix in pop_title_prefixes:
if bill.title_no_number.startswith(prefix + " "):
factors.append(
("startswith:" + prefix,
"The %s's title starts with \"%s.\"" % (bill.noun, prefix),
"Title starts with \"%s\"." % prefix))
cosponsors = list(
Cosponsor.objects.filter(bill=bill,
withdrawn=None).select_related("person"))
committees = list(bill.committees.all())
maj_party = majority_party[bill.bill_type]
if bill.sponsor_role:
# party of the sponsor
sponsor_party = bill.sponsor_role.party
if sponsor_party != maj_party:
factors.append(("sponsor_minority",
"The sponsor is a member of the minority party.",
"Sponsor is a member of the minority party."))
# is the sponsor a member/chair of a committee to which the bill has
# been referred?
for rname, rvalue in (("member", CommitteeMemberRole.member),
("rankingmember",
CommitteeMemberRole.ranking_member),
("chair", CommitteeMemberRole.vice_chair),
("chair", CommitteeMemberRole.chair)):
for committee in committees:
if committee_membership.get(bill.sponsor_id,
{}).get(committee.code) == rvalue:
if rvalue != CommitteeMemberRole.member:
factors.append((
"sponsor_committee_%s" % rname,
"The sponsor is the %s of a committee to which the %s has been referred."
% (CommitteeMemberRole.by_value(
rvalue).label.lower(), bill.noun),
"Sponsor is a relevant committee %s." %
CommitteeMemberRole.by_value(rvalue).label.lower()
))
elif sponsor_party == maj_party:
factors.append((
"sponsor_committee_member_majority",
"The sponsor is on a committee to which the %s has been referred, and the sponsor is a member of the majority party."
% bill.noun,
"Sponsor is on a relevant committee & in majority party."
))
# leadership score of the sponsor, doesn't actually seem to be helpful,
# even though leadership score of cosponsors is.
if get_leadership_score(bill.sponsor) > .8:
if sponsor_party == maj_party:
factors.append((
"sponsor_leader_majority",
"The sponsor is in the majority party and has a high leadership score.",
"Sponsor has a high leadership score (majority party)."))
else:
factors.append((
"sponsor_leader_minority",
"The sponsor has a high leadership score but is not in the majority party.",
"Sponsor has a high leadership score (minority party)."))
# count cosponsor assignments to committees by committee role and Member party
for rname, rvalue in (("committeemember", CommitteeMemberRole.member),
("rankingmember",
CommitteeMemberRole.ranking_member),
("chair", CommitteeMemberRole.vice_chair),
("chair", CommitteeMemberRole.chair)):
num_cosp = 0
for cosponsor in cosponsors:
for committee in committees:
cvalue = committee_membership.get(cosponsor.person.id,
{}).get(committee.code)
if cvalue == rvalue or (rvalue == CommitteeMemberRole.member
and cvalue != None):
num_cosp += 1
break
if rvalue == CommitteeMemberRole.member:
if num_cosp >= 2:
factors.append((
"cosponsors_%s",
"At least two cosponsors serve on a committee to which the %s has been referred."
% (bill.noun, ),
"2 or more cosponsors are on a relevant committee."))
elif num_cosp > 0:
rname2 = CommitteeMemberRole.by_value(rvalue).label.lower()
factors.append((
"cosponsor_%s" % rname,
"A cosponsor is the %s of a committee to which the %s has been referred."
% (rname2, bill.noun),
"A cosponsor is a relevant committee %s." % rname2))
# what committees is the bill assigned to? only look at committees
# in the originating chamber, since assignments in the other chamber
# indicate the bill had at least one successful vote.
for cm in committees:
if cm.committee != None: continue # skip subcommittees
if CommitteeType.by_value(
cm.committee_type).label != bill.originating_chamber:
continue
factors.append(("committee_%s" % cm.code,
"The bill was referred to %s." % cm.shortname,
"Referred to %s (incl. companion)." % cm.shortname))
# do we have cosponsors on both parties?
num_cosp_majority = 0
for cosponsor in cosponsors:
if cosponsor.role.party == maj_party:
num_cosp_majority += 1
if bill.sponsor and sponsor_party == maj_party and len(
cosponsors) >= 6 and num_cosp_majority < 2.0 * len(cosponsors) / 3:
factors.append((
"cosponsors_bipartisan",
"The sponsor is in the majority party and at least one third of the %s's cosponsors are from the minority party."
% bill.noun,
"Sponsor is in majority party and 1/3rd+ of cosponsors are in minority party."
))
elif num_cosp_majority > 0 and num_cosp_majority < len(cosponsors):
factors.append((
"cosponsors_crosspartisan",
"There is at least one cosponsor from the majority party and one cosponsor outside of the majority party.",
"Has cosponsors from both parties."))
for is_majority in (False, True):
for cosponsor in cosponsors:
if (cosponsor.role.party == maj_party) != is_majority: continue
if get_leadership_score(cosponsor.person) > .85:
if is_majority:
factors.append((
"cosponsor_leader_majority",
"A cosponsor in the majority party has a high leadership score.",
"Cosponsor has high leadership score (majority party)."
))
else:
factors.append((
"cosponsor_leader_minority",
"A cosponsor in the minority party has a high leadership score.",
"Cosponsor has high leadership score (minority party)."
))
break
# Is this bill a re-intro from last Congress, and if so was that bill reported by committee?
if bill.sponsor:
def normalize_title(title):
# remove anything that looks like a year
return re.sub(r"of \d\d\d\d$", "", title)
for reintro in Bill.objects.filter(congress=bill.congress - 1,
sponsor=bill.sponsor):
if normalize_title(bill.title_no_number) == normalize_title(
reintro.title_no_number):
if reintro.current_status != BillStatus.introduced:
factors.append((
"reintroduced_of_reported",
"This %s was reported by committee as %s in the previous session of Congress."
% (bill.noun, reintro.display_number),
"Got past committee in a previous Congress."))
else:
factors.append((
"reintroduced",
"This %s was a re-introduction of %s from the previous session of Congress."
% (bill.noun, reintro.display_number),
"Is a bill reintroduced from a previous Congress."))
break
if include_related_bills: # prevent infinite recursion
# Add factors from any CRS-identified identical bill, changing most factors'
# key into companion_KEY so that they become separate factors to consider.
# For some specific factors, lump them in with the factor for the bill itself.
for rb in RelatedBill.objects.filter(
bill=bill, relation="identical").select_related(
"related_bill", "related_bill__sponsor_role"):
# has a companion
factors.append((
"companion",
"The %s has been introduced in both chambers (the other is %s)."
% (bill.noun, rb.related_bill.display_number),
"Has a companion bill in the other chamber."))
# companion sponsor's party
if bill.sponsor_role and rb.related_bill.sponsor_role:
if bill.sponsor_role.party != rb.related_bill.sponsor_role.party:
factors.append((
"companion_bipartisan",
"The %s's companion %s was sponsored by a member of the other party."
% (bill.noun, rb.related_bill.display_number),
"Has a companion bill sponsored by a member of the other party."
))
for f in get_bill_factors(rb.related_bill,
pop_title_prefixes,
committee_membership,
majority_party,
lobbying_data,
include_related_bills=False):
if "startswith" in f[0]:
continue # don't include title factors because the title is probs the same
if f[0] in ("introduced_first90days", "introduced_last90days",
"introduced_firstyear", "reintroduced_of_reported",
"reintroduced") or f[0].startswith("committee_"):
f = (f[0], "%s (on companion bill %s)" %
(f[1], rb.related_bill.display_number), f[2])
else:
f = ("companion__" + f[0], "Companion bill " +
rb.related_bill.display_number + ": " + f[1],
"On a companion bill: " + f[2])
# Make sure not to duplicate any factors, especially if we are promoting the companion
# bill factor to a main factor, we don't want to double count or override the description
# on the main bill.
if f[0] in set(k[0] for k in factors): continue
factors.append(f)
# Are lobbyists registering that they are lobbying on this bill? Does this bill
# have more registered lobbying than the median bill? Overall this picks out
# bills NOT likely to be enacted.
#
# Two possible explanations: First, lobbying can be to defeat a bill not just
# pass it. So this would indicate that on balance lobbying is having that effect.
#
# Second it could be because lobbyists don't bother with
# the easy bills that don't need their help. Meaning, they pick out a pool of
# improbable bllls, and presumably make those bills more likely to be enacted
# but still not as likely as the easy bills. (If they truly hurt a bill's future, they
# would presumably know and stop lobbying!)
#
# Looking at lobbying might be more useful if we combined it with another
# factor that could pick out the hard bills, and then this might show that for
# hard bills, lobbying made the bills more successful. But it's a bit impossible
# because surely the lobbyists know better than we do which bills are hard,
# so it would be impossible to factor out "hard bills" entirely.
if False:
if lobbying_data["counts"].get(
(bill.bill_type, bill.number), 0) > lobbying_data["median"]:
factors.append((
"crp-lobby-many",
"The Center for Responsive Politics reports that a large number of organizations are lobbying on this %s."
% bill.noun, "Has many lobbyists."))
elif lobbying_data["counts"].get((bill.bill_type, bill.number), 0) > 0:
factors.append((
"crp-lobby",
"The Center for Responsive Politics reports that organizations are lobbying on this %s."
% bill.noun, "Has lobbyists."))
return factors
