def get_transparency_stats(person, role, stats, congress, startdate, enddate):
global transparency_bills
if not transparency_bills:
transparency_bills = []
for line in open("analysis/transparency-bills.txt"):
bill = Bill.from_congressproject_id(re.split("\s", line)[0])
if bill.congress != congress: continue
transparency_bills.append(bill)
# which bills are in the right chamber?
plausible_bills = []
for bill in transparency_bills:
if BillType.by_value(bill.bill_type).chamber == RoleType.by_value(role.role_type).congress_chamber:
plausible_bills.append(bill)
# did person sponsor any of these within this session?
sponsored = []
for bill in transparency_bills:
if startdate <= bill.introduced_date <= enddate and bill.sponsor == person:
sponsored.append(bill)
# did person cosponsor any of these within this session?
cosponsored = []
for cosp in Cosponsor.objects.filter(person=person, bill__in=transparency_bills, joined__gte=startdate, joined__lte=enddate):
cosponsored.append(cosp.bill)
stats["transparency-bills"] = {
"value": len(sponsored)*3 + len(cosponsored),
"sponsored": make_bill_entries(sponsored),
"cosponsored": make_bill_entries(cosponsored),
"num_bills": len(plausible_bills),
"chamber": RoleType.by_value(role.role_type).congress_chamber,
}
def fixup_item(item):
item = dict(item)
if item["my_ratio"] * item["other_ratio"] > .9:
item["identical"] = True
item["other"] = Bill.from_congressproject_id(item["other"])
item["other_ratio"] *= 100
return item
def fixup_item(item):
item = dict(item)
if item["my_ratio"] * item["other_ratio"] > .9:
item["identical"] = True
item["other"] = Bill.from_congressproject_id(item["other"])
item["other_ratio"] *= 100
return item
def dump_reactions(request):
from django.db.models import Count
from website.models import Reaction
from collections import defaultdict, OrderedDict
from website.models import Bill
# Get subjects with the most users reacting.
reactions = Reaction.objects.values_list("subject").annotate(
count=Count('subject')).order_by('-count')[0:100]
# Build ouptut.
def emojis(subject):
counts = defaultdict(lambda: 0)
for r in Reaction.objects.filter(subject=subject):
for e in (r.reaction or {}).get("emojis", []):
counts[e] += 1
return OrderedDict(sorted(counts.items(), key=lambda kv: -kv[1]))
ret = [
OrderedDict([
("subject", r[0]),
("title", Bill.from_congressproject_id(r[0][5:]).title),
("unique_users", r[1]),
("emojis", emojis(r[0])),
]) for r in reactions
]
return HttpResponse(json.dumps(ret, indent=2),
content_type="application/json")
def dump_reactions(request):
from django.db.models import Count
from website.models import Reaction
from collections import defaultdict, OrderedDict
from website.models import Bill
# Get subjects with the most users reacting.
reactions = Reaction.objects.values_list("subject").annotate(count=Count('subject')).order_by('-count')[0:100]
# Build ouptut.
def emojis(subject):
counts = defaultdict(lambda : 0)
for r in Reaction.objects.filter(subject=subject):
for e in (r.reaction or {}).get("emojis", []):
counts[e] += 1
return OrderedDict(sorted(counts.items(), key = lambda kv : -kv[1]))
ret = [
OrderedDict([
("subject", r[0]),
("title", Bill.from_congressproject_id(r[0][5:]).title),
("unique_users", r[1]),
("emojis", emojis(r[0])),
])
for r in reactions
]
return HttpResponse(json.dumps(ret, indent=2), content_type="application/json")
if b2_versioncode != latest_version_code[b2_id]: continue
# Does this record represent enough text similarity that it is worth
# loading into the database? We'll treat this record as indicating
# similarity if...
# For exceedingly formulaic bills, we'll only identify identical bills.
# Bills naming buildings etc. are formulaic and produce text similarity
# to other bills of the same type, because the part that differs is very
# small. So we use a very high threshold for comparison. This may not
# be needed. I added it after seeing the analysis produce false positives,
# but then I discovered that cmp_text_len was not being compared right in
# the next block so this may actually not be needed to help.
b1_ratio = round(float(b1_ratio), 3)
b2_ratio = round(float(b2_ratio), 3)
b1 = Bill.from_congressproject_id(b1_id)
if b1.title_no_number.startswith("A bill to designate ") \
or b1.title_no_number.startswith("To designate ") \
or b1.title_no_number.startswith("To name ") \
or "Commemorative Coin Act" in b1.title_no_number:
if b1_ratio * b2_ratio < .85:
continue
# For other bills...
# a) The bills are nearly identical, i.e. the ratios indicating how
# must text of each bill is in the other are both high, and
# there is some minimum amount of text in the bills so that we're
# sure there is substantative text at all.
# b) The bills are substantially similar to each other and the text
# in common is large enough to exclude cases where all of the
# substance in the bills are in the dis-similar parts.
if b2_versioncode != latest_version_code[b2_id]: continue
# Does this record represent enough text similarity that it is worth
# loading into the database? We'll treat this record as indicating
# similarity if...
# For exceedingly formulaic bills, we'll only identify identical bills.
# Bills naming buildings etc. are formulaic and produce text similarity
# to other bills of the same type, because the part that differs is very
# small. So we use a very high threshold for comparison. This may not
# be needed. I added it after seeing the analysis produce false positives,
# but then I discovered that cmp_text_len was not being compared right in
# the next block so this may actually not be needed to help.
b1_ratio = round(float(b1_ratio),3)
b2_ratio = round(float(b2_ratio),3)
b1 = Bill.from_congressproject_id(b1_id)
if b1.title_no_number.startswith("A bill to designate ") \
or b1.title_no_number.startswith("To designate ") \
or b1.title_no_number.startswith("To name ") \
or b1.title_no_number.startswith("A bill for the relief of ") \
or "Commemorative Coin Act" in b1.title_no_number:
if b1_ratio*b2_ratio < .85:
continue
# For other bills...
if is_text_incorporated(b1_ratio, b2_ratio, cmp_text_len):
# Index this information with both bills.
# For b2, we're saying that it (or parts of it) were enacted
# through these other bills...
text_incorporation[b2_id][b1_id] = {
def lookup_reps(request):
from django.contrib.humanize.templatetags.humanize import ordinal
from person.name import get_person_name
# Get the state and district from the query string.
try:
state = request.GET['state']
district = int(request.GET['district'])
if state not in stateapportionment: raise Exception()
except:
return {}
# Get the bill (optional) from the query string
from bill.models import Bill
try:
bill = Bill.from_congressproject_id(request.GET["bill"])
except:
bill = None
# Helper to get relevant committee assignments.
from committee.models import CommitteeMember, CommitteeMemberRole
from committee.util import sort_members
def mention_committees_once(committeeassignments):
# The committee assignments have been sorted first by role (i.e.
# committees that the person is the chair of come first) and then
# by committee name (which also puts subcommittees after committees).
# In order to be less verbose, only mention each full committee
# once --- take the first in each mention.
seen = set()
for c in committeeassignments:
if (c.committee in seen) or (c.committee.committee in seen):
continue
yield c
if c.committee.committee is not None:
seen.add(c.committee.committee) # add main committee
else:
seen.add(c.committee) # add this committee
bounds = get_district_bounds(state, district)
return {
"state": {
"name": statenames[state],
"isTerritory": stateapportionment[state] == "T",
},
"district": {
"ordinal": ordinal(district) if district > 0 else "At Large",
"bounds": {
"center": {
"latitude": bounds[0],
"longitude": bounds[1]
},
"zoom": bounds[2]
}
},
"members": [{
"id":
p.id,
"name":
get_person_name(p,
role_recent=True,
firstname_position="before",
show_title=True,
show_party=False,
show_district=False),
"name_formal":
p.current_role.get_title() + " " + p.lastname,
"url":
p.get_absolute_url(),
"type":
p.current_role.get_role_type_display(),
"description":
p.current_role.get_description(),
"party":
p.current_role.party,
"photo_url":
p.get_photo_url_50() if p.has_photo() else None,
"contact_url": (p.current_role.extra or {}).get("contact_form")
or p.current_role.website,
"phone":
p.current_role.phone,
"website":
p.current_role.website,
"pronouns": {
"him_her": p.him_her,
"his_her": p.his_her,
"he_she": p.he_she,
},
"bill-status": {
"cosponsor":
p in bill.cosponsors.all(),
"committee-assignments": [{
"committee":
c.committee.fullname,
"role":
c.get_role_display() if c.role in (
CommitteeMemberRole.chair,
CommitteeMemberRole.ranking_member,
CommitteeMemberRole.vice_chair) else None,
} for c in mention_committees_once(
sort_members(
CommitteeMember.objects.filter(
person=p, committee__in=bill.committees.all())))]
} if bill else None,
} for p in list(
Person.objects.filter(roles__current=True,
roles__state=state,
roles__role_type=RoleType.senator).order_by(
'roles__senator_rank')) +
list(
Person.objects.filter(
roles__current=True,
roles__state=state,
roles__district=district,
roles__role_type=RoleType.representative))]
}
# Skip if this record is for an outdated version of either bill.
if b1_versioncode != latest_version_code[b1_id]: continue
if b2_versioncode != latest_version_code[b2_id]: continue
# Does this record represent enough text similarity that it is worth
# loading into the database? We'll treat this record as indicating
# similarity if...
# For exceedingly formulaic bills, we'll only identify identical bills.
# Bills naming buildings etc. are formulaic and produce text similarity
# to other bills of the same type, because the part that differs is very
# small. So we use a very high threshold for comparison.
b1_ratio = round(float(b1_ratio),3)
b2_ratio = round(float(b2_ratio),3)
b1 = Bill.from_congressproject_id(b1_id)
if b1.title_no_number.startswith("A bill to designate ") \
or b1.title_no_number.startswith("To designate ") \
or b1.title_no_number.startswith("To name ") \
or "Commemorative Coin Act" in b1.title_no_number:
if b1_ratio*b2_ratio < .85:
continue
# For other bills...
# a) The bills are nearly identical, i.e. the ratios indicating how
# must text of each bill is in the other are both high, and
# there is some minimum amount of text in the bills so that we're
# sure there is substantative text at all.
# b) The bills are substantially similar to each other and the text
# in common is large enough to exclude cases where all of the
# substance in the bills are in the dis-similar parts.
def new_stakeholder(request):
from bill.models import Bill
related_bill = None
if request.GET.get('bill'):
related_bill = Bill.from_congressproject_id(request.GET['bill'])
class NewStakehoderForm(Form):
organization_name = CharField()
organization_website = URLField(initial="http://")
twitter_account = CharField(initial="@", required=False)
if related_bill:
position = ChoiceField(choices=[(None, '(choose)'), (1, "Support"),
(0, "Neutral"), (-1, "Oppose")],
required=False,
label="Your organization's position on " +
related_bill.display_number)
position_statement_link = URLField(
required=False,
label=
"Link to webpage or PDF containing a position statement about "
+ related_bill.display_number + " (optional)")
position_statement_content = CharField(
required=False,
widget=Textarea,
label="Paste the text of your position statement about " +
related_bill.display_number + " (optional)")
if request.method == "GET":
form = NewStakehoderForm()
else: # POST
form = NewStakehoderForm(request.POST)
if form.is_valid():
# Create a new un-verified Stakeholder object.
stk = Stakeholder()
stk.name = form.cleaned_data['organization_name']
stk.website = form.cleaned_data['organization_website'] or None
stk.twitter_handle = form.cleaned_data['twitter_account'].lstrip(
"@") or None
stk.save()
# Create a new post.
if related_bill and (
form.cleaned_data['position'] != ''
or form.cleaned_data['position_statement_link']
or form.cleaned_data['position_statement_content']):
post = Post()
post.stakeholder = stk
if form.cleaned_data[
'position_statement_link'] or form.cleaned_data[
'position_statement_content']:
post.post_type = 1 # summary
post.link = (form.cleaned_data['position_statement_link']
or None)
post.content = (
form.cleaned_data['position_statement_content']
or None)
else:
post.post_type = 0 # positions only
post.save()
bp = BillPosition()
bp.post = post
bp.bill = related_bill
if form.cleaned_data['position'] != '':
bp.position = int(form.cleaned_data['position'])
bp.save()
# Make this user an admin.
stk.admins.add(request.user)
# Go view it.
return HttpResponseRedirect(stk.get_absolute_url())
return render(request, "stakeholder/new.html", {
"form": form,
"related_bill": related_bill,
})
def lookup_reps(request):
from django.contrib.humanize.templatetags.humanize import ordinal
from person.name import get_person_name
# Get the state and district from the query string.
try:
state = request.GET['state']
district = int(request.GET['district'])
if state not in stateapportionment: raise Exception()
except:
return {
}
# Get the bill (optional) from the query string
from bill.models import Bill
try:
bill = Bill.from_congressproject_id(request.GET["bill"])
except:
bill = None
# Helper to get relevant committee assignments.
from committee.models import CommitteeMember, CommitteeMemberRole
from committee.util import sort_members
def mention_committees_once(committeeassignments):
# The committee assignments have been sorted first by role (i.e.
# committees that the person is the chair of come first) and then
# by committee name (which also puts subcommittees after committees).
# In order to be less verbose, only mention each full committee
# once --- take the first in each mention.
seen = set()
for c in committeeassignments:
if (c.committee in seen) or (c.committee.committee in seen):
continue
yield c
if c.committee.committee is not None:
seen.add(c.committee.committee) # add main committee
else:
seen.add(c.committee) # add this committee
bounds = get_district_bounds(state, district)
return {
"state": {
"name": statenames[state],
"isTerritory": stateapportionment[state] == "T",
},
"district": {
"ordinal": ordinal(district) if district > 0 else "At Large",
"bounds": {
"center": { "latitude": bounds[0], "longitude": bounds[1] },
"zoom": bounds[2]
}
},
"members": [
{
"id": p.id,
"name": get_person_name(p, role_recent=True, firstname_position="before", show_title=True, show_party=False, show_district=False),
"name_formal": p.current_role.get_title() + " " + p.lastname,
"url": p.get_absolute_url(),
"type": p.current_role.get_role_type_display(),
"description": p.current_role.get_description(),
"party": p.current_role.party,
"photo_url": p.get_photo_url_50() if p.has_photo() else None,
"contact_url": (p.current_role.extra or {}).get("contact_form") or p.current_role.website,
"phone": p.current_role.phone,
"website": p.current_role.website,
"pronouns": {
"him_her": p.him_her,
"his_her": p.his_her,
"he_she": p.he_she,
},
"bill-status": {
"cosponsor": p in bill.cosponsors.all(),
"committee-assignments": [
{
"committee": c.committee.fullname,
"role": c.get_role_display() if c.role in (CommitteeMemberRole.chair, CommitteeMemberRole.ranking_member, CommitteeMemberRole.vice_chair) else None,
}
for c in
mention_committees_once(
sort_members(
CommitteeMember.objects.filter(person=p, committee__in=bill.committees.all())))
]
} if bill else None,
}
for p in
list(Person.objects.filter(roles__current=True, roles__state=state, roles__role_type=RoleType.senator)
.order_by('roles__senator_rank'))
+
list(Person.objects.filter(roles__current=True, roles__state=state, roles__district=district, roles__role_type=RoleType.representative))
]
}
def new_stakeholder_post(request):
from bill.models import Bill
related_bill = None
if request.GET.get('bill'):
related_bill = Bill.from_congressproject_id(request.GET['bill'])
data = {}
user_admin_of_stakeholders = request.user.stakeholder_set.all()
class NewStakehoderForm(Form):
if not user_admin_of_stakeholders:
organization_name = CharField()
organization_website = URLField(initial="http://")
twitter_account = CharField(initial="@", required=False)
else:
organization = ChoiceField(choices=[
(s.id, s.name) for s in user_admin_of_stakeholders
],
label="Your organization")
if related_bill:
position = ChoiceField(choices=[(None, '(choose)'), (1, "Support"),
(0, "Neutral"), (-1, "Oppose")],
required=True,
label="Your organization's position on " +
related_bill.display_number)
position_statement_link = URLField(
required=False,
label=
"Link to webpage or PDF containing a position statement about "
+ related_bill.display_number)
position_statement_content = CharField(
required=True,
widget=Textarea,
label="Paste the text of your position statement about " +
related_bill.display_number)
#If post already exists, update it instead of making a new one. Assumes only one of user's accounts has a statement on a given bill. Should be integrated into code below for clarity. Code added by Ben, a bad coder.
if user_admin_of_stakeholders:
for s in user_admin_of_stakeholders:
for p in Post.objects.filter(stakeholder=s):
for bp in p.bill_positions.all():
if bp.bill == related_bill:
data = {
'organization': (s.id, s.name),
'position': bp.position,
'position_statement_link': p.link,
'position_statement_content': p.content
}
if request.method == 'POST':
form = NewStakehoderForm(request.POST,
initial=data)
if form.is_valid():
if form.cleaned_data['position'] != '':
bp.position = int(
form.cleaned_data['position'])
p.link = form.cleaned_data[
'position_statement_link']
p.content = form.cleaned_data[
'position_statement_content']
bp.save()
p.save()
return HttpResponseRedirect(
s.get_absolute_url())
if request.method == "GET":
form = NewStakehoderForm(initial=data)
else: # POST
form = NewStakehoderForm(request.POST)
if form.is_valid():
if not user_admin_of_stakeholders:
# Create a new un-verified Stakeholder object.
stk = Stakeholder()
stk.name = form.cleaned_data['organization_name']
stk.website = form.cleaned_data['organization_website'] or None
stk.twitter_handle = form.cleaned_data[
'twitter_account'].lstrip("@") or None
stk.save()
# Make this user an admin.
stk.admins.add(request.user)
else:
# Get an existing Stakeholder that they are the admin of.
stk = get_object_or_404(Stakeholder,
id=form.cleaned_data['organization'])
if request.user not in stk.admins.all():
# Invalid. Get out of here.
return HttpResponseRedirect(stk.get_absolute_url())
# Create a new post if this page is for a related bill and a position,
# link, or statement are provided.
if related_bill and (
form.cleaned_data['position'] != ''
or form.cleaned_data['position_statement_link']
or form.cleaned_data['position_statement_content']):
# Create a new Post object.
post = Post()
post.stakeholder = stk
if form.cleaned_data[
'position_statement_link'] or form.cleaned_data[
'position_statement_content']:
post.post_type = 1 # summary
post.link = (form.cleaned_data['position_statement_link']
or None)
post.content = (
form.cleaned_data['position_statement_content']
or None)
else:
post.post_type = 0 # positions only
post.save()
# Attach a BillPosition to the Post.
bp = BillPosition()
bp.post = post
bp.bill = related_bill
if form.cleaned_data['position'] != '':
bp.position = int(form.cleaned_data['position'])
bp.save()
# Go view it.
return HttpResponseRedirect(stk.get_absolute_url())
return render(request, "stakeholder/new.html", {
"form": form,
"related_bill": related_bill,
})
