def gatherBillDetails(self, st):
pv_dt = (datetime.datetime.today() - datetime.timedelta(int(config.default.prev_days)))
logging.debug(str('Gathering Bills from %s forward: '%pv_dt.isoformat() + time.strftime("%B %d, %Y", time.gmtime())))
session = max([x for x in openstates.state_metadata(st)['session_details']])
bill_list = [x for x in openstates.bills(state=st, subjects='tax') if (x['updated_at'][:4]==str(datetime.date.today().year))]
bill_list2 = [x for x in openstates.bills(state=st, q='tax') if (x['updated_at'][:4]==str(datetime.date.today().year)) and (x not in bill_list)]
bill_list.extend(bill_list2)
bills = [openstates.bill_detail(st, bill['session'], bill['bill_id']) for bill in bill_list]
self.bill_lines = []
for bill in bills:
if datetime.datetime.strptime(bill['updated_at'], '%Y-%m-%d %H:%M:%S') > pv_dt:
insertrow = {
'ost_id' : bill['id'],
'div' : bill['bill_id'][:1],
'st' : bill['state'],
'session' : bill['session'],
'num' : bill['bill_id'].split()[1],
'details' : bill['title'],
'title': bill['title'],
'year': time.strftime('%Y'),
'webpage' : ", ".join([x['url'] for x in bill['sources']])[:149],
'subjects' : ", ".join(bill['subjects'])[:249],
'dateadd': datetime.datetime.now(),
'email': True,
'watched' : 'New'
}
self.bill_lines.append(insertrow)
else: continue
if len(self.bill_lines)<1: return False
else: return session
def produceBillJSONFiles(files_destination):
bill_fields = "bill_id,title,alternate_titles,action_dates,actions,chamber,\
updated_at,id,scraped_subjects,type,versions,votes"
# bills = openstates.bills(state="tx", search_window="session",
# fields=bill_fields)
bills = []
pageNo = 1
while True:
billsToAdd = openstates.bills(state="tx", search_window="session",
fields=bill_fields, page=pageNo)
if len(billsToAdd) == 0:
break
bills.extend(billsToAdd)
pageNo += 1
addToBills(bills)
if not os.path.exists(files_destination):
os.makedirs(files_destination)
for bill in bills:
for subject in bill['scraped_subjects']:
if 'Education--Higher' in subject:
filePath = os.path.join(files_destination, bill['id'] + '.json')
with io.open(filePath, mode='w', encoding='utf8') as f:
jsonDump = json.dumps(obj=bill,
ensure_ascii=False,
separators=(',',':'))
f.write(jsonDump)
break
def load_bills(self, session, date, specific_bill):
search_kwargs = {
'state': 'tx',
'search_window': ('session:{}'.format(session)
if session else 'session'),
'page': 1,
'per_page': 10000,
}
if date:
search_kwargs['updated_since'] = date
if specific_bill:
search_kwargs['bill_id'] = specific_bill
while True:
bills = openstates.bills(**search_kwargs)
for bill in bills:
bill_data = self.fetch_bill(bill['id'])
self.load_bill(bill_data)
search_kwargs['page'] += 1
if len(bills) < 10000:
break
def getBills():
import sunlight
import json
from sunlight import openstates
id_bill_json = openstates.bills(state="id", search_window="session")
id_bills = byteify(json.dumps(id_bills_json))
for bill in id_bills_json:
bill_adder = Bill(bill["bill_id"], bill["session"], bill["title"], bill["id"], bill["updated_at"])
db.session.add(bill_adder)
db.session.commit()
return id_bills
def produceEnhancedDistrictJSONString(geoJSONString, chamber_string):
leg_fields = "full_name,district,offices,party,roles,leg_id,photo_url"
legislators = openstates.legislators(state="tx",
active=True,
chamber=chamber_string,
fields=leg_fields)
legislator_id_set = {}
for legislator in legislators:
legislator_id_set[legislator['leg_id']] = legislator
legislator['higher_ed_bills'] = {'primary': [], 'cosponsor': []}
legislator['photo_url'] = 'images/' + legislator['photo_url'].split(
'images/')[1]
bill_fields = "id,sponsors,scraped_subjects"
bills = openstates.bills(state="tx",
search_window="session",
fields=bill_fields)
for bill in bills:
try:
for subject in bill['scraped_subjects']:
if 'Education--Higher' in subject:
addSponsorsToSet(legislator_id_set, bill)
break
except KeyError:
print 'ERROR KeyError ' + str(bill)
decoder = json.JSONDecoder()
geoJSON = decoder.decode(geoJSONString)
chamberName = ''
if (chamber_string == 'upper'):
chamberName = 'senate'
else:
chamberName = 'house'
districts = geoJSON['objects'][chamberName]['geometries']
legislator_district_set = {}
for legislator in legislators:
try:
legislator_district_set[int(legislator['district'])] = legislator
except KeyError: # someone without a district, lt. governor, etc.
print "'legislator' without district: %s" % str(legislator)
continue
for district in districts:
if int(district['id']) in legislator_district_set:
district['properties'] = {}
district['properties']['legislator'] = legislator_district_set[int(
district['id'])]
addToLegislators(legislators)
return json.dumps(obj=geoJSON, ensure_ascii=False, separators=(',', ':'))
def generateCopyTemplate(file_destination):
bill_fields = "id,bill_id,title,scraped_subjects"
bills = []
pageNo = 1
while True:
billsToAdd = openstates.bills(state="tx", search_window="session",
fields=bill_fields, page=pageNo)
if len(billsToAdd) == 0:
break
bills.extend(billsToAdd)
pageNo += 1
billsToCopySheet = []
for bill in bills:
if 'scraped_subjects' in bill:
for subject in bill['scraped_subjects']:
if 'Education--Higher' in subject:
billsToCopySheet.append(bill)
break
if (not os.path.exists(file_destination)) and (not file_destination == ''):
os.makedirs(file_destination)
workbook = xlsxwriter.Workbook(os.path.join(file_destination, 'copysheet.xlsx'))
bills_sheet = workbook.add_worksheet('Bills')
bills_sheet.write_row(0, 0, ['key', 'Bill Summary', 'Bill ID', 'title'])
for i in xrange(1, len(billsToCopySheet) + 1):
bill = billsToCopySheet[i - 1]
rowToAdd = [bill['id'], '', bill['bill_id'], bill['title']]
bills_sheet.write_row(i, 0, rowToAdd)
leg_fields = 'full_name,leg_id'
legislators = openstates.legislators(state="tx", active=True,
chamber='upper',
fields=leg_fields)
legislators.extend(openstates.legislators(state="tx", active=True,
chamber='lower',
fields=leg_fields))
leg_sheet = workbook.add_worksheet('Legislators')
leg_sheet.write_row(0, 0, ['key', 'Higher Education', 'Name'])
for i in xrange(1, len(legislators) + 1):
legislator = legislators[i - 1]
rowToAdd = [legislator['leg_id'], '', legislator['full_name']]
leg_sheet.write_row(i, 0, rowToAdd)
workbook.close()
def produceEnhancedDistrictJSONString(geoJSONString, chamber_string):
leg_fields = "full_name,district,offices,party,roles,leg_id,photo_url"
legislators = openstates.legislators(state="tx", active=True,
chamber=chamber_string,
fields=leg_fields)
legislator_id_set={}
for legislator in legislators:
legislator_id_set[legislator['leg_id']] = legislator
legislator['higher_ed_bills'] = {'primary':[], 'cosponsor':[]}
legislator['photo_url'] = 'images/' + legislator['photo_url'].split('images/')[1]
bill_fields = "id,sponsors,scraped_subjects"
bills = openstates.bills(state="tx", search_window="session",
fields=bill_fields)
for bill in bills:
try:
for subject in bill['scraped_subjects']:
if 'Education--Higher' in subject:
addSponsorsToSet(legislator_id_set, bill)
break
except KeyError:
print 'ERROR KeyError ' + str(bill)
decoder = json.JSONDecoder()
geoJSON = decoder.decode(geoJSONString)
chamberName = ''
if (chamber_string == 'upper'):
chamberName = 'senate'
else:
chamberName = 'house'
districts = geoJSON['objects'][chamberName]['geometries']
legislator_district_set = {}
for legislator in legislators:
try:
legislator_district_set[int(legislator['district'])] = legislator
except KeyError: # someone without a district, lt. governor, etc.
print "'legislator' without district: %s" % str(legislator)
continue
for district in districts:
if int(district['id']) in legislator_district_set:
district['properties'] = {}
district['properties']['legislator'] = legislator_district_set[int(district['id'])]
addToLegislators(legislators)
return json.dumps(obj=geoJSON, ensure_ascii=False, separators=(',',':'))
def bill_metadata(self):
spec = {
'state': self.abbr,
'chamber': self.chamber,
'per_page': self.PER_PAGE}
if self.term and not self.session:
if isinstance(self.term, basestring):
# Search the specified term.
spec['search_window'] = 'term:' + self.term
else:
# Search the current term.
spec['search_window'] = 'term'
elif self.session:
if isinstance(self.session, basestring):
# Search the specified session.
spec['search_window'] = 'session:' + self.session
else:
# Search the current session.
spec['search_window'] = 'session'
logging.info('Fetching bill metadata...')
page = 1
meta = []
while True:
spec.update(page=page)
logging.debug('Fetching metadata: %r' % spec)
more_meta = openstates.bills(**spec)
if not more_meta:
break
meta += more_meta
if self.limit and self.limit < (page * self.PER_PAGE):
break
page += 1
logging.info('...done.')
if self.limit:
meta = meta[:self.limit]
if len(meta) < 300:
# If the term or session contains too few bills for a
# meaningful calculation, complain/bail.
msg = 'Too few bills found (%d); aborting. %r'
data = dict(
abbr=self.abbr,
session=self.session,
term=self.term)
raise TooFewBillsError(msg % (len(meta), data,))
return meta
def main():
detail = openstates.bill_detail(state="mn", session="upper", bill_id="SF 14")
senate = openstates.bills(state="mn", chamber="upper", search_window="term")
for bill in senate[:1]:
print bill
print bill.get("state")
print bill.get("session")
print bill.get("bill_id")
detail = openstates.bill_detail(
state=bill.get("state"), session=bill.get("session"), bill_id=bill.get("bill_id")
)
print detail
def update_bills(self, state):
prev_bills = [b.bill_id for b in PaBills.objects.all()]
bills = openstates.bills(state=state, search_window="term")
count = 0
for bill in bills:
if bill['bill_id'] not in prev_bills:
count += 1
new_bill = PaBills(state=state, session=bill['session'],
title=bill['title'], type=bill['type'][0],
chamber=bill['chamber'],
created_at=bill['created_at'],
updated_at=bill['updated_at'],
bill_id=bill['bill_id'])
new_bill.save()
print "Added a total of %s bills to database" % count
def populateBills():
import sunlight
import json
from sunlight import openstates
id_bill_json = openstates.bills(
state="id"
# search_window = 'session',
# updated_since = '2016-03-08'
)
print id_bill_json
id_bills = byteify(json.dumps(id_bill_json))
# db.session.query(Bill).update({Bill.bill_id:id_bills["bill_id"], Bill.session: id_bills["session"], Bill.title: id_bills["title"], Bill.id: id_bills["id"], Bill.updated_at: id_bills["updated_at"]})
for bill in id_bill_json:
bill_adder = Bill(bill["bill_id"], bill["session"], bill["title"], bill["id"], bill["updated_at"])
db.session.add(bill_adder)
db.session.commit()
return id_bills
def search(request):
if request.method == 'POST':
state = request.POST.get('state', None)
keyword = request.POST.get('keyword', None)
# call sunlightfoundation api for state legislation
bills = openstates.bills(state=state, q=keyword)
# call api for federal legislation data
fed_bills_response = requests.get('https://www.govtrack.us/api/v2/bill?q=' + keyword)
if fed_bills_response.status_code == 200:
fed_bills = fed_bills_response.json()
else:
fed_bills = None
context = {'bills': bills, 'fed_bills': fed_bills, 'state': state, 'keyword': keyword}
return render(request, 'billtrack/bills.html', context)
else:
return render(request, 'billtrack/form.html')
def update_bills(self, state):
prev_bills = [b.bill_id for b in PaBills.objects.all()]
bills = openstates.bills(state=state, search_window="term")
count = 0
for bill in bills:
if bill['bill_id'] not in prev_bills:
count += 1
new_bill = PaBills(state=state,
session=bill['session'],
title=bill['title'],
type=bill['type'][0],
chamber=bill['chamber'],
created_at=bill['created_at'],
updated_at=bill['updated_at'],
bill_id=bill['bill_id'])
new_bill.save()
print "Added a total of %s bills to database" % count
def data(who):
sponsored_bills = openstates.bills(sponsor_id=who, fields='subjects')
count = Counter([el for sl in map(lambda x: x.get('subjects', []),
sponsored_bills) for el in sl])
[count.pop(x) for x in blacklist if x in count]
ret = do_math(count)
additive = min(map(lambda x: x[0], ret))
if additive >= 0:
additive = 0
else:
additive = (-additive) + 1
output = []
for el in ret:
output.append([
el[0] + additive,
el[1]
])
return json.dumps(output)
def produceBillJSONFiles(files_destination):
bill_fields = "bill_id,title,alternate_titles,action_dates,actions,chamber,\
updated_at,id,scraped_subjects,type,versions,votes"
# bills = openstates.bills(state="tx", search_window="session",
# fields=bill_fields)
bills = []
pageNo = 1
while True:
billsToAdd = openstates.bills(state="tx", search_window="session",
fields=bill_fields, page=pageNo)
if len(billsToAdd) == 0:
break
bills.extend(billsToAdd)
pageNo += 1
addToBills(bills)
if not os.path.exists(files_destination):
os.makedirs(files_destination)
for bill in bills:
try:
for subject in bill['scraped_subjects']:
if 'Education--Higher' in subject:
filePath = os.path.join(files_destination, bill['id'] + '.json')
with io.open(filePath, mode='w', encoding='utf8') as f:
jsonDump = json.dumps(obj=bill,
ensure_ascii=False,
separators=(',',':'))
f.write(jsonDump)
break
except KeyError:
print 'ERROR KeyError ' + str(bill)
not_in_legis=0.0,
legis_names=tuple(self.leg_ids))
return rollcall
if __name__ == '__main__':
from sunlight import openstates, response_cache
response_cache.enable('mongo')
response_cache.logger.setLevel(10)
# Wrangle the API data into a Rollcall object.
spec = dict(state='al', chamber='lower', search_window='term:2011-2014')
valid_ids = [leg['id'] for leg in openstates.legislators(**spec)]
builder = RollcallBuilder(valid_ids)
bills = openstates.bills(**spec)
for bill in bills:
bill = openstates.bill(bill['id'])
for vote in bill['votes']:
if vote['chamber'] != bill['chamber']:
continue
builder.add_vote(vote)
rollcall = builder.get_rollcall()
wnominate = rollcall.wnominate(polarity=('ALL000086', 'ALL000085'))
wnom_values = wnominate.legislators.coord1D
ideal = rollcall.ideal()
ideal_values = ideal.xbar
import pdb; pdb.set_trace()
not_in_legis=0.0,
legis_names=tuple(self.leg_ids))
return rollcall
if __name__ == '__main__':
from sunlight import openstates, response_cache
response_cache.enable('mongo')
response_cache.logger.setLevel(10)
# Wrangle the API data into a Rollcall object.
spec = dict(state='al', chamber='lower', search_window='term:2011-2014')
valid_ids = [leg['id'] for leg in openstates.legislators(**spec)]
builder = RollcallBuilder(valid_ids)
bills = openstates.bills(**spec)
for bill in bills:
bill = openstates.bill(bill['id'])
for vote in bill['votes']:
if vote['chamber'] != bill['chamber']:
continue
builder.add_vote(vote)
rollcall = builder.get_rollcall()
wnominate = rollcall.wnominate(polarity=('ALL000086', 'ALL000085'))
wnom_values = wnominate.legislators.coord1D
ideal = rollcall.ideal()
ideal_values = ideal.xbar
import pdb
import commands
"""
Program to get the bill sponsor information and write it to db.
"""
author_db = pd.DataFrame(columns=[
'bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id'
])
billdetail_db = pd.DataFrame(columns=[
'bill_name', 'sunlight_id', 'title', 'impact_clause', 'scraped_subjects',
'official_subjects', 'summary_sunlight', 'summary_text',
'full_cleaned_text'
])
ca_bills = openstates.bills(state='ca', search_window='term')
authorindex = 0
billindex = 0
for i in range(len(ca_bills)):
thisbilldetail = openstates.bill_detail(state='ca',
session=ca_bills[i]['session'],
bill_id=ca_bills[i]['bill_id'])
keys = thisbilldetail.keys()
if (thisbilldetail['created_at'] <
'2015-06-01') and ('HR' not in thisbilldetail['bill_id']):
billname = ca_bills[i]['bill_id']
billsid = ca_bills[i]['id']
billtitle = ca_bills[i]['title']
billclause = thisbilldetail['+impact_clause']
def get_updated_bill_list(self):
return openstates.bills(
state='tx',
updated_since=self.latest_date,
search_window='session')
import urllib
import re
import pandas as pd
from bs4 import BeautifulSoup
import commands
"""
Program to get the bill sponsor information and write it to db.
"""
author_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id'])
billdetail_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'title', 'impact_clause',
'scraped_subjects', 'official_subjects', 'summary_sunlight', 'summary_text', 'full_cleaned_text'])
ca_bills = openstates.bills(state='ca', search_window='term')
authorindex = 0;
billindex = 0;
for i in range(len(ca_bills)):
thisbilldetail = openstates.bill_detail(state='ca', session=ca_bills[i]['session'], bill_id = ca_bills[i]['bill_id']);
keys = thisbilldetail.keys()
if (thisbilldetail['created_at'] < '2015-06-01') and ('HR' not in thisbilldetail['bill_id']):
billname = ca_bills[i]['bill_id'];
billsid = ca_bills[i]['id'];
billtitle = ca_bills[i]['title'];
billclause = thisbilldetail['+impact_clause']
if 'scraped_subjects' in keys:
billscraped = thisbilldetail['scraped_subjects']
else:
'Military', 'Municipal and County Issues', 'Nominations', 'Other',
'Public Services', 'Recreation', 'Reproductive Issues', 'Resolutions',
'Science and Medical Research', 'Senior Issues',
'Sexual Orientation and Gender Issues', 'Social Issues', 'State Agencies',
'Technology and Communication', 'Trade', 'Transportation',
'Welfare and Poverty'
]
states = [
"AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", "HI",
"ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN",
"MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH",
"OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA",
"WV", "WI", "WY", "PR"
]
# Grabbinga all bills from every topic and state
# and saves them to their own file.
# File name is StateAbbr-Topic.json
for topic in topics:
for state_abbr in states:
bills = openstates.bills(search_window='session',
subject=topic,
state=state_abbr)
bills_json = json.dumps(bills, sort_keys=True, indent=4)
f = open(
'../json/' + state_abbr + '-' +
topic.replace(',', '').replace(' ', '') + '.json', 'w')
f.write(bills_json)
f.close()
'Federal, State, and Local Issues', 'Gambling and Gaming', 'Government Reform',
'Guns', 'Health', 'Housing and Property', 'Immigration', 'Indigenous Peoples',
'Insurance', 'Judiciary', 'Labor and Employment', 'Legal Issues',
'Legislative Affairs', 'Military', 'Municipal and County Issues',
'Nominations', 'Other', 'Public Services', 'Recreation', 'Reproductive Issues',
'Resolutions', 'Science and Medical Research', 'Senior Issues',
'Sexual Orientation and Gender Issues', 'Social Issues', 'State Agencies',
'Technology and Communication', 'Trade', 'Transportation', 'Welfare and Poverty']
states = ["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA",
"HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
"MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
"NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
"SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY", "PR"]
# Grabbinga all bills from every topic and state
# and saves them to their own file.
# File name is StateAbbr-Topic.json
for topic in topics:
for state_abbr in states:
bills = openstates.bills(
search_window='session',
subject=topic,
state=state_abbr)
bills_json = json.dumps(bills, sort_keys=True, indent=4)
f = open('../json/' + state_abbr + '-' + topic.replace(',', '').replace(' ', '') + '.json', 'w')
f.write(bills_json)
f.close()
It also sets up pandas dataframes for each session, which can then be written
out to a sql database.
"""
# set up the sessions and databases.
sessions = ['20092010', '20112012', '20132014', '20152016']
author_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id', 'session'])
billdetail_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'title', 'impact_clause',
'scraped_subjects', 'official_subjects', 'summary_sunlight', 'summary_text', 'full_cleaned_text', 'session'])
authorindex = 0;
billindex = 0;
for session in sessions:
thissearch = 'session:'+session;
ca_bills = openstates.bills(state='ca', search_window=thissearch)
for index, ca_bill in enumerate(ca_bills):
print str(index) + 'out of ' + str(len(ca_bills))
thisbilldetail = openstates.bill_detail(state='ca', session=ca_bill['session'], bill_id = ca_bill['bill_id']);
keys = thisbilldetail.keys()
if 'HR' not in thisbilldetail['bill_id']:
billname = ca_bill['bill_id'];
billsid = ca_bill['id'];
billtitle = ca_bill['title'];
if '+impact_clause' in keys:
billclause = thisbilldetail['+impact_clause']
else:
billclause = None;
# import packages we need: openstates, csv, and regex
from sunlight import openstates
import csv
import re
# api call using openstates package.
# see example output: http://sunlightlabs.github.io/openstates-api/bills.html#examples/bill-search
oklahoma_bills = openstates.bills(
state='ok',
search_window='term:2015-2016'
)
# oklahoma legislators
ok_legislators = openstates.legislators(
state='ok',
active='true'
)
# we need an array of the legislators ids
# with the first value being bill_id for our
# csv header row
ok_legislators_array = ['bill_id', 'chamber', 'vote_id']
for legislator in ok_legislators:
ok_legislators_array.append(legislator['leg_id'])
# create or open votes.csv file, with write
with open('votes.csv', 'w') as f:
# create writer object on the file we named f
# extrasaction parameter means that if there is a missing or extra leg_id in our array
# then the writer will continue regardless
writer = csv.DictWriter(f, fieldnames=ok_legislators_array, extrasaction='ignore')
sessions = ['20092010', '20112012', '20132014', '20152016']
author_db = pd.DataFrame(columns=[
'bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id',
'session'
])
billdetail_db = pd.DataFrame(columns=[
'bill_name', 'sunlight_id', 'title', 'impact_clause', 'scraped_subjects',
'official_subjects', 'summary_sunlight', 'summary_text',
'full_cleaned_text', 'session'
])
authorindex = 0
billindex = 0
for session in sessions:
thissearch = 'session:' + session
ca_bills = openstates.bills(state='ca', search_window=thissearch)
for index, ca_bill in enumerate(ca_bills):
print str(index) + 'out of ' + str(len(ca_bills))
thisbilldetail = openstates.bill_detail(state='ca',
session=ca_bill['session'],
bill_id=ca_bill['bill_id'])
keys = thisbilldetail.keys()
if 'HR' not in thisbilldetail['bill_id']:
billname = ca_bill['bill_id']
billsid = ca_bill['id']
billtitle = ca_bill['title']
if '+impact_clause' in keys:
billclause = thisbilldetail['+impact_clause']
