def gatherBillDetails(self, st): pv_dt = (datetime.datetime.today() - datetime.timedelta(int(config.default.prev_days))) logging.debug(str('Gathering Bills from %s forward: '%pv_dt.isoformat() + time.strftime("%B %d, %Y", time.gmtime()))) session = max([x for x in openstates.state_metadata(st)['session_details']]) bill_list = [x for x in openstates.bills(state=st, subjects='tax') if (x['updated_at'][:4]==str(datetime.date.today().year))] bill_list2 = [x for x in openstates.bills(state=st, q='tax') if (x['updated_at'][:4]==str(datetime.date.today().year)) and (x not in bill_list)] bill_list.extend(bill_list2) bills = [openstates.bill_detail(st, bill['session'], bill['bill_id']) for bill in bill_list] self.bill_lines = [] for bill in bills: if datetime.datetime.strptime(bill['updated_at'], '%Y-%m-%d %H:%M:%S') > pv_dt: insertrow = { 'ost_id' : bill['id'], 'div' : bill['bill_id'][:1], 'st' : bill['state'], 'session' : bill['session'], 'num' : bill['bill_id'].split()[1], 'details' : bill['title'], 'title': bill['title'], 'year': time.strftime('%Y'), 'webpage' : ", ".join([x['url'] for x in bill['sources']])[:149], 'subjects' : ", ".join(bill['subjects'])[:249], 'dateadd': datetime.datetime.now(), 'email': True, 'watched' : 'New' } self.bill_lines.append(insertrow) else: continue if len(self.bill_lines)<1: return False else: return session
def produceBillJSONFiles(files_destination): bill_fields = "bill_id,title,alternate_titles,action_dates,actions,chamber,\ updated_at,id,scraped_subjects,type,versions,votes" # bills = openstates.bills(state="tx", search_window="session", # fields=bill_fields) bills = [] pageNo = 1 while True: billsToAdd = openstates.bills(state="tx", search_window="session", fields=bill_fields, page=pageNo) if len(billsToAdd) == 0: break bills.extend(billsToAdd) pageNo += 1 addToBills(bills) if not os.path.exists(files_destination): os.makedirs(files_destination) for bill in bills: for subject in bill['scraped_subjects']: if 'Education--Higher' in subject: filePath = os.path.join(files_destination, bill['id'] + '.json') with io.open(filePath, mode='w', encoding='utf8') as f: jsonDump = json.dumps(obj=bill, ensure_ascii=False, separators=(',',':')) f.write(jsonDump) break
def load_bills(self, session, date, specific_bill): search_kwargs = { 'state': 'tx', 'search_window': ('session:{}'.format(session) if session else 'session'), 'page': 1, 'per_page': 10000, } if date: search_kwargs['updated_since'] = date if specific_bill: search_kwargs['bill_id'] = specific_bill while True: bills = openstates.bills(**search_kwargs) for bill in bills: bill_data = self.fetch_bill(bill['id']) self.load_bill(bill_data) search_kwargs['page'] += 1 if len(bills) < 10000: break
def getBills(): import sunlight import json from sunlight import openstates id_bill_json = openstates.bills(state="id", search_window="session") id_bills = byteify(json.dumps(id_bills_json)) for bill in id_bills_json: bill_adder = Bill(bill["bill_id"], bill["session"], bill["title"], bill["id"], bill["updated_at"]) db.session.add(bill_adder) db.session.commit() return id_bills
def produceEnhancedDistrictJSONString(geoJSONString, chamber_string): leg_fields = "full_name,district,offices,party,roles,leg_id,photo_url" legislators = openstates.legislators(state="tx", active=True, chamber=chamber_string, fields=leg_fields) legislator_id_set = {} for legislator in legislators: legislator_id_set[legislator['leg_id']] = legislator legislator['higher_ed_bills'] = {'primary': [], 'cosponsor': []} legislator['photo_url'] = 'images/' + legislator['photo_url'].split( 'images/')[1] bill_fields = "id,sponsors,scraped_subjects" bills = openstates.bills(state="tx", search_window="session", fields=bill_fields) for bill in bills: try: for subject in bill['scraped_subjects']: if 'Education--Higher' in subject: addSponsorsToSet(legislator_id_set, bill) break except KeyError: print 'ERROR KeyError ' + str(bill) decoder = json.JSONDecoder() geoJSON = decoder.decode(geoJSONString) chamberName = '' if (chamber_string == 'upper'): chamberName = 'senate' else: chamberName = 'house' districts = geoJSON['objects'][chamberName]['geometries'] legislator_district_set = {} for legislator in legislators: try: legislator_district_set[int(legislator['district'])] = legislator except KeyError: # someone without a district, lt. governor, etc. print "'legislator' without district: %s" % str(legislator) continue for district in districts: if int(district['id']) in legislator_district_set: district['properties'] = {} district['properties']['legislator'] = legislator_district_set[int( district['id'])] addToLegislators(legislators) return json.dumps(obj=geoJSON, ensure_ascii=False, separators=(',', ':'))
def generateCopyTemplate(file_destination): bill_fields = "id,bill_id,title,scraped_subjects" bills = [] pageNo = 1 while True: billsToAdd = openstates.bills(state="tx", search_window="session", fields=bill_fields, page=pageNo) if len(billsToAdd) == 0: break bills.extend(billsToAdd) pageNo += 1 billsToCopySheet = [] for bill in bills: if 'scraped_subjects' in bill: for subject in bill['scraped_subjects']: if 'Education--Higher' in subject: billsToCopySheet.append(bill) break if (not os.path.exists(file_destination)) and (not file_destination == ''): os.makedirs(file_destination) workbook = xlsxwriter.Workbook(os.path.join(file_destination, 'copysheet.xlsx')) bills_sheet = workbook.add_worksheet('Bills') bills_sheet.write_row(0, 0, ['key', 'Bill Summary', 'Bill ID', 'title']) for i in xrange(1, len(billsToCopySheet) + 1): bill = billsToCopySheet[i - 1] rowToAdd = [bill['id'], '', bill['bill_id'], bill['title']] bills_sheet.write_row(i, 0, rowToAdd) leg_fields = 'full_name,leg_id' legislators = openstates.legislators(state="tx", active=True, chamber='upper', fields=leg_fields) legislators.extend(openstates.legislators(state="tx", active=True, chamber='lower', fields=leg_fields)) leg_sheet = workbook.add_worksheet('Legislators') leg_sheet.write_row(0, 0, ['key', 'Higher Education', 'Name']) for i in xrange(1, len(legislators) + 1): legislator = legislators[i - 1] rowToAdd = [legislator['leg_id'], '', legislator['full_name']] leg_sheet.write_row(i, 0, rowToAdd) workbook.close()
def produceEnhancedDistrictJSONString(geoJSONString, chamber_string): leg_fields = "full_name,district,offices,party,roles,leg_id,photo_url" legislators = openstates.legislators(state="tx", active=True, chamber=chamber_string, fields=leg_fields) legislator_id_set={} for legislator in legislators: legislator_id_set[legislator['leg_id']] = legislator legislator['higher_ed_bills'] = {'primary':[], 'cosponsor':[]} legislator['photo_url'] = 'images/' + legislator['photo_url'].split('images/')[1] bill_fields = "id,sponsors,scraped_subjects" bills = openstates.bills(state="tx", search_window="session", fields=bill_fields) for bill in bills: try: for subject in bill['scraped_subjects']: if 'Education--Higher' in subject: addSponsorsToSet(legislator_id_set, bill) break except KeyError: print 'ERROR KeyError ' + str(bill) decoder = json.JSONDecoder() geoJSON = decoder.decode(geoJSONString) chamberName = '' if (chamber_string == 'upper'): chamberName = 'senate' else: chamberName = 'house' districts = geoJSON['objects'][chamberName]['geometries'] legislator_district_set = {} for legislator in legislators: try: legislator_district_set[int(legislator['district'])] = legislator except KeyError: # someone without a district, lt. governor, etc. print "'legislator' without district: %s" % str(legislator) continue for district in districts: if int(district['id']) in legislator_district_set: district['properties'] = {} district['properties']['legislator'] = legislator_district_set[int(district['id'])] addToLegislators(legislators) return json.dumps(obj=geoJSON, ensure_ascii=False, separators=(',',':'))
def bill_metadata(self): spec = { 'state': self.abbr, 'chamber': self.chamber, 'per_page': self.PER_PAGE} if self.term and not self.session: if isinstance(self.term, basestring): # Search the specified term. spec['search_window'] = 'term:' + self.term else: # Search the current term. spec['search_window'] = 'term' elif self.session: if isinstance(self.session, basestring): # Search the specified session. spec['search_window'] = 'session:' + self.session else: # Search the current session. spec['search_window'] = 'session' logging.info('Fetching bill metadata...') page = 1 meta = [] while True: spec.update(page=page) logging.debug('Fetching metadata: %r' % spec) more_meta = openstates.bills(**spec) if not more_meta: break meta += more_meta if self.limit and self.limit < (page * self.PER_PAGE): break page += 1 logging.info('...done.') if self.limit: meta = meta[:self.limit] if len(meta) < 300: # If the term or session contains too few bills for a # meaningful calculation, complain/bail. msg = 'Too few bills found (%d); aborting. %r' data = dict( abbr=self.abbr, session=self.session, term=self.term) raise TooFewBillsError(msg % (len(meta), data,)) return meta
def main(): detail = openstates.bill_detail(state="mn", session="upper", bill_id="SF 14") senate = openstates.bills(state="mn", chamber="upper", search_window="term") for bill in senate[:1]: print bill print bill.get("state") print bill.get("session") print bill.get("bill_id") detail = openstates.bill_detail( state=bill.get("state"), session=bill.get("session"), bill_id=bill.get("bill_id") ) print detail
def update_bills(self, state): prev_bills = [b.bill_id for b in PaBills.objects.all()] bills = openstates.bills(state=state, search_window="term") count = 0 for bill in bills: if bill['bill_id'] not in prev_bills: count += 1 new_bill = PaBills(state=state, session=bill['session'], title=bill['title'], type=bill['type'][0], chamber=bill['chamber'], created_at=bill['created_at'], updated_at=bill['updated_at'], bill_id=bill['bill_id']) new_bill.save() print "Added a total of %s bills to database" % count
def populateBills(): import sunlight import json from sunlight import openstates id_bill_json = openstates.bills( state="id" # search_window = 'session', # updated_since = '2016-03-08' ) print id_bill_json id_bills = byteify(json.dumps(id_bill_json)) # db.session.query(Bill).update({Bill.bill_id:id_bills["bill_id"], Bill.session: id_bills["session"], Bill.title: id_bills["title"], Bill.id: id_bills["id"], Bill.updated_at: id_bills["updated_at"]}) for bill in id_bill_json: bill_adder = Bill(bill["bill_id"], bill["session"], bill["title"], bill["id"], bill["updated_at"]) db.session.add(bill_adder) db.session.commit() return id_bills
def search(request): if request.method == 'POST': state = request.POST.get('state', None) keyword = request.POST.get('keyword', None) # call sunlightfoundation api for state legislation bills = openstates.bills(state=state, q=keyword) # call api for federal legislation data fed_bills_response = requests.get('https://www.govtrack.us/api/v2/bill?q=' + keyword) if fed_bills_response.status_code == 200: fed_bills = fed_bills_response.json() else: fed_bills = None context = {'bills': bills, 'fed_bills': fed_bills, 'state': state, 'keyword': keyword} return render(request, 'billtrack/bills.html', context) else: return render(request, 'billtrack/form.html')
def data(who): sponsored_bills = openstates.bills(sponsor_id=who, fields='subjects') count = Counter([el for sl in map(lambda x: x.get('subjects', []), sponsored_bills) for el in sl]) [count.pop(x) for x in blacklist if x in count] ret = do_math(count) additive = min(map(lambda x: x[0], ret)) if additive >= 0: additive = 0 else: additive = (-additive) + 1 output = [] for el in ret: output.append([ el[0] + additive, el[1] ]) return json.dumps(output)
def produceBillJSONFiles(files_destination): bill_fields = "bill_id,title,alternate_titles,action_dates,actions,chamber,\ updated_at,id,scraped_subjects,type,versions,votes" # bills = openstates.bills(state="tx", search_window="session", # fields=bill_fields) bills = [] pageNo = 1 while True: billsToAdd = openstates.bills(state="tx", search_window="session", fields=bill_fields, page=pageNo) if len(billsToAdd) == 0: break bills.extend(billsToAdd) pageNo += 1 addToBills(bills) if not os.path.exists(files_destination): os.makedirs(files_destination) for bill in bills: try: for subject in bill['scraped_subjects']: if 'Education--Higher' in subject: filePath = os.path.join(files_destination, bill['id'] + '.json') with io.open(filePath, mode='w', encoding='utf8') as f: jsonDump = json.dumps(obj=bill, ensure_ascii=False, separators=(',',':')) f.write(jsonDump) break except KeyError: print 'ERROR KeyError ' + str(bill)
not_in_legis=0.0, legis_names=tuple(self.leg_ids)) return rollcall if __name__ == '__main__': from sunlight import openstates, response_cache response_cache.enable('mongo') response_cache.logger.setLevel(10) # Wrangle the API data into a Rollcall object. spec = dict(state='al', chamber='lower', search_window='term:2011-2014') valid_ids = [leg['id'] for leg in openstates.legislators(**spec)] builder = RollcallBuilder(valid_ids) bills = openstates.bills(**spec) for bill in bills: bill = openstates.bill(bill['id']) for vote in bill['votes']: if vote['chamber'] != bill['chamber']: continue builder.add_vote(vote) rollcall = builder.get_rollcall() wnominate = rollcall.wnominate(polarity=('ALL000086', 'ALL000085')) wnom_values = wnominate.legislators.coord1D ideal = rollcall.ideal() ideal_values = ideal.xbar import pdb; pdb.set_trace()
not_in_legis=0.0, legis_names=tuple(self.leg_ids)) return rollcall if __name__ == '__main__': from sunlight import openstates, response_cache response_cache.enable('mongo') response_cache.logger.setLevel(10) # Wrangle the API data into a Rollcall object. spec = dict(state='al', chamber='lower', search_window='term:2011-2014') valid_ids = [leg['id'] for leg in openstates.legislators(**spec)] builder = RollcallBuilder(valid_ids) bills = openstates.bills(**spec) for bill in bills: bill = openstates.bill(bill['id']) for vote in bill['votes']: if vote['chamber'] != bill['chamber']: continue builder.add_vote(vote) rollcall = builder.get_rollcall() wnominate = rollcall.wnominate(polarity=('ALL000086', 'ALL000085')) wnom_values = wnominate.legislators.coord1D ideal = rollcall.ideal() ideal_values = ideal.xbar import pdb
import commands """ Program to get the bill sponsor information and write it to db. """ author_db = pd.DataFrame(columns=[ 'bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id' ]) billdetail_db = pd.DataFrame(columns=[ 'bill_name', 'sunlight_id', 'title', 'impact_clause', 'scraped_subjects', 'official_subjects', 'summary_sunlight', 'summary_text', 'full_cleaned_text' ]) ca_bills = openstates.bills(state='ca', search_window='term') authorindex = 0 billindex = 0 for i in range(len(ca_bills)): thisbilldetail = openstates.bill_detail(state='ca', session=ca_bills[i]['session'], bill_id=ca_bills[i]['bill_id']) keys = thisbilldetail.keys() if (thisbilldetail['created_at'] < '2015-06-01') and ('HR' not in thisbilldetail['bill_id']): billname = ca_bills[i]['bill_id'] billsid = ca_bills[i]['id'] billtitle = ca_bills[i]['title'] billclause = thisbilldetail['+impact_clause']
def get_updated_bill_list(self): return openstates.bills( state='tx', updated_since=self.latest_date, search_window='session')
import urllib import re import pandas as pd from bs4 import BeautifulSoup import commands """ Program to get the bill sponsor information and write it to db. """ author_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id']) billdetail_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'title', 'impact_clause', 'scraped_subjects', 'official_subjects', 'summary_sunlight', 'summary_text', 'full_cleaned_text']) ca_bills = openstates.bills(state='ca', search_window='term') authorindex = 0; billindex = 0; for i in range(len(ca_bills)): thisbilldetail = openstates.bill_detail(state='ca', session=ca_bills[i]['session'], bill_id = ca_bills[i]['bill_id']); keys = thisbilldetail.keys() if (thisbilldetail['created_at'] < '2015-06-01') and ('HR' not in thisbilldetail['bill_id']): billname = ca_bills[i]['bill_id']; billsid = ca_bills[i]['id']; billtitle = ca_bills[i]['title']; billclause = thisbilldetail['+impact_clause'] if 'scraped_subjects' in keys: billscraped = thisbilldetail['scraped_subjects'] else:
'Military', 'Municipal and County Issues', 'Nominations', 'Other', 'Public Services', 'Recreation', 'Reproductive Issues', 'Resolutions', 'Science and Medical Research', 'Senior Issues', 'Sexual Orientation and Gender Issues', 'Social Issues', 'State Agencies', 'Technology and Communication', 'Trade', 'Transportation', 'Welfare and Poverty' ] states = [ "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY", "PR" ] # Grabbinga all bills from every topic and state # and saves them to their own file. # File name is StateAbbr-Topic.json for topic in topics: for state_abbr in states: bills = openstates.bills(search_window='session', subject=topic, state=state_abbr) bills_json = json.dumps(bills, sort_keys=True, indent=4) f = open( '../json/' + state_abbr + '-' + topic.replace(',', '').replace(' ', '') + '.json', 'w') f.write(bills_json) f.close()
'Federal, State, and Local Issues', 'Gambling and Gaming', 'Government Reform', 'Guns', 'Health', 'Housing and Property', 'Immigration', 'Indigenous Peoples', 'Insurance', 'Judiciary', 'Labor and Employment', 'Legal Issues', 'Legislative Affairs', 'Military', 'Municipal and County Issues', 'Nominations', 'Other', 'Public Services', 'Recreation', 'Reproductive Issues', 'Resolutions', 'Science and Medical Research', 'Senior Issues', 'Sexual Orientation and Gender Issues', 'Social Issues', 'State Agencies', 'Technology and Communication', 'Trade', 'Transportation', 'Welfare and Poverty'] states = ["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DC", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY", "PR"] # Grabbinga all bills from every topic and state # and saves them to their own file. # File name is StateAbbr-Topic.json for topic in topics: for state_abbr in states: bills = openstates.bills( search_window='session', subject=topic, state=state_abbr) bills_json = json.dumps(bills, sort_keys=True, indent=4) f = open('../json/' + state_abbr + '-' + topic.replace(',', '').replace(' ', '') + '.json', 'w') f.write(bills_json) f.close()
It also sets up pandas dataframes for each session, which can then be written out to a sql database. """ # set up the sessions and databases. sessions = ['20092010', '20112012', '20132014', '20152016'] author_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id', 'session']) billdetail_db = pd.DataFrame(columns=['bill_name', 'sunlight_id', 'title', 'impact_clause', 'scraped_subjects', 'official_subjects', 'summary_sunlight', 'summary_text', 'full_cleaned_text', 'session']) authorindex = 0; billindex = 0; for session in sessions: thissearch = 'session:'+session; ca_bills = openstates.bills(state='ca', search_window=thissearch) for index, ca_bill in enumerate(ca_bills): print str(index) + 'out of ' + str(len(ca_bills)) thisbilldetail = openstates.bill_detail(state='ca', session=ca_bill['session'], bill_id = ca_bill['bill_id']); keys = thisbilldetail.keys() if 'HR' not in thisbilldetail['bill_id']: billname = ca_bill['bill_id']; billsid = ca_bill['id']; billtitle = ca_bill['title']; if '+impact_clause' in keys: billclause = thisbilldetail['+impact_clause'] else: billclause = None;
# import packages we need: openstates, csv, and regex from sunlight import openstates import csv import re # api call using openstates package. # see example output: http://sunlightlabs.github.io/openstates-api/bills.html#examples/bill-search oklahoma_bills = openstates.bills( state='ok', search_window='term:2015-2016' ) # oklahoma legislators ok_legislators = openstates.legislators( state='ok', active='true' ) # we need an array of the legislators ids # with the first value being bill_id for our # csv header row ok_legislators_array = ['bill_id', 'chamber', 'vote_id'] for legislator in ok_legislators: ok_legislators_array.append(legislator['leg_id']) # create or open votes.csv file, with write with open('votes.csv', 'w') as f: # create writer object on the file we named f # extrasaction parameter means that if there is a missing or extra leg_id in our array # then the writer will continue regardless writer = csv.DictWriter(f, fieldnames=ok_legislators_array, extrasaction='ignore')
sessions = ['20092010', '20112012', '20132014', '20152016'] author_db = pd.DataFrame(columns=[ 'bill_name', 'sunlight_id', 'sponsor_name', 'sponsor_sunlight_id', 'session' ]) billdetail_db = pd.DataFrame(columns=[ 'bill_name', 'sunlight_id', 'title', 'impact_clause', 'scraped_subjects', 'official_subjects', 'summary_sunlight', 'summary_text', 'full_cleaned_text', 'session' ]) authorindex = 0 billindex = 0 for session in sessions: thissearch = 'session:' + session ca_bills = openstates.bills(state='ca', search_window=thissearch) for index, ca_bill in enumerate(ca_bills): print str(index) + 'out of ' + str(len(ca_bills)) thisbilldetail = openstates.bill_detail(state='ca', session=ca_bill['session'], bill_id=ca_bill['bill_id']) keys = thisbilldetail.keys() if 'HR' not in thisbilldetail['bill_id']: billname = ca_bill['bill_id'] billsid = ca_bill['id'] billtitle = ca_bill['title'] if '+impact_clause' in keys: billclause = thisbilldetail['+impact_clause']