def list(self, request): state = 'TN' updated_since = str(date.today() - timedelta(days=1)) bills = pyopenstates.search_bills(state=state, updated_since=updated_since) for b in bills: stringify_all_dates(b) return self.respond(raw=bills, status=200)
def testBillSearchSort(self): """Sorting bill search results""" sorted_bills = pyopenstates.search_bills(state="dc", search_window="term", sort="created_at") self.assertGreater(sorted_bills[0]["created_at"], sorted_bills[-1]["created_at"])
def handle_search_bills_intent(self, message): search = message.data.get("State").capitalize() name_to_abbr = us.states.mapping('name', 'abbr') name = name_to_abbr[search] response = pyopenstates.search_bills(state=name, search_window="session") for i in response: self.speak(i['bill_id']) self.speak(i['title'])
def stuff(): bills = pyopenstates.search_bills(state='az', chamber='upper') first_ten_bills = bills[0:10] print('bill search in az', json.dumps(first_ten_bills, default=serialize_datetime)) for bill in first_ten_bills: bill_details = pyopenstates.get_bill(uid=bill['id']) latest_version_text_url = bill_details['versions'][-1]['url'] print('bill', bill['bill_id'], json.dumps(bill_details, default=serialize_datetime)) print('text url', latest_version_text_url)
def getBill(self, state, chamber, tags): all_bills = [] for each in tags: bill = pyopenstates.search_bills(state=state, chamber=chamber, q=each) all_bills.append(bill) all_bills.append(each) return all_bills
def query_state(state, search_terms): """ https://openstates.github.io/pyopenstates/pyopenstates%20module.html#pyopenstates.search_bills uses keyworded argument in function """ print('Query for ' + state + ' bills') bills = pyopenstates.search_bills(state=state, q=search_terms) return bills
def testBillSearchFullText(self): """A basic full-text search returns results that contain the query string""" query = "taxi" results = pyopenstates.search_bills(state="dc", q=query) self.assertGreater(len(results), 1) match = False for result in results: if query.lower() in result["title"].lower(): match = True break self.assertTrue(match)
def main(): relevant_bills = dict() with open('keywords.json', 'r') as keywords_file: keywords = json.load(keywords_file) for state in get_states(): relevant_bills[state] = dict() for session in get_sessions_for_state(state).keys(): relevant_bills[state][session] = dict() print(state, "session", session) bills = pyopenstates.search_bills(state=state, session=session, subject='Reproductive Issues') relevant_bills[state][session].update(search_bills_for_keywords(bills, keywords)) print(json.dumps(relevant_bills, default=serialize_datetime))
def get_recent_bills(): """Get recent bills in NJ""" last_week = (datetime.today() - timedelta(days=7)).strftime("%Y-%m-%d") saved_bills = get_cached_bills() if not saved_bills: new_bills = pyopenstates.search_bills( state="nj", updated_since=last_week, type="bill", chamber="upper", sort="updated_at", search_window="term", ) bills_found = len(new_bills) curr_time = datetime.now().timestamp() data = {"timestamp": curr_time, "bills": []} for i in range(min(5, bills_found)): this_bill = new_bills[i] this_bill_sponsors = [] for sponsor in this_bill["sponsors"]: this_bill_sponsors.append(sponsor["name"]) data["bills"].append({ "title": this_bill["title"], "updated_at": this_bill["updated_at"].strftime("%Y-%m-%d"), "last_action": this_bill["actions"][-1]["action"], "sponsors": this_bill_sponsors, }) with open(BILL_CACHE_FILE, "w", encoding="utf8") as cache: cache.write(json.dumps(data)) return data return saved_bills
def query(self, session='session' ): # default 'session' returns only current session Tables.query(self) bills = pyopenstates.search_bills( state=config.STATE, search_window=session, type="bill", # chamber="upper", # lower # updated_since="YYYY-MM-DD", # subject="", # sponsor_id="000000", sort="created_at", fields='id') self.raw_dictionary = map( lambda dic: pyopenstates.get_bill( dic['id'], fields=[ 'id', 'bill_id', 'chamber', '+short_title', 'actions', 'action_dates', 'session', 'sources', 'sponsors', 'subjects', 'title', 'votes' ]), bills)
def current_session_bills(self): if self._current_session_bills is None: print('...downloading.....') results = pyos.search_bills(state=self.state, chamber=self.chamber, search_window='session') print('..parsing.......') df = pd.DataFrame(results) columns = [ 'bill_id', 'title', 'action_dates', 'actions', 'created_at', 'sources', 'sponsors', 'summary', 'updated_at', 'versions' ] if self.chamber == 'upper': df = df[df.bill_id.str.startswith('S')] self._current_session_bills = df[columns] elif self.chamber == 'lower': df = df[df.bill_id.str.startswith('A')] self._current_session_bills = df[columns] #self._current_session_bills['sponsor_count'] = df['sponsors'].map(lambda x: len(x)) return self._current_session_bills
def testPaginatedResults(self): """Paginated results""" results = pyopenstates.search_bills(state="dc") paged_results = pyopenstates.search_bills(state="dc", per_page=100) self.assertEqual(len(results), len(paged_results))
#!/usr/bin/env python import os import re import requests import pandas as pd from glob import glob from pyopenstates import search_bills # current := 111st- legislative sessions (2019-) print('Downloading data from OpenStates...') current = pd.DataFrame(search_bills(state='tn', search_window='session:111')) \ .query('bill_id.str.contains("SB|HB")', engine='python') \ .reset_index(drop=True) current['session'] = current['session'].astype(int) # Remove the 'Amends TCA Title x, Chapter y' from bill titles current['title'] = current['title'].str.replace(' - $', '') current['title'] = current['title'].str.replace(' - Amends.+$', '') # Sponsors for each bill is a list of dictionaries sponsors = [[sponsor.get('name') for sponsor in d] for d in current['sponsors']] current['sponsors'] = sponsors # Look for bills meeting outcomes of interest print('Extracting bill outcomes...')
def testTimestampConversionInList(self): """Timestamp conversion in a list""" bill = pyopenstates.search_bills(state="oh")[0] self.assertTrue(type(bill["created_at"]) == datetime)
def get_reproductive_issues_bills(state): return pyopenstates.search_bills(state=state, subject='Reproductive Issues')
def update_state_recent(state_name, openstates_state_abbrev): """ Update state information using repeated hits to the openstates API. Does not use bulk data, which appears to be aggregiously outdated (Bulk data only goes through mid 2016 (as of May 9, 2017)) """ current_year = datetime.datetime.now().year current_month = datetime.datetime.now().month one_year_ago = "{}-{}-01".format(current_year - 1, str(current_month).zfill(2)) six_months_ago = "{}-{}-01".format( current_year if current_month > 6 else current_year - 1, str(current_month - 6 if current_month > 6 else current_month + 6).zfill(2)) one_month_ago = "{}-{}-01".format( current_year if current_month > 1 else current_year - 1, str(current_month - 1 if current_month > 1 else 12).zfill(2)) num_new_legislators = 0 num_updated_legislators = 0 num_legislator_exceptions = 0 num_new_bills = 0 num_updated_bills = 0 num_bill_exceptions = 0 sys.stdout.write( "Updating state legislators and bills: {}\n".format(state_name)) try: state_geotag = cm.GeoTag.objects.get(feature_type="SP", name=state_name) all_legislators = pyopenstates.search_legislators( state=openstates_state_abbrev) for leg_json in all_legislators: result = update_legislator(leg_json, state_geotag) if result == "updated": num_updated_legislators += 1 elif result == "new": num_new_legislators += 1 sys.stdout.write( "Number of legislators added: {}\n".format(num_new_legislators)) sys.stdout.write("Number of legislators updated: {}\n".format( num_updated_legislators)) sys.stdout.write( "Number of exceptions updating a legislator: {}\n".format( num_legislator_exceptions)) recently_updated_bills = None # try to get recent bills up to 1 year old. If the API thinks it's too big of a request, try shorter periods try: recently_updated_bills = pyopenstates.search_bills( state=openstates_state_abbrev, updated_since=one_year_ago, fields=[ "title", "id", "bill_id", "subjects", "versions", "action_dates" ]) except pyopenstates.APIError as e: sys.stderr.write( "Surviving openstates APIError: {}\n trying a more recent update period\n" .format(unicode(e))) if recently_updated_bills is None: try: recently_updated_bills = pyopenstates.search_bills( state=openstates_state_abbrev, updated_since=six_months_ago, fields=[ "title", "id", "bill_id", "subjects", "versions", "action_dates" ]) except pyopenstates.APIError as e: sys.stderr.write( "Surviving openstates APIError: {}\n trying a more recent update period\n" .format(unicode(e))) if recently_updated_bills is None: recently_updated_bills = pyopenstates.search_bills( state=openstates_state_abbrev, updated_since=one_month_ago, fields=[ "title", "id", "bill_id", "subjects", "versions", "action_dates" ]) sys.stderr.write("Number of recently-updated bills: {}\n".format( len(recently_updated_bills))) for bill_json in recently_updated_bills: result = update_bill(bill_json, state_geotag) if result == "updated": num_updated_bills += 1 if num_updated_bills % 100 == 0: sys.stderr.write( "Updating bill #{}\n".format(num_updated_bills)) elif result == "new": num_new_bills += 1 if num_new_bills % 100 == 0: sys.stderr.write( "Adding new bill #{}\n".format(num_new_bills)) sys.stdout.write("Number of bills added: {}\n".format(num_new_bills)) sys.stdout.write( "Number of bills updated: {}\n".format(num_updated_bills)) sys.stdout.write("DONE\n") except cm.GeoTag.DoesNotExist: sys.stdout.write("state not in geotags\n") finally: sys.stdout.flush()
import requests from pprint import pprint as ppr import pyopenstates state = 'MN' apiKey = 'd2c0db7e-6a6e-4606-a9b0-83c18e647ff6' pyopenstates.set_api_key(apiKey) print('Starting') bills_upper = pyopenstates.search_bills(state='MN', chamber="upper", updated_since="2017-01-01") reps = pyopenstates.search_legislators(state=state) for r in reps: name = r['full_name'] first_name = r['first_name'] last_name = r['last_name'] full_name = r['first_name'] + ' ' + r['last_name'] house = r['chamber'] if house == 'lower': position = 'Representative' print(full_name) elif house == 'upper': position = 'Senator' """ for bill in bills_upper: # ppr(bill) title = bill['title']
def scrape(self): state = 'MN' session = self.jurisdiction.legislative_sessions[0] apiKey = 'd2c0db7e-6a6e-4606-a9b0-83c18e647ff6' pyopenstates.set_api_key(apiKey) bills_upper = pyopenstates.search_bills(state=state, chamber="upper", updated_since="2017-01-01") bills_lower = pyopenstates.search_bills(state=state, chamber="lower", updated_since="2017-01-01") for b in bills_lower: number = b['bill_id'] title = b['title'] bill_id = b['id'] dbill = pyopenstates.get_bill(bill_id) url = dbill['sources'][0]['url'] bill = Bill(identifier=number, legislative_session=session['identifier'], title=title, classification=b['type'][0], chamber='upper') bill.add_source(url) bill.add_identifier(bill_id, scheme='openstatesv1') subjects = b['subjects'] for s in subjects: bill.add_subject(s) sponsors = dbill['sponsors'] for sponsor in sponsors: if not sponsor['leg_id'] == None: l = pyopenstates.get_legislator(sponsor['leg_id']) full_name = l['full_name'].split(' ') if len(full_name) == 3: full_name.pop(1) full_name = (' ').join(full_name) primary = False if sponsor['type'] == 'primary': primary = True try: bill.add_sponsorship(name=full_name, classification=sponsor['type'], entity_type='person', primary=primary) except: pass actions = dbill['actions'] for act in actions: action = act['action'] actor = act['actor'] date = tz.localize(datetime.strptime(act['date'], DATE_FORMAT)) Action_Type = act['type'] bill.add_action(action, date, chamber=actor) action_dates = dbill['action_dates'] for act in action_dates.items(): k, v = act[0], act[1] if '_' in k: chamber = k.split('_')[1] elif k == 'signed': chamber = 'executive' else: chamber = None k.replace('_', ' ') if not v == None and not k in ['first', 'last']: bill.add_action(k, tz.localize(v), chamber=chamber) yield bill for b in bills_upper: number = b['bill_id'] title = b['title'] bill_id = b['id'] dbill = pyopenstates.get_bill(bill_id) url = dbill['sources'][0]['url'] bill = Bill(identifier=number, legislative_session=session['identifier'], title=title, classification=b['type'][0], chamber='upper') bill.add_source(url) bill.add_identifier(bill_id, scheme='openstatesv1') subjects = b['subjects'] for s in subjects: bill.add_subject(s) sponsors = dbill['sponsors'] for sponsor in sponsors: if not sponsor['leg_id'] == None: l = pyopenstates.get_legislator(sponsor['leg_id']) full_name = l['full_name'].split(' ') if len(full_name) == 3: full_name.pop(1) full_name = (' ').join(full_name) primary = False if sponsor['type'] == 'primary': primary = True try: bill.add_sponsorship(name=full_name, classification=sponsor['type'], entity_type='person', primary=primary) except: pass actions = dbill['actions'] for act in actions: action = act['action'] actor = act['actor'] date = tz.localize(datetime.strptime(act['date'], DATE_FORMAT)) Action_Type = act['type'] bill.add_action(action, date, chamber=actor) action_dates = dbill['action_dates'] for act in action_dates.items(): k, v = act[0], act[1] if '_' in k: chamber = k.split('_')[1] elif k == 'signed': chamber = 'executive' else: chamber = None k.replace('_', ' ') if not v == None and not k in ['first', 'last']: bill.add_action(k, tz.localize(v), chamber=chamber) yield bill
state = raw_input("Which state? ") # state = sys.argv[1] pyopenstates.set_api_key(apiKey) DIGISTATE = dx_State.objects.get(state_name__code=state) HOUSE, c1 = Org.objects.get_or_create(name="MN House of Representative", digistate=DIGISTATE) SENATE, c2 = Org.objects.get_or_create(name="MN State Senate", digistate=DIGISTATE) cmts = pyopenstates.search_committees(state=state) reps = pyopenstates.search_legislators(state=state) districts_lower = pyopoenstates.search_districts(state=state, chamber="lower") #districts_upper = pyopenstates.search_districts(state=state, chamber="upper") bills_upper = pyopenstates.search_bills(state=state, chamber="upper", updated_since="2017-01-01") bills_lower = pyopenstates.search_bills(state=state, chamber="lower", updated_since="2017-01-01") for bill in bills_upper: number = bill['bill_id'] intro = bill['title'] status = bill['id'] # try: nbill, create = PolicyRecord.objects.get_or_create(number=number, intro_text=intro[:512], digistate=DIGISTATE, primary_org=SENATE, status=status) dbill = pyopenstates.get_bill(nbill.status)
import json import sys # Useful for debugging pp = pprint.PrettyPrinter(indent=4) default_st = 'mt' st = sys.argv[1] if len(sys.argv) > 1 else default_st OUTFILE_VOTES = '../data/%s/votes-%s.csv' % (st, st) OUTFILE_SENATOR_INFO = '../data/%s/senator-info-raw-%s.json' % (st, st) # Get all general information of all bills bills = pyopenstates.search_bills(state=st, search_window='all', chamber='upper', per_page=10000, fields=['id', 'bill_id']) print('TOTAL BILLS: ', len(bills)) # Fetches extra information on each bill (voting information) legislators = set() votes = [] count = 0 for bill in bills: count = count + 1 print('Requesting: ', bill['bill_id'], 'Count: ', count) fullBill = pyopenstates.get_bill(uid=bill['id']) for vote in fullBill['votes']: if not vote: continue