def list(self, request):
state = 'TN'
updated_since = str(date.today() - timedelta(days=1))
bills = pyopenstates.search_bills(state=state, updated_since=updated_since)
for b in bills:
stringify_all_dates(b)
return self.respond(raw=bills, status=200)
def testBillSearchSort(self):
"""Sorting bill search results"""
sorted_bills = pyopenstates.search_bills(state="dc",
search_window="term",
sort="created_at")
self.assertGreater(sorted_bills[0]["created_at"],
sorted_bills[-1]["created_at"])
def handle_search_bills_intent(self, message):
search = message.data.get("State").capitalize()
name_to_abbr = us.states.mapping('name', 'abbr')
name = name_to_abbr[search]
response = pyopenstates.search_bills(state=name,
search_window="session")
for i in response:
self.speak(i['bill_id'])
self.speak(i['title'])
def stuff():
bills = pyopenstates.search_bills(state='az', chamber='upper')
first_ten_bills = bills[0:10]
print('bill search in az', json.dumps(first_ten_bills, default=serialize_datetime))
for bill in first_ten_bills:
bill_details = pyopenstates.get_bill(uid=bill['id'])
latest_version_text_url = bill_details['versions'][-1]['url']
print('bill', bill['bill_id'], json.dumps(bill_details, default=serialize_datetime))
print('text url', latest_version_text_url)
def getBill(self, state, chamber, tags):
all_bills = []
for each in tags:
bill = pyopenstates.search_bills(state=state,
chamber=chamber,
q=each)
all_bills.append(bill)
all_bills.append(each)
return all_bills
def query_state(state, search_terms):
"""
https://openstates.github.io/pyopenstates/pyopenstates%20module.html#pyopenstates.search_bills
uses keyworded argument in function
"""
print('Query for ' + state + ' bills')
bills = pyopenstates.search_bills(state=state, q=search_terms)
return bills
def testBillSearchFullText(self):
"""A basic full-text search returns results that contain the query string"""
query = "taxi"
results = pyopenstates.search_bills(state="dc", q=query)
self.assertGreater(len(results), 1)
match = False
for result in results:
if query.lower() in result["title"].lower():
match = True
break
self.assertTrue(match)
def main():
relevant_bills = dict()
with open('keywords.json', 'r') as keywords_file:
keywords = json.load(keywords_file)
for state in get_states():
relevant_bills[state] = dict()
for session in get_sessions_for_state(state).keys():
relevant_bills[state][session] = dict()
print(state, "session", session)
bills = pyopenstates.search_bills(state=state, session=session, subject='Reproductive Issues')
relevant_bills[state][session].update(search_bills_for_keywords(bills, keywords))
print(json.dumps(relevant_bills, default=serialize_datetime))
def get_recent_bills():
"""Get recent bills in NJ"""
last_week = (datetime.today() - timedelta(days=7)).strftime("%Y-%m-%d")
saved_bills = get_cached_bills()
if not saved_bills:
new_bills = pyopenstates.search_bills(
state="nj",
updated_since=last_week,
type="bill",
chamber="upper",
sort="updated_at",
search_window="term",
)
bills_found = len(new_bills)
curr_time = datetime.now().timestamp()
data = {"timestamp": curr_time, "bills": []}
for i in range(min(5, bills_found)):
this_bill = new_bills[i]
this_bill_sponsors = []
for sponsor in this_bill["sponsors"]:
this_bill_sponsors.append(sponsor["name"])
data["bills"].append({
"title":
this_bill["title"],
"updated_at":
this_bill["updated_at"].strftime("%Y-%m-%d"),
"last_action":
this_bill["actions"][-1]["action"],
"sponsors":
this_bill_sponsors,
})
with open(BILL_CACHE_FILE, "w", encoding="utf8") as cache:
cache.write(json.dumps(data))
return data
return saved_bills
def query(self,
session='session'
): # default 'session' returns only current session
Tables.query(self)
bills = pyopenstates.search_bills(
state=config.STATE,
search_window=session,
type="bill",
# chamber="upper", # lower
# updated_since="YYYY-MM-DD",
# subject="",
# sponsor_id="000000",
sort="created_at",
fields='id')
self.raw_dictionary = map(
lambda dic: pyopenstates.get_bill(
dic['id'],
fields=[
'id', 'bill_id', 'chamber', '+short_title', 'actions',
'action_dates', 'session', 'sources', 'sponsors',
'subjects', 'title', 'votes'
]), bills)
def current_session_bills(self):
if self._current_session_bills is None:
print('...downloading.....')
results = pyos.search_bills(state=self.state,
chamber=self.chamber,
search_window='session')
print('..parsing.......')
df = pd.DataFrame(results)
columns = [
'bill_id', 'title', 'action_dates', 'actions', 'created_at',
'sources', 'sponsors', 'summary', 'updated_at', 'versions'
]
if self.chamber == 'upper':
df = df[df.bill_id.str.startswith('S')]
self._current_session_bills = df[columns]
elif self.chamber == 'lower':
df = df[df.bill_id.str.startswith('A')]
self._current_session_bills = df[columns]
#self._current_session_bills['sponsor_count'] = df['sponsors'].map(lambda x: len(x))
return self._current_session_bills
def testPaginatedResults(self):
"""Paginated results"""
results = pyopenstates.search_bills(state="dc")
paged_results = pyopenstates.search_bills(state="dc", per_page=100)
self.assertEqual(len(results), len(paged_results))
#!/usr/bin/env python
import os
import re
import requests
import pandas as pd
from glob import glob
from pyopenstates import search_bills
# current := 111st- legislative sessions (2019-)
print('Downloading data from OpenStates...')
current = pd.DataFrame(search_bills(state='tn', search_window='session:111')) \
.query('bill_id.str.contains("SB|HB")', engine='python') \
.reset_index(drop=True)
current['session'] = current['session'].astype(int)
# Remove the 'Amends TCA Title x, Chapter y' from bill titles
current['title'] = current['title'].str.replace(' - $', '')
current['title'] = current['title'].str.replace(' - Amends.+$', '')
# Sponsors for each bill is a list of dictionaries
sponsors = [[sponsor.get('name') for sponsor in d]
for d in current['sponsors']]
current['sponsors'] = sponsors
# Look for bills meeting outcomes of interest
print('Extracting bill outcomes...')
def testTimestampConversionInList(self):
"""Timestamp conversion in a list"""
bill = pyopenstates.search_bills(state="oh")[0]
self.assertTrue(type(bill["created_at"]) == datetime)
def get_reproductive_issues_bills(state):
return pyopenstates.search_bills(state=state, subject='Reproductive Issues')
def update_state_recent(state_name, openstates_state_abbrev):
"""
Update state information using repeated hits to the openstates API.
Does not use bulk data, which appears to be aggregiously outdated
(Bulk data only goes through mid 2016 (as of May 9, 2017))
"""
current_year = datetime.datetime.now().year
current_month = datetime.datetime.now().month
one_year_ago = "{}-{}-01".format(current_year - 1,
str(current_month).zfill(2))
six_months_ago = "{}-{}-01".format(
current_year if current_month > 6 else current_year - 1,
str(current_month - 6 if current_month > 6 else current_month +
6).zfill(2))
one_month_ago = "{}-{}-01".format(
current_year if current_month > 1 else current_year - 1,
str(current_month - 1 if current_month > 1 else 12).zfill(2))
num_new_legislators = 0
num_updated_legislators = 0
num_legislator_exceptions = 0
num_new_bills = 0
num_updated_bills = 0
num_bill_exceptions = 0
sys.stdout.write(
"Updating state legislators and bills: {}\n".format(state_name))
try:
state_geotag = cm.GeoTag.objects.get(feature_type="SP",
name=state_name)
all_legislators = pyopenstates.search_legislators(
state=openstates_state_abbrev)
for leg_json in all_legislators:
result = update_legislator(leg_json, state_geotag)
if result == "updated":
num_updated_legislators += 1
elif result == "new":
num_new_legislators += 1
sys.stdout.write(
"Number of legislators added: {}\n".format(num_new_legislators))
sys.stdout.write("Number of legislators updated: {}\n".format(
num_updated_legislators))
sys.stdout.write(
"Number of exceptions updating a legislator: {}\n".format(
num_legislator_exceptions))
recently_updated_bills = None
# try to get recent bills up to 1 year old. If the API thinks it's too big of a request, try shorter periods
try:
recently_updated_bills = pyopenstates.search_bills(
state=openstates_state_abbrev,
updated_since=one_year_ago,
fields=[
"title", "id", "bill_id", "subjects", "versions",
"action_dates"
])
except pyopenstates.APIError as e:
sys.stderr.write(
"Surviving openstates APIError: {}\n trying a more recent update period\n"
.format(unicode(e)))
if recently_updated_bills is None:
try:
recently_updated_bills = pyopenstates.search_bills(
state=openstates_state_abbrev,
updated_since=six_months_ago,
fields=[
"title", "id", "bill_id", "subjects", "versions",
"action_dates"
])
except pyopenstates.APIError as e:
sys.stderr.write(
"Surviving openstates APIError: {}\n trying a more recent update period\n"
.format(unicode(e)))
if recently_updated_bills is None:
recently_updated_bills = pyopenstates.search_bills(
state=openstates_state_abbrev,
updated_since=one_month_ago,
fields=[
"title", "id", "bill_id", "subjects", "versions",
"action_dates"
])
sys.stderr.write("Number of recently-updated bills: {}\n".format(
len(recently_updated_bills)))
for bill_json in recently_updated_bills:
result = update_bill(bill_json, state_geotag)
if result == "updated":
num_updated_bills += 1
if num_updated_bills % 100 == 0:
sys.stderr.write(
"Updating bill #{}\n".format(num_updated_bills))
elif result == "new":
num_new_bills += 1
if num_new_bills % 100 == 0:
sys.stderr.write(
"Adding new bill #{}\n".format(num_new_bills))
sys.stdout.write("Number of bills added: {}\n".format(num_new_bills))
sys.stdout.write(
"Number of bills updated: {}\n".format(num_updated_bills))
sys.stdout.write("DONE\n")
except cm.GeoTag.DoesNotExist:
sys.stdout.write("state not in geotags\n")
finally:
sys.stdout.flush()
import requests
from pprint import pprint as ppr
import pyopenstates
state = 'MN'
apiKey = 'd2c0db7e-6a6e-4606-a9b0-83c18e647ff6'
pyopenstates.set_api_key(apiKey)
print('Starting')
bills_upper = pyopenstates.search_bills(state='MN',
chamber="upper",
updated_since="2017-01-01")
reps = pyopenstates.search_legislators(state=state)
for r in reps:
name = r['full_name']
first_name = r['first_name']
last_name = r['last_name']
full_name = r['first_name'] + ' ' + r['last_name']
house = r['chamber']
if house == 'lower':
position = 'Representative'
print(full_name)
elif house == 'upper':
position = 'Senator'
"""
for bill in bills_upper:
# ppr(bill)
title = bill['title']
def scrape(self):
state = 'MN'
session = self.jurisdiction.legislative_sessions[0]
apiKey = 'd2c0db7e-6a6e-4606-a9b0-83c18e647ff6'
pyopenstates.set_api_key(apiKey)
bills_upper = pyopenstates.search_bills(state=state,
chamber="upper",
updated_since="2017-01-01")
bills_lower = pyopenstates.search_bills(state=state,
chamber="lower",
updated_since="2017-01-01")
for b in bills_lower:
number = b['bill_id']
title = b['title']
bill_id = b['id']
dbill = pyopenstates.get_bill(bill_id)
url = dbill['sources'][0]['url']
bill = Bill(identifier=number,
legislative_session=session['identifier'],
title=title,
classification=b['type'][0],
chamber='upper')
bill.add_source(url)
bill.add_identifier(bill_id, scheme='openstatesv1')
subjects = b['subjects']
for s in subjects:
bill.add_subject(s)
sponsors = dbill['sponsors']
for sponsor in sponsors:
if not sponsor['leg_id'] == None:
l = pyopenstates.get_legislator(sponsor['leg_id'])
full_name = l['full_name'].split(' ')
if len(full_name) == 3:
full_name.pop(1)
full_name = (' ').join(full_name)
primary = False
if sponsor['type'] == 'primary':
primary = True
try:
bill.add_sponsorship(name=full_name,
classification=sponsor['type'],
entity_type='person',
primary=primary)
except:
pass
actions = dbill['actions']
for act in actions:
action = act['action']
actor = act['actor']
date = tz.localize(datetime.strptime(act['date'], DATE_FORMAT))
Action_Type = act['type']
bill.add_action(action, date, chamber=actor)
action_dates = dbill['action_dates']
for act in action_dates.items():
k, v = act[0], act[1]
if '_' in k:
chamber = k.split('_')[1]
elif k == 'signed':
chamber = 'executive'
else:
chamber = None
k.replace('_', ' ')
if not v == None and not k in ['first', 'last']:
bill.add_action(k, tz.localize(v), chamber=chamber)
yield bill
for b in bills_upper:
number = b['bill_id']
title = b['title']
bill_id = b['id']
dbill = pyopenstates.get_bill(bill_id)
url = dbill['sources'][0]['url']
bill = Bill(identifier=number,
legislative_session=session['identifier'],
title=title,
classification=b['type'][0],
chamber='upper')
bill.add_source(url)
bill.add_identifier(bill_id, scheme='openstatesv1')
subjects = b['subjects']
for s in subjects:
bill.add_subject(s)
sponsors = dbill['sponsors']
for sponsor in sponsors:
if not sponsor['leg_id'] == None:
l = pyopenstates.get_legislator(sponsor['leg_id'])
full_name = l['full_name'].split(' ')
if len(full_name) == 3:
full_name.pop(1)
full_name = (' ').join(full_name)
primary = False
if sponsor['type'] == 'primary':
primary = True
try:
bill.add_sponsorship(name=full_name,
classification=sponsor['type'],
entity_type='person',
primary=primary)
except:
pass
actions = dbill['actions']
for act in actions:
action = act['action']
actor = act['actor']
date = tz.localize(datetime.strptime(act['date'], DATE_FORMAT))
Action_Type = act['type']
bill.add_action(action, date, chamber=actor)
action_dates = dbill['action_dates']
for act in action_dates.items():
k, v = act[0], act[1]
if '_' in k:
chamber = k.split('_')[1]
elif k == 'signed':
chamber = 'executive'
else:
chamber = None
k.replace('_', ' ')
if not v == None and not k in ['first', 'last']:
bill.add_action(k, tz.localize(v), chamber=chamber)
yield bill
state = raw_input("Which state? ")
# state = sys.argv[1]
pyopenstates.set_api_key(apiKey)
DIGISTATE = dx_State.objects.get(state_name__code=state)
HOUSE, c1 = Org.objects.get_or_create(name="MN House of Representative",
digistate=DIGISTATE)
SENATE, c2 = Org.objects.get_or_create(name="MN State Senate",
digistate=DIGISTATE)
cmts = pyopenstates.search_committees(state=state)
reps = pyopenstates.search_legislators(state=state)
districts_lower = pyopoenstates.search_districts(state=state, chamber="lower")
#districts_upper = pyopenstates.search_districts(state=state, chamber="upper")
bills_upper = pyopenstates.search_bills(state=state, chamber="upper", updated_since="2017-01-01")
bills_lower = pyopenstates.search_bills(state=state, chamber="lower", updated_since="2017-01-01")
for bill in bills_upper:
number = bill['bill_id']
intro = bill['title']
status = bill['id']
# try:
nbill, create = PolicyRecord.objects.get_or_create(number=number,
intro_text=intro[:512],
digistate=DIGISTATE,
primary_org=SENATE,
status=status)
dbill = pyopenstates.get_bill(nbill.status)
import json
import sys
# Useful for debugging
pp = pprint.PrettyPrinter(indent=4)
default_st = 'mt'
st = sys.argv[1] if len(sys.argv) > 1 else default_st
OUTFILE_VOTES = '../data/%s/votes-%s.csv' % (st, st)
OUTFILE_SENATOR_INFO = '../data/%s/senator-info-raw-%s.json' % (st, st)
# Get all general information of all bills
bills = pyopenstates.search_bills(state=st,
search_window='all',
chamber='upper',
per_page=10000,
fields=['id', 'bill_id'])
print('TOTAL BILLS: ', len(bills))
# Fetches extra information on each bill (voting information)
legislators = set()
votes = []
count = 0
for bill in bills:
count = count + 1
print('Requesting: ', bill['bill_id'], 'Count: ', count)
fullBill = pyopenstates.get_bill(uid=bill['id'])
for vote in fullBill['votes']:
if not vote:
continue
