def search_bills_for_keywords(bills, keywords):
bills_to_keywords = dict()
index = 0
for bill in bills:
matched_keywords = search_for_keywords(bill['title'], keywords)
bill_id = bill['id']
print(bill_id, index, "of", len(bills))
index += 1
if len(matched_keywords) > 0:
bills_to_keywords[bill_id] = matched_keywords
print(bill_id, ":", matched_keywords, ",")
continue
bill_details = pyopenstates.get_bill(uid=bill_id)
latest_text_url = get_latest_text_url(bill_details)
if not latest_text_url:
continue
try:
bill_full_text = s.get(latest_text_url).text
except:
print("error getting:" + latest_text_url, sys.exc_info()[0])
continue
matched_keywords = search_for_keywords(bill_full_text, keywords)
if len(matched_keywords) > 0:
bills_to_keywords[bill_id] = matched_keywords
print(bill_id, ":", matched_keywords, ",")
return bills_to_keywords
def testBillDetailsByUID(self):
"""Bill details by UID"""
_id = "CAB00004148"
title = "An act to amend Section 1750.1 of the Business and Professions Code, and to amend Section 104830 " \
"of, and to add Section 104762 to, the Health and Safety Code, relating to oral health."
bill = pyopenstates.get_bill(_id)
self.assertEqual(bill["title"], title)
def stuff():
bills = pyopenstates.search_bills(state='az', chamber='upper')
first_ten_bills = bills[0:10]
print('bill search in az', json.dumps(first_ten_bills, default=serialize_datetime))
for bill in first_ten_bills:
bill_details = pyopenstates.get_bill(uid=bill['id'])
latest_version_text_url = bill_details['versions'][-1]['url']
print('bill', bill['bill_id'], json.dumps(bill_details, default=serialize_datetime))
print('text url', latest_version_text_url)
def add_bill_fields(bills, *fields):
for i, bill in enumerate(bills):
bill_details = pyopenstates.get_bill(uid=bill['id'], fields=fields)
for field in fields:
# add each field to the bill
bill[field] = bill_details[field]
# updating the specific bill within the bills array
# bills[i] = bill
return bills
def testBillDetails(self):
"""Bill details"""
state = "ca"
term = "20092010"
bill_id = "AB 667"
title = "An act to amend Section 1750.1 of the Business and Professions Code, and to amend Section 104830 " \
"of, and to add Section 104762 to, the Health and Safety Code, relating to oral health."
bill = pyopenstates.get_bill(state=state, term=term, bill_id=bill_id)
self.assertEqual(bill["bill_id"], bill_id)
self.assertEqual(bill["title"], title)
def query(self,
session='session'
): # default 'session' returns only current session
Tables.query(self)
bills = pyopenstates.search_bills(
state=config.STATE,
search_window=session,
type="bill",
# chamber="upper", # lower
# updated_since="YYYY-MM-DD",
# subject="",
# sponsor_id="000000",
sort="created_at",
fields='id')
self.raw_dictionary = map(
lambda dic: pyopenstates.get_bill(
dic['id'],
fields=[
'id', 'bill_id', 'chamber', '+short_title', 'actions',
'action_dates', 'session', 'sources', 'sponsors',
'subjects', 'title', 'votes'
]), bills)
tracked_votes = []
for line in open(input_file):
if line.startswith('#'):
continue
input_vote_info = line.strip('\n\r').split(',')
tracked_votes.append({
'vote_session' : input_vote_info[0],
'vote_bill_no' : input_vote_info[1],
'vote_id' : input_vote_info[2],
'vote_preference' : input_vote_info[3],
'vote_weight' : input_vote_info[4]
})
for vote in tracked_votes:
all_votes_data = osClient.get_bill(state=config.state, term=vote['vote_session'], bill_id=vote['vote_bill_no'], fields='votes')['votes']
tracked_vote_data = next( (x for x in all_votes_data if x['id'] == vote['vote_id']), None )
if tracked_vote_data['chamber'] == 'upper':
upper_legislators = AddVotes(upper_legislators, tracked_vote_data)
if tracked_vote_data['chamber'] == 'lower':
lower_legislators = AddVotes(lower_legislators, tracked_vote_data)
export_fh = open('output.csv', 'w')
export_fh.write('Upper' + '\n')
header_line = 'SD,Name,Party'
for v in tracked_votes:
header_line += "," + v['vote_bill_no'] + " (" + v['vote_session'] + ")"
export_fh.write(header_line + '\n')
export_fth.write( WriteVotes(upper_legislators) )
bills_upper = pyopenstates.search_bills(state=state, chamber="upper", updated_since="2017-01-01")
bills_lower = pyopenstates.search_bills(state=state, chamber="lower", updated_since="2017-01-01")
for bill in bills_upper:
number = bill['bill_id']
intro = bill['title']
status = bill['id']
# try:
nbill, create = PolicyRecord.objects.get_or_create(number=number,
intro_text=intro[:512],
digistate=DIGISTATE,
primary_org=SENATE,
status=status)
dbill = pyopenstates.get_bill(nbill.status)
url = dbill['sources'][0]['url']
nbill.link = url
nbill.save()
sponsors = dbill['sponsors']
for sponsor in sponsors:
try:
rep = PublicOfficial.objects.get(city_str=sponsor['leg_id'])
nbill.authors.add(rep.id)
nbill.save()
except:
print("Senate oops sponsor: {0}".format(sponsor))
# except:
# ppr(bill)
# Get all general information of all bills
bills = pyopenstates.search_bills(state=st,
search_window='all',
chamber='upper',
per_page=10000,
fields=['id', 'bill_id'])
print('TOTAL BILLS: ', len(bills))
# Fetches extra information on each bill (voting information)
legislators = set()
votes = []
count = 0
for bill in bills:
count = count + 1
print('Requesting: ', bill['bill_id'], 'Count: ', count)
fullBill = pyopenstates.get_bill(uid=bill['id'])
for vote in fullBill['votes']:
if not vote:
continue
# Only fetch senate votes (lower house votes on senate bills sometimes)
if 'upper' not in vote['chamber']:
continue
# Specical cases of bad data from the API.
if (vote['date'].startswith('2011')
and st == 'al') or (vote['date'].startswith('2014')
and st == 'fl'):
continue
voteRecord = {'description': fullBill['title'], 'date': vote['date']}
for yes in vote['yes_votes']:
if yes['leg_id']:
legislators.add(yes['leg_id'])
def scrape(self):
state = 'MN'
session = self.jurisdiction.legislative_sessions[0]
apiKey = 'd2c0db7e-6a6e-4606-a9b0-83c18e647ff6'
pyopenstates.set_api_key(apiKey)
bills_upper = pyopenstates.search_bills(state=state,
chamber="upper",
updated_since="2017-01-01")
bills_lower = pyopenstates.search_bills(state=state,
chamber="lower",
updated_since="2017-01-01")
for b in bills_lower:
number = b['bill_id']
title = b['title']
bill_id = b['id']
dbill = pyopenstates.get_bill(bill_id)
url = dbill['sources'][0]['url']
bill = Bill(identifier=number,
legislative_session=session['identifier'],
title=title,
classification=b['type'][0],
chamber='upper')
bill.add_source(url)
bill.add_identifier(bill_id, scheme='openstatesv1')
subjects = b['subjects']
for s in subjects:
bill.add_subject(s)
sponsors = dbill['sponsors']
for sponsor in sponsors:
if not sponsor['leg_id'] == None:
l = pyopenstates.get_legislator(sponsor['leg_id'])
full_name = l['full_name'].split(' ')
if len(full_name) == 3:
full_name.pop(1)
full_name = (' ').join(full_name)
primary = False
if sponsor['type'] == 'primary':
primary = True
try:
bill.add_sponsorship(name=full_name,
classification=sponsor['type'],
entity_type='person',
primary=primary)
except:
pass
actions = dbill['actions']
for act in actions:
action = act['action']
actor = act['actor']
date = tz.localize(datetime.strptime(act['date'], DATE_FORMAT))
Action_Type = act['type']
bill.add_action(action, date, chamber=actor)
action_dates = dbill['action_dates']
for act in action_dates.items():
k, v = act[0], act[1]
if '_' in k:
chamber = k.split('_')[1]
elif k == 'signed':
chamber = 'executive'
else:
chamber = None
k.replace('_', ' ')
if not v == None and not k in ['first', 'last']:
bill.add_action(k, tz.localize(v), chamber=chamber)
yield bill
for b in bills_upper:
number = b['bill_id']
title = b['title']
bill_id = b['id']
dbill = pyopenstates.get_bill(bill_id)
url = dbill['sources'][0]['url']
bill = Bill(identifier=number,
legislative_session=session['identifier'],
title=title,
classification=b['type'][0],
chamber='upper')
bill.add_source(url)
bill.add_identifier(bill_id, scheme='openstatesv1')
subjects = b['subjects']
for s in subjects:
bill.add_subject(s)
sponsors = dbill['sponsors']
for sponsor in sponsors:
if not sponsor['leg_id'] == None:
l = pyopenstates.get_legislator(sponsor['leg_id'])
full_name = l['full_name'].split(' ')
if len(full_name) == 3:
full_name.pop(1)
full_name = (' ').join(full_name)
primary = False
if sponsor['type'] == 'primary':
primary = True
try:
bill.add_sponsorship(name=full_name,
classification=sponsor['type'],
entity_type='person',
primary=primary)
except:
pass
actions = dbill['actions']
for act in actions:
action = act['action']
actor = act['actor']
date = tz.localize(datetime.strptime(act['date'], DATE_FORMAT))
Action_Type = act['type']
bill.add_action(action, date, chamber=actor)
action_dates = dbill['action_dates']
for act in action_dates.items():
k, v = act[0], act[1]
if '_' in k:
chamber = k.split('_')[1]
elif k == 'signed':
chamber = 'executive'
else:
chamber = None
k.replace('_', ' ')
if not v == None and not k in ['first', 'last']:
bill.add_action(k, tz.localize(v), chamber=chamber)
yield bill