def scrape_session_2009(self, chamber, session): url, type = bills_url(chamber) with self.urlopen(url) as page_html: page = lxml.html.fromstring(page_html) for element, attribute, link, pos in page.iterlinks(): if re.search("billtype=" + type + "&billnumber=[0-9]+", link) != None: bill_page_url = bill_url(link) with self.urlopen(bill_page_url) as bill_page_str: bill_page = lxml.html.fromstring(bill_page_str) splitted_link = link.split("=") bill_number = splitted_link[-1] bill_id = bill_page.cssselect('a[class="headerlink"]') bill_id = bill_id[0] bill_id = bill_id.text_content() bill_title = bill_page.cssselect('td[style="color:Black"]') bill_title = bill_title[0] bill_title = bill_title.text_content() bill = Bill(session, chamber, bill_id, bill_title) bill.add_source(bill_page_url) actions_table_list = bill_page.cssselect('table[rules="all"]') actions_table = actions_table_list[0] action_elements = actions_table.cssselect('tr') # first element is not an action element action_elements.pop(0) for ae in action_elements: action_element_parts = ae.cssselect('td') action_date = action_element_parts[0] actor_house = action_element_parts[1] action_text = action_element_parts[2] # look for acting comittees match = re.search("(committee\(s\)|committee) on ([A-Z]{3}(/|-)[A-Z]{3}|[A-Z]{3})", action_text.text_content()) if(match != None): actor = match.group(0) elif(actor_house == 'H'): actor = "lower" elif (actor_house == 'S'): actor = "upper" else: actor = chamber action_date = dt.datetime.strptime(action_date.text_content(), '%m/%d/%Y') if (re.search("The votes were as follows", action_text.text_content()) != None): self.scrape_votes(action_text.text_content(), bill_page_url, actor_house, action_date, bill) bill.add_action(actor, action_text, action_date) with self.urlopen(versions_page_url(type, bill_number)) as versions_page_html: versions_page = lxml.html.fromstring(versions_page_html) versions_elements = versions_page.cssselect('span[class="searchtitle"]') for ve in versions_elements: element_text = ve.text_content() version_name = element_text.rstrip("_.HTM") bill.add_version(version_name, bill_version_url(element_text))
def scrape2009(self, url, year, chamberName, session, number): "e.g. http://www.legis.ga.gov/legis/2009_10/sum/sum/sb1.htm" with self.lxml_context(url) as page: # Bill name = page.cssselect('#legislation h1')[0].text_content().strip() bill_id = name.split(' - ')[0].strip() bill = Bill(session, chamberName, bill_id, name) # Sponsorships for a in page.cssselect("#sponsors a"): bill.add_sponsor('', a.text_content().strip()) # Actions for row in page.cssselect('#history tr')[1:]: date = row[0].text_content().strip() action_text = row[1].text_content().strip() if '/' not in date: continue date = datetime.datetime.strptime(date, '%m/%d/%Y') if action_text.startswith('Senate'): bill.add_action('upper', action_text, date) elif action_text.startswith('House'): bill.add_action('lower', action_text, date) # Versions for row in page.cssselect('#versions a'): bill.add_version(a.text_content(), urlparse.urljoin(url, a.get('href'))) self.save_bill(bill)
def scrape2003(self, url, year, chamberName, session, number): "e.g. http://www.legis.ga.gov/legis/2003_04/sum/sum/sb1.htm" with self.lxml_context(url) as page: # Grab the interesting tables on the page. tables = page.cssselect('center table') # Bill name = tables[0].text_content().split('-', 1)[1] bill = Bill(session, chamberName, number, name) # Sponsorships for a in tables[1].cssselect('a'): bill.add_sponsor('', a.text_content().strip()) # Actions center = page.cssselect('center table center')[0] for row in center.cssselect('table')[-2].cssselect('tr')[2:]: date = row[0].text_content().strip() action_text = row[1].text_content().strip() if '/' not in date: continue if action_text.startswith('Senate'): bill.add_action('upper', action_text, date) elif action_text.startswith('House'): bill.add_action('lower', action_text, date) # Versions for row in center.cssselect('table')[-1].cssselect('a'): bill.add_version(a.text_content(), urlparse.urljoin(url, a.get('href'))) self.save_bill(bill)
def scrape1999(self, url, year, chamberName, session, number): "e.g. http://www.legis.ga.gov/legis/1999_00/leg/sum/sb1.htm" with self.lxml_context(url) as lxml: # Grab the interesting tables on the page. tables = page.cssselect('table') # Bill name = tables[1].cssselect('a')[0].text_content().split('-', 1)[1] bill = Bill(session, chamberName, number, name) # Versions bill.add_version('Current', url.replace('/sum/', '/fulltext/')) # Sponsorships for a in tables[2].cssselect('a'): bill.add_sponsor('', a.text_content().strip()) # Actions for row in tables[-1].cssselect('tr'): senate_date = row[0].text_content().strip() action_text = row[1].text_content().strip() house_date = row[2].text_content().strip() if '/' not in senate_date and '/' not in house_date: continue if senate_date: bill.add_action('upper', action_text, senate_date) if house_date: bill.add_action('lower', action_text, house_date) self.save_bill(bill)
def parse_bill(self, chamber, session, bill_id, bill_info_url): with self.urlopen(bill_info_url) as bill_info_data: bill_info = self.soup_parser(bill_info_data) version_url = '%s/bill.doc' % bill_id version_link = bill_info.find(href=version_url) if not version_link: # This bill was withdrawn return bill_title = version_link.findNext('p').contents[0].strip() bill = Bill(session, chamber, bill_id, bill_title) bill.add_version("Most Recent Version", session_url(session) + version_url) bill.add_source(bill_info_url) sponsor_links = bill_info.findAll(href=re.compile( 'legislator/[SH]\d+\.htm')) for sponsor_link in sponsor_links: bill.add_sponsor('primary', sponsor_link.contents[0].strip()) action_p = version_link.findAllNext('p')[-1] for action in action_p.findAll(text=True): action = action.strip() if (not action or action == 'last action' or 'Prefiled' in action): continue action_date = action.split('-')[0] action_date = dt.datetime.strptime(action_date, '%b %d') # Fix: action_date = action_date.replace( year=int('20' + session[2:4])) action = '-'.join(action.split('-')[1:]) if action.endswith('House') or action.endswith('(H)'): actor = 'lower' elif action.endswith('Senate') or action.endswith('(S)'): actor = 'upper' else: actor = chamber bill.add_action(actor, action, action_date) vote_link = bill_info.find(href=re.compile('.*/vote_history.pdf')) if vote_link: bill.add_document( 'vote_history.pdf', bill_info_url.replace('.htm', '') + "/vote_history.pdf") self.save_bill(bill)
def scrape_bill(self, chamber, session, billid, histurl, year): if year[0] != 'R': session = year else: session = self.metadata['session_details'][year][ 'sub_sessions'][int(year[0]) - 1] with self.urlopen(histurl) as data: soup = BeautifulSoup(cleansource(data)) basicinfo = soup.findAll('div', id='bhistleft')[0] hist = basicinfo.table sponsor = None title = None for b in basicinfo.findAll('b'): if b.next.startswith('SUMMARY'): title = b.findNextSiblings(text=True)[0].strip() elif b.next.startswith('SPONSOR'): for a in b.findNextSiblings('a'): if not issponsorlink(a): break sponsor = cleansponsor(a.contents[0]) bill = Bill(session, chamber, billid, title) if sponsor: bill.add_sponsor('primary', sponsor) for row in hist.findAll('tr'): link = row.td.a vlink = urlbase % link['href'] vname = link.contents[0].strip() bill.add_version(vname, vlink) history = soup.findAll('div', id='bhisttab')[0].table rows = history.findAll('tr')[1:] for row in rows: tds = row.findAll('td') if len(tds) < 2: # This is not actually an action continue date, action = row.findAll('td')[:2] date = dt.datetime.strptime(date.contents[0], '%m/%d/%y') action = action.contents[0].strip() if 'House' in action: actor = 'lower' elif 'Senate' in action: actor = 'upper' else: # for lack of a better actor = chamber bill.add_action(actor, action, date) self.save_bill(bill)
def get_bill_info(self, chamber, session, bill_detail_url, version_list_url): """Extracts all the requested info for a given bill. Calls the parent's methods to enter the results into JSON files. """ if chamber == "House": chamber = 'lower' else: chamber = 'upper' with self.urlopen(bill_detail_url) as bill_html: bill_soup = BeautifulSoup(bill_html) bill_id = self.extract_bill_id(bill_soup) bill_title = self.extract_bill_title(bill_soup) bill = Bill(session, chamber, bill_id, bill_title) # Get all versions of the bill. # Versions of a bill are on a separate page, linked to from the column # labeled, "Bill Text", on the search results page. with self.urlopen(version_list_url) as version_html: version_soup = BeautifulSoup(version_html) # MN bills can have multiple versions. Get them all, and loop over # the results, adding each one. self.debug("Extracting bill versions from: " + version_list_url) bill_versions = self.extract_bill_versions(version_soup) for version in bill_versions: version_name = version['name'] version_url = urlparse.urljoin(VERSION_URL_BASE, version['url']) bill.add_version(version_name, version_url) # grab primary and cosponsors # MN uses "Primary Author" to name a bill's primary sponsor. # Everyone else listed will be added as a 'cosponsor'. sponsors = self.extract_bill_sponsors(bill_soup) primary_sponsor = sponsors[0] cosponsors = sponsors[1:] bill.add_sponsor('primary', primary_sponsor) for leg in cosponsors: bill.add_sponsor('cosponsor', leg) # Add Actions performed on the bill. bill_actions = self.extract_bill_actions(bill_soup, chamber) for action in bill_actions: action_chamber = action['action_chamber'] action_date = action['action_date'] action_text = action['action_text'] bill.add_action(action_chamber, action_text, action_date) self.save_bill(bill)
def scrape1995(self, url, year, chamberName, session, number): "e.g. http://www.legis.ga.gov/legis/1995_96/leg/sum/sb1.htm" with self.lxml_context(url) as page: # Bill name = page.cssselect('h3 br')[0].tail.split('-', 1)[1].strip() bill = Bill(session, chamberName, number, name) # Versions bill.add_version('Current', url.replace('/sum/', '/fulltext/')) # Sponsorships rows = page.cssselect('center table tr') for row in rows: if row.text_content().strip() == 'Sponsor and CoSponsors': continue if row.text_content().strip() == 'Links / Committees / Status': break for a in row.cssselect('a'): bill.add_sponsor('', a.text_content().strip()) # Actions # The actions are in a pre table that looks like: """ SENATE HOUSE ------------------------------------- 1/13/95 Read 1st time 2/6/95 1/31/95 Favorably Reported 2/1/95 Read 2nd Time 2/7/95 2/3/95 Read 3rd Time 2/3/95 Passed/Adopted """ actions = page.cssselect('pre')[0].text_content().split('\n') actions = actions[2:] for action in actions: senate_date = action[:22].strip() action_text = action[23:46].strip() house_date = action[46:].strip() if '/' not in senate_date and '/' not in house_date: continue if senate_date: bill.add_action('upper', action_text, senate_date) if house_date: bill.add_action('lower', action_text, house_date) self.save_bill(bill)
def get_bill_info(self, chamber, session, bill_detail_url, version_list_url): """Extracts all the requested info for a given bill. Calls the parent's methods to enter the results into JSON files. """ if chamber == "House": chamber = 'lower' else: chamber = 'upper' with self.urlopen(bill_detail_url) as bill_html: doc = lxml.html.fromstring(bill_html) bill_id = doc.xpath('//title/text()')[0].split()[0] bill_title = doc.xpath('//font[@size=-1]/text()')[0] bill_type = {'F': 'bill', 'R':'resolution', 'C': 'concurrent resolution'}[bill_id[1]] bill = Bill(session, chamber, bill_id, bill_title, type=bill_type) bill.add_source(bill_detail_url) # grab sponsors sponsors = doc.xpath('//table[@summary="Show Authors"]/descendant::a/text()') if sponsors: primary_sponsor = sponsors[0].strip() bill.add_sponsor('primary', primary_sponsor) cosponsors = sponsors[1:] for leg in cosponsors: bill.add_sponsor('cosponsor', leg.strip()) # Add Actions performed on the bill. bill_actions = self.extract_bill_actions(doc, chamber) for action in bill_actions: bill.add_action(action['action_chamber'], action['action_text'], action['action_date'], type=action['action_type']) # Get all versions of the bill. # Versions of a bill are on a separate page, linked to from the column # labeled, "Bill Text", on the search results page. with self.urlopen(version_list_url) as version_html: version_doc = lxml.html.fromstring(version_html) for v in version_doc.xpath('//a[starts-with(@href, "/bin/getbill.php")]'): version_url = urlparse.urljoin(VERSION_URL_BASE, v.get('href')) bill.add_version(v.text.strip(), version_url) self.save_bill(bill)
def scrape(self, chamber, session): self.validate_session(session) if chamber == 'upper': other_chamber = 'lower' bill_id = 'SB 1' else: other_chamber = 'upper' bill_id = 'HB 1' b1 = Bill(session, chamber, bill_id, 'A super bill') b1.add_source('http://example.com/') b1.add_version('As Introduced', 'http://example.com/SB1.html') b1.add_document('Google', 'http://google.com') b1.add_sponsor('primary', 'Bob Smith') b1.add_sponsor('secondary', 'Johnson, Sally') d1 = datetime.datetime.strptime('1/29/2010', '%m/%d/%Y') v1 = Vote('upper', d1, 'Final passage', True, 2, 0, 0) v1.yes('Smith') v1.yes('Johnson') d2 = datetime.datetime.strptime('1/30/2010', '%m/%d/%Y') v2 = Vote('lower', d2, 'Final passage', False, 0, 1, 1) v2.no('Bob Smith') v2.other('S. Johnson') b1.add_vote(v1) b1.add_vote(v2) b1.add_action(chamber, 'introduced', d1) b1.add_action(chamber, 'read first time', d2) b1.add_action(other_chamber, 'introduced', d2) self.save_bill(b1)
def parse_bill(scraper, url): """Given a bill status URL, return a fully loaded Bill object, except for votes, which are expected to be handled externally. """ session = extract_session(url) chamber = chamber_for_doctype(extract_doctype(url)) s = get_soup(scraper, url) bill_id = extract_bill_id(s) landmark = s(text=re.compile(".*Short Description.*")) name_span = landmark[0].findParent().findNextSibling() bill_name = get_text(name_span) bill = Bill(session, chamber, bill_id, bill_name.strip(),status_url=url) actions = extract_actions(s) for chamber,action,date in actions: bill.add_action(chamber,action,date) #kwargs are permitted if we have 'em. sponsor_dict = extract_sponsors_from_actions([action[1] for action in actions]) for type,namelist in sponsor_dict.iteritems(): for name in namelist: bill.add_sponsor(type,name) for name,link in extract_versions(scraper, s): bill.add_version(name,link) return bill
def scrape_bill(self, chamber, session, bill_number, ga_num): bill_url = self.urls['info'] % (bill_number, ga_num) with self.urlopen(bill_url) as page: page = lxml.html.fromstring(page) title = page.xpath("//span[@id='lblAbstract']")[0].text bill = Bill(session, chamber, bill_number, title) bill.add_source(bill_url) # Primary Sponsor sponsor = page.xpath("//span[@id='lblBillSponsor']")[0].text_content().split("by")[-1] sponsor = sponsor.replace('*','').strip() bill.add_sponsor('primary',sponsor) # Co-sponsors unavailable for scraping (loaded into page via AJAX) # Full summary doc summary = page.xpath("//span[@id='lblBillSponsor']/a")[0] bill.add_document('Full summary', summary.get('href')) # Actions tables = page.xpath("//table[@id='tabHistoryAmendments_tabHistory_gvBillActionHistory']") actions_table = tables[0] action_rows = actions_table.xpath("tr[position()>1]") for ar in action_rows: action_taken = ar.xpath("td")[0].text action_date = datetime.datetime.strptime(ar.xpath("td")[1].text.strip(), '%m/%d/%Y') bill.add_action(chamber, action_taken, action_date) votes_link = page.xpath("//span[@id='lblBillVotes']/a") if(len(votes_link) > 0): votes_link = votes_link[0].get('href') bill = self.scrape_votes(bill, sponsor, 'http://wapp.capitol.tn.gov/apps/Billinfo/%s' % (votes_link,)) self.save_bill(bill)
def parse_bill_xml(self, chamber, session, txt): root = lxml.etree.fromstring(txt) bill_id = ' '.join(root.attrib['bill'].split(' ')[1:]) bill_title = root.findtext("caption") if session[2] == 'R': session = session[0:2] bill = Bill(session, chamber, bill_id, bill_title) for action in root.findall('actions/action'): act_date = dt.datetime.strptime(action.findtext('date'), "%m/%d/%Y") extra = {} extra['action_number'] = action.find('actionNumber').text comment = action.find('comment') if comment is not None and comment.text: extra['comment'] = comment.text.strip() actor = {'H': 'lower', 'S': 'upper', 'E': 'executive'}[extra['action_number'][0]] desc = action.findtext('description').strip() if desc == 'Amended': type = 'amendment:passed' elif desc == 'Amendment(s) offered': type = 'amendment:introduced' elif desc == 'Amendment amended': type = 'amendment:amended' elif desc == 'Amendment withdrawn': type = 'amendment:withdrawn' elif desc.startswith('Received by the Secretary of'): type = 'bill:introduced' elif desc == 'Passed': type = 'bill:passed' elif desc.startswith('Received from the'): type = 'bill:introduced' elif desc.startswith('Signed by the Governor'): type = 'bill:signed' elif desc == 'Filed': type = 'bill:introduced' else: type = 'other' bill.add_action(actor, action.findtext('description'), act_date, type=type, **extra) for author in root.findtext('authors').split(' | '): if author != "": bill.add_sponsor('author', author) for coauthor in root.findtext('coauthors').split(' | '): if coauthor != "": bill.add_sponsor('coauthor', coauthor) for sponsor in root.findtext('sponsors').split(' | '): if sponsor != "": bill.add_sponsor('sponsor', sponsor) for cosponsor in root.findtext('cosponsors').split(' | '): if cosponsor != "": bill.add_sponsor('cosponsor', cosponsor) bill['subjects'] = [] for subject in root.iterfind('subjects/subject'): bill['subjects'].append(subject.text.strip()) return bill
def scrape_session_old(self, chamber, session): if chamber == "lower": bill_abbr = "H." chamber_name = "House" other_chamber = "Senate" else: bill_abbr = "S." chamber_name = "Senate" other_chamber = "House" start_date = '1/1/%s' % session.split('-')[0] data = urllib.urlencode({'Date': start_date, 'Body': bill_abbr[0], 'Session': session.split('-')[1]}) bill_list_url = "http://www.leg.state.vt.us/database/"\ "rintro/results.cfm" bill_list = BeautifulSoup(urllib2.urlopen(bill_list_url, data)) bill_link_re = re.compile('.*?Bill=%s.\d+.*' % bill_abbr[0]) for bill_link in bill_list.findAll('a', href=bill_link_re): bill_id = bill_link.string bill_title = bill_link.parent.parent.findAll('td')[1].string bill_info_url = "http://www.leg.state.vt.us" + bill_link['href'] bill = Bill(session, chamber, bill_id, bill_title) bill.add_source(bill_info_url) info_page = BeautifulSoup(self.urlopen(bill_info_url)) text_links = info_page.findAll('blockquote')[-1].findAll('a') for text_link in text_links: bill.add_version(text_link.string, "http://www.leg.state.vt.us" + text_link['href']) sponsors = info_page.find( text='Sponsor(s):').parent.findNext('td').findAll('b') bill.add_sponsor('primary', sponsors[0].string) for sponsor in sponsors[1:]: bill.add_sponsor('cosponsor', sponsor.string) # Grab actions from the originating chamber act_table = info_page.find( text='%s Status:' % chamber_name).findNext('table') for row in act_table.findAll('tr')[3:]: action = clean_action(row.td.string.replace( ' ', '').strip(':')) act_date = row.findAll('td')[1].b.string.replace(' ', '') if act_date != "": detail = row.findAll('td')[2].b if detail and detail.string != "": action += ": %s" % detail.string.replace(' ', '') bill.add_action(chamber, action, act_date, type=action_type(action)) # Grab actions from the other chamber act_table = info_page.find( text='%s Status:' % other_chamber).findNext('table') if act_table: if chamber == 'upper': act_chamber = 'lower' else: act_chamber = 'upper' for row in act_table.findAll('tr')[3:]: action = clean_action(row.td.string.replace( ' ', '').strip(':')) act_date = row.findAll('td')[1].b.string.replace( ' ', '') if act_date != "": detail = row.findAll('td')[2].b if detail and detail.string != "": action += ": %s" % detail.string.replace( ' ', '') date = dt.datetime.strptime(act_date, '%m/%d/%Y') bill.add_action(act_chamber, action, act_date, type=action_type(action)) self.save_bill(bill)
def scrape_session_new(self, chamber, session): if chamber == "lower": bill_abbr = "H." else: bill_abbr = "S." bill_list_path = "docs/bills.cfm?Session=%s&Body=%s" % ( session.split('-')[1], bill_abbr[0]) bill_list_url = "http://www.leg.state.vt.us/" + bill_list_path bill_list = BeautifulSoup(self.urlopen(bill_list_url)) bill_link_re = re.compile('.*?Bill=%s\.\d+.*' % bill_abbr[0]) for bill_link in bill_list.findAll('a', href=bill_link_re): bill_id = bill_link.string bill_title = bill_link.parent.findNext('b').string bill_info_url = "http://www.leg.state.vt.us" + bill_link['href'] bill = Bill(session, chamber, bill_id, bill_title) bill.add_source(bill_info_url) info_page = BeautifulSoup(self.urlopen(bill_info_url)) text_links = info_page.findAll('blockquote')[1].findAll('a') for text_link in text_links: bill.add_version(text_link.string, "http://www.leg.state.vt.us" + text_link['href']) act_table = info_page.findAll('blockquote')[2].table for row in act_table.findAll('tr')[1:]: action = "" for s in row.findAll('td')[1].findAll(text=True): action += s + " " action = clean_action(action) match = re.search('Governor on (.*)$', action) if match: act_date = parse_exec_date(match.group(1).strip()) actor = 'Governor' else: if row['bgcolor'] == 'Salmon': actor = 'lower' else: actor = 'upper' if row.td.a: act_date = row.td.a.string else: act_date = row.td.string try: act_date = re.search( '\d{1,2}/\d{1,2}/\d{4,4}', act_date).group(0) except AttributeError: # No date, skip continue act_date = dt.datetime.strptime(act_date, '%m/%d/%Y') bill.add_action(actor, action, act_date, type=action_type(action)) vote_link = row.find('a', text='Details') if vote_link: vote_url = vote_link.parent['href'] self.parse_vote_new(bill, actor, vote_url) sponsors = info_page.find( text='Sponsor(s):').parent.parent.findAll('b') bill.add_sponsor('primary', sponsors[0].string) for sponsor in sponsors[1:]: bill.add_sponsor('cosponsor', sponsor.string) self.save_bill(bill)
def scrape(self, chamber, session): bill_search_url = 'http://www.camaraderepresentantes.org/cr_buscar.asp' bill_types = {'Project':'P', 'Resolution':'R', \ 'Joint Resolution':'RC', \ 'Concurrent Resolution':'RK', \ 'Appointment':'N'} #bodies = {'upper':'S', 'lower':'C'} bodies = {'upper':'S'} bill_search_page = lxml.html.parse(bill_search_url).getroot() search_form = bill_search_page.forms[0] for body in bodies.itervalues(): for bill_type in bill_types.itervalues(): search_form.fields['cuerpo'] = body search_form.fields['tipo'] = bill_type search_form.fields['autor'] = 'NA' if year_from_session(session) == '2009': search_form.fields['f2'] = '12/31/2009' elif year_from_session(session) == '2010': search_form.fields['f1'] = '01/01/2010' result = lxml.html.parse(lxml.html.submit_form(search_form)).getroot() table_elements = result.cssselect('table') table_elements.pop() bill_elements = grouper(3, table_elements) for actions, complete_data, bill_data in bill_elements: td_elements = bill_data.cssselect('td') title = td_elements[1].text_content() date = td_elements[3].text_content() description = td_elements[5].text_content() authors = td_elements[7].text_content().split('/') bill = Bill(session, chamber, title, description) for author in authors: if len(authors) == 1: bill.add_sponsor('primary', author) else: bill.add_sponsor('cosponsor', author) td_elements = actions.cssselect('td') td_elements = td_elements[4:-1] action_elements = grouper(3, td_elements) for date_element, action, empty in action_elements: # Clean unicode character date_text = date_element.text_content().replace(u'\xa0',u'') date = dt.datetime.strptime(date_text, '%m/%d/%Y') action_text = action.text_content() try: doc_link_part = action.iterlinks().next()[2] if 'voto' in doc_link_part: raise doc_link = doc_link_url(doc_link_part) bill.add_version((action_text, doc_link)) except: pass bill.add_action(chamber, action_text, date)
def scrape_session(self, chamber, year): if chamber == 'upper': bill_abbr = 'SB|SCR|SJR' elif chamber == 'lower': bill_abbr = 'HB|HCR|HJR' # Sessions last 2 years, 1993-1994 was the 18th session = str(18 + ((int(year) - 1993) / 2)) year2 = str(int(year) + 1) # Full calendar year date1 = '0101' + year[2:] date2 = '1231' + year2[2:] # Get bill list bill_list_url = 'http://www.legis.state.ak.us/'\ 'basis/range_multi.asp?session=%s&date1=%s&date2=%s' % ( session, date1, date2) self.log("Getting bill list for %s %s (this may take a long time)." % (chamber, session)) bill_list = self.soup_parser(self.urlopen(bill_list_url)) # Find bill links re_str = "bill=%s\d+" % bill_abbr links = bill_list.findAll(href=re.compile(re_str)) for link in links: bill_id = link.contents[0].replace(' ', '') bill_name = link.parent.parent.findNext('td').find( 'font').contents[0].strip() bill = Bill(session, chamber, bill_id, bill_name.strip()) # Get the bill info page and strip malformed t info_url = "http://www.legis.state.ak.us/basis/%s" % link['href'] info_page = self.soup_parser(self.urlopen(info_url)) bill.add_source(info_url) # Get sponsors spons_str = info_page.find( text="SPONSOR(s):").parent.parent.contents[1] sponsors_match = re.match( ' (SENATOR|REPRESENTATIVE)\([Ss]\) ([^,]+(,[^,]+){0,})', spons_str) if sponsors_match: sponsors = sponsors_match.group(2).split(',') bill.add_sponsor('primary', sponsors[0].strip()) for sponsor in sponsors[1:]: bill.add_sponsor('cosponsor', sponsor.strip()) else: # Committee sponsorship bill.add_sponsor('committee', spons_str.strip()) # Get actions act_rows = info_page.findAll('table', 'myth')[1].findAll('tr')[1:] for row in act_rows: cols = row.findAll('td') act_date = cols[0].font.contents[0] act_date = dt.datetime.strptime(act_date, '%m/%d/%y') if cols[2].font.string == "(H)": act_chamber = "lower" elif cols[2].font.string == "(S)": act_chamber = "upper" else: act_chamber = chamber action = cols[3].font.contents[0].strip() if re.match("\w+ Y(\d+) N(\d+)", action): try: vote = self.parse_vote(bill, action, act_chamber, act_date, cols[1].a['href']) bill.add_vote(vote) except: self.log("Failed parsing vote at %s" % cols[1].a['href']) bill.add_action(act_chamber, action, act_date) # Get subjects bill['subjects'] = [] subject_link_re = re.compile('.*subject=\w+$') for subject_link in info_page.findAll('a', href=subject_link_re): subject = subject_link.contents[0].strip() bill['subjects'].append(subject) # Get versions text_list_url = "http://www.legis.state.ak.us/"\ "basis/get_fulltext.asp?session=%s&bill=%s" % ( session, bill_id) text_list = self.soup_parser(self.urlopen(text_list_url)) bill.add_source(text_list_url) text_link_re = re.compile('^get_bill_text?') for text_link in text_list.findAll('a', href=text_link_re): text_name = text_link.parent.previousSibling.contents[0] text_name = text_name.strip() text_url = "http://www.legis.state.ak.us/basis/%s" % ( text_link['href']) bill.add_version(text_name, text_url) self.save_bill(bill)
def scrape_bill_type(self, chamber, session, bill_type, type_abbr): if chamber == 'upper': chamber_name = 'SENATE' else: chamber_name = 'ASSEMBLY' bills = self.session.query(CABill).filter_by( session_year=session).filter_by( measure_type=type_abbr) for bill in bills: bill_session = session if bill.session_num != '0': bill_session += ' Special Session %s' % bill.session_num bill_id = bill.short_bill_id fsbill = Bill(bill_session, chamber, bill_id, '') title = '' short_title = '' type = ['bill'] subject = '' for version in self.session.query(CABillVersion).filter_by( bill=bill).filter(CABillVersion.bill_xml != None): title = version.title short_title = version.short_title type = [bill_type] if version.appropriation == 'Yes': type.append('appropriation') if version.fiscal_committee == 'Yes': type.append('fiscal committee') if version.local_program == 'Yes': type.append('local program') if version.urgency == 'Yes': type.append('urgency') if version.taxlevy == 'Yes': type.append('tax levy') subject = version.subject fsbill.add_version(version.bill_version_id, '', date=version.bill_version_action_date, title=version.title, short_title=version.short_title, subject=[subject], type=type) if not title: self.warning("Couldn't find title for %s, skipping" % bill_id) continue fsbill['title'] = title fsbill['short_title'] = short_title fsbill['type'] = type fsbill['subjects'] = [subject] for author in version.authors: if author.house == chamber_name: fsbill.add_sponsor(author.contribution, author.name) for action in bill.actions: if not action.action: # NULL action text seems to be an error on CA's part, # unless it has some meaning I'm missing continue actor = action.actor or chamber actor = actor.strip() match = re.match(r'(Assembly|Senate)($| \(Floor)', actor) if match: actor = {'Assembly': 'lower', 'Senate': 'upper'}[match.group(1)] elif actor.startswith('Governor'): actor = 'executive' else: actor = re.sub('^Assembly', 'lower', actor) actor = re.sub('^Senate', 'upper', actor) type = [] act_str = action.action if act_str.startswith('Introduced'): type.append('bill:introduced') if 'To Com' in act_str: type.append('committee:referred') if 'Read third time. Passed.' in act_str: type.append('bill:passed') if 'Approved by Governor' in act_str: type.append('governor:signed') if 'Item veto' in act_str: type.append('governor:vetoed:line-item') if not type: type = ['other'] fsbill.add_action(actor, act_str, action.action_date, type=type) for vote in bill.votes: if vote.vote_result == '(PASS)': result = True else: result = False full_loc = vote.location.description first_part = full_loc.split(' ')[0].lower() if first_part in ['asm', 'assembly']: vote_chamber = 'lower' vote_location = ' '.join(full_loc.split(' ')[1:]) elif first_part.startswith('sen'): vote_chamber = 'upper' vote_location = ' '.join(full_loc.split(' ')[1:]) else: raise ScrapeError("Bad location: %s" % full_loc) motion = vote.motion.motion_text or '' if "Third Reading" in motion or "3rd Reading" in motion: vtype = 'passage' elif "Do Pass" in motion: vtype = 'passage' else: vtype = 'other' motion = motion.strip() # Why did it take until 2.7 to get a flags argument on re.sub? motion = re.compile(r'(\w+)( Extraordinary)? Session$', re.IGNORECASE).sub('', motion) motion = re.compile(r'^(Senate|Assembly) ', re.IGNORECASE).sub('', motion) motion = re.sub(r'^(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? ', '', motion) motion = re.sub(r' \(\w+\)$', '', motion) motion = re.sub(r'(SCR|SB|AB|AJR|ACR)\s?\d+ \w+\.?$', '', motion) motion = re.sub(r'(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? ' r'Urgency Clause$', '(Urgency Clause)', motion) motion = re.sub(r'\s+', ' ', motion) if not motion: self.warning("Got blank motion on vote for %s" % bill_id) continue fsvote = Vote(vote_chamber, vote.vote_date_time, motion, result, int(vote.ayes), int(vote.noes), int(vote.abstain), threshold=vote.threshold, type=vtype) if vote_location != 'Floor': fsvote['committee'] = vote_location for record in vote.votes: if record.vote_code == 'AYE': fsvote.yes(record.legislator_name) elif record.vote_code.startswith('NO'): fsvote.no(record.legislator_name) else: fsvote.other(record.legislator_name) fsbill.add_vote(fsvote) self.save_bill(fsbill)
def parse_bill_xml(self, chamber, session, txt): root = lxml.etree.fromstring(txt) bill_id = ' '.join(root.attrib['bill'].split(' ')[1:]) bill_title = root.findtext("caption") if session[2] == 'R': session = session[0:2] if bill_id[1] == 'B': bill_type = ['bill'] elif bill_id[1] == 'R': bill_type = ['resolution'] elif bill_id[1:3] == 'CR': bill_type = ['concurrent resolution'] elif bill_id[1:3] == 'JR': bill_type = ['joint resolution'] else: raise ScrapeError("Invalid bill_id: %s" % bill_id) bill = Bill(session, chamber, bill_id, bill_title, type=bill_type) for action in root.findall('actions/action'): act_date = dt.datetime.strptime(action.findtext('date'), "%m/%d/%Y").date() extra = {} extra['action_number'] = action.find('actionNumber').text comment = action.find('comment') if comment is not None and comment.text: extra['comment'] = comment.text.strip() actor = {'H': 'lower', 'S': 'upper', 'E': 'executive'}[extra['action_number'][0]] desc = action.findtext('description').strip() if desc == 'Amended': type = 'amendment:passed' elif desc == 'Amendment(s) offered': type = 'amendment:introduced' elif desc == 'Amendment amended': type = 'amendment:amended' elif desc == 'Amendment withdrawn': type = 'amendment:withdrawn' elif desc == 'Passed': type = 'bill:passed' elif desc.startswith('Received from the'): type = 'bill:introduced' elif desc.startswith('Sent to the Governor'): # But what if it gets lost in the mail? type = 'governor:received' elif desc.startswith('Signed by the Governor'): type = 'governor:signed' elif desc == 'Read first time': type = 'bill:introduced' else: type = 'other' bill.add_action(actor, action.findtext('description'), act_date, type=type, **extra) for author in root.findtext('authors').split(' | '): if author != "": bill.add_sponsor('author', author) for coauthor in root.findtext('coauthors').split(' | '): if coauthor != "": bill.add_sponsor('coauthor', coauthor) for sponsor in root.findtext('sponsors').split(' | '): if sponsor != "": bill.add_sponsor('sponsor', sponsor) for cosponsor in root.findtext('cosponsors').split(' | '): if cosponsor != "": bill.add_sponsor('cosponsor', cosponsor) bill['subjects'] = [] for subject in root.iterfind('subjects/subject'): bill['subjects'].append(subject.text.strip()) return bill
def scrape_bill_pages(self, session, year_abr): #Main Bill information main_bill_url = 'ftp://www.njleg.state.nj.us/ag/%sdata/MAINBILL.DBF' % (year_abr) MAINBILL_dbf, resp = self.urlretrieve(main_bill_url) main_bill_db = dbf.Dbf(MAINBILL_dbf) bill_dict = {} for rec in main_bill_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) title = rec["synopsis"] if bill_type[0] == 'A': chamber = "lower" else: chamber = "upper" bill = Bill(str(session), chamber, bill_id, title) bill.add_source(main_bill_url) bill_dict[bill_id] = bill #Sponsors bill_sponsors_url = 'ftp://www.njleg.state.nj.us/ag/%sdata/BILLSPON.DBF' % (year_abr) SPONSORS_dbf, resp = self.urlretrieve(bill_sponsors_url) bill_sponsors_db = dbf.Dbf(SPONSORS_dbf) for rec in bill_sponsors_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) bill = bill_dict[bill_id] name = rec["sponsor"] sponsor_type = rec["type"] if sponsor_type == 'P': sponsor_type = "Primary" else: sponsor_type = "Co-sponsor" bill.add_sponsor(sponsor_type, name) #Documents bill_document_url = 'ftp://www.njleg.state.nj.us/ag/%sdata/BILLWP.DBF' % (year_abr) DOC_dbf, resp = self.urlretrieve(bill_document_url) bill_document_db = dbf.Dbf(DOC_dbf) #print bill_document_db[2] for rec in bill_document_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) bill = bill_dict[bill_id] document = rec["document"] document = document.split('\\') doc_name = document[-1] document = document[-2] + "/" + document[-1] year = str(year_abr) + str((year_abr + 1)) doc_url = "ftp://www.njleg.state.nj.us/%s" % year doc_url = doc_url + "/" + document bill.add_document(doc_name, doc_url) #Senate Votes file1 = 'A' + str(year_abr) file2 = 'A' + str(year_abr + 1) file3 = 'S' + str(year_abr) file4 = 'S' + str(year_abr + 1) if str(year_abr) != '2010': vote_info_list = [file1, file2, file3, file4] else: vote_info_list = [file1, file3] for bill_vote_file in vote_info_list: s_vote_url = 'ftp://www.njleg.state.nj.us/votes/%s.zip' % bill_vote_file s_vote_zip, resp = self.urlretrieve(s_vote_url) zipedfile = zipfile.ZipFile(s_vote_zip) vfile = "%s.txt" % bill_vote_file vote_file = zipedfile.open(vfile, 'U') vdict_file = csv.DictReader(vote_file) votes = {} if bill_vote_file[0] == "A": chamber = "lower" else: chamber = "upper" for rec in vdict_file: bill_id = rec["Bill"] bill_id = bill_id.strip() leg = rec["Full_Name"] date = rec["Session_Date"] date = datetime.strptime(date, "%m/%d/%Y") action = rec["Action"] leg_vote = rec["Legislator_Vote"] vote_id = bill_id + "_" + action vote_id = vote_id.replace(" ", "_") passed = None if vote_id not in votes: votes[vote_id] = Vote(chamber, date, action, passed, None, None, None, bill_id = bill_id) if leg_vote == "Y": votes[vote_id].yes(leg) elif leg_vote == "N": votes[vote_id].no(leg) else: votes[vote_id].other(leg) #Counts yes/no/other votes and saves overall vote for vote in votes.itervalues(): vote_yes_count = len(vote["yes_votes"]) vote_no_count = len(vote["no_votes"]) vote_other_count = len(vote["other_votes"]) vote["yes_count"] = vote_yes_count vote["no_count"] = vote_no_count vote["other_count"] = vote_other_count if vote_yes_count > vote_no_count: vote["passed"] = True else: vote["passed"] = False vote_bill_id = vote["bill_id"] bill = bill_dict[vote_bill_id] bill.add_vote(vote) #Actions bill_action_url = 'ftp://www.njleg.state.nj.us/ag/%sdata/BILLHIST.DBF' % (year_abr) ACTION_dbf, resp = self.urlretrieve(bill_action_url) bill_action_db = dbf.Dbf(ACTION_dbf) bill.add_source(bill_sponsors_url) bill.add_source(bill_document_url) bill.add_source(bill_action_url) for rec in bill_action_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) bill = bill_dict[bill_id] action = rec["action"] date = rec["dateaction"] actor = rec["house"] comment = rec["comment"] bill.add_action(actor, action, date, comment = comment) self.save_bill(bill)
def scrape_bill(self, chamber, session, url): url = url + "&Year=%s" % session with self.urlopen(url) as page: page = page.replace(' ', ' ').replace('<br>', '\n') page = lxml.html.fromstring(page) page.make_links_absolute(url) title = page.xpath('//h3')[0].text.strip() title = re.match(r"^\w+\s+\d+:\s+(.*)$", title).group(1) bill_id = page.xpath("string(//pre[@class='billhistory']/b)") bill_id = bill_id.split()[0].strip() bill = Bill(session, chamber, bill_id, title) bill.add_source(url) hist = page.xpath("string(//pre[@class='billhistory'])").strip() act_re = re.compile(r'^ (\d\d/\d\d/\d\d) (SENATE|HOUSE)' r'(.*\n(\s{16,16}.*\n){0,})', re.MULTILINE) # Actions for match in act_re.finditer(hist): action = match.group(3).replace('\n', ' ') action = re.sub(r'\s+', ' ', action).strip() if match.group(2) == 'SENATE': actor = 'upper' else: actor = 'lower' date = match.group(1) date = datetime.datetime.strptime(date, "%m/%d/%y") for act_text in re.split(' -[HS]J \d+;? ?', action): act_text = act_text.strip() if not act_text: continue types = [] act_lower = act_text.lower() if act_lower.startswith('introduced'): types.append('bill:introduced') if 'referred to' in act_lower: types.append('committee:referred') if 'died in committee' in act_lower: types.append('committee:failed') if 'favorable by' in act_lower: types.append('committee:passed:favorable') if 'amendment(s) adopted' in act_lower: types.append('amendment:passed') bill.add_action(actor, act_text, date, type=types) # Sponsors primary_sponsor = re.search(r'by ([^;(\n]+;?|\w+)', hist).group(1).strip('; ') bill.add_sponsor('primary', primary_sponsor) cospon_re = re.compile(r'\((CO-SPONSORS|CO-AUTHORS)\) ' '([\w .]+(;[\w .\n]+){0,})', re.MULTILINE) match = cospon_re.search(hist) if match: for cosponsor in match.group(2).split(';'): cosponsor = cosponsor.replace('\n', '').strip() bill.add_sponsor('cosponsor', cosponsor) # Versions for link in page.xpath("//a[contains(@href, 'billtext/html')]"): version = link.xpath('string(../../td[1])').strip() bill.add_version(version, link.attrib['href']) # House Votes for link in page.xpath("//a[contains(@href, 'votes/html/h')]"): bill.add_vote(self.scrape_lower_vote(link.attrib['href'])) # Senate Votes for link in page.xpath("//a[contains(@href, 'votes/html/S')]"): bill.add_vote(self.scrape_upper_vote(link.attrib['href'])) self.save_bill(bill)
def scrape_new_session(self, chamber, session): """ Scrapes SD's bill data from 2009 on. """ if chamber == 'upper': bill_abbr = 'SB' elif chamber == 'lower': bill_abbr = 'HB' # Get bill list page session_url = 'http://legis.state.sd.us/sessions/%s/' % session bill_list_url = session_url + 'BillList.aspx' self.log('Getting bill list for %s %s' % (chamber, session)) bill_list = self.soup_parser(self.urlopen(bill_list_url)) # Format of bill link contents bill_re = re.compile(u'%s\xa0(\d+)' % bill_abbr) date_re = re.compile('\d{2}/\d{2}/\d{4}') for bill_link in bill_list.findAll('a'): if len(bill_link.contents) == 0: # Empty link continue #print bill_link.contents[0] bill_match = bill_re.search(bill_link.contents[0]) if not bill_match: continue # Parse bill ID and name bill_id = bill_link.contents[0].replace(u'\xa0', ' ') bill_name = bill_link.findNext().contents[0] # Download history page hist_url = session_url + bill_link['href'] history = self.soup_parser(self.urlopen(hist_url)) bill = Bill(session, chamber, bill_id, bill_name) bill.add_source(hist_url) # Get all bill versions text_table = history.findAll('table')[1] for row in text_table.findAll('tr')[2:]: #version_date = row.find('td').string version_path = row.findAll('td')[1].a['href'] version_url = "http://legis.state.sd.us/sessions/%s/%s" % ( session, version_path) version_name = row.findAll('td')[1].a.contents[0].strip() bill.add_version(version_name, version_url) # Get actions act_table = history.find('table') for act_row in act_table.findAll('tr')[6:]: if act_row.find(text='Action'): continue # Get the date (if can't find one then this isn't an action) date_match = date_re.match(act_row.td.a.contents[0]) if not date_match: continue act_date = date_match.group(0) act_date = dt.datetime.strptime(act_date, "%m/%d/%Y") # Get the action string action = "" for node in act_row.findAll('td')[1].contents: if hasattr(node, 'contents'): action += node.contents[0] if node.contents[0].startswith('YEAS'): # This is a vote! vote_url = "http://legis.state.sd.us/sessions/"\ "%s/%s" % (session, node['href']) vote = self.scrape_new_vote(vote_url) vote['date'] = act_date bill.add_vote(vote) else: action += node action = action.strip() # Add action bill.add_action(chamber, action, act_date) self.save_bill(bill)
def scrape_bill(self, chamber, session, bill_id, title, url): with self.urlopen(url) as page: page = lxml.html.fromstring(page) page.make_links_absolute(url) bill = Bill(session, chamber, bill_id, title) bill.add_source(url) actor = chamber for row in page.xpath( "//table[contains(@id, 'BillActions')]/tr")[6:]: action = row.xpath("string(td[2])").strip() if action == 'Action': continue atypes = [] if action.startswith('First read'): atypes.append('bill:introduced') elif action.startswith('Signed by Governor'): atypes.append('governor:signed') match = re.match(r'(.*) Do Pass( Amended)?, (Passed|Failed)', action) if match: if match.group(1) in ['Senate', 'House of Representatives']: first = 'bill' else: first = 'committee' atypes.append("%s:%s" % (first, match.group(3).lower())) if 'referred to' in action.lower(): atypes.append('committee:referred') if 'Motion to amend, Passed Amendment' in action: atypes.append('amendment:introduced') atypes.append('amendment:passed') if 'Veto override, Passed' in action: atypes.append('bill:veto_override:passed') elif 'Veto override, Failed' in action: atypes.append('bill:veto_override:failed') match = re.match("First read in (Senate|House)", action) if match: if match.group(1) == 'Senate': actor = 'upper' else: actor = 'lower' date = row.xpath("string(td[1])").strip() match = re.match('\d{2}/\d{2}/\d{4}', date) if not match: self.warning("Bad date: %s" % date) continue date = datetime.datetime.strptime(date, "%m/%d/%Y").date() for link in row.xpath("td[2]/a[contains(@href, 'RollCall')]"): self.scrape_vote(bill, date, link.attrib['href']) bill.add_action(actor, action, date, type=atypes) self.save_bill(bill)
def scrape_old_session(self, chamber, session): """ Scrape SD's bill data from 1997 through 2008. """ if chamber == 'upper': bill_abbr = 'SB' else: bill_abbr = 'HB' # Get bill list page (and replace malformed tags that some versions of # BeautifulSoup choke on) session_url = 'http://legis.state.sd.us/sessions/%s/' % session bill_list_url = session_url + 'billlist.htm' bill_list = self.soup_parser(self.urlopen(bill_list_url)) # Bill and text link formats bill_re = re.compile('%s (\d+)' % bill_abbr) text_re = re.compile('/sessions/%s/bills/%s.*\.htm' % ( session, bill_abbr), re.IGNORECASE) date_re = re.compile('\d{2}/\d{2}/\d{4}') for bill_link in bill_list.findAll('a', href=re.compile('\d\.htm$')): if len(bill_link.contents) == 0: # Empty link continue bill_match = bill_re.match(bill_link.contents[0]) if not bill_match: # Not bill link continue # Get the bill ID and name bill_id = bill_link.contents[0] bill_name = bill_link.findNext().contents[0] # Get history page (replacing malformed tag) hist_url = session_url + bill_link['href'] history = self.soup_parser(self.urlopen(hist_url)) # Get URL of latest verion of bill (should be listed last) bill_url = history.findAll('a', href=text_re)[-1]['href'] bill_url = 'http://legis.state.sd.us%s' % bill_url # Add bill bill = Bill(session, chamber, bill_id, bill_name) bill.add_source(hist_url) # Get bill versions text_table = history.findAll('table')[1] for row in text_table.findAll('tr')[2:]: #version_date = row.find('td').string version_path = row.findAll('td')[1].a['href'] version_url = "http://legis.state.sd.us" + version_path version_name = row.findAll('td')[1].a.contents[0].strip() bill.add_version(version_name, version_url) # Get actions act_table = history.find('table') for act_row in act_table.findAll('tr')[6:]: if act_row.find(text="Action"): continue # Get the date (if can't find one then this isn't an action) date_match = date_re.match(act_row.td.a.contents[0]) if not date_match: continue act_date = date_match.group(0) act_date = dt.datetime.strptime(act_date, "%m/%d/%Y") # Get the action string action = "" for node in act_row.findAll('td')[1].contents: if hasattr(node, 'contents'): action += node.contents[0] if node.contents[0].startswith('YEAS'): # This is a vote! if node['href'][0] == '/': vote_url = "http://legis.state.sd.us/%s" % ( node['href']) else: vote_url = "http://legis.state.sd.us/"\ "sessions/%s/%s" % (session, node['href']) vote = self.scrape_old_vote(vote_url) vote['date'] = act_date bill.add_vote(vote) else: action += node action = action.strip() # Add action bill.add_action(chamber, action, act_date) self.save_bill(bill)
def scrape_bill_info(self, session, ld, session_id, bill_id, title): bill_info_url = 'http://www.mainelegislature.org/LawMakerWeb/summary.asp?LD=%s&SessionID=%s' % (ld, session_id) with self.urlopen(bill_info_url) as bill_sum_page: root = lxml.etree.fromstring(bill_sum_page, lxml.etree.HTMLParser()) sponsor = root.xpath('string(//tr[3]/td[1]/b[1])') if bill_id[0] == "S": chamber = "upper" else: chamber = "lower" bill = Bill(str(session), chamber, bill_id, title) bill.add_source(bill_info_url) #Actions actions_url_addon = root.xpath('string(//table/tr[3]/td/a/@href)') actions_url = 'http://www.mainelegislature.org/LawMakerWeb/%s' % actions_url_addon bill.add_source(actions_url) with self.urlopen(actions_url) as actions_page: root2 = lxml.etree.fromstring(actions_page, lxml.etree.HTMLParser()) count = 2 for mr in root2.xpath("//td[2]/table[2]/tr[position() > 1]/td[1]"): date = mr.xpath('string()') date = datetime.strptime(date, "%m/%d/%Y") actor_path = "string(//td[2]/table/tr[%s]/td[2])" % count actor = root2.xpath(actor_path) action_path = "string(//td[2]/table/tr[%s]/td[3])" % count action = root2.xpath(action_path) count = count + 1 if actor == "House": actor = "lower" else: actor = "upper" bill.add_action(actor, action, date) #Votes votes_url_addon = root.xpath('string(//table/tr[9]/td/a/@href)') votes_url = 'http://www.mainelegislature.org/LawMakerWeb/%s' % votes_url_addon bill.add_source(votes_url) with self.urlopen(votes_url) as votes_page: vote_root = lxml.etree.fromstring(votes_page, lxml.etree.HTMLParser()) for mr in vote_root.xpath('//table[position() > 1]/tr/td/a'): vote_detail_addon = mr.xpath('string(@href)') vote_detail_url = 'http://www.mainelegislature.org/LawMakerWeb/%s' % vote_detail_addon bill.add_source(vote_detail_url) with self.urlopen(vote_detail_url) as vote_detail_page: detail_root = lxml.etree.fromstring(vote_detail_page, lxml.etree.HTMLParser()) date = detail_root.xpath('string(//table[2]//tr[2]/td[3])') try: date = datetime.strptime(date, "%B %d, %Y") except: date = datetime.strptime(date, "%b. %d, %Y") motion = detail_root.xpath('string(//table[2]//tr[3]/td[3])') passed = detail_root.xpath('string(//table[2]//tr[5]/td[3])') == 'PREVAILS' yes_count = detail_root.xpath('string(//table[2]//tr[6]/td[3])') no_count = detail_root.xpath('string(//table[2]//tr[7]/td[3])') absent_count = detail_root.xpath('string(//table[2]//tr[6]/td[3])') excused_count = detail_root.xpath('string(//table[2]//tr[6]/td[3])') other_count = 0 if votes_url.find('House') != -1: chamber = "lower" else: chamber = "upper" vote = Vote(chamber, date, motion, passed, int(yes_count), int(no_count), other_count, absent_count = int(absent_count), excused_count = int(excused_count)) for member in detail_root.xpath('//table[3]/tr[position() > 1]'): leg = member.xpath('string(td[2])') party = member.xpath('string(td[3])') leg_vote = member.xpath('string(td[4])') if leg_vote == "Y": vote.yes(leg) elif leg_vote == "N": vote.no(leg) else: vote.other(leg) bill.add_vote(vote) self.save_bill(bill)
def scrape_bill_type(self, chamber, session, bill_type, type_abbr): if chamber == "upper": chamber_name = "SENATE" else: chamber_name = "ASSEMBLY" bills = self.session.query(CABill).filter_by(session_year=session).filter_by(measure_type=type_abbr) for bill in bills: bill_session = session if bill.session_num != "0": bill_session += " Special Session %s" % bill.session_num bill_id = bill.short_bill_id fsbill = Bill(bill_session, chamber, bill_id, "") # Construct session for web query, going from '20092010' to '0910' source_session = session[2:4] + session[6:8] # Turn 'AB 10' into 'ab_10' source_num = "%s_%s" % (bill.measure_type.lower(), bill.measure_num) # Construct a fake source url source_url = "http://www.leginfo.ca.gov/cgi-bin/postquery?" "bill_number=%s&sess=%s" % ( source_num, source_session, ) fsbill.add_source(source_url) title = "" short_title = "" type = ["bill"] subject = "" for version in ( self.session.query(CABillVersion).filter_by(bill=bill).filter(CABillVersion.bill_xml != None) ): title = version.title short_title = version.short_title type = [bill_type] if version.appropriation == "Yes": type.append("appropriation") if version.fiscal_committee == "Yes": type.append("fiscal committee") if version.local_program == "Yes": type.append("local program") if version.urgency == "Yes": type.append("urgency") if version.taxlevy == "Yes": type.append("tax levy") subject = version.subject fsbill.add_version( version.bill_version_id, "", date=version.bill_version_action_date.date(), title=version.title, short_title=version.short_title, subject=[subject], type=type, ) if not title: self.warning("Couldn't find title for %s, skipping" % bill_id) continue fsbill["title"] = title fsbill["short_title"] = short_title fsbill["type"] = type fsbill["subjects"] = [subject] for author in version.authors: if author.house == chamber_name: fsbill.add_sponsor(author.contribution, author.name) for action in bill.actions: if not action.action: # NULL action text seems to be an error on CA's part, # unless it has some meaning I'm missing continue actor = action.actor or chamber actor = actor.strip() match = re.match(r"(Assembly|Senate)($| \(Floor)", actor) if match: actor = {"Assembly": "lower", "Senate": "upper"}[match.group(1)] elif actor.startswith("Governor"): actor = "executive" else: actor = re.sub("^Assembly", "lower", actor) actor = re.sub("^Senate", "upper", actor) type = [] act_str = action.action if act_str.startswith("Introduced"): type.append("bill:introduced") if "To Com" in act_str: type.append("committee:referred") if "Read third time. Passed." in act_str: type.append("bill:passed") if "Approved by Governor" in act_str: type.append("governor:signed") if "Item veto" in act_str: type.append("governor:vetoed:line-item") if not type: type = ["other"] fsbill.add_action(actor, act_str, action.action_date.date(), type=type) for vote in bill.votes: if vote.vote_result == "(PASS)": result = True else: result = False full_loc = vote.location.description first_part = full_loc.split(" ")[0].lower() if first_part in ["asm", "assembly"]: vote_chamber = "lower" vote_location = " ".join(full_loc.split(" ")[1:]) elif first_part.startswith("sen"): vote_chamber = "upper" vote_location = " ".join(full_loc.split(" ")[1:]) else: raise ScrapeError("Bad location: %s" % full_loc) motion = vote.motion.motion_text or "" if "Third Reading" in motion or "3rd Reading" in motion: vtype = "passage" elif "Do Pass" in motion: vtype = "passage" else: vtype = "other" motion = motion.strip() # Why did it take until 2.7 to get a flags argument on re.sub? motion = re.compile(r"(\w+)( Extraordinary)? Session$", re.IGNORECASE).sub("", motion) motion = re.compile(r"^(Senate|Assembly) ", re.IGNORECASE).sub("", motion) motion = re.sub(r"^(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? ", "", motion) motion = re.sub(r" \(\w+\)$", "", motion) motion = re.sub(r"(SCR|SB|AB|AJR|ACR)\s?\d+ \w+\.?$", "", motion) motion = re.sub(r"(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? " r"Urgency Clause$", "(Urgency Clause)", motion) motion = re.sub(r"\s+", " ", motion) if not motion: self.warning("Got blank motion on vote for %s" % bill_id) continue fsvote = Vote( vote_chamber, self._tz.localize(vote.vote_date_time), motion, result, int(vote.ayes), int(vote.noes), int(vote.abstain), threshold=vote.threshold, type=vtype, ) if vote_location != "Floor": fsvote["committee"] = vote_location for record in vote.votes: if record.vote_code == "AYE": fsvote.yes(record.legislator_name) elif record.vote_code.startswith("NO"): fsvote.no(record.legislator_name) else: fsvote.other(record.legislator_name) fsbill.add_vote(fsvote) self.save_bill(fsbill)
def scrape(self, chamber, year): session = "%s%d" % (year, int(year) + 1) if session not in [s_ for t in metadata['terms'] for s_ in t['sessions']]: raise NoDataForPeriod(year) if chamber == 'upper': measure_abbr = 'SB' chamber_name = 'SENATE' house_type = 'S' else: measure_abbr = 'AB' chamber_name = 'ASSEMBLY' house_type = 'A' bills = self.session.query(CABill).filter_by( session_year=session).filter_by( measure_type=measure_abbr) for bill in bills: bill_session = session if bill.session_num != '0': bill_session += ' Special Session %s' % bill.session_num bill_id = bill.short_bill_id version = self.session.query(CABillVersion).filter_by( bill=bill).filter(CABillVersion.bill_xml != None).first() if not version: # not enough data to import continue fsbill = Bill(bill_session, chamber, bill_id, version.title, short_title=version.short_title) for author in version.authors: if author.house == chamber_name: fsbill.add_sponsor(author.contribution, author.name) for action in bill.actions: if not action.action: # NULL action text seems to be an error on CA's part, # unless it has some meaning I'm missing continue actor = action.actor or chamber actor = actor.strip() match = re.match(r'(Assembly|Senate)($| \(Floor)', actor) if match: actor = {'Assembly': 'lower', 'Senate': 'upper'}[match.group(1)] elif actor.startswith('Governor'): actor = 'executive' else: actor = re.sub('^Assembly', 'lower', actor) actor = re.sub('^Senate', 'upper', actor) type = [] act_str = action.action if act_str.startswith('Introduced'): type.append('bill:introduced') if 'To Com' in act_str: type.append('committee:referred') if 'Read third time. Passed.' in act_str: type.append('bill:passed') if 'Approved by Governor' in act_str: type.append('bill:signed') if 'Item veto' in act_str: type.append('veto:line-item') if not type: type = ['other'] fsbill.add_action(actor, act_str, action.action_date, type=type) for vote in bill.votes: if vote.vote_result == '(PASS)': result = True else: result = False full_loc = vote.location.description first_part = full_loc.split(' ')[0].lower() if first_part in ['asm', 'assembly']: vote_chamber = 'lower' vote_location = ' '.join(full_loc.split(' ')[1:]) elif first_part.startswith('sen'): vote_chamber = 'upper' vote_location = ' '.join(full_loc.split(' ')[1:]) else: vote_chamber = '' vote_location = full_loc fsvote = Vote(vote_chamber, vote.vote_date_time, vote.motion.motion_text or '', result, vote.ayes, vote.noes, vote.abstain, threshold=vote.threshold, location=vote_location) for record in vote.votes: if record.vote_code == 'AYE': fsvote.yes(record.legislator_name) elif record.vote_code.startswith('NO'): fsvote.no(record.legislator_name) else: fsvote.other(record.legislator_name) fsbill.add_vote(fsvote) self.save_bill(fsbill)
def scrape_bill_pages(self, session, year_abr): """ assemble information on a bill from a number of DBF files """ #Main Bill information main_bill_url, main_bill_db = self.get_dbf(year_abr, 'MAINBILL') # keep a dictionary of bills (mapping bill_id to Bill obj) bill_dict = {} for rec in main_bill_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) title = rec["synopsis"] if bill_type[0] == 'A': chamber = "lower" else: chamber = "upper" bill = Bill(str(session), chamber, bill_id, title, type=self._bill_types[bill_type[1:]]) bill.add_source(main_bill_url) bill_dict[bill_id] = bill #Sponsors bill_sponsors_url, bill_sponsors_db = self.get_dbf(year_abr, 'BILLSPON') for rec in bill_sponsors_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) bill = bill_dict[bill_id] name = rec["sponsor"] sponsor_type = rec["type"] if sponsor_type == 'P': sponsor_type = "Primary" else: sponsor_type = "Co-sponsor" bill.add_sponsor(sponsor_type, name) #Documents bill_document_url, bill_document_db = self.get_dbf(year_abr, 'BILLWP') #print bill_document_db[2] for rec in bill_document_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) bill = bill_dict[bill_id] document = rec["document"] document = document.split('\\') document = document[-2] + "/" + document[-1] year = str(year_abr) + str((year_abr + 1)) #doc_url = "ftp://www.njleg.state.nj.us/%s/%s" % (year, document) htm_url = 'http://www.njleg.state.nj.us/%s/Bills/%s' % (year_abr, document.replace('.DOC', '.HTM')) # name document based _doctype doc_name = self._doctypes[rec['doctype']] if rec['comment']: doc_name += ' ' + rec['comment'] if rec['doctype'] in self._version_types: bill.add_version(doc_name, htm_url) else: bill.add_document(doc_name, htm_url) #Senate Votes file1 = 'A' + str(year_abr) file2 = 'A' + str(year_abr + 1) file3 = 'S' + str(year_abr) file4 = 'S' + str(year_abr + 1) if str(year_abr) != '2010': vote_info_list = [file1, file2, file3, file4] else: vote_info_list = [file1, file3] for bill_vote_file in vote_info_list: s_vote_url = 'ftp://www.njleg.state.nj.us/votes/%s.zip' % bill_vote_file s_vote_zip, resp = self.urlretrieve(s_vote_url) zipedfile = zipfile.ZipFile(s_vote_zip) vfile = "%s.txt" % bill_vote_file vote_file = zipedfile.open(vfile, 'U') vdict_file = csv.DictReader(vote_file) votes = {} if bill_vote_file[0] == "A": chamber = "lower" else: chamber = "upper" for rec in vdict_file: bill_id = rec["Bill"] bill_id = bill_id.strip() leg = rec["Full_Name"] date = rec["Session_Date"] date = datetime.strptime(date, "%m/%d/%Y") action = rec["Action"] leg_vote = rec["Legislator_Vote"] vote_id = bill_id + "_" + action vote_id = vote_id.replace(" ", "_") passed = None if vote_id not in votes: votes[vote_id] = Vote(chamber, date, action, passed, None, None, None, bill_id=bill_id) if leg_vote == "Y": votes[vote_id].yes(leg) elif leg_vote == "N": votes[vote_id].no(leg) else: votes[vote_id].other(leg) #Counts yes/no/other votes and saves overall vote for vote in votes.itervalues(): vote_yes_count = len(vote["yes_votes"]) vote_no_count = len(vote["no_votes"]) vote_other_count = len(vote["other_votes"]) vote["yes_count"] = vote_yes_count vote["no_count"] = vote_no_count vote["other_count"] = vote_other_count if vote_yes_count > vote_no_count: vote["passed"] = True else: vote["passed"] = False vote_bill_id = vote["bill_id"] bill = bill_dict[vote_bill_id] bill.add_vote(vote) #Actions bill_action_url, bill_action_db = self.get_dbf(year_abr, 'BILLHIST') for rec in bill_action_db: bill_type = rec["billtype"] bill_number = int(rec["billnumber"]) bill_id = bill_type + str(bill_number) bill = bill_dict[bill_id] action = rec["action"] date = rec["dateaction"] actor = rec["house"] comment = rec["comment"] action, atype = self.categorize_action(action) if comment: action += (' ' + comment) bill.add_action(actor, action, date, type=atype) # Subjects subject_url, subject_db = self.get_dbf(year_abr, 'BILLSUBJ') for rec in subject_db: bill_id = rec['billtype'] + str(int(rec['billnumber'])) bill = bill_dict.get(bill_id) if bill: bill.setdefault('subjects', []).append(rec['subjectkey']) else: self.warning('invalid bill id in BILLSUBJ.DBF: %s' % bill_id) # save all bills at the end for bill in bill_dict.itervalues(): # add sources bill.add_source(bill_sponsors_url) bill.add_source(bill_document_url) bill.add_source(bill_action_url) bill.add_source(subject_url) self.save_bill(bill)
def parse_bill_xml(self, chamber, session, txt): root = lxml.etree.fromstring(txt) bill_id = " ".join(root.attrib["bill"].split(" ")[1:]) bill_title = root.findtext("caption") if session[2] == "R": session = session[0:2] if bill_id[1] == "B": bill_type = ["bill"] elif bill_id[1] == "R": bill_type = ["resolution"] elif bill_id[1:3] == "CR": bill_type = ["concurrent resolution"] elif bill_id[1:3] == "JR": bill_type = ["joint resolution"] else: raise ScrapeError("Invalid bill_id: %s" % bill_id) bill = Bill(session, chamber, bill_id, bill_title, type=bill_type) for action in root.findall("actions/action"): act_date = dt.datetime.strptime(action.findtext("date"), "%m/%d/%Y").date() extra = {} extra["action_number"] = action.find("actionNumber").text comment = action.find("comment") if comment is not None and comment.text: extra["comment"] = comment.text.strip() actor = {"H": "lower", "S": "upper", "E": "executive"}[extra["action_number"][0]] desc = action.findtext("description").strip() if desc == "Amended": type = "amendment:passed" elif desc == "Amendment(s) offered": type = "amendment:introduced" elif desc == "Amendment amended": type = "amendment:amended" elif desc == "Amendment withdrawn": type = "amendment:withdrawn" elif desc.startswith("Received by the Secretary of"): type = "bill:introduced" elif desc == "Passed": type = "bill:passed" elif desc.startswith("Received from the"): type = "bill:introduced" elif desc.startswith("Signed by the Governor"): type = "governor:signed" elif desc == "Filed": type = "bill:introduced" else: type = "other" bill.add_action(actor, action.findtext("description"), act_date, type=type, **extra) for author in root.findtext("authors").split(" | "): if author != "": bill.add_sponsor("author", author) for coauthor in root.findtext("coauthors").split(" | "): if coauthor != "": bill.add_sponsor("coauthor", coauthor) for sponsor in root.findtext("sponsors").split(" | "): if sponsor != "": bill.add_sponsor("sponsor", sponsor) for cosponsor in root.findtext("cosponsors").split(" | "): if cosponsor != "": bill.add_sponsor("cosponsor", cosponsor) bill["subjects"] = [] for subject in root.iterfind("subjects/subject"): bill["subjects"].append(subject.text.strip()) return bill
def get_bill_info(self, session, sub, bill_id): bill_detail_url = ( "http://www.ncga.state.nc.us/gascripts/" "BillLookUp/BillLookUp.pl?bPrintable=true" "&Session=%s&BillID=%s&votesToView=all" % (session[0:4] + sub, bill_id) ) # parse the bill data page, finding the latest html text if bill_id[0] == "H": chamber = "lower" else: chamber = "upper" bill_data = self.urlopen(bill_detail_url) bill_soup = self.soup_parser(bill_data) bill_title = bill_soup.findAll( "div", style="text-align: center; font: bold" " 20px Arial; margin-top: 15px;" " margin-bottom: 8px;" )[0].contents[0] bill = Bill(session + sub, chamber, bill_id, bill_title) bill.add_source(bill_detail_url) # get all versions links = bill_soup.findAll("a", href=re.compile("/Sessions/%s/Bills/\w+/HTML" % session[0:4])) for link in links: version_name = link.parent.previousSibling.previousSibling version_name = version_name.contents[0].replace(" ", " ") version_name = version_name.replace(u"\u00a0", " ") version_url = "http://www.ncga.state.nc.us" + link["href"] bill.add_version(version_name, version_url) # figure out which table has sponsor data sponsor_table = bill_soup.findAll("th", text="Sponsors", limit=1)[0].findParents("table", limit=1)[0] sponsor_rows = sponsor_table.findAll("tr") for leg in sponsor_rows[1].td.findAll("a"): bill.add_sponsor("primary", leg.contents[0].replace(u"\u00a0", " ")) for leg in sponsor_rows[2].td.findAll("a"): bill.add_sponsor("cosponsor", leg.contents[0].replace(u"\u00a0", " ")) action_table = bill_soup.findAll("th", text="Chamber", limit=1)[0].findParents("table", limit=1)[0] for row in action_table.findAll("tr"): cells = row.findAll("td") if len(cells) != 3: continue act_date, actor, action = map(lambda x: self.flatten(x), cells) act_date = dt.datetime.strptime(act_date, "%m/%d/%Y") if actor == "Senate": actor = "upper" elif actor == "House": actor = "lower" elif action.endswith("Gov."): actor = "Governor" bill.add_action(actor, action, act_date) for vote in bill_soup.findAll("a", href=re.compile("RollCallVoteTranscript")): self.get_vote(bill, vote["href"]) self.save_bill(bill)