def test_bill_type_setting():
# default
b = Bill(identifier="some bill",
legislative_session="session",
title="the title")
assert b.classification == ["bill"]
# string -> list
b = Bill(
identifier="some bill",
legislative_session="session",
title="the title",
classification="string",
)
assert b.classification == ["string"]
# list unmodified
b = Bill(
identifier="some bill",
legislative_session="session",
title="the title",
classification=["two", "items"],
)
assert b.classification == ["two", "items"]
# tuple -> list
b = Bill(
identifier="some bill",
legislative_session="session",
title="the title",
classification=("two", "items"),
)
assert b.classification == ["two", "items"]
def test_save_related():
s = Scraper(juris, "/tmp/")
p = Bill("HB 1", "2021", "Test")
p.add_source("http://example.com")
o = Bill("HB 2", "2021", "Test")
o.add_source("http://example.com")
p._related.append(o)
with mock.patch("json.dump") as json_dump:
s.save_object(p)
assert json_dump.mock_calls == [
mock.call(p.as_dict(), mock.ANY, cls=mock.ANY),
mock.call(o.as_dict(), mock.ANY, cls=mock.ANY),
]
def test_save_object_invalid():
s = Scraper(juris, "/tmp/")
p = Bill("HB 1", "2021", "Test")
# no source, won't validate
with pytest.raises(ValueError):
s.save_object(p)
def scrape_bill(self, chamber, session, url):
html = self.get(url).content
page = lxml.html.fromstring(html)
page.make_links_absolute(self.BASE_URL)
if page.xpath('//h2[@style="font-size:1.3rem;"]/a[1]/text()'):
bill_id = page.xpath('//h2[@style="font-size:1.3rem;"]/a[1]/text()')[
0
].strip()
elif page.xpath('//h2[@style="font-size:1.3rem;"]/text()'):
bill_id = page.xpath('//h2[@style="font-size:1.3rem;"]/text()')[0].strip()
else:
self.warning("No bill id for {}".format(url))
return
title = page.xpath(
'//dt[contains(text(), "Title")]/following-sibling::dd[1]/text()'
)[0].strip()
if "B" in bill_id:
_type = ["bill"]
elif "J" in bill_id:
_type = ["joint resolution"]
elif "HS" in bill_id:
_type = ["resolution"]
else:
raise ValueError("unknown bill type " + bill_id)
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=title,
classification=_type,
)
bill.add_source(url)
self.scrape_bill_subjects(bill, page)
self.scrape_bill_sponsors(bill, page)
self.scrape_bill_actions(bill, page)
# fiscal note
if page.xpath('//dt[contains(text(), "Analysis")]/following-sibling::dd[1]/a'):
fiscal_note = page.xpath(
'//dt[contains(text(), "Analysis")]/following-sibling::dd[1]/a'
)[0]
fiscal_url = fiscal_note.get("href")
fiscal_title = fiscal_note.text_content()
bill.add_document_link(
fiscal_title, fiscal_url, media_type="application/pdf",
)
# effective date, where available
if page.xpath('//div[contains(text(), "Effective Date(s)")]'):
eff_date = page.xpath('//div[contains(text(), "Effective Date(s)")]/text()')[0].strip()
eff_date = eff_date.replace('Effective Date(s):', '').strip()
# this can contain multiple dates, eg "July 1, 2020, July 1, 2022"
bill.extras['date_effective'] = eff_date
# yield from self.parse_bill_votes_new(doc, bill)
yield bill
def scrape_bill(self, chamber, session, url):
html = self.get(url).text
page = lxml.html.fromstring(html)
# search for Titulo, accent over i messes up lxml, so use 'tulo'
title = page.xpath(
'//span[@id="ctl00_CPHBody_txtTitulo"]/text()')[0].strip()
bill_id = page.xpath(
'//span[@id="ctl00_CPHBody_txt_Medida"]/text()')[0].strip()
bill_type = self.classify_bill_type(bill_id)
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=title,
classification=bill_type,
)
start_year = session[0:4]
self.scrape_author_table(start_year, bill, bill_id)
# action table contains votes, hence the yield
yield from self.scrape_action_table(chamber, bill, page, url)
bill.add_source(url)
yield bill
def parse_bill(self, chamber, session, special, link):
bill_num = link.text.strip()
type_abbr = re.search("type=(B|R|)", link.attrib["href"]).group(1)
if type_abbr == "B":
btype = ["bill"]
elif type_abbr == "R":
btype = ["resolution"]
bill_id = "%s%s %s" % (utils.bill_abbr(chamber), type_abbr, bill_num)
url = utils.info_url(chamber, session, special, type_abbr, bill_num)
page = self.get(url).text
page = lxml.html.fromstring(page)
page.make_links_absolute(url)
xpath = "/".join(
[
'//div[contains(@class, "BillInfo-ShortTitle")]',
'div[@class="BillInfo-Section-Data"]',
]
)
if page.xpath(xpath):
title = page.xpath(xpath).pop().text_content().strip()
else:
self.warning("Skipping {} {}, No title found".format(bill_id, url))
return
bill = Bill(
bill_id,
legislative_session=session,
title=title,
chamber=chamber,
classification=btype,
)
bill.add_source(url)
self.parse_bill_versions(bill, page)
self.parse_history(
bill,
chamber,
utils.history_url(chamber, session, special, type_abbr, bill_num),
)
# only fetch votes if votes were seen in history
# if vote_count:
yield from self.parse_votes(
bill, utils.vote_url(chamber, session, special, type_abbr, bill_num)
)
# Dedupe sources.
sources = bill.sources
for source in sources:
if 1 < sources.count(source):
sources.remove(source)
yield bill
def parse_bill(self, url):
xml = self.get(url).content
xml = ET.fromstring(xml)
bill_num = self.get_xpath(xml, "bill/billNumber")
bill_type = self.get_xpath(xml, "bill/billType")
bill_id = "{} {}".format(bill_type, bill_num)
chamber_name = self.get_xpath(xml, "bill/originChamber")
chamber = self.chambers[chamber_name]
title = self.get_xpath(xml, "bill/title")
classification = self.classifications[bill_type]
session = self.get_xpath(xml, "bill/congress")
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=title,
classification=classification,
)
self.scrape_actions(bill, xml)
self.scrape_amendments(bill, xml, session, chamber, bill_id)
self.scrape_cbo(bill, xml)
self.scrape_committee_reports(bill, xml)
self.scrape_cosponsors(bill, xml)
self.scrape_laws(bill, xml)
self.scrape_related_bills(bill, xml)
self.scrape_sponsors(bill, xml)
self.scrape_subjects(bill, xml)
self.scrape_summaries(bill, xml)
self.scrape_titles(bill, xml)
self.scrape_versions(bill, xml)
# https://www.congress.gov/bill/116th-congress/house-bill/1
xml_url = "https://www.govinfo.gov/bulkdata/BILLSTATUS/{congress}/{type}/BILLSTATUS-{congress}{type}{num}.xml"
bill.add_source(
xml_url.format(congress=session,
type=bill_type.lower(),
num=bill_num))
cg_url = (
"https://congress.gov/bill/{congress}th-congress/{chamber}-{type}/{num}"
)
bill.add_source(
cg_url.format(
congress=session,
chamber=chamber_name.lower(),
type=classification.lower(),
num=bill_num,
))
yield bill
def scrape_bill(self, chamber, session):
url = "ftp://www.arkleg.state.ar.us/SessionInformation/LegislativeMeasures.txt"
page = csv.reader(get_utf_16_ftp_content(url).splitlines(),
delimiter="|")
for row in page:
bill_chamber = {"H": "lower", "S": "upper"}[row[0]]
if bill_chamber != chamber:
continue
bill_id = "%s%s %s" % (row[0], row[1], row[2])
type_spec = re.match(r"(H|S)([A-Z]+)\s", bill_id).group(2)
bill_type = {
"B": "bill",
"R": "resolution",
"JR": "joint resolution",
"CR": "concurrent resolution",
"MR": "memorial",
"CMR": "concurrent memorial",
}[type_spec]
if row[-1] != self.slug:
continue
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=row[3],
classification=bill_type,
)
bill.add_source(url)
primary = row[11]
if not primary:
primary = row[12]
if primary:
bill.add_sponsorship(
primary,
classification="primary",
entity_type="person",
primary=True,
)
version_url = ("ftp://www.arkleg.state.ar.us/Bills/"
"%s/Public/Searchable/%s.pdf" %
(self.slug, bill_id.replace(" ", "")))
bill.add_version_link(bill_id,
version_url,
media_type="application/pdf")
yield from self.scrape_bill_page(bill)
self.bills[bill_id] = bill
def scrape_bill(self, bill_page_url):
bill_page = lxml.html.fromstring(self.get(bill_page_url).text)
title = bill_page.xpath(
'//span[@id="ctl00_ContentPlaceHolder_SubjectLabel"]/text()'
)
if title:
title = title[0]
else:
self.warning("Missing bill title {}".format(bill_page_url))
return False
bill_no = bill_page.xpath(
'//span[@id="ctl00_ContentPlaceHolder_BillNumberLabel"]/a/text()'
)
if bill_no:
bill_no = bill_no[0]
else:
bill_no = bill_page.xpath(
'//span[@id="ctl00_ContentPlaceHolder_BillNumberLabel"]/text()'
)
if bill_no:
bill_no = bill_no[0]
else:
self.error("Missing bill number {}".format(bill_page_url))
return False
bill = Bill(
bill_no,
legislative_session=self.session,
chamber="legislature",
title=title,
classification="bill",
)
bill.add_source(bill_page_url)
self.parse_versions(bill, bill_page, bill_no)
self.parse_acts(bill, bill_page)
sponsors = bill_page.xpath(
'//span[@id="ctl00_ContentPlaceHolder_SponsorsLabel"]/text()'
)
if sponsors:
self.assign_sponsors(bill, sponsors[0], "primary")
cosponsors = bill_page.xpath(
'//span[@id="ctl00_ContentPlaceHolder_CoSponsorsLabel"]/text()'
)
if cosponsors:
self.assign_sponsors(bill, cosponsors[0], "cosponsor")
self.parse_date_actions(bill, bill_page)
self.parse_actions(bill, bill_page)
yield bill
def get_bill(self, bill_id, **kwargs):
if bill_id == "1":
assert kwargs == {"extra": "param"}
raise self.ContinueScraping
else:
assert bill_id == "2"
assert kwargs == {}
b = Bill("1", self.legislative_session, "title")
b.add_source("http://example.com")
return b
def toy_bill():
b = Bill(
identifier="HB 2017",
legislative_session="2012A",
title="A bill for an act to raise the cookie budget by 200%",
from_organization="Foo Senate",
classification="bill",
)
b.add_source("http://uri.example.com/", note="foo")
return b
def test_from_organization():
# none set
assert get_pseudo_id(Bill("HB 1", "2014", "Some Bill").from_organization) == {
"classification": "legislature"
}
# chamber set
assert get_pseudo_id(
Bill("SB 1", "2014", "Some Bill", chamber="upper").from_organization
) == {"classification": "upper"}
# org direct set
assert (
Bill("HB 1", "2014", "Some Bill", from_organization="test").from_organization
== "test"
)
# can't set both
with pytest.raises(ValueError):
Bill("HB 1", "2014", "Some Bill", from_organization="upper", chamber="upper")
def scrape_bill_info(self, session, chambers):
info_url = "ftp://ftp.cga.ct.gov/pub/data/bill_info.csv"
data = self.get(info_url)
page = open_csv(data)
chamber_map = {"H": "lower", "S": "upper"}
for row in page:
if row["sess_year"] != session:
continue
bill_id = row["bill_num"]
chamber = chamber_map[bill_id[0]]
if chamber not in chambers:
continue
if re.match(r"^(S|H)J", bill_id):
bill_type = "joint resolution"
elif re.match(r"^(S|H)R", bill_id):
bill_type = "resolution"
else:
bill_type = "bill"
bill = Bill(
identifier=bill_id,
legislative_session=session,
title=row["bill_title"],
classification=bill_type,
chamber=chamber,
)
bill.add_source(info_url)
for introducer in self._introducers[bill_id]:
introducer = string.capwords(
introducer.decode("utf-8").replace("Rep. ", "").replace("Sen. ", "")
)
if "Dist." in introducer:
introducer = " ".join(introducer.split()[:-2])
bill.add_sponsorship(
name=introducer,
classification="primary",
primary=True,
entity_type="person",
)
try:
for subject in self._subjects[bill_id]:
bill.subject.append(subject)
self.bills[bill_id] = [bill, chamber]
yield from self.scrape_bill_page(bill)
except SkipBill:
self.warning("no such bill: " + bill_id)
pass
class HouseSearchPage(HtmlListPage):
"""
House committee roll calls are not available on the Senate's
website. Furthermore, the House uses an internal ID system in
its URLs, making accessing those pages non-trivial.
This will fetch all the House committee votes for the
given bill, and add the votes to that object.
"""
input_type = Bill
example_input = Bill("HB 1",
"2020",
"title",
chamber="upper",
classification="bill")
selector = XPath(
'//a[contains(@href, "/Bills/billsdetail.aspx?BillId=")]/@href')
def get_source_from_input(self):
url = "https://www.myfloridahouse.gov/Sections/Bills/bills.aspx"
# Keep the digits and all following characters in the bill's ID
bill_number = re.search(r"^\w+\s(\d+\w*)$",
self.input.identifier).group(1)
session_number = {
"2022D": "96",
"2022C": "95",
"2022": "93",
"2021B": "94",
"2021A": "92",
"2021": "90",
"2020": "89",
"2019": "87",
"2018": "86",
"2017A": "85",
"2017": "83",
"2016": "80",
"2015C": "82",
"2015B": "81",
"2015A": "79",
"2015": "76",
"2014O": "78",
"2014A": "77",
"2016O": "84",
}[self.input.legislative_session]
form = {
"Chamber": "B",
"SessionId": session_number,
"BillNumber": bill_number
}
return url + "?" + urlencode(form)
def process_item(self, item):
return HouseBillPage(self.input, source=item)
class HouseBillPage(HtmlListPage):
selector = XPath('//a[text()="See Votes"]/@href', min_items=0)
example_input = Bill(
"HB 1", "2020", "title", chamber="upper", classification="bill"
)
example_source = (
"https://www.myfloridahouse.gov/Sections/Bills/billsdetail.aspx?BillId=69746"
)
def process_item(self, item):
return HouseComVote(self.input, source=item)
def _recursively_process_bills(
self, request_session, chamber, session, first_item=1
):
"""
Once a search has been initiated, this function will save a
Bill object for every Paper from the given chamber
"""
url = "http://legislature.maine.gov/LawMakerWeb/searchresults.asp"
r = request_session.get(url, params={"StartWith": first_item})
r.raise_for_status()
bills = lxml.html.fromstring(r.text).xpath("//tr/td/b/a")
seen = set()
if bills:
for bill in bills:
bill_id_slug = bill.xpath("./@href")[0]
if bill_id_slug == "summary.asp?ID=280068396":
continue
bill_url = "http://legislature.maine.gov/LawMakerWeb/{}".format(
bill_id_slug
)
bill_id = bill.text[:2] + " " + bill.text[2:]
if (
session in BLACKLISTED_BILL_IDS
and bill_id in BLACKLISTED_BILL_IDS[session]
):
continue
# avoid duplicates
if bill_id in seen:
continue
seen.add(bill_id)
bill = Bill(
identifier=bill_id,
legislative_session=session,
title="",
chamber=chamber,
)
bill.add_source(bill_url)
yield from self.scrape_bill(bill, chamber)
yield bill
# Make a recursive call to this function, for the next page
PAGE_SIZE = 25
yield from self._recursively_process_bills(
request_session=request_session,
chamber=chamber,
session=session,
first_item=first_item + PAGE_SIZE,
)
def process_page(self):
chamber = "upper" if self.input.identifier.startswith("S") else "lower"
short_title = self.get_column_div("Summary").text
long_title = CSS("#title").match_one(self.root).text
if "*" in self.input.identifier:
stars = re.search(r"\*+", self.input.identifier).group()
if (
self.input.session in CARRYOVERS
and stars in CARRYOVERS[self.input.session]
):
self.input.identifier = re.sub(
r"\*+",
"-" + CARRYOVERS[self.input.session][stars],
self.input.identifier,
)
else:
self.logger.error(
f"Unidentified carryover bill {self.input.identifier}. Update CARRYOVERS dict in bills.py"
)
return
bill = Bill(
identifier=self.input.identifier,
legislative_session=self.input.session,
title=short_title,
chamber=chamber,
)
bill.subject = self.input.subjects
# use the pretty source URL
bill.add_source(self.input.source_url)
bill.add_title(long_title)
try:
sponsors = self.get_column_div("Primary Sponsor")
self.add_sponsors(bill, CSS("a").match(sponsors), primary=True)
except SelectorError:
pass
try:
cosponsors = self.get_column_div("Co-Sponsor")
self.add_sponsors(bill, CSS("a").match(cosponsors), primary=False)
except SelectorError:
pass
# TODO: figure out cosponsor div name, can't find any as of Feb 2021
self.add_actions(bill, chamber)
bdr = extract_bdr(short_title)
if bdr:
bill.extras["BDR"] = bdr
text_url = self.source.url.replace("Overview", "Text")
yield BillTabText(bill, source=text_url)
def test_save_object_basics():
# ensure that save object dumps a file
s = Scraper(juris, "/tmp/")
p = Bill("HB 1", "2021", "Test")
p.add_source("http://example.com")
with mock.patch("json.dump") as json_dump:
s.save_object(p)
# ensure object is saved in right place
filename = "bill_" + p._id + ".json"
assert filename in s.output_names["bill"]
json_dump.assert_called_once_with(p.as_dict(), mock.ANY, cls=mock.ANY)
def test_whitespace_is_stripped():
s = Scraper(juris, "/tmp/")
b = Bill(" HB 11", "2020", " a short title ")
b.subject = [" one", "two ", " three "]
b.add_source("https://example.com/ ")
s.save_object(b)
# the simple cases, and nested lists / objects
assert b.identifier == "HB 11"
assert b.title == "a short title"
assert b.sources[0]["url"] == "https://example.com/"
assert b.subject == ["one", "two", "three"]
def scrape_bill(self, session, session_slug, chamber, url):
page = lxml.html.fromstring(self.get(url).text)
bill_no = page.xpath('//*[@id="item-header"]/text()')[0].strip()
# state bill id
internal_id = re.search(r"\/Bill\/(\d+)\/Overview", url).group(1)
# bill data gets filled in from another call
bill_data_base = (
"https://www.leg.state.nv.us/App/NELIS/REL/{}/Bill/"
"FillSelectedBillTab?selectedTab=Overview&billKey={}&_={}")
bill_data_url = bill_data_base.format(session_slug, internal_id,
time.time() * 1000)
bill_page = lxml.html.fromstring(self.get(bill_data_url).text)
short_title = self.get_header_field(bill_page, "Summary:").text
short_title = short_title.replace("\u00a0", " ")
bill = Bill(
identifier=bill_no,
legislative_session=session,
title=short_title,
chamber=chamber,
)
long_title = self.get_header_field(bill_page, "Title:").text
if long_title is not None:
bill.add_abstract(long_title, "Summary")
sponsor_div = self.get_header_field(bill_page, "Primary Sponsor")
if sponsor_div is not None:
self.add_sponsors(sponsor_div, bill, "primary")
cosponsor_div = self.get_header_field(bill_page, "Co-Sponsor")
if cosponsor_div is not None:
self.add_sponsors(cosponsor_div, bill, "cosponsor")
self.add_actions(bill_page, bill, chamber)
self.add_versions(session_slug, internal_id, bill)
bill.subject = list(set(self.subject_mapping[bill_no]))
bdr = self.extract_bdr(short_title)
if bdr:
bill.extras["BDR"] = bdr
bill.extras["NV_ID"] = internal_id
bill.add_source(url)
yield bill
def handle_page(self):
bills = self.doc.xpath('//ul[@class="linkSect"]/li')
for bill in bills:
link = bill.getchildren()[0]
bill_id = str(link.text_content())
if not bill_id.startswith(("S", "H")):
continue
# create a bill
desc = bill.xpath("text()")[0].strip()
chamber = {"H": "lower", "S": "upper"}[bill_id[0]]
bill_type = {
"B": "bill",
"J": "joint resolution",
"R": "resolution"
}[bill_id[1]]
bill = Bill(
bill_id,
self.kwargs["session"],
desc,
chamber=chamber,
classification=bill_type,
)
bill_url = link.get("href")
sponsor_url = BASE_URL + URL_PATTERNS["sponsors"].format(
self.kwargs["session_id"], bill_id.replace(" ", ""))
list(
self.scrape_page_items(BillSponsorPage,
url=sponsor_url,
obj=bill))
yield from self.scrape_page_items(BillDetailPage,
url=bill_url,
obj=bill)
bill.subject = self.kwargs["subjects"][bill_id]
bill.add_source(bill_url)
yield bill
next_url = self.doc.xpath('//a/b[text()="More..."]/../@href')
if next_url:
yield from self.scrape_page_items(BillListPage,
url=next_url[0],
**self.kwargs)
def scrape_bill(self, chamber, session, bill_id, session_id):
bill_json_url = (
"https://apps.azleg.gov/api/Bill/?billNumber={}&sessionId={}&"
"legislativeBody={}".format(bill_id, session_id, self.chamber_map[chamber])
)
response = self.get(bill_json_url, timeout=80)
page = json.loads(response.content.decode("utf-8"))
if not page:
self.warning("null page for %s", bill_id)
return
bill_title = page["ShortTitle"]
bill_id = page["Number"]
internal_id = page["BillId"]
bill_type = self.get_bill_type(bill_id)
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=bill_title,
classification=bill_type,
)
self.scrape_actions(bill, page, chamber)
self.scrape_versions_and_documents(bill, internal_id)
self.scrape_sponsors(bill, internal_id)
self.scrape_subjects(bill, internal_id)
yield from self.scrape_votes(bill, page)
bill_url = (
"https://apps.azleg.gov/BillStatus/BillOverview/{}?SessionId={}".format(
internal_id, session_id
)
)
bill.add_source(bill_url)
bill.actions = sorted(bill.actions, key=lambda action: action["date"])
yield bill
def handle_list_item(self, item):
bill_id = item.text.strip()
title = item.xpath("string(../following-sibling::td[1])").strip()
sponsor = item.xpath("string(../following-sibling::td[2])").strip()
bill_url = item.attrib["href"] + "/ByCategory"
if bill_id.startswith(("SB ", "HB ", "SPB ", "HPB ")):
bill_type = "bill"
elif bill_id.startswith(("HR ", "SR ")):
bill_type = "resolution"
elif bill_id.startswith(("HJR ", "SJR ")):
bill_type = "joint resolution"
elif bill_id.startswith(("SCR ", "HCR ")):
bill_type = "concurrent resolution"
elif bill_id.startswith(("SM ", "HM ")):
bill_type = "memorial"
else:
raise ValueError("Failed to identify bill type.")
bill = Bill(
bill_id,
self.kwargs["session"],
title,
chamber="lower" if bill_id[0] == "H" else "upper",
classification=bill_type,
)
bill.add_source(bill_url)
# normalize id from HB 0004 to H4
subj_bill_id = re.sub(r"(H|S)\w+ 0*(\d+)", r"\1\2", bill_id)
bill.subject = list(self.kwargs["subjects"][subj_bill_id])
sponsor = re.sub(r"^(?:Rep|Sen)\.\s", "", sponsor)
for sp in sponsor.split(", "):
sp = sp.strip()
bill.add_sponsorship(sp, "primary", "person", True)
yield from self.scrape_page_items(BillDetail, url=bill_url, obj=bill)
yield bill
def process_item(self, item):
bill_id = item.text.strip()
title = item.xpath("string(../following-sibling::td[1])").strip()
sponsor = item.xpath("string(../following-sibling::td[2])").strip()
bill_url = item.attrib["href"] + "/ByCategory"
if bill_id.startswith(("SB ", "HB ", "SPB ", "HPB ")):
bill_type = "bill"
elif bill_id.startswith(("HR ", "SR ")):
bill_type = "resolution"
elif bill_id.startswith(("HJR ", "SJR ")):
bill_type = "joint resolution"
elif bill_id.startswith(("SCR ", "HCR ")):
bill_type = "concurrent resolution"
elif bill_id.startswith(("SM ", "HM ")):
bill_type = "memorial"
else:
raise ValueError("Failed to identify bill type.")
bill = Bill(
bill_id,
self.input["session"],
title,
chamber="lower" if bill_id[0] == "H" else "upper",
classification=bill_type,
)
bill.add_source(bill_url)
# normalize id from HB 0004 to H4
subj_bill_id = re.sub(r"(H|S)\w+ 0*(\d+)", r"\1\2", bill_id)
bill.subject = list(self.subjects[subj_bill_id])
sponsor = re.sub(r"^(?:Rep|Sen)\.\s", "", sponsor)
sponsor = re.sub(r",\s+(Jr|Sr)\.", r" \1.", sponsor)
for sp in sponsor.split(", "):
sp = sp.strip()
sp_type = "organization" if "committee" in sp.lower() else "person"
bill.add_sponsorship(sp, "primary", sp_type, True)
return BillDetail(bill)
def scrape_prefiles(self, session):
url = 'https://www.legis.iowa.gov/legislation/billTracking/prefiledBills'
page = lxml.html.fromstring(self.get(url).content)
page.make_links_absolute(url)
for row in page.xpath('//table[contains(@class, "sortable")]/tr[td]'):
title = row.xpath('td[2]/a/text()')[0].strip()
url = row.xpath('td[2]/a/@href')[0]
bill_id = self.extract_doc_id(title)
bill = Bill(
bill_id,
legislative_session=session,
chamber='legislature',
title=title,
classification='proposed bill',
)
if (row.xpath('td[3]/a')):
document_url = row.xpath('td[3]/a/@href')[0]
if '.docx' in document_url:
media_type = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
elif '.pdf' in document_url:
media_type = 'application/pdf'
bill.add_document_link(
note="Backround Statement",
url=document_url,
media_type=media_type
)
bill.add_version_link(
note="Prefiled",
url=url,
media_type="application/pdf"
)
bill.add_source(url)
yield bill
def scrape_bill_list(self, chamber, session, url):
if "joint_resolution" in url:
bill_type = "joint resolution"
elif "resolution" in url:
bill_type = "resolution"
elif "bill" in url:
bill_type = "bill"
try:
data = self.get(url).text
except scrapelib.HTTPError:
self.warning("skipping URL %s" % url)
return
doc = lxml.html.fromstring(data)
doc.make_links_absolute(url)
bill_list = doc.xpath('//ul[@class="infoLinks"]/li/div[@class="row-fluid"]')
for b in bill_list:
bill_url = b.xpath('./div[@class="span3"]/a/@href')[0]
bill_id = bill_url.rsplit("/", 1)[-1]
bill_id = bill_id.upper()
title = (
b.xpath('./div[@class="span6"]/text()')[0]
.replace(" - Relating to: ", "")
.strip()
)
bill = Bill(
bill_id,
legislative_session=session,
title=title,
chamber=chamber,
classification=bill_type,
)
bill.subject = list(set(self.subjects[bill_id]))
yield from self.scrape_bill_history(bill, bill_url, chamber)
yield bill
def scrape_bill(self, session, chamber, bill_url):
try:
page = self.lxmlize("{}{}".format(CO_URL_BASE, bill_url))
except scrapelib.HTTPError as e:
if e.response.status_code == 503:
self.error("Skipping %s w/ 503", bill_url)
return
else:
raise
bill_number = page.xpath(
'//div[contains(@class,"field-name-field-bill-number")]'
'//div[contains(@class,"field-item even")][1]/text()'
)[0].strip()
bill_title = page.xpath('//span[@property="dc:title"]/@content')[0]
bill_summary = page.xpath(
'string(//div[contains(@class,"field-name-field-bill-summary")])'
)
bill_summary = bill_summary.replace("Read More", "").strip()
bill = Bill(
bill_number, legislative_session=session, chamber=chamber, title=bill_title
)
if bill_summary:
bill.add_abstract(bill_summary, "summary")
bill.add_source("{}{}".format(CO_URL_BASE, bill_url))
self.scrape_sponsors(bill, page)
self.scrape_actions(bill, page)
self.scrape_versions(bill, page)
self.scrape_research_notes(bill, page)
self.scrape_fiscal_notes(bill, page)
self.scrape_committee_report(bill, page)
self.scrape_amendments(bill, page)
yield bill
yield from self.scrape_votes(session, bill, page)
def parse_bill(self, chamber, session, bill_id, url):
try:
page = self.lxmlize(url)
except scrapelib.HTTPError as e:
self.logger.warning(e)
return
if self.parse_bill_field(page, "Last Action") != "":
last_action = self.parse_bill_field(
page, "Last Action").xpath("text()")[0]
if "WITHDRAWN" in last_action.upper():
self.info("{} Withdrawn, skipping".format(bill_id))
return
title = self.parse_bill_field(page, "Title").text_content()
if "CR" in bill_id:
bill_type = "concurrent resolution"
elif "JR" in bill_id:
bill_type = "joint resolution"
elif "R" in bill_id:
bill_type = "resolution"
else:
bill_type = "bill"
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=title,
classification=bill_type,
)
bill.subject = self._subjects[bill_id]
bill.add_source(url)
version_ct = self.parse_versions(page, bill)
if version_ct < 1:
# Bill withdrawn
self.logger.warning("Bill withdrawn.")
return
self.parse_actions(page, bill, chamber)
self.parse_subjects(page, bill)
self.parse_proposed_amendments(page, bill)
# LM is "Locally Mandated fiscal impact"
fiscal_notes = page.xpath('//a[contains(@href, "/LM.pdf")]')
for fiscal_note in fiscal_notes:
source_url = fiscal_note.attrib["href"]
mimetype = get_media_type(source_url)
bill.add_document_link("Fiscal Note",
source_url,
media_type=mimetype)
for link in page.xpath(
"//td/span/a[contains(@href, 'Legislator-Profile')]"):
bill.add_sponsorship(
link.text.strip(),
classification="primary",
entity_type="person",
primary=True,
)
if page.xpath("//th[contains(text(),'Votes')]"):
vote_url = page.xpath(
"//a[contains(text(),'Vote History')]/@href")[0]
yield from self.scrape_votes(vote_url, bill, chamber)
bdr_no = self.parse_bill_field(page, "Bill Request Number")
if bdr_no != "" and bdr_no.xpath("text()"):
bdr = bdr_no.xpath("text()")[0].strip()
bill.extras["BDR"] = bdr
yield bill
def scrape_bill(self, chamber, session, bill_id, title, url):
page = self.get(url).json()
api_id = page["BillId"]
if re.match(r"^(S|H)B ", bill_id):
btype = ["bill"]
elif re.match(r"(S|H)C ", bill_id):
btype = ["commemoration"]
elif re.match(r"(S|H)JR ", bill_id):
btype = ["joint resolution"]
elif re.match(r"(S|H)CR ", bill_id):
btype = ["concurrent resolution"]
else:
btype = ["bill"]
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=title,
classification=btype,
)
bill.add_source(f"https://sdlegislature.gov/Session/Bill/{api_id}")
bill.add_source(url)
version_rows = page["Documents"]
assert len(version_rows) > 0
for version in version_rows:
date = version["DocumentDate"]
if date:
match = re.match(r"\d{4}-\d{2}-\d{2}", date)
date = datetime.datetime.strptime(match.group(0),
"%Y-%m-%d").date()
html_link = f"https://sdlegislature.gov/Session/Bill/{api_id}/{version['DocumentId']}"
pdf_link = f"https://mylrc.sdlegislature.gov/api/Documents/{version['DocumentId']}.pdf"
note = version["BillVersion"]
bill.add_version_link(
note,
html_link,
date=date,
media_type="text/html",
on_duplicate="ignore",
)
bill.add_version_link(
note,
pdf_link,
date=date,
media_type="application/pdf",
on_duplicate="ignore",
)
else:
self.warning("Version listed but no date or documents")
sponsors = page["BillSponsor"]
if sponsors:
for sponsor in sponsors:
sponsor_type = "person"
member = sponsor["Member"]
# first and last name are available, but UniqueName is the old link text
# could change later?
bill.add_sponsorship(
member["UniqueName"],
classification="primary",
primary=True,
entity_type=sponsor_type,
)
else:
sponsor_type = "organization"
committee_sponsor = re.search(r">(.*)</a>",
page["BillCommitteeSponsor"])[1]
bill.add_sponsorship(
committee_sponsor,
classification="primary",
primary=True,
entity_type=sponsor_type,
)
for keyword in page["Keywords"]:
bill.add_subject(keyword["Keyword"]["Keyword"])
actions_url = f"https://sdlegislature.gov/api/Bills/ActionLog/{api_id}"
yield from self.scrape_action(bill, actions_url, chamber)
yield bill
def scrape_bill(self, session, history_url):
history_xml = self.get(history_url).text
root = etree.fromstring(history_xml)
bill_title = root.findtext("caption")
if bill_title is None or "Bill does not exist" in history_xml:
self.warning("Bill does not appear to exist")
return
bill_id = " ".join(root.attrib["bill"].split(" ")[1:])
chamber = self.CHAMBERS[bill_id[0]]
if bill_id[1] == "B":
bill_type = ["bill"]
elif bill_id[1] == "R":
bill_type = ["resolution"]
elif bill_id[1:3] == "CR":
bill_type = ["concurrent resolution"]
elif bill_id[1:3] == "JR":
bill_type = ["joint resolution"]
else:
raise ScrapeError("Invalid bill_id: %s" % bill_id)
bill = Bill(
bill_id,
legislative_session=session,
chamber=chamber,
title=bill_title,
classification=bill_type,
)
bill.add_source(history_url)
bill_id_for_url = bill_id.replace(" ", "")
bill.add_source(
f"https://capitol.texas.gov/BillLookup/History.aspx?LegSess={session}&Bill={bill_id_for_url}"
)
for subject in root.iterfind("subjects/subject"):
bill.add_subject(subject.text.strip())
for version in root.iterfind(
"billtext/docTypes/bill/versions/version"):
if not version:
continue
note = version.find("versionDescription").text
html_url = version.find("WebHTMLURL").text
bill.add_version_link(note=note,
url=html_url,
media_type="text/html")
pdf_url = version.find("WebPDFURL").text
bill.add_version_link(note=note,
url=pdf_url,
media_type="application/pdf")
for analysis in root.iterfind(
"billtext/docTypes/analysis/versions/version"):
if not analysis:
continue
description = analysis.find("versionDescription").text
html_url = analysis.find("WebHTMLURL").text
bill.add_document_link(
note="Analysis ({})".format(description),
url=html_url,
media_type="text/html",
)
for fiscal_note in root.iterfind(
"billtext/docTypes/fiscalNote/versions/version"):
if not fiscal_note:
continue
description = fiscal_note.find("versionDescription").text
html_url = fiscal_note.find("WebHTMLURL").text
bill.add_document_link(
note="Fiscal Note ({})".format(description),
url=html_url,
media_type="text/html",
)
witnesses = [x for x in self.witnesses if x[0] == bill_id]
for witness in witnesses:
bill.add_document_link(
note="Witness List ({})".format(
self.NAME_SLUGS[witness[1][-5]]),
url=witness[1],
media_type="text/html",
)
for action in root.findall("actions/action"):
act_date = datetime.datetime.strptime(action.findtext("date"),
"%m/%d/%Y").date()
action_number = action.find("actionNumber").text
actor = {
"H": "lower",
"S": "upper",
"E": "executive"
}[action_number[0]]
desc = action.findtext("description").strip()
if desc == "Scheduled for public hearing on . . .":
self.warning("Skipping public hearing action with no date")
continue
atype = _categorize_action(desc)
act = bill.add_action(
action.findtext("description"),
act_date,
chamber=actor,
classification=atype,
)
if atype and "referral-committee" in atype:
repls = ["Referred to", "Recommended to be sent to "]
ctty = desc
for r in repls:
ctty = ctty.replace(r, "").strip()
act.add_related_entity(name=ctty, entity_type="organization")
for author in root.findtext("authors").split(" | "):
if author != "":
bill.add_sponsorship(author,
classification="primary",
entity_type="person",
primary=True)
for coauthor in root.findtext("coauthors").split(" | "):
if coauthor != "":
bill.add_sponsorship(
coauthor,
classification="cosponsor",
entity_type="person",
primary=False,
)
for sponsor in root.findtext("sponsors").split(" | "):
if sponsor != "":
bill.add_sponsorship(
sponsor,
classification="primary",
entity_type="person",
primary=True,
)
for cosponsor in root.findtext("cosponsors").split(" | "):
if cosponsor != "":
bill.add_sponsorship(
cosponsor,
classification="cosponsor",
entity_type="person",
primary=False,
)
if root.findtext("companions"):
self._get_companion(bill)
yield bill