def get_session_details():
"""
We will fetch a list of available sessions from the 'bill locator' page.
We won't get legislators for all these sessions, but all bills for these
sessions are available and we want to be able to get to them.
"""
scraper = Scraper()
nm_locator_url = 'http://legis.state.nm.us/lcs/locator.aspx'
with scraper.urlopen(nm_locator_url) as page:
page = BeautifulSoup(page)
#The first `tr` is simply 'Bill Locator`. Ignoring that
data_table = page.find('table', id = 'ctl00_mainCopy_Locators')('tr')[1:]
for session in data_table:
session_tag = session.find('a')
session_name = ' '.join([tag.string.strip() for tag in session_tag('span')]).strip()
session_year, sub_session_name = SESSION_NAME_RE.match(session_name).groups()
if session_year in metadata['sessions']:
if sub_session_name not in metadata['session_details'][session_year]['sub_sessions']:
metadata['session_details'][session_year]['sub_sessions'].append(sub_session_name)
else:
metadata['sessions'].append(session_year)
metadata['session_details'][session_year] = dict(years = session_year, sub_sessions = [sub_session_name])
def newfunc(
header: typing.List[str],
retries: int,
retry_wait: int,
rpm: int,
timeout: int,
user_agent: str,
verbosity: int,
verify: bool,
fastmode: bool,
**kwargs: str,
) -> None:
scraper = Scraper(
requests_per_minute=rpm,
retry_attempts=retries,
retry_wait_seconds=retry_wait,
verify=verify,
)
scraper.timeout = timeout
scraper.user_agent = user_agent
# only update headers, don't overwrite defaults
scraper.headers.update(
{k.strip(): v.strip()
for k, v in [h.split(":") for h in header]})
if fastmode:
scraper.cache_storage = SQLiteCache("spatula-cache.db")
scraper.cache_write_only = False
if verbosity == -1:
level = logging.INFO if func.__name__ != "test" else logging.DEBUG
elif verbosity == 0: # pragma: no cover
level = logging.ERROR
elif verbosity == 1: # pragma: no cover
level = logging.INFO
elif verbosity >= 2: # pragma: no cover
level = logging.DEBUG
if verbosity < 3:
# replace parent library logging
logging.getLogger("scrapelib").setLevel(logging.ERROR)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.basicConfig(level=level)
return func(**kwargs, scraper=scraper)
def test_to_items_scout():
scraper = Scraper()
page = FirstPage()
items = list(page._to_items(scraper, scout=True))
assert len(items) == 3
assert items[0] == {
"data": {
"first": 1
},
"__next__": "SecondPage source=NullSource",
}
assert items[1] == {
"data": {
"first": 2
},
"__next__": "SecondPage source=NullSource",
}
assert items[2] == {
"data": {
"first": 3
},
"__next__": "SecondPage source=NullSource",
}
SYNONYMS = [('isis', 'isil', 'islamic state'),
('veteran', 'veterans', 'shinseki', 'va'),
('affordable care act', 'obamacare', 'healthcare', 'health care',
'insurance'), ('ukraine', 'ukrainian', 'crimea'),
('unemployed', 'unemployment')]
#('palestine', 'palestinians'),
#('israel', 'israeli', 'palestine', 'palestinians'),
#('iraq', 'iraqis', 'iraqs'),
#('executive order', 'executive action'),
#('economy', 'economic'),
ROOT_URL = 'http://www.whitehouse.gov/briefing-room/press-briefings'
CSV_PATH = 'briefing_links.csv'
s = Scraper(requests_per_minute=60)
s.cache_storage = FileCache('press_briefing_cache')
s.cache_write_only = False
@task(default=True)
def update():
"""
Stub function for updating app-specific data.
"""
#update_featured_social()
@task
def scrape_briefings():
for index in range(0, 22):
# reading in what was output on the last run can make things more efficient.
#
import unicodecsv
from lxml.html import fromstring
from urlparse import urljoin
from urllib import quote_plus
import re
import os, os.path
OUTPUT_FILE = "zctas.csv"
from scrapelib import Scraper, FileCache, HTTPError
BASE_URL = 'https://en.wikipedia.org/wiki/'
# would like to follow robots, but think that the robots parser is broken...
s = Scraper(requests_per_minute=90, follow_robots=False)
s.cache_storage = FileCache('../wikipedia_cache')
s.cache_write_only = False
def test_zips():
existing = {}
if os.path.exists(OUTPUT_FILE):
for row in unicodecsv.DictReader(open(OUTPUT_FILE)):
existing[row['zip']] = row
os.rename(OUTPUT_FILE, OUTPUT_FILE + '.bak')
r = unicodecsv.DictReader(open("2013_Gaz_zcta_national.txt"),
delimiter="\t")
f = r.fieldnames
writer = unicodecsv.DictWriter(open(OUTPUT_FILE, "w"), ['zip', 'wiki_url'])
writer.writerow({'zip': 'zip', 'wiki_url': 'wiki_url'})
def test_source_no_timeout():
source = URL("https://httpbin.org/delay/1")
assert source.get_response(Scraper()).status_code == 200
def test_source_timeout():
source = URL("https://httpbin.org/delay/1", timeout=0.1)
with pytest.raises(OSError):
source.get_response(Scraper())
