class PacerSessionTest(unittest.TestCase):
"""
Test the PacerSession wrapper class
"""
def setUp(self):
self.session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
def test_data_transformation(self):
"""
Test our data transformation routine for building out PACER-compliant
multi-part form data
"""
data = {'case_id': 123, 'case_type': 'something'}
expected = {'case_id': (None, 123), 'case_type': (None, 'something')}
output = self.session._prepare_multipart_form_data(data)
self.assertEqual(output, expected)
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_ignores_non_data_posts(self, mock_post):
"""
Test that POSTs without a data parameter just pass through as normal.
:param mock_post: mocked Session.post method
"""
data = {'name': ('filename', 'junk')}
self.session.post('https://free.law', files=data, auto_login=False)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertEqual(data, mock_post.call_args[1]['files'],
'the data should not be changed if using a files call')
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_transforms_data_on_post(self, mock_post):
"""
Test that POSTs using the data parameter get transformed into PACER's
delightfully odd multi-part form data.
:param mock_post: mocked Session.post method
"""
data = {'name': 'dave', 'age': 33}
expected = {'name': (None, 'dave'), 'age': (None, 33)}
self.session.post('https://free.law', data=data, auto_login=False)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertNotIn('data', mock_post.call_args[1],
'we should intercept data arguments')
self.assertEqual(expected, mock_post.call_args[1]['files'],
'we should transform and populate the files argument')
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_sets_default_timeout(self, mock_post):
self.session.post('https://free.law', data={}, auto_login=False)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertIn('timeout', mock_post.call_args[1],
'we should add a default timeout automatically')
self.assertEqual(300, mock_post.call_args[1]['timeout'],
'default should be 300')
@mock.patch('juriscraper.pacer.http.PacerSession.login')
@SKIP_IF_NO_PACER_LOGIN
def test_auto_login(self, mock_login):
"""Do we automatically log in if needed?"""
court_id = 'ksd'
pacer_doc_id = '07902639735'
url = make_doc1_url(court_id, pacer_doc_id, True)
pacer_case_id = '81531'
# This triggers and auto-login because we aren't logged in yet.
self.session.username = PACER_USERNAME
self.session.password = PACER_PASSWORD
_ = self.session.get(url, params={
'case_id': pacer_case_id,
'got_receipt': '1',
}, allow_redirects=True)
self.assertTrue(mock_login.called,
'PacerSession.login() should be called.')
class PacerSessionTest(unittest.TestCase):
"""
Test the PacerSession wrapper class
"""
def setUp(self):
self.session = PacerSession()
def test_data_transformation(self):
"""
Test our data transformation routine for building out PACER-compliant
multi-part form data
"""
data = {'case_id': 123, 'case_type': 'something'}
expected = {'case_id': (None, 123), 'case_type': (None, 'something')}
output = self.session._prepare_multipart_form_data(data)
self.assertEqual(output, expected)
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_ignores_non_data_posts(self, mock_post):
"""
Test that POSTs without a data parameter just pass through as normal.
:param mock_post: mocked Session.post method
"""
data = {'name': ('filename', 'junk')}
self.session.post('https://free.law', files=data)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertEqual(data, mock_post.call_args[1]['files'],
'the data should not be changed if using a files call')
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_transforms_data_on_post(self, mock_post):
"""
Test that POSTs using the data parameter get transformed into PACER's
delightfully odd multi-part form data.
:param mock_post: mocked Session.post method
"""
data = {'name': 'dave', 'age': 33}
expected = {'name': (None, 'dave'), 'age': (None, 33)}
self.session.post('https://free.law', data=data)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertNotIn('data', mock_post.call_args[1],
'we should intercept data arguments')
self.assertEqual(expected, mock_post.call_args[1]['files'],
'we should transform and populate the files argument')
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_sets_default_timeout(self, mock_post):
self.session.post('https://free.law', data={})
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertIn('timeout', mock_post.call_args[1],
'we should add a default timeout automatically')
self.assertEqual(300, mock_post.call_args[1]['timeout'],
'default should be 300')
class PacerSessionTest(unittest.TestCase):
"""
Test the PacerSession wrapper class
"""
def setUp(self):
self.session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
def test_data_transformation(self):
"""
Test our data transformation routine for building out PACER-compliant
multi-part form data
"""
data = {'case_id': 123, 'case_type': 'something'}
expected = {'case_id': (None, 123), 'case_type': (None, 'something')}
output = self.session._prepare_multipart_form_data(data)
self.assertEqual(output, expected)
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_ignores_non_data_posts(self, mock_post):
"""
Test that POSTs without a data parameter just pass through as normal.
:param mock_post: mocked Session.post method
"""
data = {'name': ('filename', 'junk')}
self.session.post('https://free.law', files=data, auto_login=False)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertEqual(
data, mock_post.call_args[1]['files'],
'the data should not be changed if using a files call')
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_transforms_data_on_post(self, mock_post):
"""
Test that POSTs using the data parameter get transformed into PACER's
delightfully odd multi-part form data.
:param mock_post: mocked Session.post method
"""
data = {'name': 'dave', 'age': 33}
expected = {'name': (None, 'dave'), 'age': (None, 33)}
self.session.post('https://free.law', data=data, auto_login=False)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertNotIn('data', mock_post.call_args[1],
'we should intercept data arguments')
self.assertEqual(
expected, mock_post.call_args[1]['files'],
'we should transform and populate the files argument')
@mock.patch('juriscraper.pacer.http.requests.Session.post')
def test_sets_default_timeout(self, mock_post):
self.session.post('https://free.law', data={}, auto_login=False)
self.assertTrue(mock_post.called,
'request.Session.post should be called')
self.assertIn('timeout', mock_post.call_args[1],
'we should add a default timeout automatically')
self.assertEqual(300, mock_post.call_args[1]['timeout'],
'default should be 300')
@mock.patch('juriscraper.pacer.http.PacerSession.login')
def test_auto_login(self, mock_login):
"""Do we automatically log in if needed?"""
court_id = 'ksd'
pacer_doc_id = '07902639735'
url = make_doc1_url(court_id, pacer_doc_id, True)
pacer_case_id = '81531'
# This triggers and auto-login because we aren't logged in yet.
self.session.username = PACER_USERNAME
self.session.password = PACER_PASSWORD
_ = self.session.get(url,
params={
'case_id': pacer_case_id,
'got_receipt': '1',
},
allow_redirects=True)
self.assertTrue(mock_login.called,
'PacerSession.login() should be called.')
def fetch(ctx, overwrite=False):
print('fetch')
session = PacerSession(username=os.environ.get('PACER_USERNAME'),
password=os.environ.get('PACER_PASSWORD'))
today = date.today().strftime('%m/%d/%Y')
citations = [
'18:922A.F',
'18:922C.F',
'18:922E.F',
'18:922G.F',
'18:924A.F',
'18:924C.F',
]
for citation in citations:
outputfile = 'data/{0}.tsv'.format(citation)
if overwrite or not os.path.exists(outputfile):
body = {
"office": (None, ""),
"case_type": (None, ""),
"case_flags": (None, ""),
"citation": (None, citation),
"pending_citations": (None, "1"),
"terminated_citations": (None, "1"),
"cvbcases": (None, "No"),
"filed_from": (None, "1/1/2007"),
"filed_to": (None, today),
"terminal_digit": (None, ""),
"pending_defendants": (None, "on"),
"terminated_defendants": (None, "on"),
"fugitive_defendants": (None, ""),
"nonfugitive_defendants": (None, "1"),
"reportable_cases": (None, "1"),
"non_reportable_cases": (None, "1"),
"sort1": (None, "case number"),
"sort2": (None, ""),
"sort3": (None, ""),
"format": (None, "data")
}
intermediate_resp = session.post(
'https://ecf.ilnd.uscourts.gov/cgi-bin/CrCaseFiled-Rpt.pl?1-L_1_0-1'
.format(randint(200000, 40000000)),
files=body)
intermediate_doc = BeautifulSoup(intermediate_resp.content, 'lxml')
form = intermediate_doc.find('form')
action = form.attrs.get('action')
action_path = action.split('/')[-1]
url = 'https://ecf.ilnd.uscourts.gov/cgi-bin/' + action_path
resp = session.post(url)
print('-' * 50)
print(citation)
print('-' * 50)
print(resp.content)
with open(outputfile, 'w') as f:
f.write(resp.content)
else:
print('skipped {0}'.format(citation))
