def get_pdfs(options):
"""Get PDFs for the results of the Free Document Report queries.
At this stage, we have rows in the PACERFreeDocumentRow table, each of
which represents a PDF we need to download and merge into our normal
tables: Docket, DocketEntry, and RECAPDocument.
In this function, we iterate over the entire table of results, merge it
into our normal tables, and then download and extract the PDF.
:return: None
"""
q = options['queue']
index = options['index']
cnt = CaseNameTweaker()
rows = PACERFreeDocumentRow.objects.filter(error_msg="").only('pk')
count = rows.count()
task_name = "downloading"
if index:
task_name += " and indexing"
logger.info("%s %s items from PACER." % (task_name, count))
throttle = CeleryThrottle(queue_name=q)
completed = 0
for row in queryset_generator(rows):
throttle.maybe_wait()
if completed % 30000 == 0:
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
pacer_session.login()
chain(
process_free_opinion_result.si(row.pk, cnt).set(queue=q),
get_and_process_pdf.s(pacer_session, row.pk,
index=index).set(queue=q),
delete_pacer_row.si(row.pk).set(queue=q),
).apply_async()
completed += 1
if completed % 1000 == 0:
logger.info("Sent %s/%s tasks to celery for %s so "
"far." % (completed, count, task_name))
def get_pacer_dockets(options, row_pks, tag=None):
"""Get the pacer dockets identified by the FJC IDB rows"""
q = options['queue']
throttle = CeleryThrottle(queue_name=q)
for i, row_pk in enumerate(row_pks):
if i >= options['count'] > 0:
break
throttle.maybe_wait()
if i % 1000 == 0:
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
pacer_session.login()
logger.info("Sent %s tasks to celery so far." % i)
row = FjcIntegratedDatabase.objects.get(pk=row_pk)
chain(
get_docket_by_pacer_case_id.s(
row.pacer_case_id, row.district_id, pacer_session, **{
'tag': tag,
'show_parties_and_counsel': True,
'show_terminated_parties': True,
'show_list_of_member_cases': True
}).set(queue=q),
add_or_update_recap_docket.s().set(queue=q),
).apply_async()
def open_session(username, password):
pacer_sess = PacerSession(username=username, password=password)
return pacer_sess
def setUpClass(cls):
pacer_session = PacerSession(username='******', password='******')
cls.report = DocketReport('psc', pacer_session)
cls.pacer_case_id = '62866' # 1:07-cr-00001-RJA-HKS USA v. Green
def setUp(self):
self.session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
def setUpClass(cls):
if PACER_USERNAME and PACER_PASSWORD:
cls.pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
cls.report = ShowCaseDocApi('dcd', cls.pacer_session)
def test_logging_in_bad_credentials(self):
session = PacerSession(username='******', password='******')
with self.assertRaises(PacerLoginException):
session.login()
def setUp(self):
self.session = PacerSession()
def get_and_save_free_document_reports(options):
"""Query the Free Doc Reports on PACER and get a list of all the free
documents. Do not download those items, as that step is done later. For now
just get the list.
Note that this uses synchronous celery chains. A previous version was more
complex and did not use synchronous chains. Unfortunately in Celery 4.2.0,
or more accurately in redis-py 3.x.x, doing it that way failed nearly every
time.
This is a simpler version, though a slower one, but it should get the job
done.
"""
# Kill any *old* logs that report they're in progress. (They've failed.)
three_hrs_ago = now() - timedelta(hours=3)
PACERFreeDocumentLog.objects.filter(
date_started__lt=three_hrs_ago,
status=PACERFreeDocumentLog.SCRAPE_IN_PROGRESS,
).update(status=PACERFreeDocumentLog.SCRAPE_FAILED, )
cl_court_ids = Court.objects.filter(
jurisdiction__in=[Court.FEDERAL_DISTRICT, Court.FEDERAL_BANKRUPTCY],
in_use=True,
end_date=None,
).exclude(pk__in=['casb', 'gub', 'innb', 'miwb', 'ohsb',
'prb'], ).values_list(
'pk',
flat=True,
)
pacer_court_ids = [map_cl_to_pacer_id(v) for v in cl_court_ids]
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
pacer_session.login()
today = now()
for pacer_court_id in pacer_court_ids:
while True:
next_start_d, next_end_d = get_next_date_range(pacer_court_id)
logger.info(
"Attempting to get latest document references for "
"%s between %s and %s", pacer_court_id, next_start_d,
next_end_d)
mark_court_in_progress(pacer_court_id, next_end_d)
try:
status = get_and_save_free_document_report(
pacer_court_id, next_start_d, next_end_d,
pacer_session.cookies)
except RequestException:
logger.error(
"Failed to get document references for %s "
"between %s and %s due to network error.", pacer_court_id,
next_start_d, next_end_d)
mark_court_done_on_date(PACERFreeDocumentLog.SCRAPE_FAILED,
pacer_court_id, next_end_d)
break
except IndexError:
logger.error(
"Failed to get document references for %s "
"between %s and %s due to PACER 6.3 bug.", pacer_court_id,
next_start_d, next_end_d)
mark_court_done_on_date(PACERFreeDocumentLog.SCRAPE_FAILED,
pacer_court_id, next_end_d)
break
else:
result = mark_court_done_on_date(status, pacer_court_id,
next_end_d)
if result == PACERFreeDocumentLog.SCRAPE_SUCCESSFUL:
if next_end_d >= today.date():
logger.info("Got all document references for '%s'.",
pacer_court_id)
# Break from while loop, onwards to next court
break
else:
# More dates to do; let it continue
continue
elif result == PACERFreeDocumentLog.SCRAPE_FAILED:
logger.error("Encountered critical error on %s "
"(network error?). Marking as failed and "
"pressing on." % pacer_court_id)
# Break from while loop, onwards to next court
break
from juriscraper.pacer.http import PacerSession
from juriscraper.pacer.hidden_api import PossibleCaseNumberApi
pacer_sess = PacerSession(username="", password="")
court_id = "dcd"
docket_id = "9000013"
office_id = "1"
docket_param = "cv"
#court_id = "nmi"
#docket_id = "8800001"
api = PossibleCaseNumberApi(court_id=court_id, pacer_session=pacer_sess)
resp = api.query(docket_id)
print resp
print api.data(office_number=office_id, docket_number_letters=docket_param)
#resp = api.query("7801294")
#print res
def setUp(self):
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
pacer_session.login()
self.report = DocketReport('cand', pacer_session)
self.pacer_case_id = '186730' # 4:06-cv-07294 Foley v. Bates
def setUpClass(cls):
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
cls.report = DocketReport('cand', pacer_session)
cls.pacer_case_id = '186730' # 4:06-cv-07294 Foley v. Bates
#!/usr/bin/env python
#
# Takes an .html file on the command line, parses it using the PACER
# Docket Report parser, and outputs json to stdout.
import jsondate3 as json
import sys
from juriscraper.pacer.http import PacerSession
from juriscraper.pacer import DocketReport
pacer_session = PacerSession(username="******", password="******")
report = DocketReport("psc", pacer_session)
for path in sys.argv[1:]:
with open(path, "r") as f:
report._parse_text(f.read().decode("utf-8"))
data = report.data
print json.dumps(data, indent=2, sort_keys=True, separators=(",", ": "))
def get_and_save_free_document_reports(options):
"""Query the Free Doc Reports on PACER and get a list of all the free
documents. Do not download those items, as that step is done later.
"""
# Kill any *old* logs that report they're in progress. (They've failed.)
twelve_hrs_ago = now() - timedelta(hours=12)
PACERFreeDocumentLog.objects.filter(
date_started__lt=twelve_hrs_ago,
status=PACERFreeDocumentLog.SCRAPE_IN_PROGRESS,
).update(status=PACERFreeDocumentLog.SCRAPE_FAILED, )
cl_court_ids = Court.objects.filter(
jurisdiction__in=[Court.FEDERAL_DISTRICT, Court.FEDERAL_BANKRUPTCY],
in_use=True,
end_date=None,
).exclude(pk__in=[
'casb', 'ganb', 'gub', 'innb', 'mieb', 'miwb', 'nmib', 'nvb', 'ohsb',
'prb', 'tnwb', 'vib'
], ).values_list(
'pk',
flat=True,
)
pacer_court_ids = {
map_cl_to_pacer_id(v): {
'until': now(),
'count': 1,
'result': None
}
for v in cl_court_ids
}
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
pacer_session.login()
# Iterate over every court, X days at a time. As courts are completed,
# remove them from the list of courts to process until none are left
tomorrow = now() + timedelta(days=1)
while len(pacer_court_ids) > 0:
court_ids_copy = pacer_court_ids.copy() # Make a copy of the list.
for pacer_court_id, delay in court_ids_copy.items():
if now() < delay['until']:
# Do other courts until the delay is up. Do not print/log
# anything since at the end there will only be one court left.
continue
next_start_date, next_end_date = get_next_date_range(
pacer_court_id)
if delay['result'] is not None:
if delay['result'].ready():
result = delay['result'].get()
if result == PACERFreeDocumentLog.SCRAPE_SUCCESSFUL:
if next_start_date >= tomorrow.date():
logger.info("Finished '%s'. Marking it complete." %
pacer_court_id)
pacer_court_ids.pop(pacer_court_id, None)
continue
elif result == PACERFreeDocumentLog.SCRAPE_FAILED:
logger.error("Encountered critical error on %s "
"(network error?). Marking as failed and "
"pressing on." % pacer_court_id)
pacer_court_ids.pop(pacer_court_id, None)
continue
else:
next_delay = min(delay['count'] * 5, 30) # backoff w/cap
logger.info(
"Court %s still in progress. Delaying at least "
"%ss." % (pacer_court_id, next_delay))
pacer_court_ids[pacer_court_id]['until'] = now(
) + timedelta(seconds=next_delay)
pacer_court_ids[pacer_court_id]['count'] += 1
continue
mark_court_in_progress(pacer_court_id, next_end_date)
pacer_court_ids[pacer_court_id]['count'] = 1 # Reset
delay['result'] = chain(
get_and_save_free_document_report.si(pacer_court_id,
next_start_date,
next_end_date,
pacer_session),
mark_court_done_on_date.s(pacer_court_id, next_end_date),
).apply_async()
def get_pacer_session():
return PacerSession(username=PACER_USERNAME, password=PACER_PASSWORD)
def test_logging_in_bad_credentials(self):
# Make sure password is more than eight characters.
session = PacerSession(username="******", password="******")
with self.assertRaises(PacerLoginException):
session.login()
def fetch(ctx, overwrite=False):
print('fetch')
session = PacerSession(username=os.environ.get('PACER_USERNAME'),
password=os.environ.get('PACER_PASSWORD'))
today = date.today().strftime('%m/%d/%Y')
citations = [
'18:922A.F',
'18:922C.F',
'18:922E.F',
'18:922G.F',
'18:924A.F',
'18:924C.F',
]
for citation in citations:
outputfile = 'data/{0}.tsv'.format(citation)
if overwrite or not os.path.exists(outputfile):
body = {
"office": (None, ""),
"case_type": (None, ""),
"case_flags": (None, ""),
"citation": (None, citation),
"pending_citations": (None, "1"),
"terminated_citations": (None, "1"),
"cvbcases": (None, "No"),
"filed_from": (None, "1/1/2007"),
"filed_to": (None, today),
"terminal_digit": (None, ""),
"pending_defendants": (None, "on"),
"terminated_defendants": (None, "on"),
"fugitive_defendants": (None, ""),
"nonfugitive_defendants": (None, "1"),
"reportable_cases": (None, "1"),
"non_reportable_cases": (None, "1"),
"sort1": (None, "case number"),
"sort2": (None, ""),
"sort3": (None, ""),
"format": (None, "data")
}
intermediate_resp = session.post(
'https://ecf.ilnd.uscourts.gov/cgi-bin/CrCaseFiled-Rpt.pl?1-L_1_0-1'
.format(randint(200000, 40000000)),
files=body)
intermediate_doc = BeautifulSoup(intermediate_resp.content, 'lxml')
form = intermediate_doc.find('form')
action = form.attrs.get('action')
action_path = action.split('/')[-1]
url = 'https://ecf.ilnd.uscourts.gov/cgi-bin/' + action_path
resp = session.post(url)
print('-' * 50)
print(citation)
print('-' * 50)
print(resp.content)
with open(outputfile, 'w') as f:
f.write(resp.content)
else:
print('skipped {0}'.format(citation))