def download_pdfs(self):
"""Download the first (presumably complaint) PDF to downlaods dir.
:return: None
"""
q = Query()
db = TinyDB("db/master.json")
fjc_table = db.table("fjc")
for row in fjc_table.search((q.JSON == "True") & (q.PDF == "False")):
logging.info(
"Collecting PDF #%s, in %s" % (row["PACER_CASE_ID"], row["TITLE"])
)
report = FreeOpinionReport(row["COURT"], self.s)
r = report.download_pdf(row["PACER_CASE_ID"], row["pacer_doc_id"])
with open(
"downloads/pdf/pacer_complaint_%s.pdf" % row["PACER_CASE_ID"], "w"
) as file:
file.write(r.content)
fjc_table.update(
{"PDF": "True"}, doc_ids=[row.doc_id],
)
logging.info(
"Collected PDF #%s, in %s" % (row["PACER_CASE_ID"], row["TITLE"])
)
def get_and_process_pdf(self, data, session, row_pk, index=False):
if data is None:
return
result = data['result']
rd = RECAPDocument.objects.get(pk=data['rd_pk'])
report = FreeOpinionReport(data['pacer_court_id'], session)
try:
r = report.download_pdf(result.pacer_case_id, result.pacer_doc_id)
except (ConnectTimeout, ConnectionError, ReadTimeout, ReadTimeoutError,
ChunkedEncodingError) as exc:
logger.warning("Unable to get PDF for %s" % result)
raise self.retry(exc=exc)
except HTTPError as exc:
if exc.response.status_code in [
HTTP_500_INTERNAL_SERVER_ERROR, HTTP_504_GATEWAY_TIMEOUT
]:
logger.warning("Ran into HTTPError: %s. Retrying." %
exc.response.status_code)
raise self.retry(exc)
else:
msg = "Ran into unknown HTTPError. %s. Aborting." % \
exc.response.status_code
logger.error(msg)
PACERFreeDocumentRow.objects.filter(pk=row_pk).update(
error_msg=msg)
self.request.callbacks = None
return
if r is None:
msg = "Unable to get PDF for %s at %s with doc id %s" % \
(result, result.court_id, result.pacer_doc_id)
logger.error(msg)
PACERFreeDocumentRow.objects.filter(pk=row_pk).update(error_msg=msg)
self.request.callbacks = None
return
file_name = get_document_filename(
result.court.pk,
result.pacer_case_id,
result.document_number,
0, # Attachment number is zero for all free opinions.
)
cf = ContentFile(r.content)
rd.filepath_local.save(file_name, cf, save=False)
rd.is_available = True # We've got the PDF.
# request.content is sometimes a str, sometimes unicode, so
# force it all to be bytes, pleasing hashlib.
rd.sha1 = hashlib.sha1(force_bytes(r.content)).hexdigest()
rd.is_free_on_pacer = True
rd.page_count = get_page_count(rd.filepath_local.path, 'pdf')
# Save and extract, skipping OCR.
rd.save(do_extraction=False, index=index)
extract_recap_pdf(rd.pk, skip_ocr=True, check_if_needed=False)
return {'result': result, 'rd_pk': rd.pk}
def get_and_process_pdf(self, data, session, row_pk, index=False):
if data is None:
return
result = data['result']
rd = RECAPDocument.objects.get(pk=data['rd_pk'])
report = FreeOpinionReport(data['pacer_court_id'], session)
try:
r = report.download_pdf(result.pacer_case_id, result.pacer_doc_id)
except (ConnectTimeout, ConnectionError, ReadTimeout, ReadTimeoutError,
ChunkedEncodingError) as exc:
logger.warning("Unable to get PDF for %s" % result)
raise self.retry(exc=exc)
except HTTPError as exc:
if exc.response.status_code in [HTTP_500_INTERNAL_SERVER_ERROR,
HTTP_504_GATEWAY_TIMEOUT]:
logger.warning("Ran into HTTPError: %s. Retrying." %
exc.response.status_code)
raise self.retry(exc)
else:
msg = "Ran into unknown HTTPError. %s. Aborting." % \
exc.response.status_code
logger.error(msg)
PACERFreeDocumentRow.objects.filter(pk=row_pk).update(error_msg=msg)
self.request.callbacks = None
return
if r is None:
msg = "Unable to get PDF for %s at %s with doc id %s" % \
(result, result.court_id, result.pacer_doc_id)
logger.error(msg)
PACERFreeDocumentRow.objects.filter(pk=row_pk).update(error_msg=msg)
self.request.callbacks = None
return
file_name = get_document_filename(
result.court.pk,
result.pacer_case_id,
result.document_number,
0, # Attachment number is zero for all free opinions.
)
cf = ContentFile(r.content)
rd.filepath_local.save(file_name, cf, save=False)
rd.is_available = True # We've got the PDF.
# request.content is sometimes a str, sometimes unicode, so
# force it all to be bytes, pleasing hashlib.
rd.sha1 = hashlib.sha1(force_bytes(r.content)).hexdigest()
rd.is_free_on_pacer = True
rd.page_count = get_page_count(rd.filepath_local.path, 'pdf')
# Save and extract, skipping OCR.
rd.save(do_extraction=False, index=index)
extract_recap_pdf(rd.pk, skip_ocr=True, check_if_needed=False)
return {'result': result, 'rd_pk': rd.pk}
def download_pacer_pdf_by_rd(self, rd_pk, pacer_case_id, pacer_doc_id,
cookies):
"""Using a RECAPDocument object ID, download the PDF if it doesn't already
exist.
:param rd_pk: The PK of the RECAPDocument to download
:param pacer_case_id: The internal PACER case ID number
:param pacer_doc_id: The internal PACER document ID to download
:param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
logged-in PACER user.
:return: requests.Response object usually containing a PDF, or None if that
wasn't possible.
"""
rd = RECAPDocument.objects.get(pk=rd_pk)
pacer_court_id = map_cl_to_pacer_id(rd.docket_entry.docket.court_id)
s = PacerSession(cookies=cookies)
report = FreeOpinionReport(pacer_court_id, s)
try:
r = report.download_pdf(pacer_case_id, pacer_doc_id)
except HTTPError as exc:
if exc.response.status_code in [
HTTP_500_INTERNAL_SERVER_ERROR, HTTP_504_GATEWAY_TIMEOUT
]:
logger.warning(
"Ran into HTTPError while getting PDF: %s. "
"Retrying.", exc.response.status_code)
if self.request.retries == self.max_retries:
self.request.callbacks = None
return
raise self.retry(exc)
else:
logger.error(
"Ran into unknown HTTPError while getting PDF: %s. "
"Aborting.", exc.response.status_code)
self.request.callbacks = None
return
except requests.RequestException as exc:
logger.warning("Unable to get PDF for %s in %s", pacer_doc_id,
pacer_case_id)
if self.request.retries == self.max_retries:
self.request.callbacks = None
return
raise self.retry(exc=exc)
return r