def get_attachment_page_by_rd(self, rd_pk, cookies):
"""Get the attachment page for the item in PACER.
:param rd_pk: The PK of a RECAPDocument object to use as a source.
:param cookies: A requests.cookies.RequestsCookieJar with the cookies of a
logged-on PACER user.
:return: The attachment report populated with the results
"""
rd = RECAPDocument.objects.get(pk=rd_pk)
if not rd.pacer_doc_id:
# Some docket entries are just text/don't have a pacer_doc_id.
self.request.callbacks = None
return
s = PacerSession(cookies=cookies)
pacer_court_id = map_cl_to_pacer_id(rd.docket_entry.docket.court_id)
att_report = AttachmentPage(pacer_court_id, s)
try:
att_report.query(rd.pacer_doc_id)
except HTTPError as exc:
if exc.response.status_code in [
HTTP_500_INTERNAL_SERVER_ERROR, HTTP_504_GATEWAY_TIMEOUT
]:
logger.warning("Ran into HTTPError: %s. Retrying.",
exc.response.status_code)
raise self.retry(exc)
else:
msg = "Ran into unknown HTTPError. %s. Aborting."
logger.error(msg, exc.response.status_code)
self.request.callbacks = None
return
except requests.RequestException as exc:
logger.warning("Unable to get attachment page for %s", rd)
raise self.retry(exc=exc)
return att_report
def get_pacer_doc_by_rd_and_description(self,
rd_pk,
description_re,
session,
fallback_to_main_doc=False,
tag=None):
"""Using a RECAPDocument object ID and a description of a document, get the
document from PACER.
This function was originally meant to get civil cover sheets, but can be
repurposed as needed.
:param rd_pk: The PK of a RECAPDocument object to use as a source.
:param description_re: A compiled regular expression to search against the
description provided by the attachment page.
:param session: The PACER session object to use.
:param fallback_to_main_doc: Should we grab the main doc if none of the
attachments match the regex?
:param tag: A tag name to apply to any downloaded content.
:return: None
"""
rd = RECAPDocument.objects.get(pk=rd_pk)
if not rd.pacer_doc_id:
# Some docket entries are just text/don't have a pacer_doc_id.
self.request.callbacks = None
return
d = rd.docket_entry.docket
pacer_court_id = map_cl_to_pacer_id(d.court_id)
att_report = AttachmentPage(pacer_court_id, session)
try:
att_report.query(rd.pacer_doc_id)
except (ConnectTimeout, ConnectionError, ReadTimeout, ReadTimeoutError,
ChunkedEncodingError) as exc:
logger.warning("Unable to get PDF for %s" % rd)
raise self.retry(exc=exc)
except HTTPError as exc:
if exc.response.status_code in [
HTTP_500_INTERNAL_SERVER_ERROR, HTTP_504_GATEWAY_TIMEOUT
]:
logger.warning("Ran into HTTPError: %s. Retrying." %
exc.response.status_code)
raise self.retry(exc)
else:
msg = "Ran into unknown HTTPError. %s. Aborting." % \
exc.response.status_code
logger.error(msg)
self.request.callbacks = None
return
att_found = None
for attachment in att_report.data.get('attachments', []):
if description_re.search(attachment['description']):
att_found = attachment.copy()
document_type = RECAPDocument.ATTACHMENT
break
if not att_found:
if fallback_to_main_doc:
logger.info("Falling back to main document for pacer_doc_id: %s" %
rd.pacer_doc_id)
att_found = att_report.data
document_type = RECAPDocument.PACER_DOCUMENT
else:
msg = "Aborting. Did not find civil cover sheet for %s." % rd
logger.error(msg)
self.request.callbacks = None
return
if not att_found.get('pacer_doc_id'):
logger.warn("No pacer_doc_id for document (is it sealed?)")
self.request.callbacks = None
return
# Try to find the attachment already in the collection
rd, _ = RECAPDocument.objects.get_or_create(
docket_entry=rd.docket_entry,
attachment_number=att_found.get('attachment_number'),
document_number=rd.document_number,
pacer_doc_id=att_found['pacer_doc_id'],
document_type=document_type,
defaults={
'date_upload': now(),
},
)
# Replace the description if we have description data. Else fallback on old.
rd.description = att_found.get('description', '') or rd.description
if tag is not None:
tag, _ = Tag.objects.get_or_create(name=tag)
rd.tags.add(tag)
if rd.is_available:
# Great. Call it a day.
rd.save(do_extraction=False, index=False)
return
# Not available. Go get it.
try:
pacer_case_id = rd.docket_entry.docket.pacer_case_id
r = att_report.download_pdf(pacer_case_id, att_found['pacer_doc_id'])
except (ConnectTimeout, ConnectionError, ReadTimeout, ReadTimeoutError,
ChunkedEncodingError) as exc:
logger.warning("Unable to get PDF for %s" % att_found['pacer_doc_id'])
raise self.retry(exc=exc)
except HTTPError as exc:
if exc.response.status_code in [
HTTP_500_INTERNAL_SERVER_ERROR, HTTP_504_GATEWAY_TIMEOUT
]:
logger.warning("Ran into HTTPError: %s. Retrying." %
exc.response.status_code)
raise self.retry(exc)
else:
msg = "Ran into unknown HTTPError. %s. Aborting." % \
exc.response.status_code
logger.error(msg)
self.request.callbacks = None
return
if r is None:
msg = "Unable to get PDF for %s at PACER court '%s' with doc id %s" % \
(rd, pacer_court_id, rd.pacer_doc_id)
logger.error(msg)
self.request.callbacks = None
return
file_name = get_document_filename(
d.court_id,
pacer_case_id,
rd.document_number,
rd.attachment_number,
)
cf = ContentFile(r.content)
rd.filepath_local.save(file_name, cf, save=False)
rd.is_available = True # We've got the PDF.
# request.content is sometimes a str, sometimes unicode, force it all to be
# bytes, pleasing hashlib.
rd.sha1 = hashlib.sha1(force_bytes(r.content)).hexdigest()
rd.page_count = get_page_count(rd.filepath_local.path, 'pdf')
# Save, extract, then save to Solr. Skip OCR for now. Don't do these async.
rd.save(do_extraction=False, index=False)
extract_recap_pdf(rd.pk, skip_ocr=True)
add_or_update_recap_document([rd.pk])
