def get_pdfs(options):
"""Get PDFs for the results of the Free Document Report queries.
At this stage, we have rows in the PACERFreeDocumentRow table, each of
which represents a PDF we need to download and merge into our normal
tables: Docket, DocketEntry, and RECAPDocument.
In this function, we iterate over the entire table of results, merge it
into our normal tables, and then download and extract the PDF.
:return: None
"""
q = options["queue"]
index = options["index"]
cnt = CaseNameTweaker()
rows = PACERFreeDocumentRow.objects.filter(error_msg="").only("pk")
count = rows.count()
task_name = "downloading"
if index:
task_name += " and indexing"
logger.info("%s %s items from PACER." % (task_name, count))
throttle = CeleryThrottle(queue_name=q)
completed = 0
for row in queryset_generator(rows):
throttle.maybe_wait()
if completed % 30000 == 0:
pacer_session = PacerSession(
username=PACER_USERNAME, password=PACER_PASSWORD
)
pacer_session.login()
c = chain(
process_free_opinion_result.si(row.pk, cnt).set(queue=q),
get_and_process_pdf.s(pacer_session.cookies, row.pk).set(queue=q),
delete_pacer_row.s(row.pk).set(queue=q),
)
if index:
c |= add_items_to_solr.s("search.RECAPDocument").set(queue=q)
c.apply_async()
completed += 1
if completed % 1000 == 0:
logger.info(
"Sent %s/%s tasks to celery for %s so "
"far." % (completed, count, task_name)
)
def get_pdfs(options: OptionsType) -> None:
"""Get PDFs for the results of the Free Document Report queries.
At this stage, we have rows in the PACERFreeDocumentRow table, each of
which represents a PDF we need to download and merge into our normal
tables: Docket, DocketEntry, and RECAPDocument.
In this function, we iterate over the entire table of results, merge it
into our normal tables, and then download and extract the PDF.
:return: None
"""
q = cast(str, options["queue"])
index = options["index"]
cnt = CaseNameTweaker()
rows = PACERFreeDocumentRow.objects.filter(error_msg="").only("pk")
count = rows.count()
task_name = "downloading"
if index:
task_name += " and indexing"
logger.info(f"{task_name} {count} items from PACER.")
throttle = CeleryThrottle(queue_name=q)
completed = 0
for row in rows.iterator():
throttle.maybe_wait()
c = chain(
process_free_opinion_result.si(
row.pk,
row.court_id,
cnt,
).set(queue=q),
get_and_process_free_pdf.s(row.pk, row.court_id).set(queue=q),
delete_pacer_row.s(row.pk).set(queue=q),
)
if index:
c |= add_items_to_solr.s("search.RECAPDocument").set(queue=q)
c.apply_async()
completed += 1
if completed % 1000 == 0:
logger.info(
f"Sent {completed}/{count} tasks to celery for {task_name} so far."
)
def get_pdfs(options):
"""Get PDFs for the results of the Free Document Report queries.
At this stage, we have rows in the PACERFreeDocumentRow table, each of
which represents a PDF we need to download and merge into our normal
tables: Docket, DocketEntry, and RECAPDocument.
In this function, we iterate over the entire table of results, merge it
into our normal tables, and then download and extract the PDF.
:return: None
"""
q = options['queue']
index = options['index']
cnt = CaseNameTweaker()
rows = PACERFreeDocumentRow.objects.filter(error_msg="").only('pk')
count = rows.count()
task_name = "downloading"
if index:
task_name += " and indexing"
logger.info("%s %s items from PACER." % (task_name, count))
throttle = CeleryThrottle(queue_name=q)
completed = 0
for row in queryset_generator(rows):
throttle.maybe_wait()
if completed % 30000 == 0:
pacer_session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
pacer_session.login()
c = chain(
process_free_opinion_result.si(row.pk, cnt).set(queue=q),
get_and_process_pdf.s(pacer_session.cookies, row.pk).set(queue=q),
delete_pacer_row.s(row.pk).set(queue=q),
)
if index:
c |= add_items_to_solr.s('search.RECAPDocument').set(queue=q)
c.apply_async()
completed += 1
if completed % 1000 == 0:
logger.info("Sent %s/%s tasks to celery for %s so "
"far." % (completed, count, task_name))
def handle(self, *args, **options):
super(Command, self).handle(*args, **options)
if options["sweep"] is False:
# Only allow one script at a time per court combination.
# Note that multiple scripts on multiple machines could still be
# run.
court_str = "-".join(sorted(options["courts"]))
with open(f"/tmp/rss-scraper-{court_str}.pid", "w") as fp:
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
print("Another instance of this program is running with "
"this combination of courts. Only one instance "
"can crawl these courts at a time: '%s'" % court_str)
sys.exit(1)
# Loop over the PACER sites that have RSS feeds and see if they're
# ready to do.
courts = Court.federal_courts.district_pacer_courts().filter(
pacer_has_rss_feed=True, )
if options["courts"] != ["all"]:
courts = courts.filter(pk__in=options["courts"])
iterations_completed = 0
last_trim_date = None
while (options["iterations"] == 0
or iterations_completed < options["iterations"]):
for court in courts:
# Check the last time we successfully got the feed
try:
feed_status = RssFeedStatus.objects.filter(
court=court,
is_sweep=options["sweep"],
status__in=[
RssFeedStatus.PROCESSING_SUCCESSFUL,
RssFeedStatus.UNCHANGED,
RssFeedStatus.PROCESSING_IN_PROGRESS,
],
).latest("date_created")
except RssFeedStatus.DoesNotExist:
# First time running it or status items have been nuked by
# an admin. Make a dummy object, but no need to actually
# save it to the DB. Make it old.
lincolns_birthday = make_aware(datetime(1809, 2, 12))
feed_status = RssFeedStatus(
date_created=lincolns_birthday,
date_last_build=lincolns_birthday,
is_sweep=options["sweep"],
)
if options["courts"] == ["all"] and options["sweep"] is False:
# If it's all courts and it's not a sweep, check if we did
# it recently.
max_visit_ago = now() - timedelta(
seconds=self.RSS_MAX_VISIT_FREQUENCY)
if feed_status.date_created > max_visit_ago:
# Processed too recently. Try next court.
continue
# Don't crawl a court if it says it's been in progress just a
# little while. It's probably queued and on the way.
processing_cutoff = now() - timedelta(
seconds=self.RSS_MAX_PROCESSING_DURATION)
if all([
options["sweep"] is False,
feed_status.status ==
RssFeedStatus.PROCESSING_IN_PROGRESS,
feed_status.date_created > processing_cutoff,
]):
continue
# The court is ripe! Crawl it if it has changed.
# Make a new object to track the attempted crawl.
new_status = RssFeedStatus.objects.create(
court_id=court.pk,
status=RssFeedStatus.PROCESSING_IN_PROGRESS,
is_sweep=options["sweep"],
)
# Check if the item needs crawling, and crawl it if so.
chain(
check_if_feed_changed.s(court.pk, new_status.pk,
feed_status.date_last_build),
merge_rss_feed_contents.s(court.pk),
send_docket_alerts.s(),
# Update recap *documents*, not *dockets*. Updating dockets
# requires much more work, and we don't expect to get much
# docket information from the RSS feeds. RSS feeds also
# have information about hundreds or thousands of
# dockets. Updating them all would be very bad.
add_items_to_solr.s("search.RECAPDocument"),
mark_status_successful.si(new_status.pk),
).apply_async()
# Trim if not too recently trimmed.
trim_cutoff_date = now() - timedelta(
seconds=self.DELAY_BETWEEN_CACHE_TRIMS)
if last_trim_date is None or trim_cutoff_date > last_trim_date:
trim_rss_data.delay()
last_trim_date = now()
# Wait, then attempt the courts again if iterations not exceeded.
iterations_completed += 1
remaining_iterations = options["iterations"] - iterations_completed
if remaining_iterations > 0:
time.sleep(self.DELAY_BETWEEN_ITERATIONS)
def handle(self, *args, **options):
super(Command, self).handle(*args, **options)
if options['sweep'] is False:
# Only allow one script at a time per court combination.
# Note that multiple scripts on multiple machines could still be
# run.
court_str = '-'.join(sorted(options['courts']))
with open('/tmp/rss-scraper-%s.pid' % court_str, 'w') as fp:
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
print("Another instance of this program is running with "
"for this combination of courts. Only one instance "
"can crawl these courts at a time: '%s'" % court_str)
sys.exit(1)
# Loop over the PACER sites that have RSS feeds and see if they're
# ready to do.
courts = Court.objects.filter(
jurisdiction__in=[
Court.FEDERAL_BANKRUPTCY,
Court.FEDERAL_DISTRICT,
],
pacer_has_rss_feed=True,
)
if options['courts'] != ['all']:
courts = courts.filter(pk__in=options['courts'])
iterations_completed = 0
last_trim_date = None
while options['iterations'] == 0 or \
iterations_completed < options['iterations']:
for court in courts:
# Check the last time we successfully got the feed
try:
feed_status = RssFeedStatus.objects.filter(
court=court,
is_sweep=options['sweep'],
status__in=[
RssFeedStatus.PROCESSING_SUCCESSFUL,
RssFeedStatus.UNCHANGED,
RssFeedStatus.PROCESSING_IN_PROGRESS,
]
).latest('date_created')
except RssFeedStatus.DoesNotExist:
# First time running it or status items have been nuked by
# an admin. Make a dummy object, but no need to actually
# save it to the DB. Make it old.
lincolns_birthday = make_aware(datetime(1809, 2, 12))
feed_status = RssFeedStatus(
date_created=lincolns_birthday,
date_last_build=lincolns_birthday,
is_sweep=options['sweep'],
)
if options['courts'] == ['all'] and options['sweep'] is False:
# If it's all courts and it's not a sweep, check if we did
# it recently.
max_visit_ago = now() - timedelta(
seconds=self.RSS_MAX_VISIT_FREQUENCY)
if feed_status.date_created > max_visit_ago:
# Processed too recently. Try next court.
continue
# Give a court some time to complete during non-sweep crawls
processing_cutoff = now() - timedelta(
seconds=self.RSS_MAX_PROCESSING_DURATION)
if all([
options['sweep'] is False,
feed_status.status == RssFeedStatus.PROCESSING_IN_PROGRESS,
feed_status.date_created > processing_cutoff
]):
continue
# The court is ripe! Crawl it if it has changed.
# Make a new object to track the attempted crawl.
new_status = RssFeedStatus.objects.create(
court_id=court.pk,
status=RssFeedStatus.PROCESSING_IN_PROGRESS,
is_sweep=options['sweep'],
)
# Check if the item needs crawling, and crawl it if so.
chain(
check_if_feed_changed.s(court.pk, new_status.pk,
feed_status.date_last_build),
merge_rss_feed_contents.s(court.pk, new_status.pk),
send_docket_alerts.s(),
# Update recap *documents*, not *dockets*. Updating dockets
# requires much more work, and we don't expect to get much
# docket information from the RSS feeds. RSS feeds also
# have information about hundreds or thousands of
# dockets. Updating them all would be very bad.
add_items_to_solr.s('search.RECAPDocument'),
mark_status_successful.si(new_status.pk),
).apply_async()
# Trim if not too recently trimmed.
trim_cutoff_date = now() - timedelta(
seconds=self.DELAY_BETWEEN_CACHE_TRIMS)
if last_trim_date is None or trim_cutoff_date > last_trim_date:
trim_rss_cache.delay()
last_trim_date = now()
# Wait, then attempt the courts again if iterations not exceeded.
iterations_completed += 1
time.sleep(self.DELAY_BETWEEN_ITERATIONS)
