def get_dockets(options, items, tags, sample_size=0, doc_num_end=""):
"""Download dockets from PACER.
:param options: Options provided by argparse
:param items: Items from our FJC IDB database
:param tags: A list of tag names to associate with the purchased content.
:param sample_size: The number of items to get. If 0, get them all. Else,
get only this many and do it randomly.
:param doc_num_end: Only get docket numbers up to this value to constrain
costs. If set to an empty string, no constraints are applied. Note that
applying this value means no unnumbered entries will be retrieved by PACER.
"""
if sample_size > 0:
items = items.order_by("?")[:sample_size]
q = options["queue"]
throttle = CeleryThrottle(queue_name=q)
session = PacerSession(username=PACER_USERNAME, password=PACER_PASSWORD)
session.login()
for i, row in enumerate(items):
if i < options["offset"]:
continue
if i >= options["limit"] > 0:
break
if i % 5000 == 0:
# Re-authenticate just in case the auto-login mechanism isn't
# working.
session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
session.login()
# All tests pass. Get the docket.
logger.info("Doing row %s: %s", i, row)
throttle.maybe_wait()
params = make_fjc_idb_lookup_params(row)
chain(
get_pacer_case_id_and_title.s(
pass_through=None,
docket_number=row.docket_number,
court_id=row.district_id,
cookies=session.cookies,
**params,
).set(queue=q),
filter_docket_by_tags.s(tags, row.district_id).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row.district_id,
cookies=session.cookies,
tag_names=tags,
**{
"show_parties_and_counsel": True,
"show_terminated_parties": True,
"show_list_of_member_cases": False,
"doc_num_end": doc_num_end,
},
).set(queue=q),
add_or_update_recap_docket.s().set(queue=q),
).apply_async()
def get_dockets(options, items, tags, sample_size=0, doc_num_end=''):
"""Download dockets from PACER.
:param options: Options provided by argparse
:param items: Items from our FJC IDB database
:param tags: A list of tag names to associate with the purchased content.
:param sample_size: The number of items to get. If 0, get them all. Else,
get only this many and do it randomly.
:param doc_num_end: Only get docket numbers up to this value to constrain
costs. If set to an empty string, no constraints are applied. Note that
applying this value means no unnumbered entries will be retrieved by PACER.
"""
if sample_size > 0:
items = items.order_by('?')[:sample_size]
q = options['queue']
throttle = CeleryThrottle(queue_name=q)
session = PacerSession(username=PACER_USERNAME, password=PACER_PASSWORD)
session.login()
for i, row in enumerate(items):
if i < options['offset']:
continue
if i >= options['limit'] > 0:
break
if i % 5000 == 0:
# Re-authenticate just in case the auto-login mechanism isn't
# working.
session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
session.login()
# All tests pass. Get the docket.
logger.info("Doing row %s: %s", i, row)
throttle.maybe_wait()
params = make_fjc_idb_lookup_params(row)
chain(
get_pacer_case_id_and_title.s(
pass_through=None,
docket_number=row.docket_number,
court_id=row.district_id,
cookies=session.cookies,
**params
).set(queue=q),
filter_docket_by_tags.s(tags, row.district_id).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row.district_id,
cookies=session.cookies,
tag_names=tags,
**{
'show_parties_and_counsel': True,
'show_terminated_parties': True,
'show_list_of_member_cases': False,
'doc_num_end': doc_num_end,
}
).set(queue=q),
add_or_update_recap_docket.s().set(queue=q),
).apply_async()
def get_dockets(options, items, tags, sample_size=0):
"""Download dockets from PACER.
:param options: Options provided by argparse
:param items: Items from our FJC IDB database
:param tags: A list of tag names to associate with the purchased content.
:param sample_size: The number of items to get. If 0, get them all. Else,
get only this many and do it randomly.
"""
if sample_size > 0:
items = items.order_by('?')[:sample_size]
q = options['queue']
throttle = CeleryThrottle(queue_name=q)
session = PacerSession(username=PACER_USERNAME, password=PACER_PASSWORD)
session.login()
for i, row in enumerate(items):
if i < options['offset']:
continue
if i >= options['limit'] > 0:
break
if i % 5000 == 0:
# Re-authenticate just in case the auto-login mechanism isn't
# working.
session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
session.login()
# All tests pass. Get the docket.
logger.info("Doing row %s: %s", i, row)
throttle.maybe_wait()
params = make_fjc_idb_lookup_params(row)
chain(
get_pacer_case_id_and_title.s(
docket_number=row.docket_number,
court_id=row.district_id,
cookies=session.cookies,
**params
).set(queue=q),
filter_docket_by_tags.s(tags, row.district_id).set(queue=q),
get_docket_by_pacer_case_id.s(
court_id=row.district_id,
cookies=session.cookies,
tag_names=tags,
**{
'show_parties_and_counsel': True,
'show_terminated_parties': True,
'show_list_of_member_cases': True
}
).set(queue=q),
add_or_update_recap_docket.s().set(queue=q),
).apply_async()
def update_any_missing_pacer_case_ids(options):
"""The network requests were making things far too slow and had to be
disabled during the first pass. With this method, we update any items
that are missing their pacer case ID value.
"""
ds = Docket.objects.filter(
idb_data__isnull=False,
pacer_case_id=None,
)
q = options['queue']
throttle = CeleryThrottle(queue_name=q)
session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
session.login()
for i, d in enumerate(queryset_generator(ds)):
if i < options['offset']:
continue
if i >= options['limit'] > 0:
break
if i % 5000 == 0:
# Re-authenticate just in case the auto-login mechanism isn't
# working.
session = PacerSession(username=PACER_USERNAME,
password=PACER_PASSWORD)
session.login()
throttle.maybe_wait()
logger.info("Getting pacer_case_id for item %s", d)
params = make_fjc_idb_lookup_params(d.idb_data)
chain(
get_pacer_case_id_and_title.s(
pass_through=d.pk,
docket_number=d.idb_data.docket_number,
court_id=d.idb_data.district_id,
cookies=session.cookies,
**params
).set(queue=q),
update_docket_from_hidden_api.s().set(queue=q),
).apply_async()
