def do_first_pass(options):
idb_rows = FjcIntegratedDatabase.objects.filter(
dataset_source=CV_2017,
).order_by("pk")
q = options["queue"]
throttle = CeleryThrottle(queue_name=q)
for i, idb_row in enumerate(queryset_generator(idb_rows)):
# Iterate over all items in the IDB and find them in the Docket
# table. If they're not there, create a new item.
if i < options["offset"]:
continue
if i >= options["limit"] > 0:
break
throttle.maybe_wait()
# TODO: See conversation in #courtlistener channel from 2019-07-11,
# In which it appears we matched a criminal case with a civil one.
# The code below doesn't protect against that, but it should (and I
# think it does in the `do_second_pass` code, below.
ds = Docket.objects.filter(
docket_number_core=idb_row.docket_number,
court=idb_row.district,
)
count = ds.count()
if count == 0:
logger.info(
"%s: Creating new docket for IDB row: %s", i, idb_row
)
create_new_docket_from_idb.apply_async(
args=(idb_row.pk,), queue=q,
)
elif count == 1:
d = ds[0]
logger.info(
"%s: Merging Docket %s with IDB row: %s", i, d, idb_row
)
merge_docket_with_idb.apply_async(
args=(d.pk, idb_row.pk), queue=q
)
elif count > 1:
logger.warning(
"%s: Unable to merge. Got %s dockets for row: %s",
i,
count,
idb_row,
)
def handle(self, *args, **options):
super(Command, self).handle(*args, **options)
idb_rows = FjcIntegratedDatabase.objects.filter(
dataset_source=CV_2017, ).order_by('pk')
q = options['queue']
throttle = CeleryThrottle(queue_name=q)
for i, idb_row in enumerate(queryset_generator(idb_rows)):
# Iterate over all items in the IDB and find them in the Docket
# table. If they're not there, create a new item.
if i < options['offset']:
continue
if i >= options['limit'] > 0:
break
throttle.maybe_wait()
docket_number_no_0s = remove_leading_zeros(idb_row.docket_number)
ds = Docket.objects.filter(
Q(docket_number_core=idb_row.docket_number)
| Q(docket_number_core=docket_number_no_0s),
court=idb_row.district,
)
count = ds.count()
if count == 0:
logger.info("%s: Creating new docket for IDB row: %s", i,
idb_row)
create_new_docket_from_idb.apply_async(
args=(idb_row.pk, ),
queue=q,
)
elif count == 1:
d = ds[0]
logger.info("%s: Merging Docket %s with IDB row: %s", i, d,
idb_row)
merge_docket_with_idb.apply_async(args=(d.pk, idb_row.pk),
queue=q)
elif count > 1:
logger.warn("%s: Unable to merge. Got %s dockets for row: %s",
i, count, idb_row)
def do_first_pass(options):
idb_rows = FjcIntegratedDatabase.objects.filter(
dataset_source=CV_2017,
).order_by('pk')
q = options['queue']
throttle = CeleryThrottle(queue_name=q)
for i, idb_row in enumerate(queryset_generator(idb_rows)):
# Iterate over all items in the IDB and find them in the Docket
# table. If they're not there, create a new item.
if i < options['offset']:
continue
if i >= options['limit'] > 0:
break
throttle.maybe_wait()
ds = Docket.objects.filter(
docket_number_core=idb_row.docket_number,
court=idb_row.district,
)
count = ds.count()
if count == 0:
logger.info("%s: Creating new docket for IDB row: %s",
i, idb_row)
create_new_docket_from_idb.apply_async(
args=(idb_row.pk,),
queue=q,
)
elif count == 1:
d = ds[0]
logger.info("%s: Merging Docket %s with IDB row: %s",
i, d, idb_row)
merge_docket_with_idb.apply_async(args=(d.pk, idb_row.pk),
queue=q)
elif count > 1:
logger.warn("%s: Unable to merge. Got %s dockets for row: %s",
i, count, idb_row)