def test_valid_query_cases_response_unavailable_doc_currently_uploading(self):
self.available_doc.available = 0
self.available_doc.save()
ppquery = PickledPut(filename=IACommon.get_docketxml_name('nysd', '1234'))
ppquery.save()
response = self.client.post('/recap/query_cases/', {'json': simplejson.dumps(self.valid_params)})
self.assertEquals("{}", response.content)
PickledPut.objects.all().delete()
def test_valid_query_cases_response_old_doc_currently_uploading(self):
self.available_doc.available = 0
two_days_ago = datetime.datetime.now() - datetime.timedelta(2)
self.available_doc.modified= two_days_ago
self.available_doc.save()
ppquery = PickledPut(filename=IACommon.get_docketxml_name('nysd', '1234'))
ppquery.save()
response = self.client.post('/recap/query_cases/', {'json': simplejson.dumps(self.valid_params)})
output= simplejson.loads(response.content)
self.assertEquals(2, len(output))
self.assertEquals(self.available_doc.lastdate.strftime("%m/%d/%y"), output['timestamp'])
self.assertEquals(IACommon.get_dockethtml_url('nysd', '1234'), output['docket_url'])
PickledPut.objects.all().delete()
def put_file(filebits, court, casenum, docnum, subdocnum, metadict={}):
""" PUT the file into a new Internet Archive bucket. """
request = IACommon.make_pdf_request(filebits, court, casenum, docnum,
subdocnum, metadict)
# If this file is already scheduled, drop this. # TK: what we want?
filename = IACommon.get_pdfname(court, casenum, docnum, subdocnum)
query = PickledPut.objects.filter(filename=filename)
if query:
logging.info("put_file: same file already pickled. %s" % filename)
return "IA PUT failed: the same file is already in the pickle bucket."
# Add a PickledPut DB entry to schedule the PUT, not yet ready
ppentry = PickledPut(filename=filename)
# Fix a race case?
try:
ppentry.save()
except IntegrityError:
logging.info("put_file: same file already pickled. %s" % filename)
return "IA PUT failed: the same file is already in the pickle bucket."
# Pickle the request object into the jar
pickle_success, message = pickle_object(request, filename)
if pickle_success:
# PickledPut now ready for processing.
ppentry.ready = 1
ppentry.save()
logging.info("put_file: ready. %s" % filename)
else:
# Could not pickle object, so remove from DB
logging.warning("put_file: could not pickle PDF. %s" % filename)
ppentry.delete()
return message
def put_file(filebits, court, casenum, docnum, subdocnum, metadict={}):
""" PUT the file into a new Internet Archive bucket. """
request = IACommon.make_pdf_request(filebits, court, casenum,
docnum, subdocnum, metadict)
# If this file is already scheduled, drop this. # TK: what we want?
filename = IACommon.get_pdfname(court, casenum, docnum, subdocnum)
query = PickledPut.objects.filter(filename=filename)
if query:
logging.info("put_file: same file already pickled. %s" % filename)
return "IA PUT failed: the same file is already in the pickle bucket."
# Add a PickledPut DB entry to schedule the PUT, not yet ready
ppentry = PickledPut(filename=filename)
# Fix a race case?
try:
ppentry.save()
except IntegrityError:
logging.info("put_file: same file already pickled. %s" % filename)
return "IA PUT failed: the same file is already in the pickle bucket."
# Pickle the request object into the jar
pickle_success, message = pickle_object(request, filename)
if pickle_success:
# PickledPut now ready for processing.
ppentry.ready = 1
ppentry.save()
logging.info("put_file: ready. %s" % filename)
else:
# Could not pickle object, so remove from DB
logging.warning("put_file: could not pickle PDF. %s" % filename)
ppentry.delete()
return message
def do_me_up(docket):
''' Download, merge and update the docket with IA. '''
# Pickle this object for do_me_up by the cron process.
court = docket.get_court()
casenum = docket.get_casenum()
docketname = IACommon.get_docketxml_name(court, casenum)
# Check if this docket is already scheduled to be processed.
query = PickledPut.objects.filter(filename=docketname)
try:
ppentry = query[0]
except IndexError:
# Not already scheduled, so schedule it now.
ppentry = PickledPut(filename=docketname, docket=1)
try:
ppentry.save()
except IntegrityError:
# Try again.
do_me_up(docket)
else:
# Pickle this object.
pickle_success, msg = IA.pickle_object(docket, docketname)
if pickle_success:
# Ready for processing.
ppentry.ready = 1
ppentry.save()
logging.info("do_me_up: ready. %s" % (docketname))
else:
# Pickle failed, remove from DB.
ppentry.delete()
logging.error("do_me_up: %s %s" % (msg, docketname))
else:
# Already scheduled.
# If there is a lock for this case, it's being uploaded. Don't merge now
locked = BucketLockManager.lock_exists(court, casenum)
if ppentry.ready and not locked:
# Docket is waiting to be processed by cron job.
# Revert state back to 'not ready' so we can do local merge.
ppentry.ready = 0
ppentry.save()
# Fetch and unpickle the waiting docket.
prev_docket, unpickle_msg = IA.unpickle_object(docketname)
if prev_docket:
# Do the local merge.
prev_docket.merge_docket(docket)
# Pickle it back
pickle_success, pickle_msg = \
IA.pickle_object(prev_docket, docketname)
if pickle_success:
# Merged and ready.
ppentry.ready = 1
ppentry.save()
logging.info("do_me_up: merged and ready. %s" %(docketname))
else:
# Re-pickle failed, delete.
ppentry.delete()
logging.error("do_me_up: re-%s %s" % (pickle_msg,
docketname))
else:
# Unpickle failed
ppentry.delete()
IA.delete_pickle(docketname)
logging.error("do_me_up: %s %s" % (unpickle_msg, docketname))
# Ignore if in any of the other three possible state...
# because another cron job is already doing work on this entity
# Don't delete DB entry or pickle file.
elif ppentry.ready and locked:
pass
#logging.debug("do_me_up: %s discarded, processing conflict." %
# (docketname))
elif not ppentry.ready and not locked:
pass
#logging.debug("do_me_up: %s discarded, preparation conflict." %
# (docketname))
else:
logging.error("do_me_up: %s discarded, inconsistent state." %
(docketname))
def do_me_up(docket):
""" Download, merge and update the docket with IA. """
# Pickle this object for do_me_up by the cron process.
court = docket.get_court()
casenum = docket.get_casenum()
docketname = IACommon.get_docketxml_name(court, casenum)
# Check if this docket is already scheduled to be processed.
query = PickledPut.objects.filter(filename=docketname)
try:
ppentry = query[0]
except IndexError:
# Not already scheduled, so schedule it now.
ppentry = PickledPut(filename=docketname, docket=1)
try:
ppentry.save()
except IntegrityError:
# Try again.
do_me_up(docket)
else:
# Pickle this object.
pickle_success, msg = IA.pickle_object(docket, docketname)
if pickle_success:
# Ready for processing.
ppentry.ready = 1
ppentry.save()
logging.info("do_me_up: ready. %s" % (docketname))
else:
# Pickle failed, remove from DB.
ppentry.delete()
logging.error("do_me_up: %s %s" % (msg, docketname))
else:
# Already scheduled.
# If there is a lock for this case, it's being uploaded. Don't merge now
locked = BucketLockManager.lock_exists(court, casenum)
if ppentry.ready and not locked:
# Docket is waiting to be processed by cron job.
# Revert state back to 'not ready' so we can do local merge.
ppentry.ready = 0
ppentry.save()
# Fetch and unpickle the waiting docket.
prev_docket, unpickle_msg = IA.unpickle_object(docketname)
if prev_docket:
# Do the local merge.
prev_docket.merge_docket(docket)
# Pickle it back
pickle_success, pickle_msg = \
IA.pickle_object(prev_docket, docketname)
if pickle_success:
# Merged and ready.
ppentry.ready = 1
ppentry.save()
logging.info(
"do_me_up: merged and ready. %s" % (docketname))
else:
# Re-pickle failed, delete.
ppentry.delete()
logging.error("do_me_up: re-%s %s" % (pickle_msg,
docketname))
else:
# Unpickle failed
ppentry.delete()
IA.delete_pickle(docketname)
logging.error("do_me_up: %s %s" % (unpickle_msg, docketname))
# Ignore if in any of the other three possible state...
# because another cron job is already doing work on this entity
# Don't delete DB entry or pickle file.
elif ppentry.ready and locked:
pass
#logging.debug("do_me_up: %s discarded, processing conflict." %
# (docketname))
elif not ppentry.ready and not locked:
pass
#logging.debug("do_me_up: %s discarded, preparation conflict." %
# (docketname))
else:
logging.error("do_me_up: %s discarded, inconsistent state." %
(docketname))
