def querylocks(request):
try:
key = request.GET["key"].strip()
except KeyError:
# Fail. Missing required arguments.
return HttpResponse("0<br>Missing arguments.")
authquery = Uploader.objects.filter(key=key)
# Authenticate the uploader.
try:
uploader = authquery[0]
except IndexError:
# Fail. No auth key match.
return HttpResponse("0<br>Authentication failed.")
else:
uploaderid = uploader.id
lock_triples = BucketLockManager.query_locks(uploaderid)
pairs = ""
numlocks = 0
for (court, casenum, nonce) in lock_triples:
pairs += "%s,%s,%s<br>" % (court, casenum, nonce)
numlocks += 1
return HttpResponse("%d<br>%s" % (numlocks, pairs))
def lock(request):
try:
key = request.GET["key"].strip()
court = request.GET["court"].strip()
casenum = request.GET["casenum"].strip()
one_per_uploader = 1 if request.GET.get('one_per_uploader') else 0
except KeyError:
# Fail. Missing required arguments.
return HttpResponse("0<br>Missing arguments.")
authquery = Uploader.objects.filter(key=key)
# Authenticate the uploader.
try:
uploader = authquery[0]
except IndexError:
# Fail. No auth key match.
return HttpResponse("0<br>Authentication failed.")
else:
uploaderid = uploader.id
# Try to grab the lock.
lock_nonce, errmsg = BucketLockManager.get_lock(court, casenum,
uploaderid, one_per_uploader)
if lock_nonce:
return HttpResponse("1<br>%s" % lock_nonce)
else:
return HttpResponse("0<br>%s" % errmsg)
def unlock(request):
try:
key = request.GET["key"].strip()
court = request.GET["court"].strip()
casenum = request.GET["casenum"].strip()
modified = bool(int(request.GET["modified"]))
ignore_nonce = bool(int(request.GET["nononce"]))
except KeyError:
# Fail. Missing required arguments.
return HttpResponse("0<br>Missing arguments.")
except ValueError:
return HttpResponse("0<br>Invalid integer boolean for 'modified'.")
authquery = Uploader.objects.filter(key=key)
# Authenticate the uploader.
try:
uploader = authquery[0]
except IndexError:
# Fail. No auth key match.
return HttpResponse("0<br>Authentication failed.")
else:
uploaderid = uploader.id
dropped, errmsg = BucketLockManager.drop_lock(court, casenum, uploaderid,
modified=modified,
ignore_nonce = ignore_nonce)
if dropped:
return HttpResponse("1")
else:
return HttpResponse("0<br>%s" % errmsg)
def querylocks(request):
try:
key = request.GET["key"].strip()
except KeyError:
# Fail. Missing required arguments.
return HttpResponse("0<br>Missing arguments.")
authquery = Uploader.objects.filter(key=key)
# Authenticate the uploader.
try:
uploader = authquery[0]
except IndexError:
# Fail. No auth key match.
return HttpResponse("0<br>Authentication failed.")
else:
uploaderid = uploader.id
lock_triples = BucketLockManager.query_locks(uploaderid)
pairs = ""
numlocks = 0
for (court, casenum, nonce) in lock_triples:
pairs += "%s,%s,%s<br>" % (court, casenum, nonce)
numlocks += 1
return HttpResponse("%d<br>%s" % (numlocks, pairs))
def unlock(request):
try:
key = request.GET["key"].strip()
court = request.GET["court"].strip()
casenum = request.GET["casenum"].strip()
modified = bool(int(request.GET["modified"]))
ignore_nonce = bool(int(request.GET["nononce"]))
except KeyError:
# Fail. Missing required arguments.
return HttpResponse("0<br>Missing arguments.")
except ValueError:
return HttpResponse("0<br>Invalid integer boolean for 'modified'.")
authquery = Uploader.objects.filter(key=key)
# Authenticate the uploader.
try:
uploader = authquery[0]
except IndexError:
# Fail. No auth key match.
return HttpResponse("0<br>Authentication failed.")
else:
uploaderid = uploader.id
dropped, errmsg = BucketLockManager.drop_lock(court,
casenum,
uploaderid,
modified=modified,
ignore_nonce=ignore_nonce)
if dropped:
return HttpResponse("1")
else:
return HttpResponse("0<br>%s" % errmsg)
def lock(request):
try:
key = request.GET["key"].strip()
court = request.GET["court"].strip()
casenum = request.GET["casenum"].strip()
one_per_uploader = 1 if request.GET.get('one_per_uploader') else 0
except KeyError:
# Fail. Missing required arguments.
return HttpResponse("0<br>Missing arguments.")
authquery = Uploader.objects.filter(key=key)
# Authenticate the uploader.
try:
uploader = authquery[0]
except IndexError:
# Fail. No auth key match.
return HttpResponse("0<br>Authentication failed.")
else:
uploaderid = uploader.id
# Try to grab the lock.
lock_nonce, errmsg = BucketLockManager.get_lock(court, casenum, uploaderid,
one_per_uploader)
if not lock_nonce or lock_nonce == 'bigdoc':
return HttpResponse("0<br>%s" % errmsg)
else:
return HttpResponse("1<br>%s" % lock_nonce)
def _cron_get_updates():
''' Async fetch and update after a lock has been unlocked. '''
# Calculate the TIMEOUT cutoff
now = datetime.datetime.now()
timeout_delta = datetime.timedelta(seconds=LOCK_TIMEOUT)
timeout_cutoff = now - timeout_delta
# Set both ready and expired locks to the 'processing' state.
readylocks = BucketLockManager.mark_ready_for_processing(timeout_cutoff)
expiredlocks = BucketLockManager.mark_expired_for_processing(timeout_cutoff)
# Then, go through the ready locks one-by-one with HTTP waiting.
for lock in readylocks:
_cron_fetch_update(lock)
for expiredlock in expiredlocks:
court = unicode(expiredlock.court)
casenum = unicode(expiredlock.casenum)
print " %s.%s lock expired." % (court, casenum)
_cron_fetch_update(expiredlock)
def _cron_fetch_update(lock):
court = unicode(lock.court)
casenum = unicode(lock.casenum)
nonce = unicode(lock.nonce)
docketstring, fetcherror = IADirect.get_docket_string(court, casenum)
if not docketstring:
# Couldn't get the docket. Try again later.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s couldn't fetch the docket: %d" % (court, casenum,
fetcherror)
return
ia_docket, message = DocketXML.parse_xml_string(docketstring)
if not ia_docket:
# Docket parsing error.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s docket parsing error: %s" % (court, casenum, message)
return
elif ia_docket.nonce == nonce or not nonce:
# Got the docket and it is either:
# 1. up-to-date (nonce match), or
# 2. expired (ignore nonce)
# In both scenarios, update the local DB.
DocumentManager.update_local_db(ia_docket, ignore_available=0)
print " %s.%s fetched and DB updated." % (court, casenum)
ia_docket_orig_hash = hash(pickle.dumps(ia_docket))
local_docket = DocumentManager.create_docket_from_local_documents(
court, casenum)
if local_docket:
ia_docket.merge_docket(local_docket)
ia_docket_after_local_merge_hash = hash(pickle.dumps(ia_docket))
if ia_docket_orig_hash != ia_docket_after_local_merge_hash:
print " After fetch, some locally stored information was " \
"missing from %s.%s. Local info addition scheduled." % (
court, casenum)
UploadHandler.do_me_up(ia_docket)
# Remove the lock.
lock.delete()
else:
# Got the docket but it is not update to date. Try again later.
BucketLockManager.try_lock_later(lock)
print " %s.%s fetched, wait more." % (court, casenum)
def _cron_fetch_update(lock):
court = unicode(lock.court)
casenum = unicode(lock.casenum)
nonce = unicode(lock.nonce)
docketstring, fetcherror = IADirect.get_docket_string(court, casenum)
if not docketstring:
# Couldn't get the docket. Try again later.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s couldn't fetch the docket: %d" % (court, casenum,
fetcherror)
return
ia_docket, message = DocketXML.parse_xml_string(docketstring)
if not ia_docket:
# Docket parsing error.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s docket parsing error: %s" % (court, casenum,
message)
return
elif ia_docket.nonce == nonce or not nonce:
# Got the docket and it is either:
# 1. up-to-date (nonce match), or
# 2. expired (ignore nonce)
# In both scenarios, update the local DB.
DocumentManager.update_local_db(ia_docket, ignore_available=0)
print " %s.%s fetched and DB updated." % (court, casenum)
ia_docket_orig_hash = hash(pickle.dumps(ia_docket))
local_docket = DocumentManager.create_docket_from_local_documents(court, casenum)
if local_docket:
ia_docket.merge_docket(local_docket)
ia_docket_after_local_merge_hash = hash(pickle.dumps(ia_docket))
if ia_docket_orig_hash != ia_docket_after_local_merge_hash:
print " After fetch, some locally stored information was missing from %s.%s. Local info addition scheduled." % (court, casenum)
UploadHandler.do_me_up(ia_docket)
# Remove the lock.
lock.delete()
else:
# Got the docket but it is not update to date. Try again later.
BucketLockManager.try_lock_later(lock)
print " %s.%s fetched, wait more." % (court, casenum)
def _cron_get_updates():
""" Async fetch and update after a lock has been unlocked. """
# Calculate the TIMEOUT cutoff
now = datetime.datetime.now()
timeout_delta = datetime.timedelta(seconds=LOCK_TIMEOUT)
timeout_cutoff = now - timeout_delta
# Set both ready and expired locks to the 'processing' state.
readylocks = BucketLockManager.mark_ready_for_processing(timeout_cutoff)
# expiredlocks = BucketLockManager.mark_expired_for_processing(timeout_cutoff)
# Then, go through the ready locks one-by-one with HTTP waiting.
for lock in readylocks:
_cron_fetch_update(lock)
def _cron_get_updates():
""" Async fetch and update after a lock has been unlocked. """
# Calculate the TIMEOUT cutoff
now = datetime.datetime.now()
timeout_delta = datetime.timedelta(seconds=LOCK_TIMEOUT)
timeout_cutoff = now - timeout_delta
# Set both ready and expired locks to the 'processing' state.
readylocks = BucketLockManager.mark_ready_for_processing(timeout_cutoff)
# expiredlocks = BucketLockManager.mark_expired_for_processing(timeout_cutoff)
# Then, go through the ready locks one-by-one with HTTP waiting.
for lock in readylocks:
_cron_fetch_update(lock)
def _cron_put_pickles():
# Get uploader credentials.
uploader_query = Uploader.objects.filter(key=AUTH_KEY)
try:
RECAP_UPLOADER_ID = uploader_query[0].id
except IndexError:
print " could not find uploader with key=%s" % AUTH_KEY
return
# Get all ready pickles
query = PickledPut.objects.filter(ready=1, processing=0) \
.order_by('-filename')
# Set all ready pickles to the processing state
#for ppentry in query:
# ppentry.processing = 1
# ppentry.save()
# Keep track of court, casenum. Only lock and unlock once for each case.
curr_court = None
curr_casenum = None
lock_nonce = None
# Process pickles one at a time.
for ppentry in query:
filename = ppentry.filename
ppmeta = IACommon.get_meta_from_filename(filename)
court = ppmeta["court"]
casenum = ppmeta["casenum"]
# Make sure we have the lock for this case.
if curr_court == court and curr_casenum == casenum:
# Same case as the previous ppentry.
if not lock_nonce:
# Skip if we don't have the lock already.
# ppentry.processing = 0
# ppentry.save()
continue
# Otherwise, we already have the lock, so continue.
else:
# Switching to a new case.
# Drop the current lock (from previous case), if necessary.
if curr_court and curr_casenum:
dropped, errmsg = BucketLockManager.drop_lock(curr_court,
curr_casenum,
RECAP_UPLOADER_ID,
nolocaldb=1)
if not dropped:
print " %s.%s someone stole my lock?" % \
(court, unicode(casenum))
# Grab new lock
curr_court = court
curr_casenum = casenum
lock_nonce, errmsg = BucketLockManager.get_lock(court, casenum,
RECAP_UPLOADER_ID,
one_per_uploader=1)
if not lock_nonce:
print " Passing on %s.%s: %s" % (court, casenum, errmsg)
# We don't have a lock, so don't drop the lock in the next loop
curr_court = None
curr_casenum = None
continue
# We'll always have the lock here.
# Unpickle the object
obj, unpickle_msg = unpickle_object(filename)
# Two cases for the unpickled object: Request or DocketXML
if obj and ppentry.docket:
_cron_process_docketXML(obj, ppentry)
elif obj:
# Dispatch the PUT request
_cron_process_PDF(obj, ppentry)
else:
# Unpickling failed
# If unpickling fails, it could mean that another cron job
# has already finished this PP - not sure how to distinguish this
print " %s %s (Another cron job completed?)" % (filename, unpickle_msg)
# Delete the entry from the DB
ppentry.delete()
# Delete the pickle file
delete_pickle(filename)
# Drop last lock
if curr_court and curr_casenum:
dropped, errmsg = BucketLockManager.drop_lock(curr_court, curr_casenum,
RECAP_UPLOADER_ID,
nolocaldb=1)
if not dropped:
print " %s.%s someone stole my lock??" % (court, unicode(casenum))
def do_me_up(docket):
''' Download, merge and update the docket with IA. '''
# Pickle this object for do_me_up by the cron process.
court = docket.get_court()
casenum = docket.get_casenum()
docketname = IACommon.get_docketxml_name(court, casenum)
# Check if this docket is already scheduled to be processed.
query = PickledPut.objects.filter(filename=docketname)
try:
ppentry = query[0]
except IndexError:
# Not already scheduled, so schedule it now.
ppentry = PickledPut(filename=docketname, docket=1)
try:
ppentry.save()
except IntegrityError:
# Try again.
do_me_up(docket)
else:
# Pickle this object.
pickle_success, msg = IA.pickle_object(docket, docketname)
if pickle_success:
# Ready for processing.
ppentry.ready = 1
ppentry.save()
logging.info("do_me_up: ready. %s" % (docketname))
else:
# Pickle failed, remove from DB.
ppentry.delete()
logging.error("do_me_up: %s %s" % (msg, docketname))
else:
# Already scheduled.
# If there is a lock for this case, it's being uploaded. Don't merge now
locked = BucketLockManager.lock_exists(court, casenum)
if ppentry.ready and not locked:
# Docket is waiting to be processed by cron job.
# Revert state back to 'not ready' so we can do local merge.
ppentry.ready = 0
ppentry.save()
# Fetch and unpickle the waiting docket.
prev_docket, unpickle_msg = IA.unpickle_object(docketname)
if prev_docket:
# Do the local merge.
prev_docket.merge_docket(docket)
# Pickle it back
pickle_success, pickle_msg = \
IA.pickle_object(prev_docket, docketname)
if pickle_success:
# Merged and ready.
ppentry.ready = 1
ppentry.save()
logging.info("do_me_up: merged and ready. %s" %(docketname))
else:
# Re-pickle failed, delete.
ppentry.delete()
logging.error("do_me_up: re-%s %s" % (pickle_msg,
docketname))
else:
# Unpickle failed
ppentry.delete()
IA.delete_pickle(docketname)
logging.error("do_me_up: %s %s" % (unpickle_msg, docketname))
# Ignore if in any of the other three possible state...
# because another cron job is already doing work on this entity
# Don't delete DB entry or pickle file.
elif ppentry.ready and locked:
pass
#logging.debug("do_me_up: %s discarded, processing conflict." %
# (docketname))
elif not ppentry.ready and not locked:
pass
#logging.debug("do_me_up: %s discarded, preparation conflict." %
# (docketname))
else:
logging.error("do_me_up: %s discarded, inconsistent state." %
(docketname))
def _cron_put_pickles():
# Get uploader credentials.
uploader_query = Uploader.objects.filter(key=AUTH_KEY)
try:
RECAP_UPLOADER_ID = uploader_query[0].id
except IndexError:
print " could not find uploader with key=%s" % AUTH_KEY
return
# Get all ready pickles
query = PickledPut.objects.filter(ready=1, processing=0) \
.order_by('-filename')
# Set all ready pickles to the processing state
# for ppentry in query:
# ppentry.processing = 1
# ppentry.save()
# Keep track of court, casenum. Only lock and unlock once for each case.
curr_court = None
curr_casenum = None
lock_nonce = None
# Process pickles one at a time.
for ppentry in query:
filename = ppentry.filename
ppmeta = IACommon.get_meta_from_filename(filename)
court = ppmeta["court"]
casenum = ppmeta["casenum"]
# Make sure we have the lock for this case.
if curr_court == court and curr_casenum == casenum:
# Same case as the previous ppentry.
if not lock_nonce:
# Skip if we don't have the lock already.
# ppentry.processing = 0
# ppentry.save()
continue
# Otherwise, we already have the lock, so continue.
else:
# Switching to a new case.
# Drop the current lock (from previous case), if necessary.
if curr_court and curr_casenum:
dropped, errmsg = BucketLockManager.drop_lock(
curr_court, curr_casenum, RECAP_UPLOADER_ID, nolocaldb=1)
if not dropped:
print " %s.%s someone stole my lock?" % \
(court, unicode(casenum))
# Grab new lock
curr_court = court
curr_casenum = casenum
lock_nonce, errmsg = BucketLockManager.get_lock(court,
casenum,
RECAP_UPLOADER_ID,
one_per_uploader=1)
if not lock_nonce:
print " Passing on %s.%s: %s" % (court, casenum, errmsg)
if not lock_nonce or lock_nonce == 'bigdoc':
# We don't have a lock, so don't drop the lock in the next loop
curr_court = None
curr_casenum = None
continue
# We'll always have the lock here.
# Unpickle the object
obj, unpickle_msg = unpickle_object(filename)
# Two cases for the unpickled object: Request or DocketXML
if obj and ppentry.docket:
print "Processing docket: %s" % filename
_cron_process_docketXML(obj, ppentry)
elif obj:
# Dispatch the PUT request
_cron_process_PDF(obj, ppentry)
else:
# Unpickling failed
# If unpickling fails, it could mean that another cron job
# has already finished this PP - not sure how to distinguish this
print " %s %s (Another cron job completed?)" % (filename,
unpickle_msg)
# Delete the entry from the DB
ppentry.delete()
# Delete the pickle file
delete_pickle(filename)
# Drop last lock
if curr_court and curr_casenum:
dropped, errmsg = BucketLockManager.drop_lock(curr_court,
curr_casenum,
RECAP_UPLOADER_ID,
nolocaldb=1)
if not dropped:
print " %s.%s someone stole my lock??" % (court, unicode(casenum))
def do_me_up(docket):
""" Download, merge and update the docket with IA. """
# Pickle this object for do_me_up by the cron process.
court = docket.get_court()
casenum = docket.get_casenum()
docketname = IACommon.get_docketxml_name(court, casenum)
# Check if this docket is already scheduled to be processed.
query = PickledPut.objects.filter(filename=docketname)
try:
ppentry = query[0]
except IndexError:
# Not already scheduled, so schedule it now.
ppentry = PickledPut(filename=docketname, docket=1)
try:
ppentry.save()
except IntegrityError:
# Try again.
do_me_up(docket)
else:
# Pickle this object.
pickle_success, msg = IA.pickle_object(docket, docketname)
if pickle_success:
# Ready for processing.
ppentry.ready = 1
ppentry.save()
logging.info("do_me_up: ready. %s" % (docketname))
else:
# Pickle failed, remove from DB.
ppentry.delete()
logging.error("do_me_up: %s %s" % (msg, docketname))
else:
# Already scheduled.
# If there is a lock for this case, it's being uploaded. Don't merge now
locked = BucketLockManager.lock_exists(court, casenum)
if ppentry.ready and not locked:
# Docket is waiting to be processed by cron job.
# Revert state back to 'not ready' so we can do local merge.
ppentry.ready = 0
ppentry.save()
# Fetch and unpickle the waiting docket.
prev_docket, unpickle_msg = IA.unpickle_object(docketname)
if prev_docket:
# Do the local merge.
prev_docket.merge_docket(docket)
# Pickle it back
pickle_success, pickle_msg = \
IA.pickle_object(prev_docket, docketname)
if pickle_success:
# Merged and ready.
ppentry.ready = 1
ppentry.save()
logging.info(
"do_me_up: merged and ready. %s" % (docketname))
else:
# Re-pickle failed, delete.
ppentry.delete()
logging.error("do_me_up: re-%s %s" % (pickle_msg,
docketname))
else:
# Unpickle failed
ppentry.delete()
IA.delete_pickle(docketname)
logging.error("do_me_up: %s %s" % (unpickle_msg, docketname))
# Ignore if in any of the other three possible state...
# because another cron job is already doing work on this entity
# Don't delete DB entry or pickle file.
elif ppentry.ready and locked:
pass
#logging.debug("do_me_up: %s discarded, processing conflict." %
# (docketname))
elif not ppentry.ready and not locked:
pass
#logging.debug("do_me_up: %s discarded, preparation conflict." %
# (docketname))
else:
logging.error("do_me_up: %s discarded, inconsistent state." %
(docketname))