def _cron_fetch_update(lock):
court = unicode(lock.court)
casenum = unicode(lock.casenum)
nonce = unicode(lock.nonce)
docketstring, fetcherror = IADirect.get_docket_string(court, casenum)
if not docketstring:
# Couldn't get the docket. Try again later.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s couldn't fetch the docket: %d" % (court, casenum,
fetcherror)
return
ia_docket, message = DocketXML.parse_xml_string(docketstring)
if not ia_docket:
# Docket parsing error.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s docket parsing error: %s" % (court, casenum, message)
return
elif ia_docket.nonce == nonce or not nonce:
# Got the docket and it is either:
# 1. up-to-date (nonce match), or
# 2. expired (ignore nonce)
# In both scenarios, update the local DB.
DocumentManager.update_local_db(ia_docket, ignore_available=0)
print " %s.%s fetched and DB updated." % (court, casenum)
ia_docket_orig_hash = hash(pickle.dumps(ia_docket))
local_docket = DocumentManager.create_docket_from_local_documents(
court, casenum)
if local_docket:
ia_docket.merge_docket(local_docket)
ia_docket_after_local_merge_hash = hash(pickle.dumps(ia_docket))
if ia_docket_orig_hash != ia_docket_after_local_merge_hash:
print " After fetch, some locally stored information was " \
"missing from %s.%s. Local info addition scheduled." % (
court, casenum)
UploadHandler.do_me_up(ia_docket)
# Remove the lock.
lock.delete()
else:
# Got the docket but it is not update to date. Try again later.
BucketLockManager.try_lock_later(lock)
print " %s.%s fetched, wait more." % (court, casenum)
def _cron_fetch_update(lock):
court = unicode(lock.court)
casenum = unicode(lock.casenum)
nonce = unicode(lock.nonce)
docketstring, fetcherror = IADirect.get_docket_string(court, casenum)
if not docketstring:
# Couldn't get the docket. Try again later.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s couldn't fetch the docket: %d" % (court, casenum,
fetcherror)
return
ia_docket, message = DocketXML.parse_xml_string(docketstring)
if not ia_docket:
# Docket parsing error.
if nonce:
BucketLockManager.try_lock_later(lock)
else:
lock.delete()
print " %s.%s docket parsing error: %s" % (court, casenum,
message)
return
elif ia_docket.nonce == nonce or not nonce:
# Got the docket and it is either:
# 1. up-to-date (nonce match), or
# 2. expired (ignore nonce)
# In both scenarios, update the local DB.
DocumentManager.update_local_db(ia_docket, ignore_available=0)
print " %s.%s fetched and DB updated." % (court, casenum)
ia_docket_orig_hash = hash(pickle.dumps(ia_docket))
local_docket = DocumentManager.create_docket_from_local_documents(court, casenum)
if local_docket:
ia_docket.merge_docket(local_docket)
ia_docket_after_local_merge_hash = hash(pickle.dumps(ia_docket))
if ia_docket_orig_hash != ia_docket_after_local_merge_hash:
print " After fetch, some locally stored information was missing from %s.%s. Local info addition scheduled." % (court, casenum)
UploadHandler.do_me_up(ia_docket)
# Remove the lock.
lock.delete()
else:
# Got the docket but it is not update to date. Try again later.
BucketLockManager.try_lock_later(lock)
print " %s.%s fetched, wait more." % (court, casenum)
def _cron_process_PDF(obj, ppentry):
filename = ppentry.filename
meta = IACommon.get_meta_from_filename(filename)
court = meta["court"]
casenum = meta["casenum"]
docnum = meta["docnum"]
subdocnum = meta["subdocnum"]
invalid_PDF = _is_invalid_pdf(obj, filename)
# We only want to check for ssns on valid PDFs
# PyPdf doesn't deal well with bad input
if not invalid_PDF:
# SSN privacy check
has_ssn = _has_ssn(obj, filename)
else:
has_ssn = False
# Blacklist file check
in_blacklist = _in_blacklist(filename)
if invalid_PDF or has_ssn or in_blacklist:
docket = DocketXML.make_docket_for_pdf("",
court,
casenum,
docnum,
subdocnum,
available=0)
UploadHandler.do_me_up(docket)
# Delete the entry from the DB
ppentry.delete()
# Quarantine the pickle file for analysis
_quarantine_pickle(filename,
ssn=has_ssn,
blacklist_file=in_blacklist,
invalid_PDF=invalid_PDF)
return
put_result, put_msg = _dispatch_put(obj, ppentry)
if put_result:
# Put success-- mark this document as available in the DB
DocumentManager.mark_as_available(filename)
docket = DocketXML.make_docket_for_pdf("",
court,
casenum,
docnum,
subdocnum,
available=1)
UploadHandler.do_me_up(docket)
print " %s %s" % (filename, put_msg)
def mark_document_as_unavailable(document):
#if not document.available:
# print "Exiting: This document isn't currently available on IA"
# print usage()
# exit()
document.available = 0
document.lastdate = datetime.datetime.now() # this ensures that the archive.recapthelaw will get the update
document.save()
docket = DocketXML.make_docket_for_pdf("", document.court, document.casenum, document.docnum,
document.subdocnum, available=0)
UploadHandler.do_me_up(docket)
def mark_document_as_unavailable(document):
# if not document.available:
# print "Exiting: This document isn't currently available on IA"
# print usage()
# exit()
document.available = 0
document.lastdate = datetime.datetime.now() # this ensures that the archive.recapthelaw will get the update
document.save()
docket = DocketXML.make_docket_for_pdf("", document.court,
document.casenum, document.docnum,
document.subdocnum, available=0)
UploadHandler.do_me_up(docket)
def _cron_process_PDF(obj, ppentry):
filename = ppentry.filename
meta = IACommon.get_meta_from_filename(filename)
court = meta["court"]
casenum = meta["casenum"]
docnum = meta["docnum"]
subdocnum = meta["subdocnum"]
invalid_PDF = _is_invalid_pdf(obj, filename)
# We only want to check for ssns on valid PDFs
# PyPdf doesn't deal well with bad input
if not invalid_PDF:
# SSN privacy check
has_ssn = _has_ssn(obj, filename)
else:
has_ssn = False
# Blacklist file check
in_blacklist = _in_blacklist(filename)
if invalid_PDF or has_ssn or in_blacklist:
docket = DocketXML.make_docket_for_pdf("", court, casenum, docnum,
subdocnum, available=0)
UploadHandler.do_me_up(docket)
# Delete the entry from the DB
ppentry.delete()
# Quarantine the pickle file for analysis
_quarantine_pickle(filename, ssn=has_ssn, blacklist_file= in_blacklist, invalid_PDF= invalid_PDF)
return
put_result, put_msg = _dispatch_put(obj, ppentry)
if put_result:
# Put success-- mark this document as available in the DB
DocumentManager.mark_as_available(filename)
docket = DocketXML.make_docket_for_pdf("", court, casenum, docnum,
subdocnum, available=1)
UploadHandler.do_me_up(docket)
print " %s %s" % (filename, put_msg)
" The contents of filename should have a single case per line, each identified by 'court casenum'\n "
)
sys.exit(1)
cases_to_repair = read_in_cases_to_repair(sys.argv[1])
for case in cases_to_repair:
court = case[0]
casenum = case[1]
print "Repairing case %s.%s...." % (court, casenum)
docket = DocumentManager.create_docket_from_local_documents(
court, casenum)
if docket:
# this will merge our docket with existing one on IA
UploadHandler.do_me_up(docket)
else:
print " Could not create docket from local documents for %s %s" % (
court, casenum)
# for each case, create docket fromlocal
# call do_me_up(docket)
# download ia_docket
#
# merge ia docket and local docket
#
# if there is a difference, schedule the docket for upload (created a pickled put?)
sys.stderr.write("Usage: %s <filename_containing_cases_to_repair>\n " % sys.argv[0])
sys.stderr.write(" The contents of filename should have a single case per line, each identified by 'court casenum'\n " )
sys.exit(1)
cases_to_repair = read_in_cases_to_repair(sys.argv[1])
for case in cases_to_repair:
court = case[0]
casenum = case[1]
print "Repairing case %s.%s...." % (court, casenum)
docket = DocumentManager.create_docket_from_local_documents(court, casenum)
if docket:
# this will merge our docket with existing one on IA
UploadHandler.do_me_up(docket)
else:
print " Could not create docket from local documents for %s %s" % (court, casenum)
# for each case, create docket fromlocal
# call do_me_up(docket)
# download ia_docket
#
# merge ia docket and local docket
#
# if there is a difference, schedule the docket for upload (created a pickled put?)
