def pruneSolr_worker(job_package, PID=False): if PID: # prune specific PID solr_handle.delete_by_key(PID) return "PRUNED" else: doc_id = job_package['doc_id'] if not fedora_handle.get_object(doc_id).exists: print "Did not find object in Fedora, pruning from Solr..." solr_handle.delete_by_key(doc_id) return "PRUNED" else: return "IGNORED"
def pruneSolr_worker(job_package): print "pruning Solr of objects not found in Fedora" # variables count = 0 pruned = [] start = 0 rows = 100 # get solr results obj solr_total = solr_handle.search(q='*:*', fl='id').total_results # iterate through while start < solr_total: # perform search solr_result = solr_handle.search(q='*:*', fl='id', rows=rows, start=start) # iterate for doc in solr_result.documents: doc_id = doc['id'] print "pruneSolr checking %s, %i / %i" % (doc_id, count, solr_total) if not fedora_handle.get_object(doc_id).exists: print "Did not find object in Fedora, pruning from Solr..." pruned.append(doc_id) solr_handle.delete_by_key(doc_id) # bump counter count+=1 # bump start start += rows # return JSON report return json.dumps(pruned)
def ingestBagAndPush(bag_dir, dest_repo, refresh_remote=True, overwrite=False, export_context='migrate'):
# DEBUG
print dir(localConfig)
# load bag_handle and ingest
print "Working on:",bag_dir
bag_handle = WSUDOR_ContentTypes.WSUDOR_Object(object_type="bag", payload=bag_dir)
if bag_dir == 'Could not load WSUDOR or Bag object.':
print "Aborting, bag_handle initiziation was unsuccessful."
return False
# validate bag for WSUDOR ingest
valid_results = bag_handle.validIngestBag()
if valid_results['verdict'] != True:
print "Bag is not valid for the following reasons, aborting.", valid_results
return False
# ingest bag & skip indexing
ingest_bag = bag_handle.ingestBag(indexObject=False)
# push to remote repo
print "sending object..."
# Use object method
obj_handle = WSUDOR_ContentTypes.WSUDOR_Object(bag_handle.pid)
obj_handle.sendObject(dest_repo, refresh_remote=refresh_remote, overwrite=overwrite, export_context=export_context)
# delete local object
print "finally, removing object"
fedora_handle.purge_object(bag_handle.pid)
# remove from Solr
solr_handle.delete_by_key(bag_handle.pid)
return json.dumps({"Ingest Results for {bag_label}, PID: {bag_pid}".format(bag_label=bag_handle.label.encode('utf-8'),bag_pid=bag_handle.pid):True})
def delete(self):
delete_response = solr_handle.delete_by_key(self.id, commit=False)
return delete_response
def bagIngest_worker(job_package):
bag_dir = job_package['bag_dir']
# load bag_handle and ingest
print "Working on:",bag_dir
bag_handle = WSUDOR_ContentTypes.WSUDOR_Object(bag_dir, object_type="bag")
if bag_dir == 'Could not load WSUDOR or Bag object.':
print "Aborting, bag_handle initiziation was unsuccessful."
return False
# validate bag for WSUDOR ingest
valid_results = bag_handle.validIngestBag()
if valid_results['verdict'] != True:
print "Bag is not valid for the following reasons, aborting.", valid_results
return False
# optional flags
###########################################################################################
# overwrite
if 'overwrite' in job_package['form_data']:
print "purging object if exists"
if fedora_handle.get_object(bag_handle.pid).exists:
fedora_handle.purge_object(bag_handle.pid)
# push to remote
if 'push_remote' in job_package['form_data']:
# get options
# set destination repo
dest_repo = job_package['form_data']['dest_repo']
# get export context
export_context = job_package['form_data']['export_context']
# overwrite
if 'overwrite' in job_package['form_data']:
overwrite = True
else:
overwrite = False
# refresh remote
if 'refresh_remote' in job_package['form_data']:
refresh_remote = True
else:
refresh_remote = False
# omit checksums
if 'omit_checksums' in job_package['form_data']:
omit_checksums = True
else:
omit_checksums = False
# ingest bag
try:
# because we're sending remotely, not indexing locally
ingest_bag = bag_handle.ingestBag(indexObject=False)
except Exception, e:
raise Exception(e)
return False
# push to remote repo
print "sending object..."
# Use object method
obj_handle = WSUDOR_ContentTypes.WSUDOR_Object(bag_handle.pid)
obj_handle.sendObject(dest_repo, refresh_remote=refresh_remote, overwrite=overwrite, export_context=export_context, omit_checksums=omit_checksums)
# delete local object (and constituent objects)
print "purging Constituents if present"
if getattr(obj_handle, 'purgeConstituents', None):
obj_handle.purgeConstituents()
print "finally, removing object"
fedora_handle.purge_object(obj_handle.pid)
# remove from Solr
solr_handle.delete_by_key(obj_handle.pid)
# fire ingestWorkspace callback if checked
if 'origin' in job_package['form_data'] and job_package['form_data']['origin'] == 'ingestWorkspace' and ingest_bag == True:
print "firing ingest callback"
actions.actions.ingestBag_callback.apply_async(kwargs={'job_package':job_package}, queue=job_package['username'])
return json.dumps({"Ingest Results for %s, PID: %s" % (bag_handle.label.encode('utf-8'), bag_handle.pid):ingest_bag})