def __createFromSelected(self):
self.vc("log").debug("Creating package from selected...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
# if modifying existing manifest, we already have an identifier,
# otherwise create a new one
manifestId = self.__getActiveManifestId()
if manifestId is None:
manifestHash = "%s.tfpackage" % uuid.uuid4()
else:
manifestHash = self.__getActiveManifestPid()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
manifest = self.__getActiveManifest()
oldType = manifest.getType()
if oldType is None:
manifest.setType(packageType)
else:
manifest.setType(oldType)
#self.vc("log").debug("Manifest: {}", manifest)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
if manifestId is None:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, and make sure they are both deployed
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
else:
# update existing object
object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId)
manifestStream = FileUtils.openInputStream(manifestFile)
StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream)
manifestStream.close()
object.close()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __activate__(self, context):
request = context["request"]
storage = context["Services"].getStorage()
auth = context["page"].authentication
log = context["log"]
username = auth.get_name()
oid = request.getParameter("oid")
approval = request.getParameter("approval")
approval_comment = request.getParameter("approval_comment")
storedObj = storage.getObject(oid)
committeeResponses = None
payloadList = storedObj.getPayloadIdList()
if payloadList.contains("committee-responses.metadata"):
committeeResponsePayload = storedObj.getPayload("committee-responses.metadata")
committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject()
else:
committeeResponses = JsonObject()
committeeResponse = JsonObject()
committeeResponse.put("approval",approval)
committeeResponse.put("approval_comment",approval_comment)
committeeResponses.put(username,committeeResponse)
log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment))
StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8"))
context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def updateLocalRecordRelations(self, jobItems):
oidIdentifierMap = HashMap()
for jobItem in jobItems:
oidIdentifierMap.put(jobItem.get("oid"),jobItem.get("required_identifiers")[0].get("identifier"))
for jobItem in jobItems:
type = jobItem.get("type");
targetSystem = self.systemConfig.getString(None, "curation", "supported-types", type);
if targetSystem == "redbox":
oid = jobItem.get("oid")
digitalObject = StorageUtils.getDigitalObject(self.services.getStorage(), oid)
tfPackagePid = self.getPackageData(digitalObject)
metadataJsonPayload = digitalObject.getPayload(tfPackagePid)
metadataJsonInstream = metadataJsonPayload.open()
metadataJson = JsonSimple(metadataJsonInstream)
metadataJsonPayload.close()
relationships = metadataJson.getArray("relationships")
if relationships is not None:
for relationship in relationships:
system = relationship.get("system")
if system != "redbox" or system != None:
url = self.systemConfig.getString("can't find it", "curation","external-system-urls","get-oid-for-identifier",system)
client = BasicHttpClient(url+ "&identifier="+relationship.get("identifier"))
get = GetMethod(url+ "&identifier="+relationship.get("identifier"))
client.executeMethod(get)
if get.getStatusCode() == 200:
response = JsonSimple(get.getResponseBodyAsString())
relationship.put("curatedPid",oidIdentifierMap.get(response.getString(None,"oid")))
relationship.put("isCurated",True)
#Now update the relationship on Mint's side
break
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
StorageUtils.createOrUpdatePayload(digitalObject,tfPackagePid,istream)
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper")
externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder")
oid = self.request.getParameter("oid")
if oid is None :
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
relationshipType = self.request.getParameter("relationship")
curatedPid = self.request.getParameter("curatedPid")
sourceId = self.request.getParameter("sourceIdentifier")
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJson = self.getTfPackage(digitalObject)
relationships = metadataJson.getArray("relationships")
found = False
for relationship in relationships:
if relationship.get("identifier") == sourceId:
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
found = True
if not found:
relationship = JsonObject()
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
relationship.put("relationship",relationshipType)
relationship.put("identifier",sourceId)
relationships.add(relationship)
self.log.info(metadataJson.toString(True))
out.println(metadataJson.toString(True))
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
for pid in digitalObject.getPayloadIdList():
if pid.endswith(".tfpackage"):
StorageUtils.createOrUpdatePayload(digitalObject,pid,istream)
out.close()
finally:
self.sessionState.remove("username")
def __checkMetadataPayload(self, identifier):
# We are just going to confirm the existance of
# 'metadata.json', or create an empty one if it
# doesn't exist. Makes curation function for this
# option and removes some log errors on the details
# screen.
try:
self.object.getPayload("metadata.json")
# all is good, the above will throw an exception if it doesn't exist
return
except Exception:
self.log.info("Creating 'metadata.json' payload for object '{}'", self.oid)
# Prep data
metadata = JsonSimple()
metadata.getJsonObject().put("recordIDPrefix", "")
metadata.writeObject("data")
# The only real data we require is the ID for curation
idHolder = metadata.writeObject("metadata")
idHolder.put("dc.identifier", identifier)
# Store it
inStream = IOUtils.toInputStream(metadata.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "metadata.json", inStream)
except StorageException, e:
self.log.error("Error creating 'metadata.json' payload for object '{}'", self.oid, e)
return
def updateRelationships(self, relationship,pid,identifier):
oid = self.findOidByIdentifier(relationship.get("identifier"))
self.writer.println(oid)
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJsonPayload = digitalObject.getPayload("metadata.json")
metadataJsonInstream = metadataJsonPayload.open()
metadataJson = JsonSimple(metadataJsonInstream)
metadataJsonPayload.close()
relationships = metadataJson.getArray("relationships")
found = False
if relationships is None:
relationships = JSONArray()
metadataJson.getJsonObject().put("relationships",relationships)
for relationship1 in relationships:
if relationship1.get("identifier") == identifier:
relationship1.put("isCurated",True)
relationship1.put("curatedPid",pid)
found = True
if not found:
newRelationship = JsonObject()
newRelationship.put("isCurated",True)
newRelationship.put("curatedPid",pid)
newRelationship.put("relationship",relationship.get("relationship"))
newRelationship.put("identifier",identifier)
relationships.add(newRelationship)
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream)
def __activate__(self, context):
self.auth = context["page"].authentication
self.errorMsg = ""
self.request = context["request"]
self.response = context["response"]
self.formData = context["formData"]
self.storage = context["Services"].getStorage()
self.log = context["log"]
self.reportManager = context["Services"].getService("reportManager")
fromOid = self.formData.get("fromOid")
fromObject = self.storage.getObject(fromOid)
if (self.auth.is_logged_in()):
if (self.auth.is_admin() == True):
pass
elif (self.__isOwner(fromObject)):
pass
else:
self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access."
else:
self.errorMsg = "Please login."
if self.errorMsg == "":
toOid = self.formData.get("toOid")
toObject = self.storage.getObject(toOid)
storeRelatedData = self.formData.get("relatedData")
fromTFPackage = self._getTFPackage(fromObject)
toTFPackage = self._getTFPackage(toObject)
fromInputStream = fromTFPackage.open()
try:
StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream)
except StorageException:
print "error setting tfPackage"
fromTFPackage.close()
fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject()
if storeRelatedData != "false" :
# add relatedOid info
fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid)
inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream)
except StorageException:
print "error setting tfPackage"
tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue")
self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue)
self._reharvestPackage()
result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid , toOid)
else:
result = '{"status": "err", "message": "%s"}' % self.errorMsg
writer = self.response.getPrintWriter("application/json; charset=UTF-8")
writer.println(result)
writer.close()
def saveResponse(self, context):
""" Save into object storage key to assessor's name
It has four keys: status, recommendation, size-agreement and comments
when status == "submitted", reviewer sees it
"""
oid = self.request.getParameter("oid")
action = self.request.getParameter("action")
if action and re.match("submit", action, re.I):
status = "submitted"
else:
status = "draft"
recommendation = self.request.getParameter("recommendation")
sizeAgreement = self.request.getParameter("size-agreement")
comments = self.request.getParameter("comments")
storedObj, fileExisted = self.hasResponses(oid)
if fileExisted:
committeeResponses = self.getResponses(storedObj)
else:
committeeResponses = JsonObject()
assessorResponse = JsonObject()
assessorResponse.put("status", status)
if status == 'submitted':
assessorResponse.put("date",self._getToday())
assessorResponse.put("recommendation",recommendation)
assessorResponse.put("size-agreement",sizeAgreement)
assessorResponse.put("comments",comments)
committeeResponses.put(self.assessor,assessorResponse)
StorageUtils.createOrUpdatePayload(storedObj,self.PAYLOAD,IOUtils.toInputStream(committeeResponses.toString(), "UTF-8"))
context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def _addPropertyValueToTFMeta(self, object, tfMetaPropertyValue):
objectMetadata = object.getMetadata()
objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue)
output = ByteArrayOutputStream();
objectMetadata.store(output, None);
input = ByteArrayInputStream(output.toByteArray());
StorageUtils.createOrUpdatePayload(object,"TF-OBJ-META",input);
def _addPropertyValueToTFMeta(self, object, tfMetaPropertyValue):
objectMetadata = object.getMetadata()
objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue)
output = ByteArrayOutputStream()
objectMetadata.store(output, None)
input = ByteArrayInputStream(output.toByteArray())
StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input)
def updateWorkFlowMetadata(self, workflowMetadata, toWorkflowId, toWorkflowStage):
workflowMetaDataJson = JsonSimple(workflowMetadata.open()).getJsonObject()
workflowMetaDataJson.put("id", toWorkflowId)
workflowMetaDataJson.put("step", toWorkflowStage)
inStream = IOUtils.toInputStream(workflowMetaDataJson.toString(), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
def _addPropertyValueToTFMeta(self, obj, tfMetaPropertyValue):
objectMetadata = obj.getMetadata()
objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue)
objectMetadata.setProperty("render-pending", "true")
output = ByteArrayOutputStream();
objectMetadata.store(output, None);
input = ByteArrayInputStream(output.toByteArray());
StorageUtils.createOrUpdatePayload(obj,"TF-OBJ-META",input);
def updatePackageType(self, tfPackage, toWorkflowId):
tfPackageJson = JsonSimple(tfPackage.open()).getJsonObject()
tfPackageJson.put("packageType", toWorkflowId)
inStream = IOUtils.toInputStream(tfPackageJson.toString(), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, tfPackage.getId(), inStream)
except StorageException:
print " ERROR updating dataset payload"
def updatePackageType(self, tfPackage, toWorkflowId):
tfPackageJson = JsonSimple(tfPackage.open()).getJsonObject()
tfPackageJson.put("packageType", toWorkflowId)
inStream = IOUtils.toInputStream(tfPackageJson.toString(), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, tfPackage.getId(),
inStream)
except StorageException:
print " ERROR updating dataset payload"
def __activate__(self, context):
self.None = context["log"]
self.systemConfig = context["systemConfig"]
self.sessionState = context["sessionState"]
self.response = context["response"]
self.request = context["request"]
self.indexer = context["Services"].getIndexer()
self.storage = context["Services"].getStorage()
self.log = context["log"]
self.sessionState.set("username", "admin")
self.writer = self.response.getPrintWriter("text/plain; charset=UTF-8")
publishedRecords = self.findPublishedRecords()
for publishedRecord in publishedRecords:
digitalObject = StorageUtils.getDigitalObject(
self.storage, publishedRecord.getString(None, "storage_id"))
tfPackage = self.getTfPackage(digitalObject)
metadata = digitalObject.getMetadata()
configObject = StorageUtils.getDigitalObject(
self.storage, metadata.getProperty("jsonConfigOid"))
payload = configObject.getPayload(
metadata.getProperty("jsonConfigPid"))
inStream = payload.open()
jsonConfig = JsonSimple(inStream)
payload.close()
requiredIdentifiers = jsonConfig.getArray("curation",
"requiredIdentifiers")
if requiredIdentifiers is not None:
pidName = self.systemConfig.getString(None, "curation",
"identifier-pids",
requiredIdentifiers[0])
pid = metadata.getProperty(pidName)
identifier = tfPackage.getString(pid, "metadata",
"dc.identifier")
relationships = tfPackage.getArray("relationships")
if relationships is not None:
for relationship in relationships:
self.writer.println(relationship)
if relationship.get("broker") is None:
if relationship.get(
"system"
) is not None and relationship.get(
"system") != self.systemConfig.getString(
None, "system"):
self.writer.println(
"notifyExternalRelationship")
self.notifyExternalRelationship(
relationship, pid,
relationship.get("system"), identifier)
else:
self.updateRelationships(relationship, pid)
self.writer.close()
self.sessionState.remove("username")
def __createOrUpdateArrayPayload(self, oid, payloadName, newObject):
"""
Create or update a payload object in storage defined by oid
The content of this digital object is a JsonArray of objects
payloadName: name of the payload
newObject: new object to be appended, e.g. a JsonObject
"""
objList = self.__getPayloadJsonArray(oid, payloadName)
objList.add(newObject)
storedObj = self.Services.getStorage().getObject(oid)
StorageUtils.createOrUpdatePayload(storedObj, payloadName,IOUtils.toInputStream(objList.toString(), "UTF-8"))
def __change(self, context, oid, new_owner):
storage = self.services.getStorage()
object = storage.getObject(oid)
objectMetadata = object.getMetadata()
owner = objectMetadata.getProperty("owner")
objectMetadata.setProperty("owner", new_owner)
self.log.debug("grantAccess.py: Changed ownership from {} to {} ",
owner, new_owner)
output = ByteArrayOutputStream()
objectMetadata.store(output, None)
input = ByteArrayInputStream(output.toByteArray())
StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input)
try:
auth = context["page"].authentication
source = context["formData"].get("source")
self.log.debug("grantAccess.py: authentication plugin: = {}",
source)
auth.set_access_plugin(source)
# special condition when setting admin as owner - revoke all viewers
if new_owner == "admin":
viewers = self.getViewers(oid, owner)
self.log.debug(
"grantAccess.py: New owner is admin, revoking all viewers")
self.log.debug("grantAccess.py: Viewers: " +
viewers.toString())
for viewer in viewers:
self.log.debug("Revoking:%s" % viewer)
auth.revoke_user_access(oid, viewer)
# when there are viewers, the previous owner somehow joins the read-only list, revoke access to the previous owner as well.
if viewers.size() > 0:
auth.revoke_user_access(oid, owner)
else:
self.log.info(
"Grant previous owner {} view access by adding them to security_execption.",
owner)
auth.grant_user_access(
oid, owner) # give previous owner read access
err = auth.get_error()
if err is None or err == 'Duplicate! That user has already been applied to this record.':
Services.indexer.index(oid)
Services.indexer.commit()
return '{"status":"ok", "new_owner": "' + new_owner + '"}'
else:
self.log.error(
"grantAccess.py: Error raised during calling authentication for changing ownership. Exception: "
+ err)
except Exception, e:
self.log.error(
"grantAccess.py: Unexpected error raised during changing ownership of data. Exception: "
+ str(e))
def updateWorkFlowMetadata(self, workflowMetadata, toWorkflowId,
toWorkflowStage):
workflowMetaDataJson = JsonSimple(
workflowMetadata.open()).getJsonObject()
workflowMetaDataJson.put("id", toWorkflowId)
workflowMetaDataJson.put("step", toWorkflowStage)
inStream = IOUtils.toInputStream(workflowMetaDataJson.toString(),
"UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
def __createOrUpdateArrayPayload(self, oid, payloadName, newObject):
"""
Create or update a payload object in storage defined by oid
The content of this digital object is a JsonArray of objects
payloadName: name of the payload
newObject: new object to be appended, e.g. a JsonObject
"""
objList = self.__getPayloadJsonArray(oid, payloadName)
objList.add(newObject)
storedObj = self.Services.getStorage().getObject(oid)
StorageUtils.createOrUpdatePayload(
storedObj, payloadName,
IOUtils.toInputStream(objList.toString(), "UTF-8"))
def getTFPackagePid(self,oid):
digitalObject = StorageUtils.getDigitalObject(self.storage,oid)
for pid in digitalObject.getPayloadIdList():
pidString = String(pid)
if pidString.endsWith("tfpackage"):
return pid
return None
def __activate__(self, context):
self.None = context["log"]
self.systemConfig = context["systemConfig"]
self.sessionState = context["sessionState"]
self.response = context["response"]
self.request = context["request"]
self.indexer = context["Services"].getIndexer()
self.storage = context["Services"].getStorage()
self.log = context["log"]
self.sessionState.set("username","admin")
self.writer = self.response.getPrintWriter("text/plain; charset=UTF-8")
publishedRecords = self.findPublishedRecords()
for publishedRecord in publishedRecords:
self.log.info("processing record "+publishedRecord.getString(None,"storage_id"))
digitalObject = StorageUtils.getDigitalObject(self.storage, publishedRecord.getString(None,"storage_id"))
tfPackage = self.getTfPackage(digitalObject)
metadata = digitalObject.getMetadata()
configObject = StorageUtils.getDigitalObject(self.storage,metadata.getProperty("jsonConfigOid"))
payload = configObject.getPayload(metadata.getProperty("jsonConfigPid"))
inStream = payload.open()
jsonConfig = JsonSimple(inStream)
payload.close()
requiredIdentifiers = jsonConfig.getArray("curation","requiredIdentifiers")
if requiredIdentifiers is not None:
pidName = self.systemConfig.getString(None,"curation","identifier-pids",requiredIdentifiers[0])
pid = metadata.getProperty(pidName)
identifier = tfPackage.getString(pid,"metadata","dc.identifier")
relationships = tfPackage.getArray("relationships")
if relationships is not None:
for relationship in relationships:
if relationship.get("broker") is None:
if relationship.get("system") is not None and relationship.get("system") != self.systemConfig.getString(None,"system"):
self.notifyExternalRelationship(relationship,pid,relationship.get("system"),identifier)
else:
self.updateRelationships(relationship,pid,identifier)
self.writer.close()
self.sessionState.remove("username")
def __change(self, context, oid, new_owner):
storage = self.services.getStorage()
object = storage.getObject(oid)
objectMetadata = object.getMetadata()
owner = objectMetadata.getProperty("owner")
objectMetadata.setProperty("owner", new_owner)
self.log.debug("grantAccess.py: Changed ownership from {} to {} ", owner, new_owner)
output = ByteArrayOutputStream()
objectMetadata.store(output, None)
input = ByteArrayInputStream(output.toByteArray())
StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input)
try:
auth = context["page"].authentication
source = context["formData"].get("source")
self.log.debug("grantAccess.py: authentication plugin: = {}", source)
auth.set_access_plugin(source)
# special condition when setting admin as owner - revoke all viewers
if new_owner == "admin":
viewers = self.getViewers(oid)
self.log.debug("grantAccess.py: New owner is admin, revoking all viewers")
self.log.debug("grantAccess.py: Viewers: " + viewers.toString())
for viewer in viewers:
self.log.debug("Revoking:%s" % viewer)
auth.revoke_user_access(oid, viewer)
# when there are viewers, the previous owner somehow joins the read-only list, revoke access to the previous owner as well.
if viewers.size() > 0:
auth.revoke_user_access(oid, owner)
else:
self.log.info("Grant previous owner {} view access by adding them to security_execption.", owner)
auth.grant_user_access(oid, owner) # give previous owner read access
err = auth.get_error()
if err is None or err == 'Duplicate! That user has already been applied to this record.':
Services.indexer.index(oid)
Services.indexer.commit()
return '{"status":"ok", "new_owner": "' + new_owner + '"}'
else:
self.log.error("grantAccess.py: Error raised during calling authentication for changing ownership. Exception: " + err)
except Exception, e:
self.log.error("grantAccess.py: Unexpected error raised during changing ownership of data. Exception: " + str(e))
def __change(self, context, oid, new_owner):
try:
storage = context["Services"].getStorage()
object = storage.getObject(oid)
objectMetadata = object.getMetadata()
owner = objectMetadata.getProperty("owner")
objectMetadata.setProperty("owner", new_owner)
self.log.debug("Changing ownership to : " + new_owner)
output = ByteArrayOutputStream()
objectMetadata.store(output, None)
input = ByteArrayInputStream(output.toByteArray())
StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input)
auth = context["page"].authentication
#special condition when setting admin as owner - revoke all viewers
if new_owner == "admin":
self.log.debug("New owner is admin, revoking all viewers")
viewers = self.getViewers(oid)
self.log.debug("Viewers: " + viewers.toString())
for viewer in viewers:
self.log.debug("Revoking:%s" % viewer)
auth.revoke_user_access(oid, viewer)
auth.revoke_user_access(oid, owner)
# don't know what's going on but this was originally set to "owner", changed it to new_owner.
auth.grant_user_access(oid, new_owner)
err = auth.get_error()
if err is None or err == 'Duplicate! That user has already been applied to this record.':
Services.indexer.index(oid)
Services.indexer.commit()
return '{"status":"ok", "new_owner": "' + new_owner + '"}'
else:
self.log.error(
"Error during changing ownership of data. Exception: " +
err)
except Exception, e:
self.log.error(
"Error during changing ownership of data. Exception: " +
str(e))
def updateObjectMetadata(self, objectMetaData, toWorkflowId):
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(
toWorkflowId)
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = File(workflowsDir, jsonConfigFile)
configObject = StorageUtils.checkHarvestFile(self.storage, configFile)
if configObject is None:
oid = StorageUtils.generateOid(configFile)
configObject = StorageUtils.getDigitalObject(self.storage, oid)
objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
objectMetaData.setProperty("jsonConfigOid", configObject.getId())
configJson = JsonSimple(configFile)
rulesFileName = configJson.getString(None, "indexer", "script",
"rules")
rulesFile = File(workflowsDir, rulesFileName)
rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile)
if rulesObject is None:
oid = StorageUtils.generateOid(rulesFile)
rulesObject = StorageUtils.getDigitalObject(self.storage, oid)
objectMetaData.setProperty("rulesPid", rulesFileName)
objectMetaData.setProperty("rulesOid", rulesObject.getId())
objectMetaData.setProperty("workflowTransitioned", "true")
objectMetaData.setProperty(
"date_transitioned",
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def updateObjectMetadata(self, objectMetaData, toWorkflowId):
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(toWorkflowId)
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = File(workflowsDir, jsonConfigFile)
configObject = StorageUtils.checkHarvestFile(self.storage, configFile);
if configObject is None:
oid = StorageUtils.generateOid(configFile);
configObject = StorageUtils.getDigitalObject(self.storage, oid);
objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
objectMetaData.setProperty("jsonConfigOid", configObject.getId())
configJson = JsonSimple(configFile)
rulesFileName = configJson.getString(None, "indexer","script","rules")
rulesFile = File(workflowsDir,rulesFileName)
rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile);
if rulesObject is None:
oid = StorageUtils.generateOid(rulesFile);
rulesObject = StorageUtils.getDigitalObject(self.storage, oid);
objectMetaData.setProperty("rulesPid", rulesFileName)
objectMetaData.setProperty("rulesOid", rulesObject.getId())
objectMetaData.setProperty("workflowTransitioned", "true")
objectMetaData.setProperty("date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def __change(self, context, oid, new_owner):
try:
storage = context["Services"].getStorage()
object = storage.getObject(oid)
objectMetadata = object.getMetadata()
owner = objectMetadata.getProperty("owner")
objectMetadata.setProperty("owner", new_owner)
self.log.debug("Changing ownership to : " + new_owner)
output = ByteArrayOutputStream()
objectMetadata.store(output, None)
input = ByteArrayInputStream(output.toByteArray())
StorageUtils.createOrUpdatePayload(object,"TF-OBJ-META",input)
auth = context["page"].authentication
#special condition when setting admin as owner - revoke all viewers
if new_owner == "admin":
self.log.debug("New owner is admin, revoking all viewers")
viewers = self.getViewers(oid)
self.log.debug("Viewers: " + viewers.toString())
for viewer in viewers:
self.log.debug("Revoking:%s" % viewer)
auth.revoke_user_access(oid, viewer)
auth.revoke_user_access(oid, owner)
# don't know what's going on but this was originally set to "owner", changed it to new_owner.
auth.grant_user_access(oid, new_owner)
err = auth.get_error()
if err is None or err == 'Duplicate! That user has already been applied to this record.':
Services.indexer.index(oid)
Services.indexer.commit()
return '{"status":"ok", "new_owner": "' + new_owner + '"}'
else:
self.log.error("Error during changing ownership of data. Exception: " + err)
except Exception, e:
self.log.error("Error during changing ownership of data. Exception: " + str(e))
def __workflow(self):
# Workflow data
WORKFLOW_ID = "Parties_People"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
initialStep = 0
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def getObjectMeta(self,oid):
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
return digitalObject.getMetadata()
def __createFromSelected(self):
self.vc("log").debug("Creating package from selected...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
# if modifying existing manifest, we already have an identifier,
# otherwise create a new one
manifestId = self.__getActiveManifestId()
if manifestId is None:
manifestHash = "%s.tfpackage" % uuid.uuid4()
else:
manifestHash = self.__getActiveManifestPid()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
manifest = self.__getActiveManifest()
oldType = manifest.getType()
if oldType is None:
manifest.setType(packageType)
else:
manifest.setType(oldType)
self.vc("log").debug("Manifest: %s" % manifest)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
if manifestId is None:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, and make sure they are both deployed
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
else:
# update existing object
object = StorageUtils.getDigitalObject(Services.getStorage(),
manifestId)
manifestStream = FileUtils.openInputStream(manifestFile)
StorageUtils.createOrUpdatePayload(object, manifestHash,
manifestStream)
manifestStream.close()
object.close()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
class AnotarData:
def __init__(self):
pass
def __activate__(self, context):
self.velocityContext = context
self.__auth = context["page"].authentication
# This gets called a lot
self.fd = self.vc("formData").get
self.action = self.fd("action")
self.rootUri = self.fd("rootUri")
self.json = self.fd("json")
self.type = self.fd("type")
self.rootUriList = self.vc("formData").getValues("rootUriList")
if self.rootUriList is None:
self.rootUriList = self.vc("formData").getValues("rootUriList[]")
self.portalPath = self.vc("portalPath")
#print "action:'%s' formData:'%s'" % (self.action, formData)
# used so that ajax requests don't cache
if self.rootUri and self.rootUri.find("?ticks") > -1:
self.rootUri = self.rootUri[:self.rootUri.find("?ticks")]
# Portal path info
portalPath = self.portalPath + "/"
self.oid = self.rootUri
if self.oid and self.oid.startswith(portalPath):
self.oid = self.oid[len(portalPath):]
# oid for packaged items
if self.oid:
hashIndex = self.oid.find("#")
if hashIndex > -1:
self.oid = self.oid[hashIndex + 1:]
result = ""
if self.action == "getList":
# Response is a list of object (nested)
#print "**** anotar.py : GET_SOLR : " + self.rootUri
result = self.search_solr()
elif self.action == "put":
result = self.__authenticate()
if result is None:
# Response is an ID
#print "**** anotar.py : PUT : " + self.rootUri
result = self.put()
elif self.action == "delete":
result = self.__authenticate()
if result is None:
# Response is empty
result = self.delete()
if result != "":
self.vc("response").setStatus(500)
elif self.action == "get-image":
# Response is the JSON format expected by image annotation plugin
result = self.get_image()
elif self.action == "save-image":
result = self.__authenticate()
if result is None:
# Response is anotar JSON
result = self.save_image()
elif self.action == "delete-image":
result = self.__authenticate()
if result is None:
result = self.delete_image()
writer = self.vc("response").getPrintWriter(
"text/plain; charset=UTF-8")
writer.println(result)
writer.close()
def __authenticate(self):
if not self.__auth.is_logged_in():
self.vc("response").setStatus(500)
return "Only registered users can access this API"
return None
# Get from velocity context
def vc(self, index):
if self.velocityContext[index] is not None:
return self.velocityContext[index]
else:
log.error("ERROR: Requested context entry '" + index +
"' doesn't exist")
return None
def generate_id(self):
counter = 0
fileName = "anotar." + str(counter)
payloadList = self.obj.getPayloadIdList()
while fileName in payloadList:
counter = counter + 1
fileName = "anotar." + str(counter)
self.pid = fileName
print "New ID (" + self.pid + ")"
def modify_json(self):
#print "**** anotar.py : add_json() : adding json : " + json
jsonSimple = JsonSimple(self.json)
jsonObj = jsonSimple.getJsonObject()
jsonObj.put("id", self.pid)
rootUri = jsonSimple.getString(None, ["annotates", "rootUri"])
if rootUri is not None:
baseUrl = "http://%s:%s/" % (self.vc("request").serverName,
self.vc("serverPort"))
myUri = baseUrl + rootUri + "#" + self.pid
jsonObj.put("uri", myUri)
jsonObj.put("schemaVersionUri",
"http://www.purl.org/anotar/schema/0.1")
self.json = jsonSimple.toString()
def process_response(self, result):
#print " ******** result =", result
docs = []
rootDocs = []
docsDict = {}
# Build a dictionary of the annotations
for doc in result:
#hack is done here to replace [] with null as json.py does not properly parse
jsonStr = unicode(doc.get("jsonString").replace(
"[]", "null")).encode("utf-8")
doc = json.read(jsonStr)
doc["replies"] = []
docs.append(doc)
docsDict[doc["uri"]] = doc
if doc["annotates"]["uri"] == doc["annotates"]["rootUri"]:
rootDocs.append(doc)
# Now process the dictionary
for doc in docs:
# If we are NOT a top level annotation
if doc["annotates"]["uri"] != doc["annotates"]["rootUri"]:
# Find what we are annotating
try:
d = docsDict[doc["annotates"]["uri"]]
d["replies"].append(doc) # Add ourselves to its reply list
except:
# TODO KeyError
pass
return json.write(rootDocs)
def process_tags(self, result):
tags = []
tagsDict = {}
# Build a dictionary of the tags
for doc in result:
# Get Anotar data from Solr data
doc = JsonSimple(doc.get("jsonString"))
# Get actual tag text
tag = doc.getString(None, ["content", "literal"])
# Find out if they have locators
locs = doc.getJsonSimpleList(["annotates", "locators"]).size()
if locs == 0:
# Basic tags, just aggregate counts
if tag in tagsDict:
# We've seen it before, just increment the counter
existing = tagsDict[tag]
count = existing.getInteger(0, ["tagCount"])
existing.getJsonObject().put("tagCount", str(count + 1))
else:
# First time, store this object
doc.getJsonObject().put("tagCount", str(1))
tagsDict[tag] = doc
else:
# Tags with a locator, special case for images etc.
tags.append(doc.toString())
# Push all the 'basic' counts into the list to return
for tag in tagsDict:
tags.append(tagsDict[tag].toString())
return "[" + ",".join(tags) + "]"
def put(self, pid=None):
try:
self.obj = Services.storage.getObject(self.oid)
except StorageException, e:
print " * anotar.py : Error creating object : ", e
return e.getMessage()
if pid:
self.pid = pid
else:
self.generate_id()
self.modify_json()
try:
p = StorageUtils.createOrUpdatePayload(
self.obj, self.pid, IOUtils.toInputStream(self.json, "UTF-8"))
except StorageException, e:
print " * anotar.py : Error creating payload :", e
return e.getMessage()
wfMetaObj.put("step", "pending")
wfMetaObj.put("pageTitle", "Uploaded Files - Management")
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == "pending":
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
jsonString = String(wfMeta.toString())
inStream = ByteArrayInputStream(jsonString.getBytes("UTF-8"))
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException, e:
print " * workflow-harvester.py : Error updating workflow payload"
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfMeta.getString(None, ["step"]))
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
def __filePath(self):
baseFilePath = self.params["base.file.path"]
filePath = self.object.getMetadata().getProperty("file.path")
if baseFilePath:
def getTfPackage(self,oid, pid):
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
tfPackageInputStream = digitalObject.getPayload(pid).open()
return JsonSimple(tfPackageInputStream)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
if self.owner == "guest":
pageTitle = "Submission Request"
displayType = "submission-request"
initialStep = 0
else:
pageTitle = "Metadata Record"
displayType = "package-dataset"
initialStep = 1
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
#Used to make sure we have a created date
createdDateFlag = False
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime)
self.utils.add(self.index, "date_created", solrTime)
self.log.debug("Set created date to :%s" % solrTime)
createdDateFlag = True
elif field == "redbox:embargo.dc:date":
self.embargoedDate = value
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
# check if this is an array field
fnameparts = field.split(":")
if fnameparts is not None and len(fnameparts) >= 3:
if field.startswith("bibo") or field.startswith("skos"):
arrParts = fnameparts[1].split(".")
else:
arrParts = fnameparts[2].split(".")
# we're not interested in: Relationship, Type and some redbox:origin
if arrParts is not None and len(arrParts) >= 2 and field.find(":Relationship.") == -1 and field.find("dc:type") == -1 and field.find("redbox:origin") == -1 and arrParts[1].isdigit():
# we've got an array field
fldPart = ":%s" % arrParts[0]
prefixEndIdx = field.find(fldPart) + len(fldPart)
suffixStartIdx = prefixEndIdx+len(arrParts[1])+1
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[suffixStartIdx:]
if field.endswith("Name"):
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx]
self.log.debug("Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value))
if field.endswith("Name"):
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
idx = arrParts[1]
fullField = fullFieldMap.get(idx)
if (fullField is None):
fullField = ""
if (field.endswith("givenName")):
fullField = "%s, %s" % (fullField, value)
if (field.endswith("familyName")):
fullField = "%s%s" % (value, fullField)
self.log.debug("fullname now is :%s" % fullField)
fullFieldMap.put(idx, fullField)
else:
fieldlist = self.arrayBucket.get(arrFldName)
if fieldlist is None:
fieldlist = []
self.arrayBucket.put(arrFldName, fieldlist)
fieldlist.append(value)
for compfield in self.compFields:
if field.startswith(compfield):
arrFldName = self.reportingFieldPrefix +compfield
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
fullField = fullFieldMap.get("1")
if fullField is None:
fullField = ""
if field.endswith(self.compFieldsConfig[compfield]["end"]):
fullField = "%s%s%s" % (fullField, self.compFieldsConfig[compfield]["delim"] ,value)
if field.endswith(self.compFieldsConfig[compfield]["start"]):
fullField = "%s%s" % (value, fullField)
self.log.debug("full field now is :%s" % fullField)
fullFieldMap.put("1", fullField)
self.utils.add(self.index, "display_type", displayType)
# Make sure we have a creation date
if not createdDateFlag:
self.utils.add(self.index, "date_created", self.last_modified)
self.log.debug("Forced creation date to %s because it was not explicitly set." % self.last_modified)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def getWorkflowMeta(self,oid):
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
workflowMetaInputStream = digitalObject.getPayload("workflow.metadata").open()
return JsonSimple(workflowMetaInputStream)
def getPayloadJson(self,oid,payloadName):
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
workflowMetaInputStream = digitalObject.getPayload(payloadName).open()
return JsonSimple(workflowMetaInputStream)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
#if self.owner == "guest":
# pageTitle = "Submission Request"
# displayType = "submission-request"
# initialStep = 0
#else:
# pageTitle = "Metadata Record"
# displayType = "package-dataset"
# initialStep = 1
## Harvesting straight into the 'Published' stage
pageTitle = "Metadata Record"
displayType = "package-dataset"
#initialStep = 4
initialStep = 3
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(
None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put(
"label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException:
print(" ERROR updating dataset payload")
# Form processing
coreFields = [
"title", "description", "manifest", "metaList", "relationships",
"responses"
]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ",
parsedTime)
self.utils.add(self.index, "date_created", solrTime)
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
self.utils.add(self.index, "display_type", displayType)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
wfMetaObj.put("pageTitle", "Uploaded Files - Management")
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == "pending":
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
jsonString = String(wfMeta.toString())
inStream = ByteArrayInputStream(jsonString.getBytes("UTF-8"))
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException, e:
print " * workflow-harvester.py : Error updating workflow payload"
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step",
wfMeta.getString(None, ["step"]))
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
def __filePath(self):
baseFilePath = self.params["base.file.path"]
def getPayloadJsonByExtension(self,oid,payloadExtension):
digitalObject = StorageUtils.getDigitalObject(self.storage,oid)
pid = self.findPidForExtenstion(digitalObject, payloadExtension)
tfPackageInputStream = digitalObject.getPayload(pid).open()
return JsonSimple(tfPackageInputStream)
def __attachFile(self):
try:
# WebKit/IE prefixes C:\fakepath\ with javascript manipulated file inputs
uploadFile = self.formData.get("uploadFile")
uploadFile = uploadFile.replace("C:\\fakepath\\", "")
fileDetails = self.vc("sessionState").get(uploadFile)
# Establish that we do have details on the uploaded file
if fileDetails is None:
uploadFile = uploadFile.rsplit("\\", 1)[-1]
fileDetails = self.vc("sessionState").get(uploadFile)
if fileDetails is None:
self.log.error("**** fileDetails is None!!! ***")
return self.__toJson({
"error": "fileDetails is None (no upload file!)"
})
self.log.debug("Attach Upload: fileDetails: '{}'", fileDetails)
errorDetails = fileDetails.get("error")
if errorDetails:
self.log.error("ERROR: %s" % errorDetails)
return self.__toJson({"error": errorDetails})
# Look for the storage info we need
jsonFormData = JsonSimple(self.formData.get("json"))
oid = jsonFormData.getString(None, "oid")
fname = fileDetails.get("name")
foid = fileDetails.get("oid")
self.log.debug("attach oid='{}', filename='{}', foid='{}'", [oid, fname, foid])
# Make sure it was actually stored
try:
attachObj = self.Services.getStorage().getObject(foid)
except StorageException, e:
return JsonSimple({"error": "Attached file - '%s'" % str(e)})
# Build up some metadata to store alongside the file
attachFormData = JsonSimple(self.formData.get("json", "{}"))
attachMetadata = {
"type": "attachment",
"created_by": "workflow.py",
"formData": {
"oid": foid,
"attached_to": oid,
"filename": fname,
"access_rights": attachFormData.getString("private", ["accessRights"]),
"attachment_type": attachFormData.getString("supporting-material", ["attachmentType"])
}
}
# We are going to send an update on all attachments back with our response
attachedFiles = self.__getAttachedFiles(oid)
attachedFiles.append(dict(attachMetadata["formData"]))
# Now store our metadata for this file
try:
jsonMetadata = self.__toJson(attachMetadata)
jsonIn = ByteArrayInputStream(jsonMetadata.toString())
StorageUtils.createOrUpdatePayload(attachObj, "workflow.metadata", jsonIn)
jsonIn.close();
attachObj.close();
except StorageException, e:
self.log.error("Failed to create attachment metadata!", e)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
if self.owner == "guest":
pageTitle = "Submission Request"
displayType = "submission-request"
initialStep = 0
else:
pageTitle = "Metadata Record"
displayType = "package-dataset"
initialStep = 1
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(
None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put(
"label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = [
"title", "description", "manifest", "metaList", "relationships",
"responses"
]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
#Used to make sure we have a created date
createdDateFlag = False
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ",
parsedTime)
self.utils.add(self.index, "date_created", solrTime)
self.log.debug("Set created date to :%s" % solrTime)
createdDateFlag = True
elif field == "redbox:embargo.dc:date":
self.embargoedDate = value
elif field == "create_timestamp":
self.createTimeStamp = value
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
# check if this is an array field
fnameparts = field.split(":")
if fnameparts is not None and len(fnameparts) >= 3:
if field.startswith("bibo") or field.startswith(
"skos"):
arrParts = fnameparts[1].split(".")
else:
arrParts = fnameparts[2].split(".")
# we're not interested in: Relationship, Type and some redbox:origin
if arrParts is not None and len(
arrParts) >= 2 and field.find(
":Relationship.") == -1 and field.find(
"dc:type") == -1 and field.find(
"redbox:origin"
) == -1 and arrParts[1].isdigit():
# we've got an array field
fldPart = ":%s" % arrParts[0]
prefixEndIdx = field.find(fldPart) + len(fldPart)
suffixStartIdx = prefixEndIdx + len(
arrParts[1]) + 1
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[
suffixStartIdx:]
if field.endswith("Name"):
arrFldName = self.reportingFieldPrefix + field[:
prefixEndIdx]
self.log.debug(
"Array Field name is:%s from: %s, with value:%s"
% (arrFldName, field, value))
if field.endswith("Name"):
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(
arrFldName, fullFieldMap)
idx = arrParts[1]
fullField = fullFieldMap.get(idx)
if (fullField is None):
fullField = ""
if (field.endswith("givenName")):
fullField = "%s, %s" % (fullField, value)
if (field.endswith("familyName")):
fullField = "%s%s" % (value, fullField)
self.log.debug("fullname now is :%s" %
fullField)
fullFieldMap.put(idx, fullField)
else:
fieldlist = self.arrayBucket.get(arrFldName)
if fieldlist is None:
fieldlist = []
self.arrayBucket.put(arrFldName, fieldlist)
fieldlist.append(value)
for compfield in self.compFields:
if field.startswith(compfield):
arrFldName = self.reportingFieldPrefix + compfield
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
fullField = fullFieldMap.get("1")
if fullField is None:
fullField = ""
if field.endswith(
self.compFieldsConfig[compfield]["end"]):
fullField = "%s%s%s" % (
fullField,
self.compFieldsConfig[compfield]["delim"],
value)
if field.endswith(
self.compFieldsConfig[compfield]["start"]):
fullField = "%s%s" % (value, fullField)
self.log.debug("full field now is :%s" % fullField)
fullFieldMap.put("1", fullField)
self.utils.add(self.index, "display_type", displayType)
# Make sure we have a creation date
if not createdDateFlag:
self.utils.add(self.index, "date_created", self.last_modified)
self.log.debug(
"Forced creation date to %s because it was not explicitly set."
% self.last_modified)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __activate__(self, context):
self.auth = context["page"].authentication
self.errorMsg = ""
self.request = context["request"]
self.response = context["response"]
self.formData = context["formData"]
self.storage = context["Services"].getStorage()
self.log = context["log"]
self.reportManager = context["Services"].getService("reportManager")
fromOid = self.formData.get("fromOid")
fromObject = self.storage.getObject(fromOid)
if (self.auth.is_logged_in()):
if (self.auth.is_admin() == True):
pass
elif (self.__isOwner(fromObject)):
pass
else:
self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access."
else:
self.errorMsg = "Please login."
if self.errorMsg == "":
toOid = self.formData.get("toOid")
toObject = self.storage.getObject(toOid)
storeRelatedData = self.formData.get("relatedData")
fromTFPackage = self._getTFPackage(fromObject)
toTFPackage = self._getTFPackage(toObject)
# get relevant dc:description from new object before you overwrite it with 'from' data, as this should exist from form data created in initial object packaging (packaging.py)
toTFPackageJson = JsonSimple(toTFPackage.open()).getJsonObject()
relevant_description = toTFPackageJson.get("dc:description")
fromInputStream = fromTFPackage.open()
try:
StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream)
except StorageException:
print "error setting tfPackage"
finally:
fromTFPackage.close()
tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue")
if tfMetaPropertyValue == 'dmpToSelfSub':
# fetch recently created new 'to' object from storage to get all data
toTFPackage = self._getTFPackage(toObject)
toTFPackageJson = JsonSimple(toTFPackage.open()).getJsonObject()
self.setMultiDescription(toTFPackageJson, relevant_description)
inStream = IOUtils.toInputStream(toTFPackageJson.toJSONString(), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), inStream)
except StorageException:
print "error setting description text in tfPackage"
finally:
inStream.close()
self.log.info(
"Completed migrating 'dc:description' to dc:decription.1.text' for oid: %s." % toTFPackage.getId())
self.log.debug("Result: %r" % toTFPackageJson)
fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject()
self.log.debug('from json is: %r' % fromTFPackageJson)
if storeRelatedData != "false":
# add relatedOid info
fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid)
self.log.debug('from tfPackage json is now: %r' % fromTFPackageJson)
inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream)
except StorageException:
print "error setting tfPackage"
finally:
inStream.close()
self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue)
self._reharvestPackage()
result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid, toOid)
else:
result = '{"status": "err", "message": "%s"}' % self.errorMsg
writer = self.response.getPrintWriter("application/json; charset=UTF-8")
writer.println(result)
writer.close()
def __workflow(self):
# Workflow data
WORKFLOW_ID = "servicesUI2"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
pageTitle = "Services Record"
displayType = "package-service"
initialStep = 0
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime)
self.utils.add(self.index, "date_created", solrTime)
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
self.utils.add(self.index, "display_type", displayType)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __attachFile(self):
try:
# WebKit/IE prefixes C:\fakepath\ with javascript manipulated file inputs
uploadFile = self.formData.get("uploadFile")
self.log.debug(uploadFile)
self.log.debug(self.vc("sessionState").toString())
self.log.debug(self.formData.toString())
uploadFile = uploadFile.replace("C:\\fakepath\\", "")
fileDetails = self.vc("sessionState").get(uploadFile)
# Establish that we do have details on the uploaded file
if fileDetails is None:
#uploadFile = uploadFile.rsplit("\\", 1)[-1]
fileDetails = self.vc("sessionState").get(uploadFile)
if fileDetails is None:
self.log.error("**** fileDetails is None!!! ***")
return self.__toJson(
{"error": "fileDetails is None (no upload file!)"})
self.log.debug("Attach Upload: fileDetails: '{}'", fileDetails)
errorDetails = fileDetails.get("error")
if errorDetails:
self.log.error("ERROR: %s" % errorDetails)
return self.__toJson({"error": errorDetails})
# Look for the storage info we need
jsonFormData = JsonSimple(self.formData.get("json"))
oid = jsonFormData.getString(None, "oid")
fname = fileDetails.get("name")
foid = fileDetails.get("oid")
attachType = jsonFormData.getString("supporting-material",
"attachmentType")
self.log.debug("attach oid='{}', filename='{}', foid='{}'",
[oid, fname, foid])
# Make sure it was actually stored
try:
attachObj = self.Services.getStorage().getObject(foid)
except StorageException, e:
return JsonSimple({"error": "Attached file - '%s'" % str(e)})
# Build up some metadata to store alongside the file
attachFormData = JsonSimple(self.formData.get("json", "{}"))
attachMetadata = {
"type": "attachment",
"created_by": "attachments.py",
"formData": {
"oid":
foid,
"attached_to":
oid,
"filename":
fname,
"access_rights":
attachFormData.getString("private", ["accessRights"]),
"attachment_type":
attachType,
"description":
attachFormData.getString("", ["description"]),
"owner":
self.vc("page").authentication.get_username()
}
}
# Record an attachment in attachments.metadata
attachment = self.__createAttachmentJson(
foid, attachMetadata["formData"]["attachment_type"],
attachMetadata["formData"]["description"],
attachMetadata["formData"]["filename"])
# print "Attaching %s" % attachment.toString().encode('utf-8')
self.__createOrUpdateArrayPayload(oid, "attachments.metadata",
attachment)
# We are going to send an update on all attachments back with our response
attachedFiles = self.__getAttachedFiles(oid, attachType)
attachedFiles.append(dict(attachMetadata["formData"]))
# Now store our metadata for this file
try:
jsonMetadata = self.__toJson(attachMetadata)
jsonIn = ByteArrayInputStream(jsonMetadata.toString())
StorageUtils.createOrUpdatePayload(attachObj,
"attachments.metadata",
jsonIn)
jsonIn.close()
attachObj.close()
except StorageException, e:
self.log.error("Failed to create attachment metadata!", e)
