def getLabel(self, jsonFile, key):
value = self.metadata.get(key)
jsonLabelFile = System.getProperty("fascinator.home") + jsonFile
entries = JsonSimple(File(jsonLabelFile)).getJsonArray()
for entry in entries:
entryJson = JsonSimple(entry)
if value == entryJson.getString("", "value"):
return entryJson.getString("", "label")
return None
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
self.vc("log").debug("packageType = '{}'", packageType)
self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
if self.vc("formData").getValues("sequencesMetaList") != None:
sequenceService = ApplicationContextProvider.getApplicationContext().getBean("sequenceService")
sequencesMetaList = list(self.vc("formData").getValues("sequencesMetaList"))
for sequenceInfo in sequencesMetaList:
sequenceInfoJson = JsonSimple(sequenceInfo)
sequenceIndex = sequenceService.getSequence(sequenceInfoJson.getString(None,"sequenceName"))
jsonObj.put(sequenceInfoJson.getString(None,"metadataName"), String.format(sequenceInfoJson.getString(None,"stringFormat"),sequenceIndex))
self.vc("log").debug("json object created is: %r" % jsonObj)
outWriter.write(manifest.toString(True))
outWriter.close()
# adding ability to set access plugin
accessPlugin = self.vc("formData").get("access_plugin", "derby")
if accessPlugin is not None:
self.vc("page").authentication.set_access_plugin(accessPlugin)
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def getLabel(self, jsonFile, key):
value = self.metadata.get(key)
jsonLabelFile = System.getProperty("fascinator.home") + jsonFile
entries = JsonSimple(File(jsonLabelFile)).getJsonArray()
for entry in entries:
entryJson = JsonSimple(entry)
if value == entryJson.getString("", "value"):
return entryJson.getString("", "label")
return None
def execProcessors(self, procId, configJson, dataMap, stageName):
for procObj in configJson.getArray(stageName):
procJson = JsonSimple(procObj)
procClassName = procJson.getString("", "class")
procConfigPath = procJson.getString("", "config")
procInputKey = procJson.getString("", "inputKey")
procOutputKey = procJson.getString("", "outputKey")
procClass = Class.forName(procClassName)
procInst = procClass.newInstance()
procMethod = procClass.getMethod("process", self.get_class("java.lang.String"),self.get_class("java.lang.String"), self.get_class("java.lang.String"),self.get_class("java.lang.String"),self.get_class("java.lang.String"), self.get_class("java.util.HashMap"))
procMethod.invoke(procInst, procId, procInputKey, procOutputKey, stageName, procConfigPath, dataMap)
def __activate__(self, context):
self.velocityContext = context
formData = self.vc("formData")
# build the URL and query parameters to retrieve
proxyUrls = JsonSimple(self.vc("systemConfig").getObject("proxy-urls"))
url = ""
key = formData.get("ns", "")
if proxyUrls.getJsonObject().containsKey(key):
url = proxyUrls.getString("", [key])
queryStr = formData.get("qs")
if queryStr == "searchTerms={searchTerms}":
queryStr = None
if queryStr:
if formData.get("jaffa2autocomplete", "false") == "true":
url += "?searchTerms=%s" % queryStr.lower()
else:
url += "?%s" % queryStr
self.vc("log").debug("Proxy URL = '{}'", url)
data = None
try:
data = self.__wget(url)
except Exception, e:
data = '{"error":"%s"}' % str(e)
self.vc("log").error("ERROR accessing URL:", e)
def __activate__(self, context):
self.velocityContext = context
formData = self.vc("formData")
# build the URL and query parameters to retrieve
proxyUrls = JsonSimple(self.vc("systemConfig").getObject("proxy-urls"))
url = ""
key = formData.get("ns", "")
if proxyUrls.getJsonObject().containsKey(key):
url = proxyUrls.getString("", [key])
queryStr = formData.get("qs")
if queryStr == "searchTerms={searchTerms}":
queryStr = None
if queryStr:
if formData.get("jaffa2autocomplete", "false") == "true":
url += "?searchTerms=%s" % queryStr
else:
url += "?%s" % queryStr
self.vc("log").debug("Proxy URL = '{}'", url)
data = None
try:
data = self.__wget(url)
except Exception, e:
data = '{"error":"%s"}' % str(e)
self.vc("log").error("ERROR accessing URL:", e)
def updateLocalRecordRelations(self, jobItems):
oidIdentifierMap = HashMap()
for jobItem in jobItems:
oidIdentifierMap.put(jobItem.get("oid"),jobItem.get("required_identifiers")[0].get("identifier"))
for jobItem in jobItems:
type = jobItem.get("type");
targetSystem = self.systemConfig.getString(None, "curation", "supported-types", type);
if targetSystem == "redbox":
oid = jobItem.get("oid")
digitalObject = StorageUtils.getDigitalObject(self.services.getStorage(), oid)
tfPackagePid = self.getPackageData(digitalObject)
metadataJsonPayload = digitalObject.getPayload(tfPackagePid)
metadataJsonInstream = metadataJsonPayload.open()
metadataJson = JsonSimple(metadataJsonInstream)
metadataJsonPayload.close()
relationships = metadataJson.getArray("relationships")
if relationships is not None:
for relationship in relationships:
system = relationship.get("system")
if system != "redbox" or system != None:
url = self.systemConfig.getString("can't find it", "curation","external-system-urls","get-oid-for-identifier",system)
client = BasicHttpClient(url+ "&identifier="+relationship.get("identifier"))
get = GetMethod(url+ "&identifier="+relationship.get("identifier"))
client.executeMethod(get)
if get.getStatusCode() == 200:
response = JsonSimple(get.getResponseBodyAsString())
relationship.put("curatedPid",oidIdentifierMap.get(response.getString(None,"oid")))
relationship.put("isCurated",True)
#Now update the relationship on Mint's side
break
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
StorageUtils.createOrUpdatePayload(digitalObject,tfPackagePid,istream)
def __activate__(self, context):
self.log = context["log"]
self.request = context["request"]
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
processingSet = self.request.getParameter("processingSet")
self.procMsg = None
# read configuration and trigger processing stream sets
# storing the return object on the map
configFilePath = FascinatorHome.getPath(
"process") + "/processConfig.json"
procConfigFile = File(configFilePath)
if procConfigFile.exists() == True:
self.dataMap = HashMap()
self.dataMap.put("indexer", context['Services'].getIndexer())
self.procConfigJson = JsonSimple(procConfigFile)
for configObj in self.procConfigJson.getJsonArray():
configJson = JsonSimple(configObj)
procId = configJson.getString("", "id")
if processingSet is not None:
if procId == processingSet:
self.execProcSet(procId, configJson)
else:
self.execProcSet(procId, configJson)
if self.procMsg is None:
self.procMsg = "Processing complete!"
else:
self.procMsg = "Configuration file does not exist: " + configFilePath
def __hotfix(self, formData):
oldType = formData.getString(None, ["dc:type"])
newType = formData.getString(None, ["dc:type.rdf:PlainLiteral"])
if oldType != newType or newType is None:
self.log.debug("Bugged Type?: v1.4: '{}', OLD: '{}'", newType, oldType)
else:
## No fix required
return None
## Get Backup data
## NOTE: The only known production system affected by this bug
## was caught during a v1.4 upgrade. Alter this line if required.
pid = "1.4.workflow.backup"
oldData = None
try:
payload = self.object.getPayload(pid)
try:
oldData = JsonSimple(payload.open())
except Exception:
self.log.error("Error parsing JSON '{}'", pid)
finally:
payload.close()
except StorageException:
self.log.error("Error accessing '{}'", pid)
return None
oldType = oldData.getString(None, ["dc:type"])
self.log.debug("Old Type: '{}' => 'dc:type.rdf:PlainLiteral'", oldType)
formData.getJsonObject().put("dc:type.rdf:PlainLiteral", oldType);
return formData
def process_tags(self, result):
tags = []
tagsDict = {}
# Build a dictionary of the tags
for doc in result:
# Get Anotar data from Solr data
doc = JsonSimple(doc.get("jsonString"))
# Get actual tag text
tag = doc.getString(None, ["content", "literal"])
# Find out if they have locators
locs = doc.getJsonSimpleList(["annotates", "locators"]).size()
if locs == 0:
# Basic tags, just aggregate counts
if tag in tagsDict:
# We've seen it before, just increment the counter
existing = tagsDict[tag]
count = existing.getInteger(0, ["tagCount"])
existing.getJsonObject().put("tagCount", str(count + 1))
else:
# First time, store this object
doc.getJsonObject().put("tagCount", str(1))
tagsDict[tag] = doc
else:
# Tags with a locator, special case for images etc.
tags.append(doc.toString())
# Push all the 'basic' counts into the list to return
for tag in tagsDict:
tags.append(tagsDict[tag].toString())
return "[" + ",".join(tags) + "]"
def __activate__(self, context):
self.log = context["log"]
self.request = context["request"]
self.sessionState = context["sessionState"]
self.sessionState.set("username","admin")
processingSet = self.request.getParameter("processingSet")
self.procMsg = None
# read configuration and trigger processing stream sets
# storing the return object on the map
configFilePath = FascinatorHome.getPath("process")+"/processConfig.json"
procConfigFile = File(configFilePath)
if procConfigFile.exists() == True:
self.dataMap = HashMap()
self.dataMap.put("indexer", context['Services'].getIndexer())
self.procConfigJson = JsonSimple(procConfigFile)
for configObj in self.procConfigJson.getJsonArray():
configJson = JsonSimple(configObj)
procId = configJson.getString("", "id")
if processingSet is not None:
if procId == processingSet:
self.execProcSet(procId, configJson)
else:
self.execProcSet(procId, configJson)
if self.procMsg is None:
self.procMsg = "Processing complete!"
else:
self.procMsg = "Configuration file does not exist: " + configFilePath
def __hotfix(self, formData):
oldType = formData.getString(None, ["dc:type"])
newType = formData.getString(None, ["dc:type.rdf:PlainLiteral"])
if oldType != newType or newType is None:
self.log.debug("Bugged Type?: v1.4: '{}', OLD: '{}'", newType,
oldType)
else:
## No fix required
return None
## Get Backup data
## NOTE: The only known production system affected by this bug
## was caught during a v1.4 upgrade. Alter this line if required.
pid = "1.4.workflow.backup"
oldData = None
try:
payload = self.object.getPayload(pid)
try:
oldData = JsonSimple(payload.open())
except Exception:
self.log.error("Error parsing JSON '{}'", pid)
finally:
payload.close()
except StorageException:
self.log.error("Error accessing '{}'", pid)
return None
oldType = oldData.getString(None, ["dc:type"])
self.log.debug("Old Type: '{}' => 'dc:type.rdf:PlainLiteral'", oldType)
formData.getJsonObject().put("dc:type.rdf:PlainLiteral", oldType)
return formData
def __formData(self):
# Find our workflow form data
packagePid = None
try:
self.pidList = self.object.getPayloadIdList()
for pid in self.pidList:
if pid.endswith(self.packagePidSuffix):
packagePid = pid
except StorageException:
self.log.error("Error accessing object PID list for object '{}' ", self.oid)
return
if packagePid is None:
self.log.debug("Object '{}' has no form data", self.oid)
return
# Retrieve our form data
workflowData = None
try:
payload = self.object.getPayload(packagePid)
try:
workflowData = JsonSimple(payload.open())
except Exception:
self.log.error("Error parsing JSON '{}'", packagePid)
finally:
payload.close()
except StorageException:
self.log.error("Error accessing '{}'", packagePid)
return
# Test our version data
self.version = workflowData.getString("{NO VERSION}", ["redbox:formVersion"])
oldData = String(workflowData.toString(True))
if self.version != self.redboxVersion:
self.log.info("OID '{}' requires an upgrade: '{}' => '{}'", [self.oid, self.version, self.redboxVersion])
# The version data is old, run our upgrade
# function to see if any alterations are
# required. Most likely at least the
# version number will change.
newWorkflowData = self.__upgrade(workflowData)
else:
newWorkflowData = self.__hotfix(workflowData)
if newWorkflowData is not None:
self.log.debug("OID '{}' was hotfixed for v1.2 'dc:type' bug", self.oid)
else:
self.log.debug("OID '{}' requires no work, skipping", self.oid)
return
# Backup our data first
backedUp = self.__backup(oldData)
if not backedUp:
self.log.error("Upgrade aborted, data backup failed!")
return
# Save the newly modified data
jsonString = String(newWorkflowData.toString(True))
inStream = ByteArrayInputStream(jsonString.getBytes("UTF-8"))
try:
self.object.updatePayload(packagePid, inStream)
except StorageException, e:
self.log.error("Error updating workflow payload: ", e)
def getLabel(self, jsonFile, key, listKey, valKey, labelKey):
value = self.metadata.get(key)
jsonLabelFile = System.getProperty("fascinator.home") + jsonFile
jsonLabel = JsonSimple(File(jsonLabelFile))
entries = jsonLabel.getJsonArray()
# the structure of the json file is fun and complicated
if entries is None:
entries = jsonLabel.getArray(listKey)
else:
valKey = "value"
labelKey = "label"
for entry in entries:
entryJson = JsonSimple(entry)
if value == entryJson.getString("", valKey):
return entryJson.getString("", labelKey)
return None
def getLabel(self, jsonFile, key, listKey, valKey, labelKey):
value = self.metadata.get(key)
jsonLabelFile = System.getProperty("fascinator.home") + jsonFile
jsonLabel = JsonSimple(File(jsonLabelFile))
entries = jsonLabel.getJsonArray()
# the structure of the json file is fun and complicated
if entries is None:
entries = jsonLabel.getArray(listKey)
else:
valKey = "value"
labelKey = "label"
for entry in entries:
entryJson = JsonSimple(entry)
if value == entryJson.getString("", valKey):
return entryJson.getString("", labelKey)
return None
def updateObjectMetadata(self, objectMetaData, toWorkflowId):
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(
toWorkflowId)
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = File(workflowsDir, jsonConfigFile)
configObject = StorageUtils.checkHarvestFile(self.storage, configFile)
if configObject is None:
oid = StorageUtils.generateOid(configFile)
configObject = StorageUtils.getDigitalObject(self.storage, oid)
objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
objectMetaData.setProperty("jsonConfigOid", configObject.getId())
configJson = JsonSimple(configFile)
rulesFileName = configJson.getString(None, "indexer", "script",
"rules")
rulesFile = File(workflowsDir, rulesFileName)
rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile)
if rulesObject is None:
oid = StorageUtils.generateOid(rulesFile)
rulesObject = StorageUtils.getDigitalObject(self.storage, oid)
objectMetaData.setProperty("rulesPid", rulesFileName)
objectMetaData.setProperty("rulesOid", rulesObject.getId())
objectMetaData.setProperty("workflowTransitioned", "true")
objectMetaData.setProperty(
"date_transitioned",
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def process_tags(self, result):
tags = []
tagsDict = {}
# Build a dictionary of the tags
for doc in result:
# Get Anotar data from Solr data
doc = JsonSimple(doc.get("jsonString"))
# Get actual tag text
tag = doc.getString(None, ["content", "literal"])
# Find out if they have locators
locs = doc.getJsonSimpleList(["annotates", "locators"]).size()
if locs == 0:
# Basic tags, just aggregate counts
if tag in tagsDict:
# We've seen it before, just increment the counter
existing = tagsDict[tag]
count = existing.getInteger(0, ["tagCount"])
existing.getJsonObject().put("tagCount", str(count + 1))
else:
# First time, store this object
doc.getJsonObject().put("tagCount", str(1))
tagsDict[tag] = doc
else:
# Tags with a locator, special case for images etc.
tags.append(doc.toString())
# Push all the 'basic' counts into the list to return
for tag in tagsDict:
tags.append(tagsDict[tag].toString())
return "[" + ",".join(tags) + "]"
def updateObjectMetadata(self, objectMetaData, toWorkflowId):
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(toWorkflowId)
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = File(workflowsDir, jsonConfigFile)
configObject = StorageUtils.checkHarvestFile(self.storage, configFile);
if configObject is None:
oid = StorageUtils.generateOid(configFile);
configObject = StorageUtils.getDigitalObject(self.storage, oid);
objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
objectMetaData.setProperty("jsonConfigOid", configObject.getId())
configJson = JsonSimple(configFile)
rulesFileName = configJson.getString(None, "indexer","script","rules")
rulesFile = File(workflowsDir,rulesFileName)
rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile);
if rulesObject is None:
oid = StorageUtils.generateOid(rulesFile);
rulesObject = StorageUtils.getDigitalObject(self.storage, oid);
objectMetaData.setProperty("rulesPid", rulesFileName)
objectMetaData.setProperty("rulesOid", rulesObject.getId())
objectMetaData.setProperty("workflowTransitioned", "true")
objectMetaData.setProperty("date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def __getUser(self):
self.log.debug("baserule.py: Assign ownership: mapping id to user")
mapConfig = self.config.getObject(["user-assignment"])
if mapConfig is None:
self.log.debug("baserule.py: no configuration has been set for mapping id to user.")
return None
uname = None
try:
userAssignment = JsonSimple(mapConfig)
self.log.debug("baserule.py: user_id to user mapping configuration")
modulePath = userAssignment.getString(None, ["module-path"])
self.log.debug("baserule.py: module-path = {}", modulePath)
className = userAssignment.getString(None, ["class-name"])
self.log.debug("baserule.py: className = {}", className)
initParams = userAssignment.getArray(["init-params"])
self.log.debug("baserule.py: init-params (list) = {}", ' ,'.join(initParams))
actionMethodName = userAssignment.getString(None, ["action-method-name"])
self.log.debug("baserule.py: action-method-name = {}", actionMethodName)
# JsonArray
actionMethodParams = userAssignment.getArray(["action-method-params"])
# This block may be used to mapping method params to internal variables?
try:
self.log.debug("baserule.py: read parameters from tfpackage.")
params = self.__getItems(actionMethodParams)
except Exception, e:
self.log.debug("baserule.py: read parameters failed. Reason: {}", str(e))
if params is None or len(params) == 0:
self.log.debug("baserule.py: read parameters returned None. Cannot carry on.")
return None
lookupPk = imp.load_source('', modulePath)
lookupClass = getattr(lookupPk, className)
lookupObj = lookupClass(*initParams)
lookupMethod = getattr(lookupObj, actionMethodName)
uname = lookupMethod(*params)
self.log.debug("baserule.py: external lookup module returns: {}", uname)
def getLabel(self, jsonFile, key):
value = self.metadata.get(key)
jsonLabelFile = System.getProperty("fascinator.home") + jsonFile
jsonF = JsonSimple(File(jsonLabelFile))
entries = jsonF.getJsonArray()
if entries is None:
entries = jsonF.getArray('results')
if entries is None:
self.log.debug("Unknown data source format: JSON file {} or its 'results' has no array.", jsonLabelFile)
return None
for entry in entries:
entryJson = JsonSimple(entry)
if value == entryJson.getString("", "id"):
return entryJson.getString("", "label")
elif value == entryJson.getString("", "value"):
return entryJson.getString("", "label")
return None
class TimNotificationData:
def __init__(self):
self.messaging = MessagingServices.getInstance()
def __activate__(self, context):
self.log = context["log"]
self.request = context["request"]
self.sessionState = context["sessionState"]
self.setting = JsonSimple(context["systemConfig"].getObject("tim.notification"))
self.sessionState.set("username","admin")
# read configuration and trigger processing stream sets
# storing the return object on the map
self.dataMap = HashMap()
self.dataMap.put("indexer", context['Services'].getIndexer())
url = self.setting.getString("","url")
data = self.__wget(url)
json = JsonSimple(data)
if json.getInteger(0,["response","numFound"]) > 0 :
username = self.setting.getString("",["email","username"])
password = self.setting.getString("",["email","password"])
body = self.setting.getString("",["email","body"])
to = self.setting.getString("",["email","to"])
if self.setting.getString("",["email","testmode"]) == "true" :
body = body + "<p>TESTMODE: Was sent to " + to
to = self.setting.getString("",["email","redirect"])
email = HtmlEmail()
email.setAuthenticator(DefaultAuthenticator(username, password))
email.setHostName(self.setting.getString("localhost",["email","host"]))
email.setSmtpPort(self.setting.getInteger(25,["email","port"]))
email.setSSL(self.setting.getBoolean(False,["email","ssl"]))
email.setTLS(self.setting.getBoolean(False,["email","tls"]))
email.setFrom(self.setting.getString("",["email","from"]))
email.setSubject(self.setting.getString("Action Required in TIM",["email","subject"]))
email.addTo(to)
email.setHtmlMsg(body)
email.send()
def __wget(self, url):
client = BasicHttpClient(url)
m = GetMethod(url)
client.executeMethod(m)
return IOUtils.toString(m.getResponseBodyAsStream(), "UTF-8")
def execProcessors(self, procId, configJson, dataMap, stageName):
for procObj in configJson.getArray(stageName):
procJson = JsonSimple(procObj)
procClassName = procJson.getString("", "class")
procConfigPath = procJson.getString("", "config")
procInputKey = procJson.getString("", "inputKey")
procOutputKey = procJson.getString("", "outputKey")
procClass = Class.forName(procClassName)
procInst = procClass.newInstance()
procMethod = procClass.getMethod(
"process", self.get_class("java.lang.String"),
self.get_class("java.lang.String"),
self.get_class("java.lang.String"),
self.get_class("java.lang.String"),
self.get_class("java.lang.String"),
self.get_class("java.util.HashMap"))
procMethod.invoke(procInst, procId, procInputKey, procOutputKey,
stageName, procConfigPath, dataMap)
def getLabel(self, jsonFile, key):
value = self.metadata.get(key)
jsonLabelFile = System.getProperty("fascinator.home") + jsonFile
jsonF = JsonSimple(File(jsonLabelFile))
entries = jsonF.getJsonArray()
if entries is None:
entries = jsonF.getArray('results')
if entries is None:
self.log.debug(
"Unknown data source format: JSON file {} or its 'results' has no array.",
jsonLabelFile)
return None
for entry in entries:
entryJson = JsonSimple(entry)
if value == entryJson.getString("", "id"):
return entryJson.getString("", "label")
elif value == entryJson.getString("", "value"):
return entryJson.getString("", "label")
return None
def modify_json(self):
# print "**** anotar.py : add_json() : adding json : " + json
jsonSimple = JsonSimple(self.json)
jsonObj = jsonSimple.getJsonObject()
jsonObj.put("id", self.pid)
rootUri = jsonSimple.getString(None, ["annotates", "rootUri"])
if rootUri is not None:
baseUrl = "http://%s:%s/" % (self.vc("request").serverName, self.vc("serverPort"))
myUri = baseUrl + rootUri + "#" + self.pid
jsonObj.put("uri", myUri)
jsonObj.put("schemaVersionUri", "http://www.purl.org/anotar/schema/0.1")
self.json = jsonSimple.toString()
def modify_json(self):
#print "**** anotar.py : add_json() : adding json : " + json
jsonSimple = JsonSimple(self.json)
jsonObj = jsonSimple.getJsonObject()
jsonObj.put("id", self.pid)
rootUri = jsonSimple.getString(None, ["annotates", "rootUri"])
if rootUri is not None:
baseUrl = "http://%s:%s/" % (self.vc("request").serverName,
self.vc("serverPort"))
myUri = baseUrl + rootUri + "#" + self.pid
jsonObj.put("uri", myUri)
jsonObj.put("schemaVersionUri",
"http://www.purl.org/anotar/schema/0.1")
self.json = jsonSimple.toString()
def __init__(self, file, config, baseline):
AlertHandler.__init__(self, file, config, baseline)
docFactory = DocumentFactory()
self.saxReader = SAXReader(docFactory)
self.xmlMapFile = StrSubstitutor.replaceSystemProperties(config['xmlMap'])
if not os.path.exists(self.xmlMapFile):
raise AlertException("Requested xmlMap file %s does not exist." % self.xmlMapFile)
## Make sure we can see our mappings
inStream = FileInputStream(File(self.xmlMapFile))
xmlMappings = JsonSimple(inStream)
self.map = xmlMappings.getObject(["mappings"])
self.exceptions = xmlMappings.getObject(["exceptions"])
self.defaultNamespace = xmlMappings.getString(None, ["defaultNamespace"])
self.mappedExceptionCount = 0
def getMintLabelByLookup(self, urlName, key, resKey, valKey):
mapIds = HashMap()
value = String(self.metadata.get(key)).replace(":", "\:")
if value is None:
return None
labels = ArrayList()
mapIds.put("searchTerms", value)
labelsMint = MintLookupHelper.get(self.systemConfig, urlName, mapIds)
self.log.debug(labelsMint.toString())
resultsArr = labelsMint.getArray(resKey)
if resultsArr is None:
return None
for result in resultsArr:
labelJson = JsonSimple(result)
labels.add(labelJson.getString("", valKey))
return labels
def __init__(self, file, config):
AlertHandler.__init__(self, file, config)
docFactory = DocumentFactory()
self.saxReader = SAXReader(docFactory)
self.xmlMapFile = config['xmlMap']
if not os.path.exists(self.xmlMapFile):
raise AlertException("Requested xmlMap file %s does not exist." %
self.file)
## Make sure we can see our mappings
inStream = FileInputStream(File(self.xmlMapFile))
xmlMappings = JsonSimple(inStream)
self.map = xmlMappings.getObject(["mappings"])
self.exceptions = xmlMappings.getObject(["exceptions"])
self.defaultNamespace = xmlMappings.getString(None,
["defaultNamespace"])
self.mappedExceptionCount = 0
def getMintLabelByLookup(self, urlName, key, resKey, valKey):
mapIds = HashMap()
value = String(self.metadata.get(key)).replace(":", "\:")
if value is None:
return None
labels = ArrayList()
mapIds.put("searchTerms", value)
labelsMint = MintLookupHelper.get(self.systemConfig, urlName, mapIds)
self.log.debug(labelsMint.toString())
resultsArr = labelsMint.getArray(resKey)
if resultsArr is None:
return None
for result in resultsArr:
labelJson = JsonSimple(result)
labels.add(labelJson.getString("", valKey))
return labels
def getMintLabels(self, urlName, key, suffix):
mapIds = HashMap()
valList = self.getList(key)
ids = ""
for eKey in valList.keySet():
entry = valList.get(eKey)
if len(ids) > 0:
ids = "%s,"%ids
ids = "%s%s" % (ids,entry.get(suffix))
if ids == "":
return None
else:
labels = ArrayList()
mapIds.put("id", ids)
labelsMint = MintLookupHelper.get(self.systemConfig, urlName, mapIds)
self.log.debug(labelsMint.getJsonArray().toString())
for label in labelsMint.getJsonArray():
labelJson = JsonSimple(label)
labels.add(labelJson.getString("", "label"))
return labels
def getMintLabels(self, urlName, key, suffix):
mapIds = HashMap()
valList = self.getList(key)
ids = ""
for eKey in valList.keySet():
entry = valList.get(eKey)
if len(ids) > 0:
ids = "%s," % ids
ids = "%s%s" % (ids, entry.get(suffix))
if ids == "":
return None
else:
labels = ArrayList()
mapIds.put("id", ids)
labelsMint = MintLookupHelper.get(self.systemConfig, urlName,
mapIds)
self.log.debug(labelsMint.getJsonArray().toString())
for label in labelsMint.getJsonArray():
labelJson = JsonSimple(label)
labels.add(labelJson.getString("", "label"))
return labels
def __updateMetadataPayload(self, data):
# Get and parse
payload = self.object.getPayload("formData.tfpackage")
json = JsonSimple(payload.open())
payload.close()
# Basic test for a mandatory field
title = json.getString(None, ["dc:title"])
if title is not None:
# We've done this before
return
# Merge
json.getJsonObject().putAll(data)
# Store it
inStream = IOUtils.toInputStream(json.toString(True), "UTF-8")
try:
self.object.updatePayload("formData.tfpackage", inStream)
except StorageException, e:
self.log.error("Error updating 'formData.tfpackage' payload for object '{}'", self.oid, e)
def __updateMetadataPayload(self, data):
# Get and parse
payload = self.object.getPayload("formData.tfpackage")
json = JsonSimple(payload.open())
payload.close()
# Basic test for a mandatory field
title = json.getString(None, ["dc:title"])
if title is not None:
# We've done this before
return
# Merge
json.getJsonObject().putAll(data)
# Store it
inStream = IOUtils.toInputStream(json.toString(True), "UTF-8")
try:
self.object.updatePayload("formData.tfpackage", inStream)
except StorageException, e:
self.log.error(
"Error updating 'formData.tfpackage' payload for object '{}'",
self.oid, e)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
if self.owner == "guest":
pageTitle = "Submission Request"
displayType = "submission-request"
initialStep = 0
else:
pageTitle = "Metadata Record"
displayType = "package-dataset"
initialStep = 1
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(
None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put(
"label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = [
"title", "description", "manifest", "metaList", "relationships",
"responses"
]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
#Used to make sure we have a created date
createdDateFlag = False
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ",
parsedTime)
self.utils.add(self.index, "date_created", solrTime)
self.log.debug("Set created date to :%s" % solrTime)
createdDateFlag = True
elif field == "redbox:embargo.dc:date":
self.embargoedDate = value
elif field == "create_timestamp":
self.createTimeStamp = value
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
# check if this is an array field
fnameparts = field.split(":")
if fnameparts is not None and len(fnameparts) >= 3:
if field.startswith("bibo") or field.startswith(
"skos"):
arrParts = fnameparts[1].split(".")
else:
arrParts = fnameparts[2].split(".")
# we're not interested in: Relationship, Type and some redbox:origin
if arrParts is not None and len(
arrParts) >= 2 and field.find(
":Relationship.") == -1 and field.find(
"dc:type") == -1 and field.find(
"redbox:origin"
) == -1 and arrParts[1].isdigit():
# we've got an array field
fldPart = ":%s" % arrParts[0]
prefixEndIdx = field.find(fldPart) + len(fldPart)
suffixStartIdx = prefixEndIdx + len(
arrParts[1]) + 1
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[
suffixStartIdx:]
if field.endswith("Name"):
arrFldName = self.reportingFieldPrefix + field[:
prefixEndIdx]
self.log.debug(
"Array Field name is:%s from: %s, with value:%s"
% (arrFldName, field, value))
if field.endswith("Name"):
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(
arrFldName, fullFieldMap)
idx = arrParts[1]
fullField = fullFieldMap.get(idx)
if (fullField is None):
fullField = ""
if (field.endswith("givenName")):
fullField = "%s, %s" % (fullField, value)
if (field.endswith("familyName")):
fullField = "%s%s" % (value, fullField)
self.log.debug("fullname now is :%s" %
fullField)
fullFieldMap.put(idx, fullField)
else:
fieldlist = self.arrayBucket.get(arrFldName)
if fieldlist is None:
fieldlist = []
self.arrayBucket.put(arrFldName, fieldlist)
fieldlist.append(value)
for compfield in self.compFields:
if field.startswith(compfield):
arrFldName = self.reportingFieldPrefix + compfield
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
fullField = fullFieldMap.get("1")
if fullField is None:
fullField = ""
if field.endswith(
self.compFieldsConfig[compfield]["end"]):
fullField = "%s%s%s" % (
fullField,
self.compFieldsConfig[compfield]["delim"],
value)
if field.endswith(
self.compFieldsConfig[compfield]["start"]):
fullField = "%s%s" % (value, fullField)
self.log.debug("full field now is :%s" % fullField)
fullFieldMap.put("1", fullField)
self.utils.add(self.index, "display_type", displayType)
# Make sure we have a creation date
if not createdDateFlag:
self.utils.add(self.index, "date_created", self.last_modified)
self.log.debug(
"Forced creation date to %s because it was not explicitly set."
% self.last_modified)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
if self.owner == "guest":
pageTitle = "Submission Request"
displayType = "submission-request"
initialStep = 0
else:
pageTitle = "Metadata Record"
displayType = "package-dataset"
initialStep = 1
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
#Used to make sure we have a created date
createdDateFlag = False
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime)
self.utils.add(self.index, "date_created", solrTime)
self.log.debug("Set created date to :%s" % solrTime)
createdDateFlag = True
elif field == "redbox:embargo.dc:date":
self.embargoedDate = value
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
# check if this is an array field
fnameparts = field.split(":")
if fnameparts is not None and len(fnameparts) >= 3:
if field.startswith("bibo") or field.startswith("skos"):
arrParts = fnameparts[1].split(".")
else:
arrParts = fnameparts[2].split(".")
# we're not interested in: Relationship, Type and some redbox:origin
if arrParts is not None and len(arrParts) >= 2 and field.find(":Relationship.") == -1 and field.find("dc:type") == -1 and field.find("redbox:origin") == -1 and arrParts[1].isdigit():
# we've got an array field
fldPart = ":%s" % arrParts[0]
prefixEndIdx = field.find(fldPart) + len(fldPart)
suffixStartIdx = prefixEndIdx+len(arrParts[1])+1
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[suffixStartIdx:]
if field.endswith("Name"):
arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx]
self.log.debug("Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value))
if field.endswith("Name"):
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
idx = arrParts[1]
fullField = fullFieldMap.get(idx)
if (fullField is None):
fullField = ""
if (field.endswith("givenName")):
fullField = "%s, %s" % (fullField, value)
if (field.endswith("familyName")):
fullField = "%s%s" % (value, fullField)
self.log.debug("fullname now is :%s" % fullField)
fullFieldMap.put(idx, fullField)
else:
fieldlist = self.arrayBucket.get(arrFldName)
if fieldlist is None:
fieldlist = []
self.arrayBucket.put(arrFldName, fieldlist)
fieldlist.append(value)
for compfield in self.compFields:
if field.startswith(compfield):
arrFldName = self.reportingFieldPrefix +compfield
fullFieldMap = self.arrayBucket.get(arrFldName)
if fullFieldMap is None:
fullFieldMap = HashMap()
self.arrayBucket.put(arrFldName, fullFieldMap)
fullField = fullFieldMap.get("1")
if fullField is None:
fullField = ""
if field.endswith(self.compFieldsConfig[compfield]["end"]):
fullField = "%s%s%s" % (fullField, self.compFieldsConfig[compfield]["delim"] ,value)
if field.endswith(self.compFieldsConfig[compfield]["start"]):
fullField = "%s%s" % (value, fullField)
self.log.debug("full field now is :%s" % fullField)
fullFieldMap.put("1", fullField)
self.utils.add(self.index, "display_type", displayType)
# Make sure we have a creation date
if not createdDateFlag:
self.utils.add(self.index, "date_created", self.last_modified)
self.log.debug("Forced creation date to %s because it was not explicitly set." % self.last_modified)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __attachFile(self):
try:
# WebKit/IE prefixes C:\fakepath\ with javascript manipulated file inputs
uploadFile = self.formData.get("uploadFile")
uploadFile = uploadFile.replace("C:\\fakepath\\", "")
fileDetails = self.vc("sessionState").get(uploadFile)
# Establish that we do have details on the uploaded file
if fileDetails is None:
uploadFile = uploadFile.rsplit("\\", 1)[-1]
fileDetails = self.vc("sessionState").get(uploadFile)
if fileDetails is None:
self.log.error("**** fileDetails is None!!! ***")
return self.__toJson({
"error": "fileDetails is None (no upload file!)"
})
self.log.debug("Attach Upload: fileDetails: '{}'", fileDetails)
errorDetails = fileDetails.get("error")
if errorDetails:
self.log.error("ERROR: %s" % errorDetails)
return self.__toJson({"error": errorDetails})
# Look for the storage info we need
jsonFormData = JsonSimple(self.formData.get("json"))
oid = jsonFormData.getString(None, "oid")
fname = fileDetails.get("name")
foid = fileDetails.get("oid")
self.log.debug("attach oid='{}', filename='{}', foid='{}'", [oid, fname, foid])
# Make sure it was actually stored
try:
attachObj = self.Services.getStorage().getObject(foid)
except StorageException, e:
return JsonSimple({"error": "Attached file - '%s'" % str(e)})
# Build up some metadata to store alongside the file
attachFormData = JsonSimple(self.formData.get("json", "{}"))
attachMetadata = {
"type": "attachment",
"created_by": "workflow.py",
"formData": {
"oid": foid,
"attached_to": oid,
"filename": fname,
"access_rights": attachFormData.getString("private", ["accessRights"]),
"attachment_type": attachFormData.getString("supporting-material", ["attachmentType"])
}
}
# We are going to send an update on all attachments back with our response
attachedFiles = self.__getAttachedFiles(oid)
attachedFiles.append(dict(attachMetadata["formData"]))
# Now store our metadata for this file
try:
jsonMetadata = self.__toJson(attachMetadata)
jsonIn = ByteArrayInputStream(jsonMetadata.toString())
StorageUtils.createOrUpdatePayload(attachObj, "workflow.metadata", jsonIn)
jsonIn.close();
attachObj.close();
except StorageException, e:
self.log.error("Failed to create attachment metadata!", e)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "Parties_People"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
initialStep = 0
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def getWorkflowStep(self):
workflow = JsonSimple(File(os.path.join(self.storedObj.getPath(),"workflow.metadata")))
# it always should have label field
return workflow.getString("Check Code Please","label")
def __formData(self):
# Find our workflow form data
packagePid = None
try:
self.pidList = self.object.getPayloadIdList()
for pid in self.pidList:
if pid.endswith(self.packagePidSuffix):
packagePid = pid
except StorageException:
self.log.error("Error accessing object PID list for object '{}' ",
self.oid)
return
if packagePid is None:
self.log.debug("Object '{}' has no form data", self.oid)
return
# Retrieve our form data
workflowData = None
try:
payload = self.object.getPayload(packagePid)
try:
workflowData = JsonSimple(payload.open())
except Exception:
self.log.error("Error parsing JSON '{}'", packagePid)
finally:
payload.close()
except StorageException:
self.log.error("Error accessing '{}'", packagePid)
return
# Test our version data
self.version = workflowData.getString("{NO VERSION}",
["redbox:formVersion"])
oldData = String(workflowData.toString(True))
if self.version != self.redboxVersion:
self.log.info("OID '{}' requires an upgrade: '{}' => '{}'",
[self.oid, self.version, self.redboxVersion])
# The version data is old, run our upgrade
# function to see if any alterations are
# required. Most likely at least the
# version number will change.
newWorkflowData = self.__upgrade(workflowData)
else:
newWorkflowData = self.__hotfix(workflowData)
if newWorkflowData is not None:
self.log.debug("OID '{}' was hotfixed for v1.2 'dc:type' bug",
self.oid)
else:
self.log.debug("OID '{}' requires no work, skipping", self.oid)
return
# Backup our data first
backedUp = self.__backup(oldData)
if not backedUp:
self.log.error("Upgrade aborted, data backup failed!")
return
# Save the newly modified data
jsonString = String(newWorkflowData.toString(True))
inStream = ByteArrayInputStream(jsonString.getBytes("UTF-8"))
try:
self.object.updatePayload(packagePid, inStream)
except StorageException, e:
self.log.error("Error updating workflow payload: ", e)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "dataset"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
#if self.owner == "guest":
# pageTitle = "Submission Request"
# displayType = "submission-request"
# initialStep = 0
#else:
# pageTitle = "Metadata Record"
# displayType = "package-dataset"
# initialStep = 1
## Harvesting straight into the 'Published' stage
pageTitle = "Metadata Record"
displayType = "package-dataset"
#initialStep = 4
initialStep = 3
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(
None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put(
"label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object,
"workflow.metadata",
inStream)
except StorageException:
print(" ERROR updating dataset payload")
# Form processing
coreFields = [
"title", "description", "manifest", "metaList", "relationships",
"responses"
]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ",
parsedTime)
self.utils.add(self.index, "date_created", solrTime)
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
self.utils.add(self.index, "display_type", displayType)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id",
wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label",
wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
self.vc("log").debug("packageType = '{}'", packageType)
self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
if self.vc("formData").getValues("sequencesMetaList") != None:
sequenceService = ApplicationContextProvider.getApplicationContext(
).getBean("sequenceService")
sequencesMetaList = list(
self.vc("formData").getValues("sequencesMetaList"))
for sequenceInfo in sequencesMetaList:
sequenceInfoJson = JsonSimple(sequenceInfo)
sequenceIndex = sequenceService.getSequence(
sequenceInfoJson.getString(None, "sequenceName"))
jsonObj.put(
sequenceInfoJson.getString(None, "metadataName"),
String.format(
sequenceInfoJson.getString(None, "stringFormat"),
sequenceIndex))
self.vc("log").debug("json object created is: %r" % jsonObj)
outWriter.write(manifest.toString(True))
outWriter.close()
# adding ability to set access plugin
accessPlugin = self.vc("formData").get("access_plugin", "derby")
if accessPlugin is not None:
self.vc("page").authentication.set_access_plugin(accessPlugin)
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __activate__(self, context):
self.velocityContext = context
self.log = self.vc("log")
self.systemConfig = self.vc("systemConfig")
self.formData = context["formData"]
self.assertionText = self.formData.get("assertion")
self.session = self.vc("sessionState")
self.response = self.vc("response")
self.request = self.vc("request")
method = self.request.getMethod()
#checking access method
if method != "POST":
self.log.error("Page not accessed by a POST, method:%s" % method)
return
self.sharedKey = String(self.systemConfig.getString("", "rapidAafSso", "sharedKey"))
self.aud = self.systemConfig.getString("", "rapidAafSso", "aud")
self.iss = self.systemConfig.getString("", "rapidAafSso", "iss")
#checking signature
jwsObject = JWSObject.parse(self.assertionText)
verifier = MACVerifier(self.sharedKey.getBytes())
verifiedSignature = jwsObject.verify(verifier)
if (verifiedSignature):
self.log.debug("Verified JWS signature!")
else:
self.log.error("Invalid JWS signature!")
return
self.log.debug(jwsObject.getPayload().toString())
self.log.debug(self.session.toString())
json = JsonSimple(jwsObject.getPayload().toString())
aud = json.getString("", "aud")
iss = json.getString("", "iss")
nbf = json.getInteger(None, "nbf")
exp = json.getInteger(None, "exp")
jti = json.getString("", "jti")
#checking aud
if self.aud != aud:
self.log.error("Invalid aud: '%s' expecting: '%s'" % (aud, self.aud))
return
#checking iss
if self.iss != iss:
self.log.error("Invalid iss: '%s' expecting: '%s'" % (iss, self.iss))
return
#checking times
now = Date().getTime() / 1000
if now < nbf or now > exp:
self.log.error("Invalid timings.")
return
#checking jti
attributeDao = ApplicationContextProvider.getApplicationContext().getBean("hibernateAuthUserAttributeDao")
params = HashMap()
params.put("key", "jti")
params.put("value", jti)
attrList = attributeDao.query("getUserAttributeByKeyAndValue", params)
if attrList.size() > 0:
self.log.error("Possible replay attack, jti:'%s' found in DB." % jti)
return
self.session.put("jwt", jwsObject.getPayload().toString())
self.session.put("jwt_json", json)
self.session.put("jwt_assertion", self.assertionText)
self.session.put("jwt_exp", exp)
self.returnAddress = self.session.get("returnAddress")
if self.returnAddress is None:
self.log.debug("No return address, using portalPath.")
self.returnAddress = self.vc("portalPath")
self.log.debug("RapidAAF SSO login complete, redirect to: %s" % self.returnAddress)
self.response.sendRedirect(self.returnAddress)
def __workflow(self):
# Workflow data
WORKFLOW_ID = "servicesUI2"
wfChanged = False
workflow_security = []
self.message_list = None
stages = self.config.getJsonSimpleList(["stages"])
pageTitle = "Services Record"
displayType = "package-service"
initialStep = 0
try:
wfMeta = self.__getJsonPayload("workflow.metadata")
wfMeta.getJsonObject().put("pageTitle", pageTitle)
# Are we indexing because of a workflow progression?
targetStep = wfMeta.getString(None, ["targetStep"])
if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]):
wfChanged = True
# Step change
wfMeta.getJsonObject().put("step", targetStep)
wfMeta.getJsonObject().remove("targetStep")
# This must be a re-index then
else:
targetStep = wfMeta.getString(None, ["step"])
# Security change
for stage in stages:
if stage.getString(None, ["name"]) == targetStep:
wfMeta.getJsonObject().put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
if wfChanged == True:
self.message_list = stage.getStringList(["message"])
except StorageException:
# No workflow payload, time to create
initialStage = stages.get(initialStep).getString(None, ["name"])
wfChanged = True
wfMeta = JsonSimple()
wfMetaObj = wfMeta.getJsonObject()
wfMetaObj.put("id", WORKFLOW_ID)
wfMetaObj.put("step", initialStage)
wfMetaObj.put("pageTitle", pageTitle)
stages = self.config.getJsonSimpleList(["stages"])
for stage in stages:
if stage.getString(None, ["name"]) == initialStage:
wfMetaObj.put("label", stage.getString(None, ["label"]))
self.item_security = stage.getStringList(["visibility"])
workflow_security = stage.getStringList(["security"])
self.message_list = stage.getStringList(["message"])
# Has the workflow metadata changed?
if wfChanged == True:
inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8")
try:
StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream)
except StorageException:
print " ERROR updating dataset payload"
# Form processing
coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"]
formData = wfMeta.getObject(["formData"])
if formData is not None:
formData = JsonSimple(formData)
# Core fields
description = formData.getStringList(["description"])
if description:
self.descriptionList = description
# Non-core fields
data = formData.getJsonObject()
for field in data.keySet():
if field not in coreFields:
self.customFields[field] = formData.getStringList([field])
# Manifest processing (formData not present in wfMeta)
manifest = self.__getJsonPayload(self.packagePid)
formTitles = manifest.getStringList(["title"])
if formTitles:
for formTitle in formTitles:
if self.title is None:
self.title = formTitle
self.descriptionList = [manifest.getString("", ["description"])]
formData = manifest.getJsonObject()
for field in formData.keySet():
if field not in coreFields:
value = formData.get(field)
if value is not None and value.strip() != "":
self.utils.add(self.index, field, value)
# We want to sort by date of creation, so it
# needs to be indexed as a date (ie. 'date_*')
if field == "dc:created":
parsedTime = time.strptime(value, "%Y-%m-%d")
solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime)
self.utils.add(self.index, "date_created", solrTime)
# try to extract some common fields for faceting
if field.startswith("dc:") and \
not (field.endswith(".dc:identifier.rdf:PlainLiteral") \
or field.endswith(".dc:identifier") \
or field.endswith(".rdf:resource")):
# index dublin core fields for faceting
basicField = field.replace("dc:", "dc_")
dot = field.find(".")
if dot > 0:
facetField = basicField[:dot]
else:
facetField = basicField
#print "Indexing DC field '%s':'%s'" % (field, facetField)
if facetField == "dc_title":
if self.title is None:
self.title = value
elif facetField == "dc_type":
if self.dcType is None:
self.dcType = value
elif facetField == "dc_creator":
if basicField.endswith("foaf_name"):
self.utils.add(self.index, "dc_creator", value)
else:
self.utils.add(self.index, facetField, value)
# index keywords for lookup
if field.startswith("dc:subject.vivo:keyword."):
self.utils.add(self.index, "keywords", value)
self.utils.add(self.index, "display_type", displayType)
# Workflow processing
wfStep = wfMeta.getString(None, ["step"])
self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"]))
self.utils.add(self.index, "workflow_step", wfStep)
self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"]))
for group in workflow_security:
self.utils.add(self.index, "workflow_security", group)
if self.owner is not None:
self.utils.add(self.index, "workflow_security", self.owner)
# set OAI-PMH status to deleted
if wfStep == "retired":
self.utils.add(self.index, "oai_deleted", "true")
def __attachFile(self):
try:
# WebKit/IE prefixes C:\fakepath\ with javascript manipulated file inputs
uploadFile = self.formData.get("uploadFile")
self.log.debug(uploadFile)
self.log.debug(self.vc("sessionState").toString())
self.log.debug(self.formData.toString())
uploadFile = uploadFile.replace("C:\\fakepath\\", "")
fileDetails = self.vc("sessionState").get(uploadFile)
# Establish that we do have details on the uploaded file
if fileDetails is None:
#uploadFile = uploadFile.rsplit("\\", 1)[-1]
fileDetails = self.vc("sessionState").get(uploadFile)
if fileDetails is None:
self.log.error("**** fileDetails is None!!! ***")
return self.__toJson(
{"error": "fileDetails is None (no upload file!)"})
self.log.debug("Attach Upload: fileDetails: '{}'", fileDetails)
errorDetails = fileDetails.get("error")
if errorDetails:
self.log.error("ERROR: %s" % errorDetails)
return self.__toJson({"error": errorDetails})
# Look for the storage info we need
jsonFormData = JsonSimple(self.formData.get("json"))
oid = jsonFormData.getString(None, "oid")
fname = fileDetails.get("name")
foid = fileDetails.get("oid")
attachType = jsonFormData.getString("supporting-material",
"attachmentType")
self.log.debug("attach oid='{}', filename='{}', foid='{}'",
[oid, fname, foid])
# Make sure it was actually stored
try:
attachObj = self.Services.getStorage().getObject(foid)
except StorageException, e:
return JsonSimple({"error": "Attached file - '%s'" % str(e)})
# Build up some metadata to store alongside the file
attachFormData = JsonSimple(self.formData.get("json", "{}"))
attachMetadata = {
"type": "attachment",
"created_by": "attachments.py",
"formData": {
"oid":
foid,
"attached_to":
oid,
"filename":
fname,
"access_rights":
attachFormData.getString("private", ["accessRights"]),
"attachment_type":
attachType,
"description":
attachFormData.getString("", ["description"]),
"owner":
self.vc("page").authentication.get_username()
}
}
# Record an attachment in attachments.metadata
attachment = self.__createAttachmentJson(
foid, attachMetadata["formData"]["attachment_type"],
attachMetadata["formData"]["description"],
attachMetadata["formData"]["filename"])
# print "Attaching %s" % attachment.toString().encode('utf-8')
self.__createOrUpdateArrayPayload(oid, "attachments.metadata",
attachment)
# We are going to send an update on all attachments back with our response
attachedFiles = self.__getAttachedFiles(oid, attachType)
attachedFiles.append(dict(attachMetadata["formData"]))
# Now store our metadata for this file
try:
jsonMetadata = self.__toJson(attachMetadata)
jsonIn = ByteArrayInputStream(jsonMetadata.toString())
StorageUtils.createOrUpdatePayload(attachObj,
"attachments.metadata",
jsonIn)
jsonIn.close()
attachObj.close()
except StorageException, e:
self.log.error("Failed to create attachment metadata!", e)
