def __createFromSelected(self):
self.vc("log").debug("Creating package from selected...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
# if modifying existing manifest, we already have an identifier,
# otherwise create a new one
manifestId = self.__getActiveManifestId()
if manifestId is None:
manifestHash = "%s.tfpackage" % uuid.uuid4()
else:
manifestHash = self.__getActiveManifestPid()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
manifest = self.__getActiveManifest()
oldType = manifest.getType()
if oldType is None:
manifest.setType(packageType)
else:
manifest.setType(oldType)
#self.vc("log").debug("Manifest: {}", manifest)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
if manifestId is None:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, and make sure they are both deployed
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
else:
# update existing object
object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId)
manifestStream = FileUtils.openInputStream(manifestFile)
StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream)
manifestStream.close()
object.close()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
self.vc("log").debug("packageType = '{}'", packageType)
self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
if self.vc("formData").getValues("sequencesMetaList") != None:
sequenceService = ApplicationContextProvider.getApplicationContext().getBean("sequenceService")
sequencesMetaList = list(self.vc("formData").getValues("sequencesMetaList"))
for sequenceInfo in sequencesMetaList:
sequenceInfoJson = JsonSimple(sequenceInfo)
sequenceIndex = sequenceService.getSequence(sequenceInfoJson.getString(None,"sequenceName"))
jsonObj.put(sequenceInfoJson.getString(None,"metadataName"), String.format(sequenceInfoJson.getString(None,"stringFormat"),sequenceIndex))
self.vc("log").debug("json object created is: %r" % jsonObj)
outWriter.write(manifest.toString(True))
outWriter.close()
# adding ability to set access plugin
accessPlugin = self.vc("formData").get("access_plugin", "derby")
if accessPlugin is not None:
self.vc("page").authentication.set_access_plugin(accessPlugin)
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __activate__(self, context):
self.log = context["log"]
self.request = context["request"]
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
processingSet = self.request.getParameter("processingSet")
self.procMsg = None
# read configuration and trigger processing stream sets
# storing the return object on the map
configFilePath = FascinatorHome.getPath(
"process") + "/processConfig.json"
procConfigFile = File(configFilePath)
if procConfigFile.exists() == True:
self.dataMap = HashMap()
self.dataMap.put("indexer", context['Services'].getIndexer())
self.procConfigJson = JsonSimple(procConfigFile)
for configObj in self.procConfigJson.getJsonArray():
configJson = JsonSimple(configObj)
procId = configJson.getString("", "id")
if processingSet is not None:
if procId == processingSet:
self.execProcSet(procId, configJson)
else:
self.execProcSet(procId, configJson)
if self.procMsg is None:
self.procMsg = "Processing complete!"
else:
self.procMsg = "Configuration file does not exist: " + configFilePath
def storeToken(self, tokenObject):
self.resetErrors()
index = "resumptionTokens-STORE"
table = "resumptionTokens"
fields = {
"token": tokenObject.getToken(),
"metadataPrefix": tokenObject.getMetadataPrefix(),
"expiry": Timestamp(tokenObject.getExpiry()),
"nextToken": tokenObject.getNextToken(),
"resultJson": ""
}
FileUtils.writeStringToFile(File(FascinatorHome.getPath("oaipmh-results")+ "/"+tokenObject.getToken()),tokenObject.getResultJson())
#self.log.debug("=== storeToken()")
#self.log.debug("=== TOKEN: '{}'", tokenObject.getToken())
#self.log.debug("=== METADATAPREFIX: '{}'", tokenObject.getMetadataPrefix())
#self.log.debug("=== EXPIRY: '{}'", tokenObject.getExpiry())
#self.log.debug("=== TOTALFOUND: '{}'", tokenObject.getTotalFound())
#self.log.debug("=== START: '{}'", tokenObject.getStart())
try:
self.db.insert(self.dbName, index, table, fields)
except Exception, e:
msg = self.parseError(e)
if msg == "Duplicate record!":
# Expected failure
self.log.error("Duplicate record already exists in table!")
else:
# Something is wrong
self.log.error("ERROR: ", e)
self.error = True
self.errorMsg = msg
def __activate__(self, context):
self.log = context["log"]
self.config = context["systemConfig"]
response = context["response"]
try:
## Variable prep
defaultPath = FascinatorHome.getPath("alerts")
self.alertsPath = self.config.getString(defaultPath,
["alerts", "path"])
self.configFile = None # We'll allocate this later... if needed
self.redboxVersion = self.config.getString(
"", "redbox.version.string")
self.csvDialect = csv.excel
self.csvDialect.skipinitialspace = True
## XML Parsing
docFactory = DocumentFactory()
##docFactory.setXPathNamespaceURIs(namespaces)
self.saxReader = SAXReader(docFactory)
## Do our job
(success, failed) = self.__processDir()
## Send response to the client (if debugging in browser)
writer = response.getPrintWriter("text/plain; charset=UTF-8")
writer.println("%s successful, %s failed" % (success, failed))
writer.close()
except Exception, e:
response.setStatus(500)
writer = response.getPrintWriter("text/plain; charset=UTF-8")
writer.println("Unexpected error during script execution:\n%s" %
str(e))
writer.close()
def __activate__(self, context):
self.log = context["log"]
self.config = context["systemConfig"]
response = context["response"]
try:
## Variable prep
defaultPath = FascinatorHome.getPath("alerts")
self.alertsPath = self.config.getString(defaultPath, ["alerts", "path"])
self.configFile = None # We'll allocate this later... if needed
self.redboxVersion = self.config.getString("", "redbox.version.string")
self.csvDialect = csv.excel
self.csvDialect.skipinitialspace = True
## XML Parsing
docFactory = DocumentFactory()
##docFactory.setXPathNamespaceURIs(namespaces)
self.saxReader = SAXReader(docFactory)
## Do our job
(success, failed) = self.__processDir()
## Send response to the client (if debugging in browser)
writer = response.getPrintWriter("text/plain; charset=UTF-8")
writer.println("%s successful, %s failed" % (success, failed))
writer.close()
except Exception,e:
response.setStatus(500)
writer = response.getPrintWriter("text/plain; charset=UTF-8")
writer.println("Unexpected error during script execution:\n%s" % str(e))
writer.close()
def processdocx(self, inputfilepath):
"""
Convert a docx to html format, and calling
"""
zipFile = ZipFile(inputfilepath)
entry = zipFile.getEntry("word/document.xml")
stream = zipFile.getInputStream(entry)
text = StreamSource(stream)
factory = TransformerFactoryImpl()
xslt = StreamSource(File(join(FascinatorHome.getPath(), "lib", "xslt", "docx2html.xsl")))
transformer = factory.newTransformer(xslt)
tf = "/tmp/%s.html" % uuid.uuid4()
transformer.transform(text, StreamResult(File(tf)))
parser = DocxHtmlParser()
parser.init()
f = open(tf, "r")
parser.feed(unescape(f.read()))
f.close()
try:
remove(tf)
except Exception, e:
self.log.error("Failed to remove uploaded word file: %s." % tf)
self.log.error(str(e))
def __activate__(self, context):
self.None = context["log"]
self.systemConfig = context["systemConfig"]
self.sessionState = context["sessionState"]
self.response = context["response"]
self.request = context["request"]
self.services = context["Services"]
self.sessionState.set("username", "admin")
self.writer = self.response.getPrintWriter("text/plain; charset=UTF-8")
curationJobDao = ApplicationContextProvider.getApplicationContext().getBean("curationJobDao")
publicationHandler = ApplicationContextProvider.getApplicationContext().getBean("publicationHandler")
jobs = JsonSimple(File(FascinatorHome.getPath() + "/curation-status-responses/inProgressJobs.json")).getArray("inProgressJobs")
self.writer.println(jobs.size())
for curationJob in jobs:
curationJob
jobStatus = self.queryJobStatus(curationJob)
self.writer.println(jobStatus.toString())
status = jobStatus.getString("failed", "status")
self.writeResponseToStatusResponseCache(jobStatus.getInteger(None, "job_id"), jobStatus)
self.writer.println(status)
if "complete" == status:
publicationHandler.publishRecords(jobStatus.getArray("job_items"))
self.updateLocalRecordRelations(jobStatus.getArray("job_items"))
self.writer.close()
self.sessionState.remove("username")
def storeToken(self, tokenObject):
self.resetErrors()
index = "resumptionTokens-STORE"
table = "resumptionTokens"
fields = {
"token": tokenObject.getToken(),
"metadataPrefix": tokenObject.getMetadataPrefix(),
"expiry": Timestamp(tokenObject.getExpiry()),
"nextToken": tokenObject.getNextToken(),
"resultJson": ""
}
FileUtils.writeStringToFile(
File(
FascinatorHome.getPath("oaipmh-results") + "/" +
tokenObject.getToken()), tokenObject.getResultJson())
#self.log.debug("=== storeToken()")
#self.log.debug("=== TOKEN: '{}'", tokenObject.getToken())
#self.log.debug("=== METADATAPREFIX: '{}'", tokenObject.getMetadataPrefix())
#self.log.debug("=== EXPIRY: '{}'", tokenObject.getExpiry())
#self.log.debug("=== TOTALFOUND: '{}'", tokenObject.getTotalFound())
#self.log.debug("=== START: '{}'", tokenObject.getStart())
try:
self.db.insert(self.dbName, index, table, fields)
except Exception, e:
msg = self.parseError(e)
if msg == "Duplicate record!":
# Expected failure
self.log.error("Duplicate record already exists in table!")
else:
# Something is wrong
self.log.error("ERROR: ", e)
self.error = True
self.errorMsg = msg
def __ingestJson(self, fileName, jsonObject, move):
if self.configFile is None:
self.configFile = FascinatorHome.getPathFile("harvest/workflows/dataset.json")
harvester = None
try:
## Cache the file out to disk... although requires it
## .tfpackage extension due to jsonVelocity transformer
jsonPath = self.pTemp(fileName)
jsonFile = open(jsonPath, "wb")
jsonFile.write(jsonObject.toString(True))
jsonFile.close()
## Now instantiate a HarvestClient just for this File.
harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
harvester.start()
## And cleanup afterwards
oid = harvester.getUploadOid()
self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
if move:
shutil.move(self.pBase(fileName), self.pDone(fileName))
return True
except Exception, e:
## TODO: This block looks to just be a copy of the
## top-level one, yet it runs per ROW, not for the
## whole File. Just the JSON data should be stored
## Move the CSV to the 'failed' directory
shutil.move(self.pBase(fileName), self.pFail(fileName))
## And write our error data to disk beside it
self.writeError(fileName, e)
return False
def __ingestJson(self, fileName, jsonObject, move):
if self.configFile is None:
self.configFile = FascinatorHome.getPathFile(
"harvest/workflows/dataset.json")
harvester = None
try:
## Cache the file out to disk... although requires it
## .tfpackage extension due to jsonVelocity transformer
jsonPath = self.pTemp(fileName)
jsonFile = open(jsonPath, "wb")
jsonFile.write(jsonObject.toString(True))
jsonFile.close()
## Now instantiate a HarvestClient just for this File.
harvester = HarvestClient(self.configFile, File(jsonPath), "guest")
harvester.start()
## And cleanup afterwards
oid = harvester.getUploadOid()
self.log.info("Harvested alert '{}' to '{}'", fileName, oid)
if move:
shutil.move(self.pBase(fileName), self.pDone(fileName))
return True
except Exception, e:
## TODO: This block looks to just be a copy of the
## top-level one, yet it runs per ROW, not for the
## whole File. Just the JSON data should be stored
## Move the CSV to the 'failed' directory
shutil.move(self.pBase(fileName), self.pFail(fileName))
## And write our error data to disk beside it
self.writeError(fileName, e)
return False
def __activate__(self, context):
self.log = context["log"]
self.request = context["request"]
self.sessionState = context["sessionState"]
self.sessionState.set("username","admin")
processingSet = self.request.getParameter("processingSet")
self.procMsg = None
# read configuration and trigger processing stream sets
# storing the return object on the map
configFilePath = FascinatorHome.getPath("process")+"/processConfig.json"
procConfigFile = File(configFilePath)
if procConfigFile.exists() == True:
self.dataMap = HashMap()
self.dataMap.put("indexer", context['Services'].getIndexer())
self.procConfigJson = JsonSimple(procConfigFile)
for configObj in self.procConfigJson.getJsonArray():
configJson = JsonSimple(configObj)
procId = configJson.getString("", "id")
if processingSet is not None:
if procId == processingSet:
self.execProcSet(procId, configJson)
else:
self.execProcSet(procId, configJson)
if self.procMsg is None:
self.procMsg = "Processing complete!"
else:
self.procMsg = "Configuration file does not exist: " + configFilePath
def updateObjectMetadata(self, objectMetaData, toWorkflowId):
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(toWorkflowId)
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = File(workflowsDir, jsonConfigFile)
configObject = StorageUtils.checkHarvestFile(self.storage, configFile);
if configObject is None:
oid = StorageUtils.generateOid(configFile);
configObject = StorageUtils.getDigitalObject(self.storage, oid);
objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
objectMetaData.setProperty("jsonConfigOid", configObject.getId())
configJson = JsonSimple(configFile)
rulesFileName = configJson.getString(None, "indexer","script","rules")
rulesFile = File(workflowsDir,rulesFileName)
rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile);
if rulesObject is None:
oid = StorageUtils.generateOid(rulesFile);
rulesObject = StorageUtils.getDigitalObject(self.storage, oid);
objectMetaData.setProperty("rulesPid", rulesFileName)
objectMetaData.setProperty("rulesOid", rulesObject.getId())
objectMetaData.setProperty("workflowTransitioned", "true")
objectMetaData.setProperty("date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def __activate__(self, context):
self.log = context["log"]
config = context["systemConfig"]
response = context["response"]
## Variable prep
self.redboxVersion = self.config.getString("", "redbox.version.string")
self.defaultPath = FascinatorHome.getPath("alerts")
## This is the older-style config that allowed 1 folder for alert
alertsPath = self.config.getString(None, ["alerts", "path"])
if alertsPath is None:
## The newer config allows for alerts to come from several folders
self.alertSet = self.config.getJsonSimpleList(defaultPath, ["alerts", "alert-set"])
for alertItem in self.alertSet:
self.log.info("Processing alert: {}. Log file: {}", alertItem["name"], )
try:
alert = Alert(redboxVersion, alertItem)
alert.processAlert()
except Exception as e:
#The Alert class will log this for us so continue to the next alert
#Some exceptions stop an alert from running at all so log them just in case
self.log.error("Alert [{}] encountered problems - please review the log files in the associated .processed directory. Exception was: {}", alertItem["name"], e.message)
else:
try:
alertItem = self.__prepareAlertFromOldConfig(alertsPath)
alert = Alert(redboxVersion, alertItem)
alert.processAlert()
except Exception as e:
#The Alert class will log this for us
pass
return
def updateObjectMetadata(self, objectMetaData, toWorkflowId):
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(
toWorkflowId)
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = File(workflowsDir, jsonConfigFile)
configObject = StorageUtils.checkHarvestFile(self.storage, configFile)
if configObject is None:
oid = StorageUtils.generateOid(configFile)
configObject = StorageUtils.getDigitalObject(self.storage, oid)
objectMetaData.setProperty("jsonConfigPid", jsonConfigFile)
objectMetaData.setProperty("jsonConfigOid", configObject.getId())
configJson = JsonSimple(configFile)
rulesFileName = configJson.getString(None, "indexer", "script",
"rules")
rulesFile = File(workflowsDir, rulesFileName)
rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile)
if rulesObject is None:
oid = StorageUtils.generateOid(rulesFile)
rulesObject = StorageUtils.getDigitalObject(self.storage, oid)
objectMetaData.setProperty("rulesPid", rulesFileName)
objectMetaData.setProperty("rulesOid", rulesObject.getId())
objectMetaData.setProperty("workflowTransitioned", "true")
objectMetaData.setProperty(
"date_transitioned",
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
log.error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __prepareAlertFromOldConfig(self, alertsPath):
return {
"name":
"Default alert",
"path":
alertsPath,
"harvestConfig":
FascinatorHome.getPathFile("harvest/workflows/dataset.json"),
"handlers": {
"csv": "CSVAlertHandler",
"xml": "XMLAlertHandler",
"rif": "XMLAlertHandler"
},
"CSVAlertHandler-params": {
"configMap": {
"csv": {
"Dialect": {
"skipinitialspace": true,
"quotechar": "\"",
"delimiter": ","
},
"hasHeaderRow": true,
"FieldMap": {
0: ["title", "redbox:submissionProcess.dc:title"],
1: [
"description",
"redbox:submissionProcess.dc:description"
],
2:
"redbox:submissionProcess.locrel:prc.foaf:Person.foaf:name",
3:
"redbox:submissionProcess.locrel:prc.foaf:Person.foaf:phone",
4:
"redbox:submissionProcess.locrel:prc.foaf:Person.foaf:mbox",
5:
"workflow_source",
6:
"redbox:submissionProcess.skos:note"
}
}
}
},
"XMLAlertHandler-params": {
"configMap": {
"xml": {
"xmlMap":
self.config.getObject(None,
["alerts", "xmlMaps", "xml"])
},
"rif": {
"xmlMap":
self.config.getObject(None,
["alerts", "xmlMaps", "rif"])
},
}
}
}
def writeResponseToStatusResponseCache(self, jobId, jobStatus):
curationStatusRespones = File(FascinatorHome.getPath() +
"/curation-status-responses")
if curationStatusRespones.exists():
FileUtils.forceMkdir(curationStatusRespones)
FileUtils.writeStringToFile(
File(curationStatusRespones.getPath() +
"/" + Integer(jobId).toString() + ".json"),
jobStatus.toString(True))
def __activate__(self, context):
self.velocityContext = context
self.vc("sessionState").remove("fq")
self.__myPlans = None
self.__sharedPlans = None
self.__myDrafts = None
self.__myDatasets = None
self.__stages = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/dataset.json")).getArray("stages")
self.__search()
def __activate__(self, context):
self.velocityContext = context
self.vc("sessionState").remove("fq")
self.__myPlans = None
self.__sharedPlans = None
self.__myDrafts = None
self.__myDatasets = None
self.__stages = JsonSimple(
FascinatorHome.getPathFile(
"harvest/workflows/dataset.json")).getArray("stages")
self.__search()
def getToken(self, tokenId):
self.resetErrors()
index = "resumptionTokens-GET"
sql = """
SELECT *
FROM resumptionTokens
WHERE token = ?
"""
fields = [tokenId]
try:
result = self.db.select(self.dbName, index, sql, fields)
# Make sure we got a response
if result is None or result.isEmpty():
return None
# Create the new token to return
metadataPrefix = result.get(0).get("METADATAPREFIX")
expiryStr = result.get(0).get("EXPIRY")
# Jython does not support %f microseconds in time parsing, makes
# this more awkward then it should be in 2.6+ Python
# 1: split into basic time + micro seconds
(basicTime, mSecs) = expiryStr.strip().split(".")
# 2: Parse the basic time
expiryDt = datetime.strptime(basicTime, "%Y-%m-%d %H:%M:%S")
# 3: Convert into a 'epoch' long and then to a string (has an extra ".0" on the end)
epoch = "%s" % time.mktime(expiryDt.timetuple())
# 4: Remove the extraneous trailing zero and re-attach microseconds
expiry = "%s%s" % (epoch.replace(".0", ""), mSecs)
nextToken = result.get(0).get("NEXTTOKEN")
file = File(
FascinatorHome.getPath("oaipmh-results") + "/" + tokenId)
resultJson = FileUtils.readFileToString(file)
FileUtils.deleteQuietly(file)
token = ResumptionToken(tokenId, metadataPrefix, nextToken,
resultJson)
token.setExpiry(expiry)
return token
except Exception, e:
# Something is wrong
self.log.error("ERROR: ", e)
self.error = True
self.errorMsg = self.parseError(e)
return None
def __activate__(self, context):
self.velocityContext = context
self.request = context["request"]
self.response = context["response"]
self.sessionState = context["sessionState"]
self.errorMsg = ""
action = self.request.getParameter("action")
homeConfigFilePath = FascinatorHome.getPath("process")+"/notification/homeInstitutions.json"
self.homeConfigFile = File(homeConfigFilePath)
if self.homeConfigFile.exists() == False:
self.errorMsg = "Configuration path does not exist: %s" % homeConfigFilePath
return
if action is None or action == "list":
self.listHomes()
else:
self.errorMsg = "Invalid action."
def getToken(self, tokenId):
self.resetErrors()
index = "resumptionTokens-GET"
sql = """
SELECT *
FROM resumptionTokens
WHERE token = ?
"""
fields = [tokenId]
try:
result = self.db.select(self.dbName, index, sql, fields)
# Make sure we got a response
if result is None or result.isEmpty():
return None
# Create the new token to return
metadataPrefix = result.get(0).get("METADATAPREFIX")
expiryStr = result.get(0).get("EXPIRY")
# Jython does not support %f microseconds in time parsing, makes
# this more awkward then it should be in 2.6+ Python
# 1: split into basic time + micro seconds
(basicTime, mSecs) = expiryStr.strip().split(".")
# 2: Parse the basic time
expiryDt = datetime.strptime(basicTime, "%Y-%m-%d %H:%M:%S")
# 3: Convert into a 'epoch' long and then to a string (has an extra ".0" on the end)
epoch = "%s" % time.mktime(expiryDt.timetuple())
# 4: Remove the extraneous trailing zero and re-attach microseconds
expiry = "%s%s" % (epoch.replace(".0", ""), mSecs)
nextToken = result.get(0).get("NEXTTOKEN")
file = File(FascinatorHome.getPath("oaipmh-results")+ "/"+tokenId)
resultJson = FileUtils.readFileToString(file)
FileUtils.deleteQuietly(file)
token = ResumptionToken(tokenId, metadataPrefix,nextToken,resultJson)
token.setExpiry(expiry)
return token
except Exception, e:
# Something is wrong
self.log.error("ERROR: ", e)
self.error = True
self.errorMsg = self.parseError(e)
return None
def removeToken(self, tokenObject):
self.resetErrors()
index = "resumptionTokens-DELETE"
table = "resumptionTokens"
fields = {
"token": tokenObject.getToken()
}
try:
self.db.delete(self.dbName, index, table, fields)
file = File(FascinatorHome.getPath("oaipmh-results")+ "/"+tokenObject.getToken())
FileUtils.deleteQuietly(file)
self.log.info("Delete successful! TOKEN='{}'", tokenObject.getToken())
return True
except Exception, e:
# Something is wrong
self.log.error("Delete failed! TOKEN='{}'", tokenObject.getToken())
self.log.error("ERROR: ", e)
self.error = True
self.errorMsg = self.parseError(e)
return False
def __activate__(self, context):
self.velocityContext = context
self.request = context["request"]
self.response = context["response"]
self.sessionState = context["sessionState"]
self.errorMsg = ""
action = self.request.getParameter("action")
homeConfigFilePath = FascinatorHome.getPath(
"process") + "/notification/homeInstitutions.json"
self.homeConfigFile = File(homeConfigFilePath)
if self.homeConfigFile.exists() == False:
self.errorMsg = "Configuration path does not exist: %s" % homeConfigFilePath
return
if action is None or action == "list":
self.listHomes()
else:
self.errorMsg = "Invalid action."
def removeToken(self, tokenObject):
self.resetErrors()
index = "resumptionTokens-DELETE"
table = "resumptionTokens"
fields = {"token": tokenObject.getToken()}
try:
self.db.delete(self.dbName, index, table, fields)
file = File(
FascinatorHome.getPath("oaipmh-results") + "/" +
tokenObject.getToken())
FileUtils.deleteQuietly(file)
self.log.info("Delete successful! TOKEN='{}'",
tokenObject.getToken())
return True
except Exception, e:
# Something is wrong
self.log.error("Delete failed! TOKEN='{}'", tokenObject.getToken())
self.log.error("ERROR: ", e)
self.error = True
self.errorMsg = self.parseError(e)
return False
def __prepareAlertFromOldConfig(self, alertsPath):
return {
"name": "Default alert",
"path": alertsPath,
"harvestConfig": FascinatorHome.getPathFile("harvest/workflows/dataset.json"),
"handlers": {
"csv": "CSVAlertHandler",
"xml": "XMLAlertHandler",
"rif": "XMLAlertHandler"
},
"CSVAlertHandler-params": {
"configMap": {
"csv": {
"Dialect": {
"skipinitialspace": true,
"quotechar": "\"",
"delimiter": ","
},
"hasHeaderRow": true,
"FieldMap": {
0: ["title","redbox:submissionProcess.dc:title"],
1: ["description", "redbox:submissionProcess.dc:description"],
2: "redbox:submissionProcess.locrel:prc.foaf:Person.foaf:name",
3: "redbox:submissionProcess.locrel:prc.foaf:Person.foaf:phone",
4: "redbox:submissionProcess.locrel:prc.foaf:Person.foaf:mbox",
5: "workflow_source",
6: "redbox:submissionProcess.skos:note"
}
}
}
},
"XMLAlertHandler-params": {
"configMap": {
"xml": {"xmlMap": self.config.getObject(None, ["alerts", "xmlMaps", "xml"])},
"rif": {"xmlMap": self.config.getObject(None, ["alerts", "xmlMaps", "rif"])},
}
}
}
def __activate__(self, context):
self.log = context["log"]
config = context["systemConfig"]
response = context["response"]
## Variable prep
self.redboxVersion = self.config.getString("", "redbox.version.string")
self.defaultPath = FascinatorHome.getPath("alerts")
## This is the older-style config that allowed 1 folder for alert
alertsPath = self.config.getString(None, ["alerts", "path"])
if alertsPath is None:
## The newer config allows for alerts to come from several folders
self.alertSet = self.config.getJsonSimpleList(
defaultPath, ["alerts", "alert-set"])
for alertItem in self.alertSet:
self.log.info(
"Processing alert: {}. Log file: {}",
alertItem["name"],
)
try:
alert = Alert(redboxVersion, alertItem)
alert.processAlert()
except Exception as e:
#The Alert class will log this for us so continue to the next alert
#Some exceptions stop an alert from running at all so log them just in case
self.log.error(
"Alert [{}] encountered problems - please review the log files in the associated .processed directory. Exception was: {}",
alertItem["name"], e.message)
else:
try:
alertItem = self.__prepareAlertFromOldConfig(alertsPath)
alert = Alert(redboxVersion, alertItem)
alert.processAlert()
except Exception as e:
#The Alert class will log this for us
pass
return
def __metadata(self):
jsonPayload = self.object.getPayload("metadata.json")
json = self.utils.getJsonObject(jsonPayload.open())
jsonPayload.close()
metadata = json.getObject("metadata")
self.utils.add(self.index, "dc_identifier", metadata.get("dc.identifier"))
data = json.getObject("data")
####Global setting for processing data
####These will need to be changed based on you system installation.
theMintHost = "http://*****:*****@example.edu.au"
sock = urllib.urlopen(theMintHost + "/mint/default/opensearch/lookup?count=999&searchTerms=Email:" + email)
mintData = sock.read()
sock.close()
jsonSimple = JsonSimple(mintData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#Ensuring that the Email identified a Party from The Mint
if results:
resultMetadata = JsonObject(results.get(0))
allData = resultMetadata.get("result-metadata")
creator = allData.get("all")
whoType = party.get("who").get("type")
if ((creator is not None) and (whoType == 'people')):
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".dc_identifier", creator.get("dc_identifier")[0])
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".foaf_name", creator.get("dc_title"))
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".foaf_title", creator.get("Honorific")[0])
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".redbox_isCoPrimaryInvestigator", "off")
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".redbox_isPrimaryInvestigator", "on")
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".foaf_givenName", creator.get("Given_Name")[0])
self.utils.add(self.index, "dc_creator.foaf_Person." + str(i) + ".foaf_familyName", creator.get("Family_Name")[0])
###Processing 'contactInfo.email' metadata
contactInfoEmail = data.get("contactInfo").get("email")
#Using the email address to obtain details from The Mint
#For testing, hard coded email address
#contactInfoEmail = "*****@*****.**"
sock = urllib.urlopen(theMintHost + "/mint/default/opensearch/lookup?count=999&searchTerms=Email:" + contactInfoEmail)
mintData = sock.read()
sock.close()
jsonSimple = JsonSimple(mintData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#Ensuring that the Email identified a Party from The Mint
if results:
resultMetadata = JsonObject(results.get(0))
allData = resultMetadata.get("result-metadata")
creator = allData.get("all")
if (creator is not None):
self.utils.add(self.index, "locrel_prc.foaf_Person.dc_identifier", creator.get("dc_identifier").toString())
self.utils.add(self.index, "locrel_prc.foaf_Person.foaf_name", creator.get("dc_title"))
self.utils.add(self.index, "locrel_prc.foaf_Person.foaf_title", creator.get("Honorific").toString())
self.utils.add(self.index, "locrel_prc.foaf_Person.foaf_givenName", creator.get("Given_Name").toString())
self.utils.add(self.index, "locrel_prc.foaf_Person.foaf_familyName", creator.get("Family_Name").toString())
###Processing 'coinvestigators' metadata
coinvestigators = data.get("coinvestigators")
for i in range(len(coinvestigators)):
self.utils.add(self.index, "dc_contributor.loclrel_clb." + str(i) + ".foaf_Agent" , coinvestigators[i])
###Processing 'anzsrcFOR' metadata
anzsrcFOR = data.get("anzsrcFOR")
for i in range(len(anzsrcFOR)):
anzsrc = anzsrcFOR[i]
#Querying against The Mint, but only using the first 4 numbers from anzsrc, this ensure a result
sock = urllib.urlopen(theMintHost + "/mint/ANZSRC_FOR/opensearch/lookup?count=999&level=http://purl.org/asc/1297.0/2008/for/" + anzsrc[:4])
mintData = sock.read()
sock.close()
jsonSimple = JsonSimple(mintData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring that anzsrc identified a record in The Mint
if results:
for j in range(len(results)):
result = JsonObject(results.get(j))
rdfAbout = result.get("rdf:about")
target = "http://purl.org/asc/1297.0/2008/for/" + anzsrc
if (rdfAbout == target):
self.utils.add(self.index, "dc_subject.anzsrc_for." + str(i) + ".skos_prefLabel" , result.get("skos:prefLabel"))
self.utils.add(self.index, "dc_subject.anzsrc_for." + str(i) + ".rdf:resource" , rdfAbout)
###Processing 'anzsrcSEO' metadata
anzsrcSEO = data.get("anzsrcSEO")
for i in range(len(anzsrcSEO)):
anzsrc = anzsrcSEO[i]
#Querying against The Mint, but only using the first 4 numbers from anzsrc, this ensure a result
sock = urllib.urlopen(theMintHost + "/mint/ANZSRC_SEO/opensearch/lookup?count=999&level=http://purl.org/asc/1297.0/2008/seo/" + anzsrc[:4])
mintData = sock.read()
sock.close()
jsonSimple = JsonSimple(mintData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring that anzsrc identified a record in The Mint
if results:
for j in range(len(results)):
result = JsonObject(results.get(j))
rdfAbout = result.get("rdf:about")
target = "http://purl.org/asc/1297.0/2008/seo/" + anzsrc
if (rdfAbout == target):
self.utils.add(self.index, "dc_subject.anzsrc_seo." + str(i) + ".skos_prefLabel" , result.get("skos:prefLabel"))
self.utils.add(self.index, "dc_subject.anzsrc_seo." + str(i) + ".rdf:resource" , rdfAbout)
###Processing 'keyword' metadata
keyword = data.get("keyword")
for i in range(len(keyword)):
self.utils.add(self.index, "dc_subject.vivo_keyword." + str(i) + ".rdf_PlainLiteral", keyword[i])
self.utils.add(self.index, "dc_accessRights.skos_prefLabel", data.get("accessRights"))
self.utils.add(self.index, "dc_license.dc_identifier", data.get("license").get("url"))
self.utils.add(self.index, "dc_license.skos_prefLabel", data.get("license").get("label"))
self.utils.add(self.index, "dc_identifier.redbox_origin", "internal")
dataLocation = data.get("dataLocation")
dataLocation = dataLocation.replace("%NAME_OF_FOLDER%", species)
self.utils.add(self.index, "bibo_Website.1.dc_identifier", dataLocation)
#The following have been intentionally set to blank. No mapping is required for these fields.
self.utils.add(self.index, "vivo_Location", "")
self.utils.add(self.index, "redbox_retentionPeriod", data.get("retentionPeriod"))
self.utils.add(self.index, "dc_extent", "unknown")
self.utils.add(self.index, "redbox_disposalDate", "")
self.utils.add(self.index, "locrel_own.foaf_Agent.1_foaf_name", "")
self.utils.add(self.index, "locrel_dtm.foaf_Agent.foaf_name", "")
###Processing 'organizationalGroup' metadata
organisationalGroup = data.get("organizationalGroup")
for i in range(len(organisationalGroup)):
organisation = organisationalGroup[i]
#Querying against The Mint
sock = urllib.urlopen(theMintHost + "/mint/Parties_Groups/opensearch/lookup?count=9999&searchTerms=ID:" + organisation)
mintData = sock.read()
sock.close()
jsonSimple = JsonSimple(mintData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring that anzsrc identified a record in The Mint
if results:
resultMetadata = JsonObject(results.get(0))
allData = resultMetadata.get("result-metadata")
orgGroup = allData.get("all")
self.utils.add(self.index, "foaf_Organization.dc_identifier", orgGroup.get("dc_identifier")[0])
self.utils.add(self.index, "foaf_Organization.skos_prefLabel", orgGroup.get("Name")[0])
self.utils.add(self.index, "foaf_fundedBy.foaf_Agent", "")
self.utils.add(self.index, "foaf_fundedBy.vivo_Grant", "")
self.utils.add(self.index, "swrc_ResearchProject.dc_title", "")
self.utils.add(self.index, "locrel_dpt.foaf_Person.foaf_name", "")
self.utils.add(self.index, "dc_SizeOrDuration", "")
self.utils.add(self.index, "dc_Policy", "")
self.utils.add(self.index, "redbox_ManagementPlan", "")
def __search(self):
indexer = Services.getIndexer()
portalQuery = Services.getPortalManager().get(self.vc("portalId")).getQuery()
portalSearchQuery = Services.getPortalManager().get(self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
req.setParam("facet", "true")
req.setParam("facet.field", "workflow_step")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
steps = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__steps = steps.getFacets().get("workflow_step")
wfConfig = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/dataset.json"))
jsonStageList = wfConfig.getJsonSimpleList(["stages"])
stages = []
for jsonStage in jsonStageList:
wfStage = WorkflowStage(jsonStage, self.__steps)
stages.append(wfStage)
self.__stages = stages
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "25")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
req.addParam("fq", "workflow_step:%s" % stages[0].getName())
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__alerts = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest("last_modified:[NOW-1MONTH TO *] AND workflow_step:live")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
self._searchEmbargoes()
self.vc("sessionState").set("fq", 'item_type:"object"')
def __metadata(self):
self.title = None
self.dcType = None
self.__checkMetadataPayload()
jsonPayload = self.object.getPayload("metadata.json")
json = self.utils.getJsonObject(jsonPayload.open())
jsonPayload.close()
metadata = json.getObject("metadata")
identifier = metadata.get("dc.identifier")
self.utils.add(self.index, "dc:identifier", identifier)
self.__storeIdentifier(identifier)
self.utils.add(self.index, "institution", "James Cook University")
self.utils.add(self.index, "source", "http://spatialecology.jcu.edu.au/Edgar/")
data = json.getObject("data")
####Global setting for processing data
####These will need to be changed based on you system installation.
theMintHost = java.lang.System.getProperty("mint.proxy.url")
collectionRelationTypesFilePath = FascinatorHome.getPath() + "/../portal/default/redbox/workflows/forms/data/"
servicesRelationTypesFilePath = FascinatorHome.getPath() + "/../portal/default/redbox/workflows/forms/data/"
descriptionTypesFilePath = FascinatorHome.getPath() + "/../portal/default/local/workflows/forms/data/"
relationshipTypesFilePath = FascinatorHome.getPath() + "/../portal/default/local/workflows/forms/data/"
###Allocating space to create the formData.tfpackage
tfpackageData = {}
###Using the species name, obtained from the directory name, to replace the text in the Title
species = data.get("species")
title = data.get("title")
title = title.replace("%NAME_OF_FOLDER%", species)
self.utils.add(self.index, "dc_title", title)
tfpackageData["dc:title"] = title
tfpackageData["title"] = title
self.utils.add(self.index, "dc_type", data.get("type"))
tfpackageData["dc:type.rdf:PlainLiteral"] = data.get("type")
tfpackageData["dc:type.skos:prefLabel"] = data.get("type")
tfpackageData["dc:created"] = time.strftime("%Y-%m-%d", time.gmtime())
tfpackageData["dc:modified"] = ""
tfpackageData["dc:language.skos:prefLabel"] = "English"
tfpackageData["dc:coverage.vivo:DateTimeInterval.vivo:start"] = data.get("temporalCoverage").get("dateFrom")
dateTo = data.get("temporalCoverage").get("dateTo")
if dateTo is not None:
tfpackageData["dc:coverage.vivo:DateTimeInterval.vivo:end"] = dateTo
tfpackageData["dc:coverage.redbox:timePeriod"] = ""
###Processing the 'spatialCoverage' metadata.
spatialCoverage = data.get("spatialCoverage")
for i in range(len(spatialCoverage)):
location = spatialCoverage[i]
if location["type"] == "text":
tfpackageData["dc:coverage.vivo:GeographicLocation." + str(i + 1) + ".dc:type"] = location["type"]
if (location["value"].startswith("POLYGON")):
tfpackageData["dc:coverage.vivo:GeographicLocation." + str(i + 1) + ".redbox:wktRaw"] = location["value"]
tfpackageData["dc:coverage.vivo:GeographicLocation." + str(i + 1) + ".rdf:PlainLiteral"] = location["value"]
###Processing the 'description' metadata.
#Reading the file here, so we only do it once.
file = open(descriptionTypesFilePath + "descriptionTypes.json")
descriptionData = file.read()
file.close()
description = data.get("description")
for i in range(len(description)):
desc = description[i]
tempDesc = desc.get("value")
tempDesc = tempDesc.replace("%NAME_OF_FOLDER%", species)
if (desc["type"] == "brief"):
tfpackageData["dc:description"] = tempDesc
tfpackageData["rif:description." + str(i + 1) + ".type"] = desc["type"]
tfpackageData["rif:description." + str(i + 1) + ".value"] = tempDesc
jsonSimple = JsonSimple(descriptionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Description Type exist
if results:
for j in range(len(results)):
descriptionType = results[j]
if (desc["type"] == descriptionType.get("id")):
tfpackageData["rif:description." + str(i + 1) + ".label"] = descriptionType.get("label")
###Processing the 'relatedPublication' metadata
relatedPublication = data.get("relatedPublication")
if relatedPublication is not None:
for i in range(len(relatedPublication)):
publication = relatedPublication[i]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) + ".dc:identifier"] = publication["doi"]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) + ".dc:title"] = publication["title"]
###Processing the 'relatedWebsite' metadata
relatedWebsite = data.get("relatedWebsite")
count = 0
for i in range(len(relatedWebsite)):
website = relatedWebsite[i]
tfpackageData["dc:relation.bibo:Website." + str(i + 1) + ".dc:identifier"] = website["url"]
tfpackageData["dc:relation.bibo:Website." + str(i + 1) + ".dc:title"] = website["notes"]
count = i + 1
###Processing the 'data_source_website' metadata (override metadata)
dataSourceWebsites = data.get("data_source_website")
if dataSourceWebsites is not None:
for i in range(len(dataSourceWebsites)):
website = dataSourceWebsites[i]
type = website.get("identifier").get("type")
if type == "uri":
count += 1
tfpackageData["dc:relation.bibo:Website." + str(count) + ".dc:identifier"] = website.get("identifier").get("value")
tfpackageData["dc:relation.bibo:Website." + str(count) + ".dc:title"] = website["notes"]
###Processing the 'relatedCollection' metadata
#Reading the file here, so we only do it once.
file = open(collectionRelationTypesFilePath + "collectionRelationTypes.json")
collectionData = file.read()
file.close()
relatedCollection = data.get("relatedCollection")
recordIdentifier = ""
if relatedCollection is not None:
for i in range(len(relatedCollection)):
collection = relatedCollection[i]
tempIdentifier = collection["identifier"]
if tempIdentifier is not None:
tempIdentifier = tempIdentifier.replace("%NAME_OF_FOLDER%", species)
recordIdentifier = tempIdentifier
else:
tempIdentifier = ""
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".dc:identifier"] = tempIdentifier
tempTitle = collection.get("title")
tempTitle = tempTitle.replace("%NAME_OF_FOLDER%", species)
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".dc:title"] = tempTitle
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".vivo:Relationship.rdf:PlainLiteral"] = collection["relationship"]
if tempIdentifier == "":
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".redbox:origin"] = "on"
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".redbox:publish"] = "on"
#Using the collection data as a lookup to obtain the 'label'
relationShip = collection.get("relationship")
jsonSimple = JsonSimple(collectionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Collection Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".vivo:Relationship.skos:prefLabel"] = relation.get("label")
###Processing the 'relatedService' metadata
#Reading the file here, so we only do it once.
file = open(servicesRelationTypesFilePath + "serviceRelationTypes.json")
servicesData = file.read()
file.close()
relatedServices = data.get("relatedService")
recordIdentifier = ""
if relatedServices is not None:
for i in range(len(relatedServices)):
service = relatedServices[i]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".dc:identifier"] = service["identifier"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".dc:title"] = service["title"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".vivo:Relationship.rdf:PlainLiteral"] = service["relationship"]
#Using the services data as a lookup to obtain the 'label'
relationShip = service.get("relationship")
jsonSimple = JsonSimple(servicesData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Service Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".vivo:Relationship.skos:prefLabel"] = relation.get("label")
###Processing the 'associatedParty' metadata
#Reading the file here so we only read it once.
file = open(relationshipTypesFilePath + "relationshipTypes.json")
relationshipData = file.read()
file.close()
associatedParty = data.get("associatedParty")
for i in range(len(associatedParty)):
party = associatedParty[i]
email = party.get("who").get("value")
if email is not None:
whoType = party.get("who").get("type")
if (whoType == 'people'):
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".dc:identifier"] = party.get("who").get("identifier")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:name"] = party.get("who").get("name")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:title"] = party.get("who").get("title")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".redbox:isCoPrimaryInvestigator"] = "off"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".redbox:isPrimaryInvestigator"] = "on"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:givenName"] = party.get("who").get("givenName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:familyName"] = party.get("who").get("familyName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".jcu:relationshipType"] = party.get("relationship")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:Organization.dc:identifier"] = party.get("affiliation").get("id")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:Organization.skos:prefLabel"] = party.get("affiliation").get("label")
jsonSimple = JsonSimple(relationshipData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Relationship Type exists
if results:
for j in range(len(results)):
relationshipType = results[j]
if (party.get("relationship") == relationshipType.get("id")):
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".jcu:relationshipLabel"] = relationshipType.get("label")
###Processing 'contactInfo.email' metadata
tfpackageData["locrel:prc.foaf:Person.dc:identifier"] = data.get("contactInfo").get("identifier")
tfpackageData["locrel:prc.foaf:Person.foaf:name"] = data.get("contactInfo").get("name")
tfpackageData["locrel:prc.foaf:Person.foaf:title"] = data.get("contactInfo").get("title")
tfpackageData["locrel:prc.foaf:Person.foaf:givenName"] = data.get("contactInfo").get("givenName")
tfpackageData["locrel:prc.foaf:Person.foaf:familyName"] = data.get("contactInfo").get("familyName")
tfpackageData["locrel:prc.foaf:Person.foaf:email"] = data.get("contactInfo").get("email")
##Stored At (on the Data Management page)
tfpackageData["vivo:Location.vivo:GeographicLocation.gn:name"] = data.get("contactInfo").get("streetAddress")
###Processing 'coinvestigators' metadata
coinvestigators = data.get("coinvestigators")
for i in range(len(coinvestigators)):
tfpackageData["dc:contributor.locrel:clb." + str(i + 1) + ".foaf:Agent"] = coinvestigators[i]
###Processing 'anzsrcFOR' metadata
anzsrcFOR = data.get("anzsrcFOR")
for i in range(len(anzsrcFOR)):
anzsrc = anzsrcFOR[i]
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) + ".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) + ".rdf:resource"] = anzsrc.get("resource")
###Processing 'anzsrcSEO' metadata
anzsrcSEO = data.get("anzsrcSEO")
for i in range(len(anzsrcSEO)):
anzsrc = anzsrcSEO[i]
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) + ".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) + ".rdf:resource"] = anzsrc.get("resource")
###Processing 'keyword' metadata
keyword = data.get("keyword")
for i in range(len(keyword)):
tfpackageData["dc:subject.vivo:keyword." + str(i + 1) + ".rdf:PlainLiteral"] = keyword[i]
###Research Themes
theme = data.get("researchTheme")
if (theme == "Tropical Ecosystems, Conservation and Climate Change"):
tfpackageData["jcu:research.themes.tropicalEcoSystems"] = "true"
elif (theme == "Industries and Economies in the Tropics"):
tfpackageData["jcu:research.themes.industriesEconomies"] = "true"
elif (theme == "People and Societies in the Tropics"):
tfpackageData["jcu:research.themes.peopleSocieties"] = "true"
elif (theme == "Tropical Health, Medicine and Biosecurity"):
tfpackageData["jcu:research.themes.tropicalHealth"] = "true"
elif (theme == "Not aligned to a University theme"):
tfpackageData["jcu:research.themes.notAligned"] = "true"
tfpackageData["dc:accessRights.skos:prefLabel"] = data.get("accessRights")
tfpackageData["dc:license.dc:identifier"] = data.get("license").get("url")
tfpackageData["dc:license.skos:prefLabel"] = data.get("license").get("label")
#identifier
additionalId = data.get("additionalIdentifier")
if additionalId is not None:
additionalId = additionalId.replace("%NAME_OF_FOLDER%", species)
tfpackageData["dc:identifier.rdf:PlainLiteral"] = additionalId
tfpackageData["dc:identifier.redbox:origin"] = "external"
tfpackageData["dc:identifier.dc:type.rdf:PlainLiteral"] = "local"
tfpackageData["dc:identifier.dc:type.skos:prefLabel"] = "Local Identifier"
else:
tfpackageData["dc:identifier.redbox:origin"] = "internal"
dataLocation = data.get("dataLocation")
dataLocation = dataLocation.replace("%NAME_OF_FOLDER%", species)
tfpackageData["bibo:Website.1.dc:identifier"] = dataLocation
#The following have been intentionally set to blank. No mapping is required for these fields.
tfpackageData["redbox:retentionPeriod"] = data.get("retentionPeriod")
tfpackageData["dc:extent"] = "unknown"
tfpackageData["redbox:disposalDate"] = ""
tfpackageData["locrel:own.foaf:Agent.1.foaf:name"] = ""
tfpackageData["locrel:dtm.foaf:Agent.foaf:name"] = ""
###Processing 'organizationalGroup' metadata
organisationalGroup = data.get("organizationalGroup")
for i in range(len(organisationalGroup)):
organisation = organisationalGroup[i]
tfpackageData["foaf:Organization.dc:identifier"] = organisation.get("identifier")
tfpackageData["foaf:Organization.skos:prefLabel"] = organisation.get("prefLabel")
tfpackageData["swrc:ResearchProject.dc:title"] = ""
tfpackageData["locrel:dpt.foaf:Person.foaf:name"] = ""
tfpackageData["dc:SizeOrDuration"] = ""
tfpackageData["dc:Policy"] = ""
#Citations
citations = data.get("citations")
for i in range(len(citations)):
citation = citations[i]
tfpackageData["dc:biblioGraphicCitation.redbox:sendCitation"] = citation.get("sendCitation")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:identifier.skos:note"] = citation.get("curationIdentifier")
paperTitle = citation.get("paperTitle")
paperTitle = paperTitle.replace("%NAME_OF_FOLDER%", species)
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:title"] = paperTitle
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." + str(i + 1) + ".foaf:familyName"] = citation.get("familyName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." + str(i + 1) + ".foaf:givenName"] = citation.get("givenName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." + str(i + 1) + ".foaf:title"] = title = citation.get("title")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:publisher.rdf:PlainLiteral"] = citation.get("publisher")
url = citation.get("url")
url = url.replace("%NAME_OF_FOLDER%", species)
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.bibo:Website.dc:identifier"] = url
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.1.rdf:PlainLiteral"] = tfpackageData["dc:created"]
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.rdf:PlainLiteral"] = "publicationDate"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.skos:prefLabel"] = "Publication Date"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.rdf:PlainLiteral"] = "created"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.skos:prefLabel"] = "Date Created"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.2.rdf:PlainLiteral"] = tfpackageData["dc:created"]
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.jcu:dataType"] = citation.get("dataType")
tfpackageData["dc:biblioGraphicCitation.skos:prefLabel"] = citation.get("familyName") + ", " + citation.get("givenName") + ". (" + time.strftime("%Y", time.gmtime()) + "). " + paperTitle + ". " + citation.get("publisher") + ". [" + citation.get("dataType") + "] {ID_WILL_BE_HERE}"
self.__updateMetadataPayload(tfpackageData)
self.__workflow()
def __search(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("last_modified:[NOW-1MONTH TO *]")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest(owner_query)
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest('workflow_security:"' + current_user + '"')
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.vc("sessionState").set("fq", 'item_type:"object"')
#sessionState.set("query", portalQuery.replace("\"", "'"))
# Load in the services UI workflow
selfSubmitWfConfig = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/servicesUI.json"))
selfSubmitJsonStageList = selfSubmitWfConfig.getJsonSimpleList(["stages"])
servicesStages = []
for jsonStage in selfSubmitJsonStageList:
wfStage = WorkflowStage(jsonStage, self.__steps)
servicesStages.append(wfStage)
self.__selfservicesStages = servicesStages
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
import sys
import os
import traceback
from com.googlecode.fascinator.common import FascinatorHome
from com.googlecode.fascinator.api.indexer import SearchRequest
from com.googlecode.fascinator.common.solr import SolrResult
from java.io import ByteArrayInputStream, ByteArrayOutputStream
from java.lang import Integer
sys.path.append(
os.path.join(FascinatorHome.getPath(), "lib", "jython", "workflowlib"))
from TransitionWorkflow import TransitionWorkflow
"""
Handy info:
- This script is usually launched by Housekeeping
- com.googlecode.fascinator.portal.quartz.ExternalJob calls this script via HTTP
"""
class TransitionWorkflowData:
def __activate__(self, context):
response = context["response"]
self.indexer = context["Services"].getIndexer()
self.systemConfig = context["systemConfig"]
self.log = context["log"]
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
writer = response.getPrintWriter("text/plain; charset=UTF-8")
try:
writer.println("Transition workflow script has been started")
count = 0
def writeResponseToStatusResponseCache(self, jobId, jobStatus):
curationStatusRespones = File(FascinatorHome.getPath()+ "/curation-status-responses")
if curationStatusRespones.exists():
FileUtils.forceMkdir(curationStatusRespones)
FileUtils.writeStringToFile(File(curationStatusRespones.getPath()+ "/" + Integer(jobId).toString() + ".json"), jobStatus.toString(True))
def __metadata(self):
self.title = None
self.dcType = None
self.__checkMetadataPayload()
jsonPayload = self.object.getPayload("metadata.json")
json = self.utils.getJsonObject(jsonPayload.open())
jsonPayload.close()
metadata = json.getObject("metadata")
identifier = metadata.get("dc.identifier")
self.utils.add(self.index, "dc:identifier", identifier)
self.__storeIdentifier(identifier)
self.utils.add(self.index, "institution", "James Cook University")
self.utils.add(self.index, "source",
"http://spatialecology.jcu.edu.au/Edgar/")
data = json.getObject("data")
####Global setting for processing data
####These will need to be changed based on you system installation.
theMintHost = java.lang.System.getProperty("mint.proxy.url")
collectionRelationTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/redbox/workflows/forms/data/"
servicesRelationTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/redbox/workflows/forms/data/"
descriptionTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/local/workflows/forms/data/"
relationshipTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/local/workflows/forms/data/"
###Allocating space to create the formData.tfpackage
tfpackageData = {}
###Using the species name, obtained from the directory name, to replace the text in the Title
species = data.get("species")
title = data.get("title")
title = title.replace("%NAME_OF_FOLDER%", species)
self.utils.add(self.index, "dc_title", title)
tfpackageData["dc:title"] = title
tfpackageData["title"] = title
self.utils.add(self.index, "dc_type", data.get("type"))
tfpackageData["dc:type.rdf:PlainLiteral"] = data.get("type")
tfpackageData["dc:type.skos:prefLabel"] = data.get("type")
tfpackageData["dc:created"] = time.strftime("%Y-%m-%d", time.gmtime())
tfpackageData["dc:modified"] = ""
tfpackageData["dc:language.skos:prefLabel"] = "English"
tfpackageData[
"dc:coverage.vivo:DateTimeInterval.vivo:start"] = data.get(
"temporalCoverage").get("dateFrom")
dateTo = data.get("temporalCoverage").get("dateTo")
if dateTo is not None:
tfpackageData[
"dc:coverage.vivo:DateTimeInterval.vivo:end"] = dateTo
tfpackageData["dc:coverage.redbox:timePeriod"] = ""
###Processing the 'spatialCoverage' metadata.
spatialCoverage = data.get("spatialCoverage")
for i in range(len(spatialCoverage)):
location = spatialCoverage[i]
if location["type"] == "text":
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) + ".dc:type"] = location["type"]
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) + ".dc:typeLabel"] = "Free Text"
if (location["value"].startswith("POLYGON")):
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) +
".redbox:wktRaw"] = location["value"]
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) +
".rdf:PlainLiteral"] = location["value"]
###Processing the 'description' metadata.
#Reading the file here, so we only do it once.
file = open(descriptionTypesFilePath + "descriptionTypes.json")
descriptionData = file.read()
file.close()
description = data.get("description")
for i in range(len(description)):
desc = description[i]
tempDesc = desc.get("value")
tempDesc = tempDesc.replace("%NAME_OF_FOLDER%", species)
if (desc["type"] == "brief"):
tfpackageData["dc:description"] = tempDesc
tfpackageData["rif:description." + str(i + 1) +
".type"] = desc["type"]
tfpackageData["rif:description." + str(i + 1) +
".value"] = tempDesc
jsonSimple = JsonSimple(descriptionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Description Type exist
if results:
for j in range(len(results)):
descriptionType = results[j]
if (desc["type"] == descriptionType.get("id")):
tfpackageData["rif:description." + str(i + 1) +
".label"] = descriptionType.get("label")
###Processing the 'relatedPublication' metadata
relatedPublication = data.get("relatedPublication")
if relatedPublication is not None:
for i in range(len(relatedPublication)):
publication = relatedPublication[i]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) +
".dc:identifier"] = publication["doi"]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) +
".dc:title"] = publication["title"]
###Processing the 'relatedWebsite' metadata
relatedWebsite = data.get("relatedWebsite")
count = 0
for i in range(len(relatedWebsite)):
website = relatedWebsite[i]
tfpackageData["dc:relation.bibo:Website." + str(i + 1) +
".dc:identifier"] = website["url"]
tfpackageData["dc:relation.bibo:Website." + str(i + 1) +
".dc:title"] = website["notes"]
count = i + 1
###Processing the 'data_source_website' metadata (override metadata)
dataSourceWebsites = data.get("data_source_website")
if dataSourceWebsites is not None:
for i in range(len(dataSourceWebsites)):
website = dataSourceWebsites[i]
type = website.get("identifier").get("type")
if type == "uri":
count += 1
tfpackageData["dc:relation.bibo:Website." + str(count) +
".dc:identifier"] = website.get(
"identifier").get("value")
tfpackageData["dc:relation.bibo:Website." + str(count) +
".dc:title"] = website["notes"]
###Processing the 'relatedCollection' metadata
#Reading the file here, so we only do it once.
file = open(collectionRelationTypesFilePath +
"collectionRelationTypes.json")
collectionData = file.read()
file.close()
relatedCollection = data.get("relatedCollection")
recordIdentifier = ""
if relatedCollection is not None:
for i in range(len(relatedCollection)):
collection = relatedCollection[i]
tempIdentifier = collection["identifier"]
if tempIdentifier is not None:
tempIdentifier = tempIdentifier.replace(
"%NAME_OF_FOLDER%", species)
recordIdentifier = tempIdentifier
else:
tempIdentifier = ""
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".dc:identifier"] = tempIdentifier
tempTitle = collection.get("title")
tempTitle = tempTitle.replace("%NAME_OF_FOLDER%", species)
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".dc:title"] = tempTitle
tfpackageData[
"dc:relation.vivo:Dataset." + str(i + 1) +
".vivo:Relationship.rdf:PlainLiteral"] = collection[
"relationship"]
if tempIdentifier == "":
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".redbox:origin"] = "on"
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".redbox:publish"] = "on"
#Using the collection data as a lookup to obtain the 'label'
relationShip = collection.get("relationship")
jsonSimple = JsonSimple(collectionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Collection Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData[
"dc:relation.vivo:Dataset." + str(i + 1) +
".vivo:Relationship.skos:prefLabel"] = relation.get(
"label")
###Processing the 'relatedService' metadata
#Reading the file here, so we only do it once.
file = open(servicesRelationTypesFilePath +
"serviceRelationTypes.json")
servicesData = file.read()
file.close()
relatedServices = data.get("relatedService")
recordIdentifier = ""
if relatedServices is not None:
for i in range(len(relatedServices)):
service = relatedServices[i]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) +
".dc:identifier"] = service["identifier"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) +
".dc:title"] = service["title"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) +
".vivo:Relationship.rdf:PlainLiteral"] = service[
"relationship"]
#Using the services data as a lookup to obtain the 'label'
relationShip = service.get("relationship")
jsonSimple = JsonSimple(servicesData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Service Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData[
"dc:relation.vivo:Service." + str(i + 1) +
".vivo:Relationship.skos:prefLabel"] = relation.get(
"label")
###Processing the 'associatedParty' metadata
#Reading the file here so we only read it once.
file = open(relationshipTypesFilePath + "relationshipTypes.json")
relationshipData = file.read()
file.close()
associatedParty = data.get("associatedParty")
if associatedParty is not None:
for i in range(len(associatedParty)):
party = associatedParty[i]
email = party.get("who").get("value")
if email is not None:
whoType = party.get("who").get("type")
if (whoType == 'people'):
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".dc:identifier"] = party.get("who").get(
"identifier")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:name"] = party.get("who").get(
"name")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:title"] = party.get("who").get(
"title")
tfpackageData[
"dc:creator.foaf:Person." + str(i + 1) +
".redbox:isCoPrimaryInvestigator"] = "off"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".redbox:isPrimaryInvestigator"] = "on"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:givenName"] = party.get(
"who").get("givenName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:familyName"] = party.get(
"who").get("familyName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".jcu:relationshipType"] = party.get(
"relationship")
tfpackageData[
"dc:creator.foaf:Person." + str(i + 1) +
".foaf:Organization.dc:identifier"] = party.get(
"affiliation").get("id")
tfpackageData[
"dc:creator.foaf:Person." + str(i + 1) +
".foaf:Organization.skos:prefLabel"] = party.get(
"affiliation").get("label")
jsonSimple = JsonSimple(relationshipData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Relationship Type exists
if results:
for j in range(len(results)):
relationshipType = results[j]
if (party.get("relationship") ==
relationshipType.get("id")):
tfpackageData[
"dc:creator.foaf:Person." +
str(i + 1) +
".jcu:relationshipLabel"] = relationshipType.get(
"label")
###Processing 'contactInfo.email' metadata
tfpackageData["locrel:prc.foaf:Person.dc:identifier"] = data.get(
"contactInfo").get("identifier")
tfpackageData["locrel:prc.foaf:Person.foaf:name"] = data.get(
"contactInfo").get("name")
tfpackageData["locrel:prc.foaf:Person.foaf:title"] = data.get(
"contactInfo").get("title")
tfpackageData["locrel:prc.foaf:Person.foaf:givenName"] = data.get(
"contactInfo").get("givenName")
tfpackageData["locrel:prc.foaf:Person.foaf:familyName"] = data.get(
"contactInfo").get("familyName")
tfpackageData["locrel:prc.foaf:Person.foaf:email"] = data.get(
"contactInfo").get("email")
##Stored At (on the Data Management page)
tfpackageData[
"vivo:Location.vivo:GeographicLocation.gn:name"] = data.get(
"contactInfo").get("streetAddress")
###Processing 'coinvestigators' metadata
coinvestigators = data.get("coinvestigators")
for i in range(len(coinvestigators)):
tfpackageData["dc:contributor.locrel:clb." + str(i + 1) +
".foaf:Agent"] = coinvestigators[i]
###Processing 'anzsrcFOR' metadata
anzsrcFOR = data.get("anzsrcFOR")
for i in range(len(anzsrcFOR)):
anzsrc = anzsrcFOR[i]
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) +
".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) +
".rdf:resource"] = anzsrc.get("resource")
###Processing 'anzsrcSEO' metadata
anzsrcSEO = data.get("anzsrcSEO")
for i in range(len(anzsrcSEO)):
anzsrc = anzsrcSEO[i]
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) +
".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) +
".rdf:resource"] = anzsrc.get("resource")
###Processing 'keyword' metadata
keyword = data.get("keyword")
for i in range(len(keyword)):
tfpackageData["dc:subject.vivo:keyword." + str(i + 1) +
".rdf:PlainLiteral"] = keyword[i]
###Research Themes
theme = data.get("researchTheme")
if (theme == "Tropical Ecosystems, Conservation and Climate Change"):
tfpackageData["jcu:research.themes.tropicalEcoSystems"] = "true"
elif (theme == "Industries and Economies in the Tropics"):
tfpackageData["jcu:research.themes.industriesEconomies"] = "true"
elif (theme == "People and Societies in the Tropics"):
tfpackageData["jcu:research.themes.peopleSocieties"] = "true"
elif (theme == "Tropical Health, Medicine and Biosecurity"):
tfpackageData["jcu:research.themes.tropicalHealth"] = "true"
elif (theme == "Not aligned to a University theme"):
tfpackageData["jcu:research.themes.notAligned"] = "true"
tfpackageData["dc:accessRights.skos:prefLabel"] = data.get(
"accessRights")
tfpackageData["dc:license.dc:identifier"] = data.get("license").get(
"url")
tfpackageData["dc:license.skos:prefLabel"] = data.get("license").get(
"label")
#identifier
additionalId = data.get("additionalIdentifier")
if additionalId is not None:
additionalId = additionalId.replace("%NAME_OF_FOLDER%", species)
tfpackageData["dc:identifier.rdf:PlainLiteral"] = additionalId
tfpackageData["dc:identifier.redbox:origin"] = "external"
tfpackageData["dc:identifier.dc:type.rdf:PlainLiteral"] = "local"
tfpackageData[
"dc:identifier.dc:type.skos:prefLabel"] = "Local Identifier"
else:
tfpackageData["dc:identifier.redbox:origin"] = "internal"
dataLocation = data.get("dataLocation")
dataLocation = dataLocation.replace("%NAME_OF_FOLDER%", species)
tfpackageData["bibo:Website.1.dc:identifier"] = dataLocation
#The following have been intentionally set to blank. No mapping is required for these fields.
tfpackageData["redbox:retentionPeriod"] = data.get("retentionPeriod")
tfpackageData["dc:extent"] = "unknown"
tfpackageData["redbox:disposalDate"] = ""
tfpackageData["locrel:own.foaf:Agent.1.foaf:name"] = ""
tfpackageData["locrel:dtm.foaf:Agent.foaf:name"] = ""
###Processing 'organizationalGroup' metadata
organisationalGroup = data.get("organizationalGroup")
for i in range(len(organisationalGroup)):
organisation = organisationalGroup[i]
tfpackageData[
"foaf:Organization.dc:identifier"] = organisation.get(
"identifier")
tfpackageData[
"foaf:Organization.skos:prefLabel"] = organisation.get(
"prefLabel")
tfpackageData["swrc:ResearchProject.dc:title"] = ""
tfpackageData["locrel:dpt.foaf:Person.foaf:name"] = ""
tfpackageData["dc:SizeOrDuration"] = ""
tfpackageData["dc:Policy"] = ""
#Citations
citations = data.get("citations")
for i in range(len(citations)):
citation = citations[i]
tfpackageData[
"dc:biblioGraphicCitation.redbox:sendCitation"] = citation.get(
"sendCitation")
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:identifier.skos:note"] = citation.get(
"curationIdentifier")
paperTitle = citation.get("paperTitle")
paperTitle = paperTitle.replace("%NAME_OF_FOLDER%", species)
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:title"] = paperTitle
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." +
str(i + 1) +
".foaf:familyName"] = citation.get("familyName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." +
str(i + 1) +
".foaf:givenName"] = citation.get("givenName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." +
str(i + 1) +
".foaf:title"] = title = citation.get("title")
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:publisher.rdf:PlainLiteral"] = citation.get(
"publisher")
url = citation.get("url")
url = url.replace("%NAME_OF_FOLDER%", species)
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.bibo:Website.dc:identifier"] = url
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.1.rdf:PlainLiteral"] = tfpackageData[
"dc:created"]
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.rdf:PlainLiteral"] = "publicationDate"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.skos:prefLabel"] = "Publication Date"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.rdf:PlainLiteral"] = "created"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.skos:prefLabel"] = "Date Created"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.2.rdf:PlainLiteral"] = tfpackageData[
"dc:created"]
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.jcu:dataType"] = citation.get(
"dataType")
tfpackageData[
"dc:biblioGraphicCitation.skos:prefLabel"] = citation.get(
"familyName"
) + ", " + citation.get("givenName") + ". (" + time.strftime(
"%Y",
time.gmtime()) + "). " + paperTitle + ". " + citation.get(
"publisher") + ". [" + citation.get(
"dataType") + "] {ID_WILL_BE_HERE}"
self.__updateMetadataPayload(tfpackageData)
self.__workflow()
def __search(self):
indexer = Services.getIndexer()
portalQuery = Services.getPortalManager().get(
self.vc("portalId")).getQuery()
portalSearchQuery = Services.getPortalManager().get(
self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
req.setParam("facet", "true")
req.setParam("facet.field", "workflow_step")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
steps = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__steps = steps.getFacets().get("workflow_step")
wfConfig = JsonSimple(
FascinatorHome.getPathFile("harvest/workflows/dataset.json"))
jsonStageList = wfConfig.getJsonSimpleList(["stages"])
stages = []
for jsonStage in jsonStageList:
wfStage = WorkflowStage(jsonStage, self.__steps)
stages.append(wfStage)
self.__stages = stages
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "25")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
req.addParam("fq", "workflow_step:%s" % stages[0].getName())
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__alerts = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest(
"last_modified:[NOW-1MONTH TO *] AND workflow_step:live")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.vc("sessionState").set("fq", 'item_type:"object"')
def __createFromSelected(self):
self.vc("log").debug("Creating package from selected...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
#self.vc("log").debug("packageType = '{}'", packageType)
#self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
# if modifying existing manifest, we already have an identifier,
# otherwise create a new one
manifestId = self.__getActiveManifestId()
if manifestId is None:
manifestHash = "%s.tfpackage" % uuid.uuid4()
else:
manifestHash = self.__getActiveManifestPid()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
manifest = self.__getActiveManifest()
oldType = manifest.getType()
if oldType is None:
manifest.setType(packageType)
else:
manifest.setType(oldType)
self.vc("log").debug("Manifest: %s" % manifest)
outWriter.write(manifest.toString(True))
outWriter.close()
try:
if manifestId is None:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, and make sure they are both deployed
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
rulesFile = self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
else:
# update existing object
object = StorageUtils.getDigitalObject(Services.getStorage(),
manifestId)
manifestStream = FileUtils.openInputStream(manifestFile)
StorageUtils.createOrUpdatePayload(object, manifestHash,
manifestStream)
manifestStream.close()
object.close()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __createNew(self):
self.vc("log").debug("Creating a new package...")
packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile()
self.vc("log").debug("packageType = '{}'", packageType)
self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile)
manifestHash = "%s.tfpackage" % uuid.uuid4()
# store the manifest file for harvesting
packageDir = FascinatorHome.getPathFile("packages")
packageDir.mkdirs()
manifestFile = File(packageDir, manifestHash)
outStream = FileOutputStream(manifestFile)
outWriter = OutputStreamWriter(outStream, "UTF-8")
self.vc("sessionState").set("package/active", None)
manifest = self.__getActiveManifest()
manifest.setType(packageType)
metaList = list(self.vc("formData").getValues("metaList"))
jsonObj = manifest.getJsonObject()
for metaName in metaList:
value = self.vc("formData").get(metaName)
jsonObj.put(metaName, value)
if self.vc("formData").getValues("sequencesMetaList") != None:
sequenceService = ApplicationContextProvider.getApplicationContext(
).getBean("sequenceService")
sequencesMetaList = list(
self.vc("formData").getValues("sequencesMetaList"))
for sequenceInfo in sequencesMetaList:
sequenceInfoJson = JsonSimple(sequenceInfo)
sequenceIndex = sequenceService.getSequence(
sequenceInfoJson.getString(None, "sequenceName"))
jsonObj.put(
sequenceInfoJson.getString(None, "metadataName"),
String.format(
sequenceInfoJson.getString(None, "stringFormat"),
sequenceIndex))
self.vc("log").debug("json object created is: %r" % jsonObj)
outWriter.write(manifest.toString(True))
outWriter.close()
# adding ability to set access plugin
accessPlugin = self.vc("formData").get("access_plugin", "derby")
if accessPlugin is not None:
self.vc("page").authentication.set_access_plugin(accessPlugin)
try:
# harvest the package as an object
username = self.vc("sessionState").get("username")
if username is None:
username = "******" # necessary?
harvester = None
# set up config files, creating if necessary
workflowsDir = FascinatorHome.getPathFile("harvest/workflows")
configFile = self.__getFile(workflowsDir, jsonConfigFile)
self.__getFile(workflowsDir, "packaging-rules.py")
# run the harvest client with our packaging workflow config
harvester = HarvestClient(configFile, manifestFile, username)
harvester.start()
manifestId = harvester.getUploadOid()
harvester.shutdown()
except Exception, ex:
error = "Packager workflow failed: %s" % str(ex)
self.vc("log").error(error, ex)
if harvester is not None:
harvester.shutdown()
return '{ "status": "failed" }'
def __metadata(self):
self.title = None
self.dcType = None
self.__checkMetadataPayload()
jsonPayload = self.object.getPayload("metadata.json")
json = self.utils.getJsonObject(jsonPayload.open())
jsonPayload.close()
metadata = json.getObject("metadata")
identifier = metadata.get("dc.identifier")
self.utils.add(self.index, "dc:identifier", identifier)
self.__storeIdentifier(identifier)
self.utils.add(self.index, "institution", "James Cook University")
self.utils.add(self.index, "source",
"http://spatialecology.jcu.edu.au/Edgar/")
data = json.getObject("data")
####Global setting for processing data
####These will need to be changed based on you system installation.
theMintHost = java.lang.System.getProperty("mint.proxy.url")
collectionRelationTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/redbox/workflows/forms/data/"
servicesRelationTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/redbox/workflows/forms/data/"
descriptionTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/local/workflows/forms/data/"
relationshipTypesFilePath = FascinatorHome.getPath(
) + "/../portal/default/local/workflows/forms/data/"
###Allocating space to create the formData.tfpackage
tfpackageData = {}
# We will do string substitutions on data that we get from the default json.
# We always replace ${NAME_OF_FOLDER} with the name of the folder; if the
# override json contains a key "DATA_SUBSTITUTIONS", then we also substitute
# stuff we find there.
# so: start with just wanting ${NAME_OF_FOLDER} replaced with the actual directory name
dirName = data.get("harvest_dir_name")
replacements = {'NAME_OF_FOLDER': dirName}
# is there a DATA_SUBSTITUTIONS key? If so, add those in.
additionalReplacements = data.get("DATA_SUBSTITUTIONS")
if additionalReplacements:
replacements.update(additionalReplacements)
# now there's a replacements dictionary with the replacements we want
# to do on our incoming JSON strings.
# FANCY PART---------------------------------------------
# Now it gets a bit fancy: Here's a method that does a
# get-and-replace all in one go. That makes the rest of
# this __metdata() method much simpler and more readable.
#
# Because this method is defined inside this __metadata()
# method, it already knows about the replacements var we
# just made.
# dataBucket is the thing that has the data. key is the
# name of the field you want to get.
def getAndReplace(dataBucket, key):
temp = dataBucket.get(key) # fetch the value
if isinstance(key, str): # if it's a string, do our replacements
return Template(temp).safe_substitute(replacements)
else: # not a string, then just hand it back
return temp
# END OF FANCY PART -------------------------------------
title = getAndReplace(data, "title")
self.utils.add(self.index, "dc_title", title)
tfpackageData["dc:title"] = title
tfpackageData["title"] = title
self.utils.add(self.index, "dc_type", data.get("type"))
tfpackageData["dc:type.rdf:PlainLiteral"] = data.get("type")
tfpackageData["dc:type.skos:prefLabel"] = data.get("type")
tfpackageData["dc:created"] = time.strftime("%Y-%m-%d", time.gmtime())
tfpackageData["dc:modified"] = ""
tfpackageData["dc:language.skos:prefLabel"] = "English"
tfpackageData[
"dc:coverage.vivo:DateTimeInterval.vivo:start"] = data.get(
"temporalCoverage").get("dateFrom")
dateTo = data.get("temporalCoverage").get("dateTo")
if dateTo is not None:
tfpackageData[
"dc:coverage.vivo:DateTimeInterval.vivo:end"] = dateTo
tfpackageData["dc:coverage.redbox:timePeriod"] = ""
###Processing the 'spatialCoverage' metadata.
spatialCoverage = data.get("spatialCoverage")
for i in range(len(spatialCoverage)):
location = spatialCoverage[i]
if location["type"] == "text":
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) + ".dc:type"] = location["type"]
location_value = getAndReplace(location, "value")
if location_value.startswith("POLYGON"):
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) +
".redbox:wktRaw"] = location_value
tfpackageData["dc:coverage.vivo:GeographicLocation." +
str(i + 1) +
".rdf:PlainLiteral"] = location_value
###Processing the 'description' metadata.
#Reading the file here, so we only do it once.
file = open(descriptionTypesFilePath + "descriptionTypes.json")
descriptionData = file.read()
file.close()
description = data.get("description")
for i in range(len(description)):
desc = description[i]
tempDesc = getAndReplace(desc, "value")
if (desc["type"] == "brief"):
tfpackageData["dc:description"] = tempDesc
tfpackageData["rif:description." + str(i + 1) +
".type"] = desc["type"]
tfpackageData["rif:description." + str(i + 1) +
".value"] = tempDesc
jsonSimple = JsonSimple(descriptionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Description Type exist
if results:
for j in range(len(results)):
descriptionType = results[j]
if (desc["type"] == descriptionType.get("id")):
tfpackageData["rif:description." + str(i + 1) +
".label"] = descriptionType.get("label")
###Processing the 'relatedPublication' metadata
relatedPublication = data.get("relatedPublication")
if relatedPublication is not None:
for i in range(len(relatedPublication)):
publication = relatedPublication[i]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) +
".dc:identifier"] = publication["doi"]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) +
".dc:title"] = publication["title"]
###Processing the 'relatedWebsite' metadata
relatedWebsite = data.get("relatedWebsite")
count = 0
for i in range(len(relatedWebsite)):
website = relatedWebsite[i]
tfpackageData["dc:relation.bibo:Website." + str(i + 1) +
".dc:identifier"] = getAndReplace(website, "url")
tfpackageData["dc:relation.bibo:Website." + str(i + 1) +
".dc:title"] = getAndReplace(website, "notes")
count = i + 1
###Processing the 'data_source_website' metadata (override metadata)
dataSourceWebsites = data.get("data_source_website")
if dataSourceWebsites is not None:
for i in range(len(dataSourceWebsites)):
website = dataSourceWebsites[i]
type = website.get("identifier").get("type")
if type == "uri":
count += 1
tfpackageData["dc:relation.bibo:Website." + str(count) +
".dc:identifier"] = getAndReplace(
website.get("identifier"), "value")
tfpackageData["dc:relation.bibo:Website." + str(count) +
".dc:title"] = getAndReplace(
website, "notes")
###Processing the 'relatedCollection' metadata
#Reading the file here, so we only do it once.
file = open(collectionRelationTypesFilePath +
"collectionRelationTypes.json")
collectionData = file.read()
file.close()
relatedCollection = data.get("relatedCollection")
recordIdentifier = ""
if relatedCollection is not None:
for i in range(len(relatedCollection)):
collection = relatedCollection[i]
tempIdentifier = collection["identifier"]
if tempIdentifier is not None:
tempIdentifier = Template(tempIdentifier).safe_substitute(
replacements)
recordIdentifier = tempIdentifier
else:
tempIdentifier = ""
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".dc:identifier"] = tempIdentifier
tempTitle = collection.get("title")
tempTitle = Template(tempTitle).safe_substitute(replacements)
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".dc:title"] = tempTitle
tfpackageData[
"dc:relation.vivo:Dataset." + str(i + 1) +
".vivo:Relationship.rdf:PlainLiteral"] = collection[
"relationship"]
if tempIdentifier == "":
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".redbox:origin"] = "on"
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) +
".redbox:publish"] = "on"
#Using the collection data as a lookup to obtain the 'label'
relationShip = collection.get("relationship")
jsonSimple = JsonSimple(collectionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Collection Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData[
"dc:relation.vivo:Dataset." + str(i + 1) +
".vivo:Relationship.skos:prefLabel"] = relation.get(
"label")
###Processing the 'relatedService' metadata
#Reading the file here, so we only do it once.
file = open(servicesRelationTypesFilePath +
"serviceRelationTypes.json")
servicesData = file.read()
file.close()
relatedServices = data.get("relatedService")
recordIdentifier = ""
if relatedServices is not None:
for i in range(len(relatedServices)):
service = relatedServices[i]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) +
".dc:identifier"] = service["identifier"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) +
".dc:title"] = service["title"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) +
".vivo:Relationship.rdf:PlainLiteral"] = service[
"relationship"]
#Using the services data as a lookup to obtain the 'label'
relationShip = service.get("relationship")
jsonSimple = JsonSimple(servicesData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Service Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData[
"dc:relation.vivo:Service." + str(i + 1) +
".vivo:Relationship.skos:prefLabel"] = relation.get(
"label")
###Processing the 'associatedParty' metadata
#Reading the file here so we only read it once.
file = open(relationshipTypesFilePath + "relationshipTypes.json")
relationshipData = file.read()
file.close()
associatedParty = data.get("associatedParty")
for i in range(len(associatedParty)):
party = associatedParty[i]
email = party.get("who").get("value")
if email is not None:
whoType = party.get("who").get("type")
if (whoType == 'people'):
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".dc:identifier"] = party.get("who").get(
"identifier")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:name"] = party.get("who").get("name")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:title"] = party.get("who").get(
"title")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".redbox:isCoPrimaryInvestigator"] = "off"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".redbox:isPrimaryInvestigator"] = "on"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:givenName"] = party.get("who").get(
"givenName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".foaf:familyName"] = party.get("who").get(
"familyName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) +
".jcu:relationshipType"] = party.get(
"relationship")
tfpackageData[
"dc:creator.foaf:Person." + str(i + 1) +
".foaf:Organization.dc:identifier"] = party.get(
"affiliation").get("id")
tfpackageData[
"dc:creator.foaf:Person." + str(i + 1) +
".foaf:Organization.skos:prefLabel"] = party.get(
"affiliation").get("label")
jsonSimple = JsonSimple(relationshipData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Relationship Type exists
if results:
for j in range(len(results)):
relationshipType = results[j]
if (party.get("relationship") ==
relationshipType.get("id")):
tfpackageData[
"dc:creator.foaf:Person." + str(i + 1) +
".jcu:relationshipLabel"] = relationshipType.get(
"label")
###Processing 'contactInfo.email' metadata
tfpackageData["locrel:prc.foaf:Person.dc:identifier"] = data.get(
"contactInfo").get("identifier")
tfpackageData["locrel:prc.foaf:Person.foaf:name"] = data.get(
"contactInfo").get("name")
tfpackageData["locrel:prc.foaf:Person.foaf:title"] = data.get(
"contactInfo").get("title")
tfpackageData["locrel:prc.foaf:Person.foaf:givenName"] = data.get(
"contactInfo").get("givenName")
tfpackageData["locrel:prc.foaf:Person.foaf:familyName"] = data.get(
"contactInfo").get("familyName")
tfpackageData["locrel:prc.foaf:Person.foaf:email"] = data.get(
"contactInfo").get("email")
##Stored At (on the Data Management page)
tfpackageData[
"vivo:Location.vivo:GeographicLocation.gn:name"] = data.get(
"contactInfo").get("streetAddress")
###Processing 'coinvestigators' metadata
coinvestigators = data.get("coinvestigators")
for i in range(len(coinvestigators)):
tfpackageData["dc:contributor.locrel:clb." + str(i + 1) +
".foaf:Agent"] = coinvestigators[i]
###Processing 'anzsrcFOR' metadata
anzsrcFOR = data.get("anzsrcFOR")
for i in range(len(anzsrcFOR)):
anzsrc = anzsrcFOR[i]
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) +
".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) +
".rdf:resource"] = anzsrc.get("resource")
###Processing 'anzsrcSEO' metadata
anzsrcSEO = data.get("anzsrcSEO")
for i in range(len(anzsrcSEO)):
anzsrc = anzsrcSEO[i]
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) +
".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) +
".rdf:resource"] = anzsrc.get("resource")
###Processing 'keyword' metadata
keyword = data.get("keyword")
for i in range(len(keyword)):
tfpackageData["dc:subject.vivo:keyword." + str(i + 1) +
".rdf:PlainLiteral"] = keyword[i]
###Research Themes
theme = data.get("researchTheme")
if (theme == "Tropical Ecosystems, Conservation and Climate Change"):
tfpackageData["jcu:research.themes.tropicalEcoSystems"] = "true"
elif (theme == "Industries and Economies in the Tropics"):
tfpackageData["jcu:research.themes.industriesEconomies"] = "true"
elif (theme == "People and Societies in the Tropics"):
tfpackageData["jcu:research.themes.peopleSocieties"] = "true"
elif (theme == "Tropical Health, Medicine and Biosecurity"):
tfpackageData["jcu:research.themes.tropicalHealth"] = "true"
elif (theme == "Not aligned to a University theme"):
tfpackageData["jcu:research.themes.notAligned"] = "true"
tfpackageData["dc:accessRights.skos:prefLabel"] = data.get(
"accessRights")
tfpackageData["dc:license.dc:identifier"] = data.get("license").get(
"url")
tfpackageData["dc:license.skos:prefLabel"] = data.get("license").get(
"label")
#identifier
additionalId = data.get("additionalIdentifier")
if additionalId is not None:
additionalId = Template(additionalId).safe_substitute(replacements)
tfpackageData["dc:identifier.rdf:PlainLiteral"] = additionalId
tfpackageData["dc:identifier.redbox:origin"] = "external"
tfpackageData["dc:identifier.dc:type.rdf:PlainLiteral"] = "local"
tfpackageData[
"dc:identifier.dc:type.skos:prefLabel"] = "Local Identifier"
else:
tfpackageData["dc:identifier.redbox:origin"] = "internal"
dataLocation = getAndReplace(data, "dataLocation")
tfpackageData["bibo:Website.1.dc:identifier"] = dataLocation
#The following have been intentionally set to blank. No mapping is required for these fields.
tfpackageData["redbox:retentionPeriod"] = data.get("retentionPeriod")
tfpackageData["dc:extent"] = "unknown"
tfpackageData["redbox:disposalDate"] = ""
tfpackageData["locrel:own.foaf:Agent.1.foaf:name"] = ""
tfpackageData["locrel:dtm.foaf:Agent.foaf:name"] = ""
###Processing 'organizationalGroup' metadata
organisationalGroup = data.get("organizationalGroup")
for i in range(len(organisationalGroup)):
organisation = organisationalGroup[i]
tfpackageData[
"foaf:Organization.dc:identifier"] = organisation.get(
"identifier")
tfpackageData[
"foaf:Organization.skos:prefLabel"] = organisation.get(
"prefLabel")
tfpackageData["swrc:ResearchProject.dc:title"] = ""
tfpackageData["locrel:dpt.foaf:Person.foaf:name"] = ""
tfpackageData["dc:SizeOrDuration"] = ""
tfpackageData["dc:Policy"] = ""
#Citations
citations = data.get("citations")
for i in range(len(citations)):
citation = citations[i]
tfpackageData[
"dc:biblioGraphicCitation.redbox:sendCitation"] = citation.get(
"sendCitation")
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:identifier.skos:note"] = citation.get(
"curationIdentifier")
paperTitle = getAndReplace(citation, "paperTitle")
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:title"] = paperTitle
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." +
str(i + 1) +
".foaf:familyName"] = citation.get("familyName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." +
str(i + 1) +
".foaf:givenName"] = citation.get("givenName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." +
str(i + 1) +
".foaf:title"] = title = citation.get("title")
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:publisher.rdf:PlainLiteral"] = getAndReplace(
citation, "publisher")
url = getAndReplace(citation, "url")
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.bibo:Website.dc:identifier"] = url
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.1.rdf:PlainLiteral"] = tfpackageData[
"dc:created"]
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.rdf:PlainLiteral"] = "publicationDate"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.skos:prefLabel"] = "Publication Date"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.rdf:PlainLiteral"] = "created"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.skos:prefLabel"] = "Date Created"
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.dc:date.2.rdf:PlainLiteral"] = tfpackageData[
"dc:created"]
tfpackageData[
"dc:biblioGraphicCitation.dc:hasPart.jcu:dataType"] = citation.get(
"dataType")
tfpackageData[
"dc:biblioGraphicCitation.skos:prefLabel"] = citation.get(
"familyName"
) + ", " + citation.get("givenName") + ". (" + time.strftime(
"%Y",
time.gmtime()) + "). " + paperTitle + ". " + citation.get(
"publisher") + ". [" + citation.get(
"dataType") + "] {ID_WILL_BE_HERE}"
self.__updateMetadataPayload(tfpackageData)
self.__workflow()
def getDisplayList(self):
return JsonSimple(
FascinatorHome.getPathFile(
os.path.join("system-files", "package-arms",
"preview-fields.json")))
def __metadata(self):
self.title = None
self.dcType = None
self.__checkMetadataPayload()
jsonPayload = self.object.getPayload("metadata.json")
json = self.utils.getJsonObject(jsonPayload.open())
jsonPayload.close()
metadata = json.getObject("metadata")
identifier = metadata.get("dc.identifier")
self.utils.add(self.index, "dc:identifier", identifier)
self.__storeIdentifier(identifier)
self.utils.add(self.index, "institution", "James Cook University")
self.utils.add(self.index, "source", "http://spatialecology.jcu.edu.au/Edgar/")
data = json.getObject("data")
####Global setting for processing data
####These will need to be changed based on you system installation.
theMintHost = java.lang.System.getProperty("mint.proxy.url")
collectionRelationTypesFilePath = FascinatorHome.getPath() + "/../portal/default/redbox/workflows/forms/data/"
servicesRelationTypesFilePath = FascinatorHome.getPath() + "/../portal/default/redbox/workflows/forms/data/"
descriptionTypesFilePath = FascinatorHome.getPath() + "/../portal/default/local/workflows/forms/data/"
relationshipTypesFilePath = FascinatorHome.getPath() + "/../portal/default/local/workflows/forms/data/"
###Allocating space to create the formData.tfpackage
tfpackageData = {}
# We will do string substitutions on data that we get from the default json.
# We always replace ${NAME_OF_FOLDER} with the name of the folder; if the
# override json contains a key "DATA_SUBSTITUTIONS", then we also substitute
# stuff we find there.
# so: start with just wanting ${NAME_OF_FOLDER} replaced with the actual directory name
dirName = data.get("harvest_dir_name")
replacements = { 'NAME_OF_FOLDER': dirName }
# is there a DATA_SUBSTITUTIONS key? If so, add those in.
additionalReplacements = data.get("DATA_SUBSTITUTIONS")
if additionalReplacements:
replacements.update(additionalReplacements)
# now there's a replacements dictionary with the replacements we want
# to do on our incoming JSON strings.
# FANCY PART---------------------------------------------
# Now it gets a bit fancy: Here's a method that does a
# get-and-replace all in one go. That makes the rest of
# this __metdata() method much simpler and more readable.
#
# Because this method is defined inside this __metadata()
# method, it already knows about the replacements var we
# just made.
# dataBucket is the thing that has the data. key is the
# name of the field you want to get.
def getAndReplace(dataBucket, key):
temp = dataBucket.get(key) # fetch the value
if isinstance(key, str): # if it's a string, do our replacements
return Template(temp).safe_substitute(replacements)
else: # not a string, then just hand it back
return temp
# END OF FANCY PART -------------------------------------
title = getAndReplace(data, "title")
self.utils.add(self.index, "dc_title", title)
tfpackageData["dc:title"] = title
tfpackageData["title"] = title
self.utils.add(self.index, "dc_type", data.get("type"))
tfpackageData["dc:type.rdf:PlainLiteral"] = data.get("type")
tfpackageData["dc:type.skos:prefLabel"] = data.get("type")
tfpackageData["dc:created"] = time.strftime("%Y-%m-%d", time.gmtime())
tfpackageData["dc:modified"] = ""
tfpackageData["dc:language.skos:prefLabel"] = "English"
tfpackageData["dc:coverage.vivo:DateTimeInterval.vivo:start"] = data.get("temporalCoverage").get("dateFrom")
dateTo = data.get("temporalCoverage").get("dateTo")
if dateTo is not None:
tfpackageData["dc:coverage.vivo:DateTimeInterval.vivo:end"] = dateTo
tfpackageData["dc:coverage.redbox:timePeriod"] = ""
###Processing the 'spatialCoverage' metadata.
spatialCoverage = data.get("spatialCoverage")
for i in range(len(spatialCoverage)):
location = spatialCoverage[i]
if location["type"] == "text":
tfpackageData["dc:coverage.vivo:GeographicLocation." + str(i + 1) + ".dc:type"] = location["type"]
location_value = getAndReplace(location, "value")
if location_value.startswith("POLYGON"):
tfpackageData["dc:coverage.vivo:GeographicLocation." + str(i + 1) + ".redbox:wktRaw"] = location_value
tfpackageData["dc:coverage.vivo:GeographicLocation." + str(i + 1) + ".rdf:PlainLiteral"] = location_value
###Processing the 'description' metadata.
#Reading the file here, so we only do it once.
file = open(descriptionTypesFilePath + "descriptionTypes.json")
descriptionData = file.read()
file.close()
description = data.get("description")
for i in range(len(description)):
desc = description[i]
tempDesc = getAndReplace(desc, "value")
if (desc["type"] == "brief"):
tfpackageData["dc:description"] = tempDesc
tfpackageData["rif:description." + str(i + 1) + ".type"] = desc["type"]
tfpackageData["rif:description." + str(i + 1) + ".value"] = tempDesc
jsonSimple = JsonSimple(descriptionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Description Type exist
if results:
for j in range(len(results)):
descriptionType = results[j]
if (desc["type"] == descriptionType.get("id")):
tfpackageData["rif:description." + str(i + 1) + ".label"] = descriptionType.get("label")
###Processing the 'relatedPublication' metadata
relatedPublication = data.get("relatedPublication")
if relatedPublication is not None:
for i in range(len(relatedPublication)):
publication = relatedPublication[i]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) + ".dc:identifier"] = publication["doi"]
tfpackageData["dc:relation.swrc:Publication." + str(i + 1) + ".dc:title"] = publication["title"]
###Processing the 'relatedWebsite' metadata
relatedWebsite = data.get("relatedWebsite")
count = 0
for i in range(len(relatedWebsite)):
website = relatedWebsite[i]
tfpackageData["dc:relation.bibo:Website." + str(i + 1) + ".dc:identifier"] = getAndReplace(website, "url")
tfpackageData["dc:relation.bibo:Website." + str(i + 1) + ".dc:title"] = getAndReplace(website, "notes")
count = i + 1
###Processing the 'data_source_website' metadata (override metadata)
dataSourceWebsites = data.get("data_source_website")
if dataSourceWebsites is not None:
for i in range(len(dataSourceWebsites)):
website = dataSourceWebsites[i]
type = website.get("identifier").get("type")
if type == "uri":
count += 1
tfpackageData["dc:relation.bibo:Website." + str(count) + ".dc:identifier"] = getAndReplace(website.get("identifier"), "value")
tfpackageData["dc:relation.bibo:Website." + str(count) + ".dc:title"] = getAndReplace(website, "notes")
###Processing the 'relatedCollection' metadata
#Reading the file here, so we only do it once.
file = open(collectionRelationTypesFilePath + "collectionRelationTypes.json")
collectionData = file.read()
file.close()
relatedCollection = data.get("relatedCollection")
recordIdentifier = ""
if relatedCollection is not None:
for i in range(len(relatedCollection)):
collection = relatedCollection[i]
tempIdentifier = collection["identifier"]
if tempIdentifier is not None:
tempIdentifier = Template( tempIdentifier ).safe_substitute(replacements)
recordIdentifier = tempIdentifier
else:
tempIdentifier = ""
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".dc:identifier"] = tempIdentifier
tempTitle = collection.get("title")
tempTitle = Template( tempTitle ).safe_substitute(replacements)
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".dc:title"] = tempTitle
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".vivo:Relationship.rdf:PlainLiteral"] = collection["relationship"]
if tempIdentifier == "":
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".redbox:origin"] = "on"
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".redbox:publish"] = "on"
#Using the collection data as a lookup to obtain the 'label'
relationShip = collection.get("relationship")
jsonSimple = JsonSimple(collectionData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Collection Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData["dc:relation.vivo:Dataset." + str(i + 1) + ".vivo:Relationship.skos:prefLabel"] = relation.get("label")
###Processing the 'relatedService' metadata
#Reading the file here, so we only do it once.
file = open(servicesRelationTypesFilePath + "serviceRelationTypes.json")
servicesData = file.read()
file.close()
relatedServices = data.get("relatedService")
recordIdentifier = ""
if relatedServices is not None:
for i in range(len(relatedServices)):
service = relatedServices[i]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".dc:identifier"] = service["identifier"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".dc:title"] = service["title"]
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".vivo:Relationship.rdf:PlainLiteral"] = service["relationship"]
#Using the services data as a lookup to obtain the 'label'
relationShip = service.get("relationship")
jsonSimple = JsonSimple(servicesData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Service Relation Types exist
if results:
for j in range(len(results)):
relation = results[j]
if (relationShip == relation.get("id")):
tfpackageData["dc:relation.vivo:Service." + str(i + 1) + ".vivo:Relationship.skos:prefLabel"] = relation.get("label")
###Processing the 'associatedParty' metadata
#Reading the file here so we only read it once.
file = open(relationshipTypesFilePath + "relationshipTypes.json")
relationshipData = file.read()
file.close()
associatedParty = data.get("associatedParty")
for i in range(len(associatedParty)):
party = associatedParty[i]
email = party.get("who").get("value")
if email is not None:
whoType = party.get("who").get("type")
if (whoType == 'people'):
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".dc:identifier"] = party.get("who").get("identifier")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:name"] = party.get("who").get("name")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:title"] = party.get("who").get("title")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".redbox:isCoPrimaryInvestigator"] = "off"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".redbox:isPrimaryInvestigator"] = "on"
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:givenName"] = party.get("who").get("givenName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:familyName"] = party.get("who").get("familyName")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".jcu:relationshipType"] = party.get("relationship")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:Organization.dc:identifier"] = party.get("affiliation").get("id")
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".foaf:Organization.skos:prefLabel"] = party.get("affiliation").get("label")
jsonSimple = JsonSimple(relationshipData)
jsonObj = jsonSimple.getJsonObject()
results = jsonObj.get("results")
#ensuring the Relationship Type exists
if results:
for j in range(len(results)):
relationshipType = results[j]
if (party.get("relationship") == relationshipType.get("id")):
tfpackageData["dc:creator.foaf:Person." + str(i + 1) + ".jcu:relationshipLabel"] = relationshipType.get("label")
###Processing 'contactInfo.email' metadata
tfpackageData["locrel:prc.foaf:Person.dc:identifier"] = data.get("contactInfo").get("identifier")
tfpackageData["locrel:prc.foaf:Person.foaf:name"] = data.get("contactInfo").get("name")
tfpackageData["locrel:prc.foaf:Person.foaf:title"] = data.get("contactInfo").get("title")
tfpackageData["locrel:prc.foaf:Person.foaf:givenName"] = data.get("contactInfo").get("givenName")
tfpackageData["locrel:prc.foaf:Person.foaf:familyName"] = data.get("contactInfo").get("familyName")
tfpackageData["locrel:prc.foaf:Person.foaf:email"] = data.get("contactInfo").get("email")
##Stored At (on the Data Management page)
tfpackageData["vivo:Location.vivo:GeographicLocation.gn:name"] = data.get("contactInfo").get("streetAddress")
###Processing 'coinvestigators' metadata
coinvestigators = data.get("coinvestigators")
for i in range(len(coinvestigators)):
tfpackageData["dc:contributor.locrel:clb." + str(i + 1) + ".foaf:Agent"] = coinvestigators[i]
###Processing 'anzsrcFOR' metadata
anzsrcFOR = data.get("anzsrcFOR")
for i in range(len(anzsrcFOR)):
anzsrc = anzsrcFOR[i]
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) + ".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:for." + str(i + 1) + ".rdf:resource"] = anzsrc.get("resource")
###Processing 'anzsrcSEO' metadata
anzsrcSEO = data.get("anzsrcSEO")
for i in range(len(anzsrcSEO)):
anzsrc = anzsrcSEO[i]
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) + ".skos:prefLabel"] = anzsrc.get("prefLabel")
tfpackageData["dc:subject.anzsrc:seo." + str(i + 1) + ".rdf:resource"] = anzsrc.get("resource")
###Processing 'keyword' metadata
keyword = data.get("keyword")
for i in range(len(keyword)):
tfpackageData["dc:subject.vivo:keyword." + str(i + 1) + ".rdf:PlainLiteral"] = keyword[i]
###Research Themes
theme = data.get("researchTheme")
if (theme == "Tropical Ecosystems, Conservation and Climate Change"):
tfpackageData["jcu:research.themes.tropicalEcoSystems"] = "true"
elif (theme == "Industries and Economies in the Tropics"):
tfpackageData["jcu:research.themes.industriesEconomies"] = "true"
elif (theme == "People and Societies in the Tropics"):
tfpackageData["jcu:research.themes.peopleSocieties"] = "true"
elif (theme == "Tropical Health, Medicine and Biosecurity"):
tfpackageData["jcu:research.themes.tropicalHealth"] = "true"
elif (theme == "Not aligned to a University theme"):
tfpackageData["jcu:research.themes.notAligned"] = "true"
tfpackageData["dc:accessRights.skos:prefLabel"] = data.get("accessRights")
tfpackageData["dc:license.dc:identifier"] = data.get("license").get("url")
tfpackageData["dc:license.skos:prefLabel"] = data.get("license").get("label")
#identifier
additionalId = data.get("additionalIdentifier")
if additionalId is not None:
additionalId = Template( additionalId ).safe_substitute(replacements)
tfpackageData["dc:identifier.rdf:PlainLiteral"] = additionalId
tfpackageData["dc:identifier.redbox:origin"] = "external"
tfpackageData["dc:identifier.dc:type.rdf:PlainLiteral"] = "local"
tfpackageData["dc:identifier.dc:type.skos:prefLabel"] = "Local Identifier"
else:
tfpackageData["dc:identifier.redbox:origin"] = "internal"
dataLocation = getAndReplace(data, "dataLocation")
tfpackageData["bibo:Website.1.dc:identifier"] = dataLocation
#The following have been intentionally set to blank. No mapping is required for these fields.
tfpackageData["redbox:retentionPeriod"] = data.get("retentionPeriod")
tfpackageData["dc:extent"] = "unknown"
tfpackageData["redbox:disposalDate"] = ""
tfpackageData["locrel:own.foaf:Agent.1.foaf:name"] = ""
tfpackageData["locrel:dtm.foaf:Agent.foaf:name"] = ""
###Processing 'organizationalGroup' metadata
organisationalGroup = data.get("organizationalGroup")
for i in range(len(organisationalGroup)):
organisation = organisationalGroup[i]
tfpackageData["foaf:Organization.dc:identifier"] = organisation.get("identifier")
tfpackageData["foaf:Organization.skos:prefLabel"] = organisation.get("prefLabel")
tfpackageData["swrc:ResearchProject.dc:title"] = ""
tfpackageData["locrel:dpt.foaf:Person.foaf:name"] = ""
tfpackageData["dc:SizeOrDuration"] = ""
tfpackageData["dc:Policy"] = ""
#Citations
citations = data.get("citations")
for i in range(len(citations)):
citation = citations[i]
tfpackageData["dc:biblioGraphicCitation.redbox:sendCitation"] = citation.get("sendCitation")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:identifier.skos:note"] = citation.get("curationIdentifier")
paperTitle = getAndReplace(citation, "paperTitle")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:title"] = paperTitle
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." + str(i + 1) + ".foaf:familyName"] = citation.get("familyName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." + str(i + 1) + ".foaf:givenName"] = citation.get("givenName")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.locrel:ctb." + str(i + 1) + ".foaf:title"] = title = citation.get("title")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:publisher.rdf:PlainLiteral"] = getAndReplace(citation, "publisher")
url = getAndReplace(citation, "url")
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.bibo:Website.dc:identifier"] = url
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.1.rdf:PlainLiteral"] = tfpackageData["dc:created"]
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.rdf:PlainLiteral"] = "publicationDate"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.1.dc:type.skos:prefLabel"] = "Publication Date"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.rdf:PlainLiteral"] = "created"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.2.dc:type.skos:prefLabel"] = "Date Created"
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.dc:date.2.rdf:PlainLiteral"] = tfpackageData["dc:created"]
tfpackageData["dc:biblioGraphicCitation.dc:hasPart.jcu:dataType"] = citation.get("dataType")
tfpackageData["dc:biblioGraphicCitation.skos:prefLabel"] = citation.get("familyName") + ", " + citation.get("givenName") + ". (" + time.strftime("%Y", time.gmtime()) + "). " + paperTitle + ". " + citation.get("publisher") + ". [" + citation.get("dataType") + "] {ID_WILL_BE_HERE}"
self.__updateMetadataPayload(tfpackageData)
self.__workflow()
def getDisplayList(self):
return JsonSimple(FascinatorHome.getPathFile(os.path.join("system-files", "package-arms", "preview-fields.json")))
import sys
import os
import traceback
from com.googlecode.fascinator.common import FascinatorHome
from com.googlecode.fascinator.api.indexer import SearchRequest
from com.googlecode.fascinator.common.solr import SolrResult
from java.io import ByteArrayInputStream, ByteArrayOutputStream
from java.lang import Integer
sys.path.append(os.path.join(FascinatorHome.getPath(), "lib", "jython", "workflowlib"))
from TransitionWorkflow import TransitionWorkflow
"""
Handy info:
- This script is usually launched by Housekeeping
- com.googlecode.fascinator.portal.quartz.ExternalJob calls this script via HTTP
"""
class TransitionWorkflowData:
def __activate__(self, context):
response = context["response"]
self.indexer = context["Services"].getIndexer()
self.systemConfig = context["systemConfig"]
self.log = context["log"]
self.sessionState = context["sessionState"]
self.sessionState.set("username","admin")
writer = response.getPrintWriter("text/plain; charset=UTF-8")
try:
writer.println("Transition workflow script has been started")
count = 0
import sys
import os
import traceback
from com.googlecode.fascinator.common import FascinatorHome
sys.path.append(
os.path.join(FascinatorHome.getPath(), "lib", "jython", "alertlib"))
from NewAlerts import NewAlerts
"""
Handy info:
- This script is usually launched by Housekeeping
- com.googlecode.fascinator.portal.quartz.ExternalJob calls this script via HTTP
"""
class NewalertsData:
def __activate__(self, context):
response = context["response"]
log = context["log"]
writer = response.getPrintWriter("text/plain; charset=UTF-8")
try:
writer.println("Alert script has been started")
alerts = NewAlerts()
alerts.run(context)
writer.println("Alert script has completed")
except Exception, e:
writer.println("The alert system had a problem - check logs")
log.error("Exception in alerts code: %s" % (e.message))
raise
AlertsData = alerts["AlertsData"]
if __name__ == "__main__":
argParse = ArgumentParser(description="Process the JCU RIF-CS imports")
argParse.add_argument('-a','--alerts', dest='alerts', action='store', help='The alerts location.')
argParse.add_argument('-m','--rifcs-map-file', nargs=1, dest='rmap', action='store', default=None, help='The location of the rifcs map file.')
argParse.add_argument('-x','--xml-map-file', nargs=1, dest='xmap', action='store', default=None, help='The location of the xml map file.')
argParse.add_argument('-p','--redbox-path', dest='path', nargs=1, action='store', help='The base path of the ReDBoX Install')
args = argParse.parse_args()
if args.path is not None and os.path.exists(args.path[0]):
fascinator_home = os.path.join(args.path[0], "home")
FascinatorHome.setPath(fascinator_home)
import json
system = open(os.path.join(fascinator_home, "system-config.json"))
system_data = json.load(system)
system.close()
_config = config(fascinator_home, system_data)
output = _config.getString(None, ["alerts", "path"])
if args.rmap is not None:
system_data["alerts"]["xmlMaps"]["rif"] = args.rmap[0]
if args.xmap is not None:
system_data["alerts"]["xmlMaps"]["xml"] = args.xmap[0]
if args.alerts is not None:
system_data['alerts']['path'] = args.alerts
import sys
import os
from com.googlecode.fascinator.common import FascinatorHome
sys.path.append(os.path.join(FascinatorHome.getPath(),"harvest", "workflows"))
from baserules import BaseIndexData
class IndexData(BaseIndexData):
def __activate__(self, context):
BaseIndexData.__activate__(self,context)
import sys
import os
from com.googlecode.fascinator.common import FascinatorHome
sys.path.append(os.path.join(FascinatorHome.getPath(), "harvest", "workflows"))
from baserules import BaseIndexData
class IndexData(BaseIndexData):
def __activate__(self, context):
BaseIndexData.__activate__(self, context)
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from com.googlecode.fascinator.common import FascinatorHome
import sys, os
sys.path.append(os.path.join(FascinatorHome.getPath(), "lib", "jython", "display"))
from Dashboard import Dashboard
class RecordsData(Dashboard):
"""
Used in AJAX call to get paged search results of ARMS records
It returns results of predefined types of search:
submitted, shared and etc. Default: requestor
"""
def __init__(self):
pass
def __activate__(self, context):
self.activate(context, context["page"].getPortal().recordsPerPage)
from com.googlecode.fascinator.common import FascinatorHome
from com.googlecode.fascinator.common import JsonSimple
import sys
import time
pathToWorkflows = FascinatorHome.getPath("harvest/workflows")
if sys.path.count(pathToWorkflows) == 0:
sys.path.append(pathToWorkflows)
class IndexData:
def __activate__(self, context):
try:
# Prepare variables
self.index = context["fields"]
self.object = context["object"]
self.payload = context["payload"]
self.params = context["params"]
self.utils = context["pyUtils"]
self.config = context["jsonConfig"]
self.log = context["log"]
# Common data
self.__newDoc() # sets: self.oid, self.pid, self.itemType
self.item_security = []
self.owner = self.params.getProperty("owner", "system")
self.log.debug("Running attachment-rules.py... itemType='{}'",
self.itemType)
# Real metadata
if self.itemType == "object":
import sys
import os
import traceback
from com.googlecode.fascinator.common import FascinatorHome
sys.path.append(os.path.join(FascinatorHome.getPath(),"lib", "jython", "alertlib"))
from NewAlerts import NewAlerts
"""
Handy info:
- This script is usually launched by Housekeeping
- com.googlecode.fascinator.portal.quartz.ExternalJob calls this script via HTTP
"""
class NewalertsData:
def __activate__(self, context):
response = context["response"]
log = context["log"]
writer = response.getPrintWriter("text/plain; charset=UTF-8")
try:
writer.println("Alert script has been started")
alerts = NewAlerts()
alerts.run(context)
writer.println("Alert script has completed")
except Exception, e:
writer.println("The alert system had a problem - check logs")
log.error("Exception in alerts code: %s" % (e.message))
raise
finally:
from com.googlecode.fascinator.common import FascinatorHome
from com.googlecode.fascinator.common import JsonSimple
import sys
import time
pathToWorkflows = FascinatorHome.getPath("harvest/workflows")
if sys.path.count(pathToWorkflows) == 0:
sys.path.append(pathToWorkflows)
class IndexData:
def __activate__(self, context):
try:
# Prepare variables
self.index = context["fields"]
self.object = context["object"]
self.payload = context["payload"]
self.params = context["params"]
self.utils = context["pyUtils"]
self.config = context["jsonConfig"]
self.log = context["log"]
# Common data
self.__newDoc() # sets: self.oid, self.pid, self.itemType
self.item_security = []
self.owner = self.params.getProperty("owner", "system")
self.log.debug("Running attachment-file-rules.py... itemType='{}'", self.itemType)
# Real metadata
if self.itemType == "object":
self.__index("repository_name", self.params["repository.name"])
self.__index("repository_type", self.params["repository.type"])
def __activate__(self, context):
self.auth = context["page"].authentication
self.errorMsg = ""
self.request = context["request"]
self.response = context["response"]
self.formData = context["formData"]
self.log = context["log"]
self.reportManager = context["Services"].getService("reportManager")
self.reportName = None
if (self.auth.is_logged_in()):
if (self.auth.is_admin() == True):
pass
else:
self.errorMsg = "Requires Admin / Librarian / Reviewer access."
else:
self.errorMsg = "Please login."
if self.errorMsg == "":
self.reportName = self.formData.get("reportName")
if (self.reportName):
self.report = self.reportManager.getReport(self.reportName)
self.func = self.formData.get("func", "")
if self.func == "" and self.request.getParameter("func"):
self.func = self.request.getParameter("func")
if self.func == "action":
self.action = self.request.getParameter("action")
if self.action == "create":
self.createReport()
out = self.response.getPrintWriter(
"text/plain; charset=UTF-8")
out.println("{\"id\":\"" + self.report.getReportName() +
"\"}")
out.close()
return
if self.action == "edit":
self.editReport()
out = self.response.getPrintWriter(
"text/plain; charset=UTF-8")
out.println("{\"id\":\"" + self.report.getReportName() +
"\"}")
out.close()
return
if self.action == "options":
out = self.response.getPrintWriter(
"text/plain; charset=UTF-8")
out.println(
FileUtils.readFileToString(
File(
FascinatorHome.getPath("reports") +
"/reportCriteriaOptions.json")))
out.close()
return
if self.action == "get-json":
out = self.response.getPrintWriter(
"text/plain; charset=UTF-8")
report = self.reportManager.getReports().get(
self.request.getParameter("reportName"))
queryFilters = report.config.getObject("query", "filter")
jsonMap = HashMap()
elementIds = ArrayList()
for elementId in queryFilters:
elementIds.add(elementId)
Collections.sort(elementIds)
for elementId in elementIds:
jsonMap.put(elementId,
queryFilters.get(elementId).get("value"))
jsonMap.put("reportName", report.getLabel())
JsonObject.writeJSONString(jsonMap, out)
out.close()
return
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from com.googlecode.fascinator.common import FascinatorHome
import sys, os
sys.path.append(os.path.join(FascinatorHome.getPath(), "lib", "jython", "data"))
from PaginatedDataRetriever import PaginatedDataRetriever
import glob
from java.lang import String
class GetRecordsData(PaginatedDataRetriever):
"""
Used in AJAX call to get paged search results of researcher dashboard records
"""
def __init__(self):
pass
def getRecordsPerPage(self):
return 10
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
from com.googlecode.fascinator.common import FascinatorHome
import sys, os
sys.path.append(
os.path.join(FascinatorHome.getPath(), "lib", "jython", "display"))
from Dashboard import Dashboard
class RequesterData(Dashboard):
def __init__(self):
pass
def __activate__(self, context):
self.activate(context, context["page"].getPortal().recordsPerPage)
self.selected = "requester"
def __activate__(self, context):
self.auth = context["page"].authentication
self.errorMsg = ""
self.request = context["request"]
self.response = context["response"]
self.formData = context["formData"]
self.log = context["log"]
self.reportManager = context["Services"].getService("reportManager")
self.reportName = None
if (self.auth.is_logged_in()):
if (self.auth.is_admin()==True):
pass
else:
self.errorMsg = "Requires Admin / Librarian / Reviewer access."
else:
self.errorMsg = "Please login."
if self.errorMsg == "":
self.reportName = self.formData.get("reportName")
if (self.reportName):
self.report = self.reportManager.getReport(self.reportName)
self.func = self.formData.get("func", "")
if self.func == "" and self.request.getParameter("func"):
self.func = self.request.getParameter("func")
if self.func == "action":
self.action = self.request.getParameter("action")
if self.action == "create":
self.createReport()
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
out.println("{\"id\":\""+self.report.getReportName()+"\"}")
out.close()
return
if self.action == "edit":
self.editReport()
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
out.println("{\"id\":\""+self.report.getReportName()+"\"}")
out.close()
return
if self.action == "options":
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
out.println(FileUtils.readFileToString(File(FascinatorHome.getPath("reports")+"/reportCriteriaOptions.json")))
out.close()
return
if self.action == "get-json":
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
report = self.reportManager.getReports().get(self.request.getParameter("reportName"))
queryFilters = report.config.getObject("query", "filter")
jsonMap = HashMap()
elementIds = ArrayList()
for elementId in queryFilters:
elementIds.add(elementId)
Collections.sort(elementIds)
for elementId in elementIds:
jsonMap.put(elementId,queryFilters.get(elementId).get("value"))
jsonMap.put("reportName",report.getLabel())
JsonObject.writeJSONString(jsonMap,out)
out.close()
return
