def process(self):
#We'll return a list with 1 JsonSimple object
jsonList = []
data = None
reader = None
inStream = None
document = None
# Run the XML through our parser
try:
inStream = FileInputStream(File(self.file))
reader = InputStreamReader(inStream, "UTF-8")
document = self.saxReader.read(reader)
# Parse fails
except:
raise
# Close our file access objects
finally:
if reader is not None:
reader.close()
if inStream is not None:
inStream.close()
# Now go looking for all our data
data = JsonObject()
data.put("workflow_source", "XML Alert") # Default
self.__mapXpathToFields(document, self.map, data)
if data is None:
return None
jsonList.append(JsonSimple(data))
return jsonList
def __getJson(self):
rvtMap = JsonObject()
try:
oid = self.vc("formData").get("oid")
object = Services.storage.getObject(oid)
payload = object.getPayload("imsmanifest.xml")
try:
from xml.etree import ElementTree
xmlStr = IOUtils.toString(payload.open(), "UTF-8")
payload.close()
xml = ElementTree.XML(xmlStr.encode("UTF-8"))
ns = xml.tag[:xml.tag.find("}")+1]
resources = {}
for res in xml.findall(ns+"resources/"+ns+"resource"):
resources[res.attrib.get("identifier")] = res.attrib.get("href")
organizations = xml.find(ns+"organizations")
defaultName = organizations.attrib.get("default")
organizations = organizations.findall(ns+"organization")
organizations = [o for o in organizations if o.attrib.get("identifier")==defaultName]
organization = organizations[0]
title = organization.find(ns+"title").text
rvtMap.put("title", title)
items = organization.findall(ns+"item")
rvtMap.put("toc", self.__getJsonItems(ns, items, resources))
except Exception, e:
data["error"] = "Error - %s" % str(e)
print data["error"]
object.close()
def send_message(self, oid):
message = JsonObject()
message.put("oid", oid)
message.put("task", "curation-confirm")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID,
message.toString())
def __getUserInfo(self, username):
"""
Query HibernateUser to get a user information
There are users managed by internal auth manager with no attributes other than password
There are users managed by external auth managers e.g. shibboleth who have attributes
Each user, at the time of writing: 20140904, each user has multiple identical attribute sets,
so, only the first one is used
We put all available attributes of a user in to return value
"""
username = username.strip()
authUserDao = ApplicationContextProvider.getApplicationContext().getBean("hibernateAuthUserDao")
parameters = HashMap()
parameters.put("username", username)
userObjectList = authUserDao.query("getUser", parameters)
userJson = JsonObject()
userJson.put("username", username)
try:
if userObjectList.size() > 0:
# One hit will be enough to get user object
userJson = self.__constructUserAttribs(userObjectList.get(0), self.ATTRIB_FILTER)
else:
# This should not be reached with external sourced users
self.log.warn("Wrong username or internal user is queried")
except Exception, e:
self.log.error("%s: cannot construct user attribute JSON, detail = %s" % (self.__class__.__name__ , str(e)))
def __activate__(self, context):
request = context["request"]
storage = context["Services"].getStorage()
auth = context["page"].authentication
log = context["log"]
username = auth.get_name()
oid = request.getParameter("oid")
approval = request.getParameter("approval")
approval_comment = request.getParameter("approval_comment")
storedObj = storage.getObject(oid)
committeeResponses = None
payloadList = storedObj.getPayloadIdList()
if payloadList.contains("committee-responses.metadata"):
committeeResponsePayload = storedObj.getPayload("committee-responses.metadata")
committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject()
else:
committeeResponses = JsonObject()
committeeResponse = JsonObject()
committeeResponse.put("approval",approval)
committeeResponse.put("approval_comment",approval_comment)
committeeResponses.put(username,committeeResponse)
log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment))
StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8"))
context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def sendMessage(self, oid):
message = JsonObject()
message.put("oid", oid)
message.put("task", "reharvest")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID,
message.toString())
def __getPackageTypes(self):
object = self.sysConfig.getObject(["portal", "packageTypes"])
packageTypes = JsonSimple.toJavaMap(object)
if packageTypes.isEmpty():
defaultPackage = JsonObject()
defaultPackage.put("jsonconfig", "packaging-config.json")
packageTypes.put("default", JsonSimple(defaultPackage))
return packageTypes
def sendMessage(self, oid, eventType):
message = JsonObject()
message.put("oid", oid)
message.put("eventType", eventType)
message.put("username", self.vc("page").authentication.get_username())
message.put("context", "Workflow")
message.put("task", "workflow")
self.messaging.queueMessage(TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def sendMessage(self, oid):
# Fake a workflow reindex. ReDBox doesn't need to reindex,
# the VITAL subscriber just needs to think we did.
message = JsonObject()
message.put("oid", oid)
message.put("task", "publish")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID,
message.toString())
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper")
externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder")
oid = self.request.getParameter("oid")
if oid is None :
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
relationshipType = self.request.getParameter("relationship")
curatedPid = self.request.getParameter("curatedPid")
sourceId = self.request.getParameter("sourceIdentifier")
system = self.request.getParameter("system")
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJsonPayload = digitalObject.getPayload("metadata.json")
metadataJsonInstream = metadataJsonPayload.open()
metadataJson = JsonSimple(metadataJsonInstream)
metadataJsonPayload.close()
relationships = metadataJson.getArray("relationships")
found = False
for relationship in relationships:
if relationship.get("identifier") == sourceId:
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
found = True
if not found:
relationship = JsonObject()
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
relationship.put("relationship",relationshipType)
relationship.put("identifier",sourceId)
relationship.put("system",system)
relationships.add(relationship)
out.println(metadataJson.toString(True))
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream)
out.close()
finally:
self.sessionState.remove("username")
def _addRelatedOid(self, tfPackageJson, relatedOid):
relatedOids = tfPackageJson.getArray("related.datasets")
if relatedOids is None:
relatedOids = JSONArray()
relatedOidJsonObject = JsonObject()
relatedOidJsonObject.put("oid", relatedOid)
relatedOids.add(relatedOidJsonObject)
jsonObject = tfPackageJson.getJsonObject()
jsonObject.put("related.datasets", relatedOids)
return jsonObject
def _readReviewers(self, storedObj, tfpackage):
"""Read from TFPACKAGE for reviewer's recommendation and map to a json with short keys:
reviewer-recommend-for : for
reviewer-recommended-storage : storage
"""
reviewersPayload = storedObj.getPayload(tfpackage)
reviewersRecommends = JsonSimple(reviewersPayload.open()).getJsonObject()
reviewers = JsonObject()
reviewers.put("for", reviewersRecommends.get("reviewer-recommend-for"))
reviewers.put("storage", reviewersRecommends.get("reviewer-recommended-storage"))
return reviewers
def transformLegacyArrays(self, originalObject):
outputJsonObject = JsonObject()
dataUtil = StorageDataUtil()
outputJsonObject = JsonObject()
jsonObject = originalObject.getJsonObject()
for keyString in jsonObject.keySet():
if self.isAnArrayKey(keyString):
prefix = self.getPrefix(keyString);
outputJsonObject.put(prefix, dataUtil.getJavaList(json,prefix));
else:
outputJsonObject.put(keyString, jsonObject.get(keyString));
return JsonSimple(outputJsonObject)
def __activate__(self, context):
self.log = context["log"]
self.services = context["Services"]
auth = context["page"].authentication
username = auth.get_username() # get system username
try:
result = self.__constructInfoJson(username)
except Exception, e:
self.log.error("%s: cannot get user attributes, detail = %s" % (self.__class__.__name__ , str(e)))
result = JsonObject()
result.put("realName", auth.get_name()) # default value: user's realName
def getCurationData(self, oid):
json = JsonObject()
try:
# Get the object from storage
storage = self.Services.getStorage()
object = storage.getObject(oid)
# Find the package payload
payload = object.getPayload("metadata.json")
# Not found?
if payload is None:
self.log.error(" * detail.py => Can't find package data!")
json.put("error", True)
return json
# Parse the data
data = JsonSimple(payload.open())
payload.close()
# Return it
json.put("error", False)
json.put("relationships", data.writeArray("relationships"))
return json
except StorageException, ex:
self.log.error(" * detail.py => Storage Error accessing data: ",
ex)
json.put("error", True)
return json
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper")
externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder")
oid = self.request.getParameter("oid")
if oid is None :
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
relationshipType = self.request.getParameter("relationship")
curatedPid = self.request.getParameter("curatedPid")
sourceId = self.request.getParameter("sourceIdentifier")
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJson = self.getTfPackage(digitalObject)
relationships = metadataJson.getArray("relationships")
found = False
for relationship in relationships:
if relationship.get("identifier") == sourceId:
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
found = True
if not found:
relationship = JsonObject()
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
relationship.put("relationship",relationshipType)
relationship.put("identifier",sourceId)
relationships.add(relationship)
self.log.info(metadataJson.toString(True))
out.println(metadataJson.toString(True))
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
for pid in digitalObject.getPayloadIdList():
if pid.endswith(".tfpackage"):
StorageUtils.createOrUpdatePayload(digitalObject,pid,istream)
out.close()
finally:
self.sessionState.remove("username")
def __constructUserJson(self, username):
"""
There are users managed by internal auth manager with no attributes
There are users managed by external auth manages e.g. shibboleth who have attributes
These users username is not necessarily the same as there normal display name
This function currently solves this issue by checking commonName attribute for shibboleth users
"""
username = username.strip()
userJson = JsonObject()
userJson.put("userName", username)
parameters = HashMap()
# print "Checking user info for %s" % username
parameters.put("username", username)
userObjectList = self.authUserDao.query("getUser", parameters)
# print "Returned size = %d" % userObjectList.size()
if userObjectList.size() > 0:
userObject = userObjectList.get(0)
#Check if this is a user with attributes?
attrb = userObject.getAttributes().get("commonName")
if attrb is None:
# print "We cannot find so called commonName, use %s instead" % username
userJson.put("realName", username)
else:
# print "We found so called commonName, use %s instead of %s" % (attrb.getValStr(), username)
userJson.put("realName", attrb.getValStr().strip())
else:
# This should not be reached
self.log.warn("What is going on here? why ends up here?")
userJson.put("realName", username)
return userJson
def getCurationData(self, oid):
json = JsonObject()
try:
# Get the object from storage
storage = self.Services.getStorage()
object = storage.getObject(oid)
# Find the package payload
payload = object.getPayload("metadata.json")
# Not found?
if payload is None:
self.log.error(" * detail.py => Can't find package data!")
json.put("error", True)
return json
# Parse the data
data = JsonSimple(payload.open())
payload.close()
# Return it
json.put("error", False)
json.put("relationships", data.writeArray("relationships"))
return json
except StorageException, ex:
self.log.error(" * detail.py => Storage Error accessing data: ", ex)
json.put("error", True)
return json
def __constructUserJson(self, username):
"""
There are users managed by internal auth manager with no attributes
There are users managed by external auth manages e.g. shibboleth who have attributes
These users username is not necessarily the same as there normal display name
This function currently solves this issue by checking commonName attribute for shibboleth users
"""
username = username.strip()
userJson = JsonObject()
userJson.put("userName", username)
parameters = HashMap()
# print "Checking user info for %s" % username
parameters.put("username", username)
userObjectList = self.authUserDao.query("getUser", parameters)
# print "Returned size = %d" % userObjectList.size()
if userObjectList.size() > 0:
userObject = userObjectList.get(0)
#Check if this is a user with attributes?
attrb = userObject.getAttributes().get("commonName")
if attrb is None:
# print "We cannot find so called commonName, use %s instead" % username
userJson.put("realName", username)
else:
# print "We found so called commonName, use %s instead of %s" % (attrb.getValStr(), username)
userJson.put("realName", attrb.getValStr().strip())
else:
# This should not be reached
self.log.warn("What is going on here? why ends up here?")
userJson.put("realName", username)
return userJson
def regenerate_key(self):
clientArray = self.getKeysArray()
name = self.formData.get("name")
clientToBeReplaced = None
index = 0
for client in clientArray:
if client.get("name") == name:
clientToBeReplaced = client
break
index = index + 1
if clientToBeReplaced is not None:
clientObject = JsonObject()
clientObject.put("name", self.formData.get("name"))
clientObject.put("key", self.get_random_key())
clientArray.set(index,clientObject)
self.apiKeyService.updateAndSaveKeys(clientArray)
def getFullHistory(self, oids):
"""Query the history and save the latest full set of workflow steps to the return JsonObject"""
req = SearchRequest('context:"Workflow" AND newStep:[* TO *] AND oid:' + oids + '')
req.setParam("fl",'eventTime,newStep,oid')
req.setParam("rows", Dashboard.MAX_ROWS)
req.setParam("sort", "oid asc, eventTime asc")
events = self._packageResults(req, "eventLog")
latest = JsonObject()
for e in events:
oid = e.get("oid")
if oid not in latest:
jObj = JsonObject()
latest.put(oid,jObj)
else:
jObj = latest.get(oid)
jObj.put('%s_eventTime' % e.get("newStep"), self.formatDate(e.get("eventTime")))
return latest
def __createAttachmentJson(self, oid, attachment_type, description, filename, access_right="public"):
"""
Create a JsonObject which contains basic information of an attachment
# An attachment is a JsonObject with keys:
# {
# "access_rights": "public",
# "attachment_type": "review-attachments",
# "description": "fileA",
# "filename": "boto.config",
# "oid": "ad8fa8238ce10813ef156a0a88262174"
# }
"""
attachment = JsonObject()
attachment.put("oid", oid)
attachment.put("attachment_type", attachment_type)
attachment.put("description", description)
attachment.put("filename", filename)
attachment.put("access_right", access_right)
return attachment
def __activate__(self, context):
self.sysConfig = JsonSimpleConfig()
self.velocityContext = context
self.log = context["log"]
self.__meta = {}
formData = self.vc("formData")
self.isAjax = formData.get("ajax") != None
if self.isAjax:
ok = JsonObject()
ok.put("ok", "OK")
self.json = ok.toString()
else:
self.json = ""
self.__selectedPackageType = formData.get("packageType", "default")
self.log.debug("formData = %s" % self.vc("formData"))
self.log.debug("selectedPackageType = '%s'" % self.__selectedPackageType)
self.__meta["packageType"] = formData.get("packageType", "default")
self.__meta["description"] = formData.get("description", "")
def __activate__(self, context):
self.sysConfig = JsonSimpleConfig()
self.velocityContext = context
self.log = context["log"]
self.__meta = {}
formData = self.vc("formData")
self.isAjax = formData.get("ajax") != None
if self.isAjax:
ok = JsonObject()
ok.put("ok", "OK")
self.json = ok.toString()
else:
self.json = ""
self.__selectedPackageType = formData.get("packageType", "default")
self.log.debug("formData = %s" % self.vc("formData"))
self.log.debug("selectedPackageType = '%s'" %
self.__selectedPackageType)
self.__meta["packageType"] = formData.get("packageType", "default")
self.__meta["description"] = formData.get("description", "")
def updateRelationships(self, relationship,pid,identifier):
oid = self.findOidByIdentifier(relationship.get("identifier"))
self.writer.println(oid)
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJsonPayload = digitalObject.getPayload("metadata.json")
metadataJsonInstream = metadataJsonPayload.open()
metadataJson = JsonSimple(metadataJsonInstream)
metadataJsonPayload.close()
relationships = metadataJson.getArray("relationships")
found = False
if relationships is None:
relationships = JSONArray()
metadataJson.getJsonObject().put("relationships",relationships)
for relationship1 in relationships:
if relationship1.get("identifier") == identifier:
relationship1.put("isCurated",True)
relationship1.put("curatedPid",pid)
found = True
if not found:
newRelationship = JsonObject()
newRelationship.put("isCurated",True)
newRelationship.put("curatedPid",pid)
newRelationship.put("relationship",relationship.get("relationship"))
newRelationship.put("identifier",identifier)
relationships.add(newRelationship)
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream)
def __constructInfoJson(self, username):
"""
There are users managed by internal auth manager with no attributes
There are users managed by external auth managers e.g. shibboleth who have attributes
We put all available attributes of a user in to return value
"""
# print "Query username = %s" % username
username = username.strip()
authUserDao = ApplicationContextProvider.getApplicationContext().getBean("hibernateAuthUserDao")
parameters = HashMap()
parameters.put("username", username)
userObjectList = authUserDao.query("getUser", parameters)
# print "Returned object = %s" % str(userObjectList)
# print "Returned size = %d" % userObjectList.size()
userJson = JsonObject()
try:
if userObjectList.size() > 0:
# One hit will be enough to get user object
userObj = userObjectList.get(0)
attrbs = userObj.getAttributes()
for attrb in attrbs.keySet():
# print "Attribute %s = %s) % (attrb, attrbs.get(attrb).getValStr())
userJson.put(attrb, attrbs.get(attrb).getValStr())
else:
# This should not be reached with external sourced users
self.log.warn("Wrong username? Every user should have a record")
userJson.put("userName", username)
except Exception, e:
self.log.error("%s: cannot construct user attribute JSON, detail = %s" % (self.__class__.__name__ , str(e)))
userJson.put("userName", username)
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
responseObject = JsonObject()
responseObject.put("oid", oid)
out.println(JsonSimple(responseObject).toString(True))
out.close()
finally:
self.sessionState.remove("username")
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
responseObject = JsonObject()
responseObject.put("oid",oid)
out.println(JsonSimple(responseObject).toString(True))
out.close()
finally:
self.sessionState.remove("username")
def __activate__(self, context):
self.request = context['request']
self.response = context['response']
self.services = context['Services']
self.sessionState = context["sessionState"]
#Assumption for time being is that users have admin access
try:
self.sessionState.set("username", "admin")
self.storage = self.services.storage
oid = self.request.getParameter("oid")
tfPackage = self.getTfPackage(oid, self.getTFPackagePid(oid))
objectMeta = self.getObjectMeta(oid)
workflowMeta = self.getWorkflowMeta(oid)
jsonObject = JsonObject()
jsonObject.put("tfPackage", tfPackage.getJsonObject())
jsonObject.put("workflow.metadata", workflowMeta.getJsonObject())
jsonObject.put("TF-OBJ-META", self.getObjectMetaJson(objectMeta))
writer = self.response.getPrintWriter("application/json; charset=UTF-8")
writer.println(JsonSimple(jsonObject).toString(True))
writer.close()
finally:
self.sessionState.remove("username")
def __createAttachmentJson(self,
oid,
attachment_type,
description,
filename,
access_right="public"):
"""
Create a JsonObject which contains basic information of an attachment
# An attachment is a JsonObject with keys:
# {
# "access_rights": "public",
# "attachment_type": "review-attachments",
# "description": "fileA",
# "filename": "boto.config",
# "oid": "ad8fa8238ce10813ef156a0a88262174"
# }
"""
attachment = JsonObject()
attachment.put("oid", oid)
attachment.put("attachment_type", attachment_type)
attachment.put("description", description)
attachment.put("filename", filename)
attachment.put("access_right", access_right)
return attachment
def __activate__(self, context):
self.log = context["log"]
self.services = context["Services"]
auth = context["page"].authentication
if not auth.is_admin():
result = JsonObject()
# only in this case, returning JSON has status to get attention
result.put("status", "403")
result.put("message","Error: Only administrative users can access this feature")
self.__respond(context["response"], result)
return
qterm = context["formData"].get("qt")
qvalue = context["formData"].get("qv")
if not qterm:
result = self.listUserAttributes()
else:
try:
result = self.getUsers(qterm, qvalue)
except Exception, e:
self.log.error("%s: cannot get user attributes, detail = %s" % (self.__class__.__name__ , str(e)))
result = JsonObject()
def getAttachedFiles(self, oid):
# Build a query
req = SearchRequest("attached_to:%s" % oid)
req.setParam("rows", "1000")
# Run a search
out = ByteArrayOutputStream()
self.Services.getIndexer().search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
# Process results
docs = JSONArray()
for doc in result.getResults():
attachmentType = self.escapeHtml(WordUtils.capitalizeFully(doc.getFirst("attachment_type").replace("-", " ")))
accessRights = self.escapeHtml(WordUtils.capitalizeFully(doc.getFirst("access_rights")))
entry = JsonObject()
entry.put("filename", self.escapeHtml(doc.getFirst("filename")))
entry.put("attachment_type", attachmentType)
entry.put("access_rights", accessRights)
entry.put("id", self.escapeHtml(doc.getFirst("id")))
docs.add(entry)
return docs
def getCurationData(self, oid):
json = JsonObject()
try:
# Get the object from storage
storage = self.Services.getStorage()
object = storage.getObject(oid)
# Find the package payload
payload = None
pidList = object.getPayloadIdList()
for pid in pidList:
if (pid.endswith(".tfpackage")):
payload = object.getPayload(pid)
# Not found?
if payload is None:
self.log.error(" * detail.py => Can't find package data!")
json.put("error", True)
return json
# Parse the data
data = JsonSimple(payload.open())
payload.close()
# Some basic cosmetic fixes
relations = data.writeArray("relationships")
for relation in relations:
if not relation.containsKey("field"):
relation.put("field", "From Object " + relation.get("oid"))
# Return it
json.put("error", False)
json.put("relationships", relations)
return json
except StorageException, ex:
self.log.error(" * detail.py => Storage Error accessing data: ",
ex)
json.put("error", True)
return json
def getAttachedFiles(self, oid):
# Build a query
req = SearchRequest("attached_to:%s" % oid)
req.setParam("rows", "1000")
# Run a search
out = ByteArrayOutputStream()
self.Services.getIndexer().search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
# Process results
docs = JSONArray()
for doc in result.getResults():
attachmentType = self.escapeHtml(
WordUtils.capitalizeFully(
doc.getFirst("attachment_type").replace("-", " ")))
accessRights = self.escapeHtml(
WordUtils.capitalizeFully(doc.getFirst("access_rights")))
entry = JsonObject()
entry.put("filename", self.escapeHtml(doc.getFirst("filename")))
entry.put("attachment_type", attachmentType)
entry.put("access_rights", accessRights)
entry.put("id", self.escapeHtml(doc.getFirst("id")))
docs.add(entry)
return docs
def add_key(self):
clientArray = self.getKeysArray()
name = self.formData.get("name")
for client in clientArray:
if client.get("name") == name:
return
clientObject = JsonObject()
clientObject.put("username", self.formData.get("name"))
if(self.formData.get("generateKey") == "true"):
clientObject.put("apiKey", self.get_random_key())
else:
clientObject.put("apiKey", self.formData.get("key"))
clientArray.add(clientObject)
self.apiKeyService.updateAndSaveKeys(clientArray)
def getCurationData(self, oid):
json = JsonObject()
try:
# Get the object from storage
storage = self.Services.getStorage()
object = storage.getObject(oid)
# Find the package payload
payload = None
pidList = object.getPayloadIdList()
for pid in pidList:
if (pid.endswith(".tfpackage")):
payload = object.getPayload(pid)
# Not found?
if payload is None:
self.log.error(" * detail.py => Can't find package data!")
json.put("error", True)
return json
# Parse the data
data = JsonSimple(payload.open())
payload.close()
# Some basic cosmetic fixes
relations = data.writeArray("relationships")
for relation in relations:
if not relation.containsKey("field"):
relation.put("field", "From Object "+relation.get("oid"))
# Return it
json.put("error", False)
json.put("relationships", relations)
return json
except StorageException, ex:
self.log.error(" * detail.py => Storage Error accessing data: ", ex)
json.put("error", True)
return json
def __constructUserAttribs(self, authUser, filter=["password"]):
"""
Construct a JSON object to store all attributes of a user object
Filter out do not wanted
"""
userJson = JsonObject()
# Get ID and username first
userJson.put("id", authUser.getId())
userJson.put("username", authUser.getUsername())
attrbs = authUser.getAttributes()
filtered = [attribName for attribName in attrbs.keySet() if attribName not in filter]
for attribName in filtered:
attribValue = attrbs.get(attribName).getValStr()
userJson.put(attribName, attribValue)
return userJson
def __constructUserAttribs(self, authUser, filter=["password"]):
"""
Construct a JSON object to store all attributes of a user object
Filter out do not wanted
"""
userJson = JsonObject()
# Get ID and username first
userJson.put("id", authUser.getId())
userJson.put("username", authUser.getUsername())
attrbs = authUser.getAttributes()
filtered = [
attribName for attribName in attrbs.keySet()
if attribName not in filter
]
for attribName in filtered:
attribValue = attrbs.get(attribName).getValStr()
userJson.put(attribName, attribValue)
return userJson
def getJson(self, state="open"):
title = "%s (%s)" % (self.getName(), self.getCount())
json = JsonSimple()
jsonObj = json.getJsonObject()
attributes = JsonObject()
attributes.put("id", self.getId())
attributes.put("fq", self.getFacetQuery())
attributes.put("title", title)
jsonObj.put("data", title)
jsonObj.put("attributes", attributes)
hasSubFacets = not self.getSubFacets().isEmpty()
if hasSubFacets:
jsonObj.put("state", state)
subFacetList = ArrayList()
for subFacet in self.getSubFacets():
subFacetList.add(subFacet.getJson("closed"))
children = JSONArray()
children.addAll(subFacetList)
jsonObj.put("children", children)
return json
def getJson(self, state="open"):
title = "%s (%s)" % (self.getName(), self.getCount())
json = JsonSimple()
jsonObj = json.getJsonObject()
attributes = JsonObject()
attributes.put("id", self.getId())
attributes.put("fq", self.getFacetQuery())
attributes.put("title", title)
jsonObj.put("data", title)
jsonObj.put("attributes", attributes)
hasSubFacets = not self.getSubFacets().isEmpty()
if hasSubFacets:
jsonObj.put("state", state)
subFacetList = ArrayList()
for subFacet in self.getSubFacets():
subFacetList.add(subFacet.getJson("closed"))
children = JSONArray()
children.addAll(subFacetList)
jsonObj.put("children", children)
return json
def searchNla(self):
# Start row
start = self.formData.get("start")
if start is None:
start = "1"
# Row limit
rows = self.formData.get("rows")
if rows is None:
rows = "10"
# Query
query = ""
# Test searches
#query = "pa.surname=\"Smith\""
#query = "cql.anywhere=\"monash\""
if query == "":
# Surname
surname = self.formData.get("surname")
if surname is not None:
query = "pa.surname=\"%s\"" % (surname)
# Firstname
firstName = self.formData.get("firstName")
if firstName is not None:
if query != "":
query += " AND "
query += "pa.firstname=\"%s\"" % (firstName)
if query != "":
query += " AND "
query += "pa.type=\"person\""
# Some basic metadata we already know
self.metadata.put("searchString", query)
self.metadata.put("startRecord", start)
self.metadata.put("rowsRequested", rows)
# Search NLA
self.log.debug("Submitting query to NLA: '{}', Start: '{}', Rows: '{}'", [query, start, rows])
sru = SRUClient()
# If using the NLA's test server, comment out the line above and uncomment the line below
# sru = SRUClient("http://www-test.nla.gov.au/apps/srw/search/peopleaustralia")
response = sru.nlaGetResponseBySearch(query, start, rows)
if response is None:
raise Exception("Error searching NLA. Please check system logs for details.")
self.metadata.put("rowsReturned", response.getRows())
self.metadata.put("totalHits", response.getTotalResults())
identities = NLAIdentity.convertNodesToIdentities(response.getResults())
for id in identities:
idEntry = JsonObject()
idEntry.put("nlaId", id.getId())
idEntry.put("displayName", id.getDisplayName())
idEntry.put("firstName", id.getFirstName())
idEntry.put("surname", id.getSurame())
idEntry.put("institution", id.getInstitution())
knownIds = JSONArray()
idEntry.put("knownIdentities", knownIds)
for knownId in id.getKnownIdentities():
thisId = JsonObject()
thisId.put("displayName", knownId.get("displayName"))
thisId.put("institution", knownId.get("institution"))
knownIds.add(thisId)
self.results.add(idEntry)
return True
except Exception, e:
## Move the XML to the 'failed' directory
shutil.move(filePath, self.pFail(fileName))
## And write our error data to disk beside it
self.writeError(fileName, e)
return None
# Close our file access objects
finally:
if reader is not None:
reader.close()
if inStream is not None:
inStream.close()
# Now go looking for all our data
json = JsonObject()
json.put("workflow_source", "XML Ingest") # Default
self.__mapXpathToFields(document, xmlMappings, xmlExceptions, json)
# Operational fields
json.put("viewId", "default")
json.put("packageType", "dataset")
json.put("redbox:formVersion", self.redboxVersion)
json.put("redbox:newForm", "true")
json.put("redbox:submissionProcess.redbox:submitted", "true")
json.put("redbox:submissionProcess.dc:date",
time.strftime("%Y-%m-%d %H:%M:%S", timestamp))
return JsonSimple(json)
## Used recursively
def __mapXpathToFields(self,
sourceData,
def __activate__(self, context):
response = context["response"]
json = JsonObject()
auth = context["page"].authentication
if auth.is_logged_in():
formData = context["formData"]
oid = formData.get("oid")
if oid:
# TODO check security on object
json.put("oid", oid)
try:
metaNode = JsonObject()
json.put("meta", metaNode)
object = context["Services"].storage.getObject(oid)
metadata = object.getMetadata()
for key in metadata.keySet():
metaNode.put(key, metadata.get(key))
except StorageException:
response.setStatus(500)
json.put("error", "Object '%s' not found" % oid)
else:
response.setStatus(500)
json.put("error", "An object identifier is required")
else:
response.setStatus(500)
json.put("error", "Only registered users can access this API")
writer = response.getPrintWriter("text/plain; charset=UTF-8")
writer.println(json.toString())
writer.close()
def getObjectMetaJson(self,objectMeta):
objMetaJson = JsonObject()
propertyNames = objectMeta.stringPropertyNames()
for propertyName in propertyNames:
objMetaJson.put(propertyName,objectMeta.get(propertyName))
return objMetaJson
def __sendMessage(self, oid, step):
message = JsonObject()
message.put("oid", oid)
if step is None:
message.put("eventType", "ReIndex")
else:
message.put("eventType", "NewStep : %s" % step)
message.put("newStep", step)
message.put("username", "admin")
message.put("context", "Workflow")
message.put("task", "workflow")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __activate__(self, context):
self.log = context["log"]
response = context["response"]
writer = response.getPrintWriter("text/plain; charset=UTF-8")
auth = context["page"].authentication
sessionState = context["sessionState"]
result = JsonObject()
result.put("status", "error")
result.put("message", "An unknown error has occurred")
if auth.is_admin():
services = context["Services"]
formData = context["formData"]
func = formData.get("func")
oid = formData.get("oid")
portalId = formData.get("portalId")
portalManager = services.portalManager
if func == "reharvest":
if oid:
self.log.debug("Reharvesting object '{}'", oid)
self.sendMessage(oid)
result.put("status", "ok")
result.put("message", "Object '%s' queued for reharvest")
elif portalId:
self.log.debug("Reharvesting view '{}'", portalId)
sessionState.set("reharvest/running/" + portalId, "true")
# TODO security filter - not necessary because this requires admin anyway?
portal = portalManager.get(portalId)
query = "*:*"
if portal.query != "":
query = portal.query
if portal.searchQuery != "":
if query == "*:*":
query = portal.searchQuery
else:
query = query + " AND " + portal.searchQuery
# query solr to get the objects to reharvest
rows = 25
req = SearchRequest(query)
req.setParam("fq", 'item_type:"object"')
req.setParam("rows", str(rows))
req.setParam("fl", "id")
done = False
count = 0
while not done:
req.setParam("start", str(count))
out = ByteArrayOutputStream()
services.indexer.search(req, out)
json = SolrResult(
ByteArrayInputStream(out.toByteArray()))
objectIds = HashSet(json.getFieldList("id"))
if not objectIds.isEmpty():
for oid in objectIds:
self.sendMessage(oid)
count = count + rows
total = json.getNumFound()
self.log.debug("Queued {} of {}...", min(count, total),
total)
done = (count >= total)
sessionState.remove("reharvest/running/" + portalId)
result.put("status", "ok")
result.put(
"message",
"Objects in '%s' queued for reharvest" % portalId)
else:
response.setStatus(500)
result.put("message",
"No object or view specified for reharvest")
elif func == "reindex":
if oid:
self.log.debug("Reindexing object '{}'", oid)
self.sendMessage(oid)
result.put("status", "ok")
result.put("message",
"Object '%s' queued for reindex" % oid)
else:
response.setStatus(500)
result.put("message", "No object specified to reindex")
else:
response.setStatus(500)
result.put("message", "Unknown action '%s'" % func)
else:
response.setStatus(500)
result.put("message",
"Only administrative users can access this API")
writer.println(result.toString())
writer.close()
def sendMessage(self, oid):
message = JsonObject()
message.put("oid", oid)
message.put("task", "reharvest")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def send_message(self, oid):
message = JsonObject()
message.put("oid", oid)
message.put("task", "curation-confirm")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def _reharvestPackage(self):
message = JsonObject()
message.put("oid", self.formData.get("toOid"))
message.put("task", "reharvest")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def process(self):
self.log.debug(" * settings.py: formData={}", self.vc("formData"))
valid = self.vc("page").csrfSecurePage()
if not valid:
self.throw_error("Invalid request")
return
result = "{}"
portalManager = self.vc("Services").getPortalManager()
portal = portalManager.get(self.vc("portalId"))
func = self.vc("formData").get("func")
if func == "view-update":
portal.setDescription(self.vc("formData").get("view-description"))
portal.setQuery(self.vc("formData").get("view-query"))
portal.setSearchQuery(self.vc("formData").get("view-search-query"))
portal.setRecordsPerPage(
int(self.vc("formData").get("view-records-per-page")))
portal.setFacetCount(
int(self.vc("formData").get("view-facet-count")))
portal.setFacetDisplay(
int(self.vc("formData").get("view-facet-display")))
portal.setFacetSort(
self.vc("formData").get("view-facet-sort") is not None)
portalManager.save(portal)
elif func == "general-update":
config = JsonSimpleConfig()
email = StringUtils.trimToEmpty(
self.vc("formData").get("general-email"))
systemEmail = StringUtils.trimToEmpty(
config.getString(None, ["email"]))
if systemEmail != email:
obj = config.writableSystemConfig()
obj.put("email", self.vc("formData").get("general-email"))
obj.put("configured", "true")
config.storeSystemConfig()
# mark restart
self.vc("Services").getHouseKeepingManager(
).requestUrgentRestart()
else:
self.log.debug(
" * settings.py: email not updated: did not change")
self.throw_error("Email address is the same! No change saved.")
elif func == "facets-update":
portal.getObject(["portal"]).remove("facet-fields")
fields = self.vc("formData").getValues("field")
labels = self.vc("formData").getValues("label")
displays = self.vc("formData").getValues("display")
deletes = self.vc("formData").getValues("delete")
for i in range(0, len(fields)):
field = fields[i]
if deletes[i] == "false":
node = portal.writeObject(
["portal", "facet-fields", field])
node.put("label", labels[i])
node.put("display", displays[i])
portalManager.save(portal)
elif func == "sort-update":
portal.getObject(["portal"]).remove("sort-fields")
default = self.vc("formData").get("default")
if default:
portal.setSortFieldDefault(default)
order = self.vc("formData").get("order")
if order:
portal.setSortFieldDefaultOrder(order)
fields = self.vc("formData").getValues("field")
labels = self.vc("formData").getValues("label")
deletes = self.vc("formData").getValues("delete")
for i in range(0, len(fields)):
field = fields[i]
if deletes[i] == "false":
node = portal.writeObject(["portal", "sort-fields"])
node.put(field, labels[i])
portalManager.save(portal)
elif func == "watcher-update":
configFile = self.getWatcherFile()
if configFile is not None:
json = JsonSimpleConfig(configFile)
pathIds = self.vc("formData").get("pathIds").split(",")
actives = self.vc("formData").getValues("watcher-active")
if actives is None:
actives = []
deletes = self.vc("formData").getValues("watcher-delete")
if deletes is None:
deletes = []
for pathId in pathIds:
if pathId not in deletes:
path = self.vc("formData").get("%s-path" % pathId)
stopped = str(pathId not in actives).lower()
watchDir = json.writeObject(
["watcher", "watchDirs", path])
watchDir.put(
"ignoreFileFilter",
self.vc("formData").get("%s-file" % pathId))
watchDir.put(
"ignoreDirectories",
self.vc("formData").get("%s-dir" % pathId))
watchDir.put("stopped", stopped)
json.writeArray(
["watcher", "watchDirs", path, "cxtTags"])
writer = FileWriter(configFile)
writer.write(json.toString(True))
writer.close()
else:
result = "The Watcher is not installed properly."
elif func == "restore-default-config":
# backup the file
JsonSimpleConfig.backupSystemFile()
# delete the file
JsonSimpleConfig.getSystemFile().delete()
# restore default
JsonSimpleConfig.getSystemFile()
# mark restart
self.vc("Services").getHouseKeepingManager().requestUrgentRestart()
elif func == "housekeeping-update":
config = JsonSimpleConfig()
freq = StringUtils.trimToEmpty(
self.vc("formData").get("housekeeping-timeout"))
systemFreq = StringUtils.trimToEmpty(
config.getString(
None, ["portal", "houseKeeping", "config", "frequency"]))
result = "House Keeper refreshed"
if systemFreq != freq:
# Get writeable access to underlying system
sysConfig = JsonSimple(config.writableSystemConfig())
# Modify the house keeping node
hkConfig = sysConfig.writeObject(
["portal", "houseKeeping", "config"])
hkConfig.put("frequency", freq)
# Write the underlying config back to disk
config.storeSystemConfig()
result = "Frequency updated, refreshing House Keeper"
# Refresh the HouseKeeper
message = JsonObject()
message.put("type", "refresh")
self.vc("Services").getHouseKeepingManager().sendMessage(
message.toString())
self.writer.println(result)
self.writer.close()
