def __activate__(self, context):
request = context["request"]
storage = context["Services"].getStorage()
auth = context["page"].authentication
log = context["log"]
username = auth.get_name()
oid = request.getParameter("oid")
approval = request.getParameter("approval")
approval_comment = request.getParameter("approval_comment")
storedObj = storage.getObject(oid)
committeeResponses = None
payloadList = storedObj.getPayloadIdList()
if payloadList.contains("committee-responses.metadata"):
committeeResponsePayload = storedObj.getPayload("committee-responses.metadata")
committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject()
else:
committeeResponses = JsonObject()
committeeResponse = JsonObject()
committeeResponse.put("approval",approval)
committeeResponse.put("approval_comment",approval_comment)
committeeResponses.put(username,committeeResponse)
log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment))
StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8"))
context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def __activate__(self, context):
self.log = context["log"]
self.services = context["Services"]
auth = context["page"].authentication
if not auth.is_admin():
result = JsonObject()
# only in this case, returning JSON has status to get attention
result.put("status", "403")
result.put(
"message",
"Error: Only administrative users can access this feature")
self.__respond(context["response"], result)
return
qterm = context["formData"].get("qt")
qvalue = context["formData"].get("qv")
if not qterm:
result = self.listUserAttributes()
else:
try:
result = self.getUsers(qterm, qvalue)
except Exception, e:
self.log.error("%s: cannot get user attributes, detail = %s" %
(self.__class__.__name__, str(e)))
result = JsonObject()
def __activate__(self, context):
response = context["response"]
json = JsonObject()
auth = context["page"].authentication
if auth.is_logged_in():
formData = context["formData"]
oid = formData.get("oid")
if oid:
# TODO check security on object
json.put("oid", oid)
try:
metaNode = JsonObject()
json.put("meta", metaNode)
object = context["Services"].storage.getObject(oid)
metadata = object.getMetadata()
for key in metadata.keySet():
metaNode.put(key, metadata.get(key))
except StorageException:
response.setStatus(500)
json.put("error", "Object '%s' not found" % oid)
else:
response.setStatus(500)
json.put("error", "An object identifier is required")
else:
response.setStatus(500)
json.put("error", "Only registered users can access this API")
writer = response.getPrintWriter("text/plain; charset=UTF-8")
writer.println(json.toString())
writer.close()
def transformLegacyArrays(self, originalObject):
outputJsonObject = JsonObject()
dataUtil = StorageDataUtil()
outputJsonObject = JsonObject()
jsonObject = originalObject.getJsonObject()
for keyString in jsonObject.keySet():
if self.isAnArrayKey(keyString):
prefix = self.getPrefix(keyString);
outputJsonObject.put(prefix, dataUtil.getJavaList(json,prefix));
else:
outputJsonObject.put(keyString, jsonObject.get(keyString));
return JsonSimple(outputJsonObject)
def __checkMetadataPayload(self):
try:
# Simple check for its existance
self.object.getPayload("formData.tfpackage")
self.firstHarvest = False
except Exception:
self.firstHarvest = True
# We need to create it
self.log.info(
"Creating 'formData.tfpackage' payload for object '{}'",
self.oid)
# Prep data
data = {
"viewId": "default",
"workflow_source": "Edgar Import",
"packageType": "dataset",
"redbox:formVersion": self.redboxVersion,
"redbox:newForm": "true"
}
package = JsonSimple(JsonObject(data))
# Store it
inStream = IOUtils.toInputStream(package.toString(True), "UTF-8")
try:
self.object.createStoredPayload("formData.tfpackage", inStream)
self.packagePid = "formData.tfpackage"
except StorageException, e:
self.log.error(
"Error creating 'formData.tfpackage' payload for object '{}'",
self.oid, e)
raise Exception("Error creating package payload: ", e)
def process(self):
#We'll return a list with 1 JsonSimple object
jsonList = []
data = None
reader = None
inStream = None
document = None
# Run the XML through our parser
try:
inStream = FileInputStream(File(self.file))
reader = InputStreamReader(inStream, "UTF-8")
document = self.saxReader.read(reader)
# Parse fails
except:
raise
# Close our file access objects
finally:
if reader is not None:
reader.close()
if inStream is not None:
inStream.close()
# Now go looking for all our data
data = JsonObject()
data.put("workflow_source", "XML Alert") # Default
self.__mapXpathToFields(document, self.map, data)
if data is None:
return None
jsonList.append(JsonSimple(data))
return jsonList
def __constructUserJson(self, username):
"""
There are users managed by internal auth manager with no attributes
There are users managed by external auth manages e.g. shibboleth who have attributes
These users username is not necessarily the same as there normal display name
This function currently solves this issue by checking commonName attribute for shibboleth users
"""
username = username.strip()
userJson = JsonObject()
userJson.put("userName", username)
parameters = HashMap()
# print "Checking user info for %s" % username
parameters.put("username", username)
userObjectList = self.authUserDao.query("getUser", parameters)
# print "Returned size = %d" % userObjectList.size()
if userObjectList.size() > 0:
userObject = userObjectList.get(0)
#Check if this is a user with attributes?
attrb = userObject.getAttributes().get("commonName")
if attrb is None:
# print "We cannot find so called commonName, use %s instead" % username
userJson.put("realName", username)
else:
# print "We found so called commonName, use %s instead of %s" % (attrb.getValStr(), username)
userJson.put("realName", attrb.getValStr().strip())
else:
# This should not be reached
self.log.warn("What is going on here? why ends up here?")
userJson.put("realName", username)
return userJson
def getCurationData(self, oid):
json = JsonObject()
try:
# Get the object from storage
storage = self.Services.getStorage()
object = storage.getObject(oid)
# Find the package payload
payload = object.getPayload("metadata.json")
# Not found?
if payload is None:
self.log.error(" * detail.py => Can't find package data!")
json.put("error", True)
return json
# Parse the data
data = JsonSimple(payload.open())
payload.close()
# Return it
json.put("error", False)
json.put("relationships", data.writeArray("relationships"))
return json
except StorageException, ex:
self.log.error(" * detail.py => Storage Error accessing data: ",
ex)
json.put("error", True)
return json
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper")
externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder")
oid = self.request.getParameter("oid")
if oid is None :
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
relationshipType = self.request.getParameter("relationship")
curatedPid = self.request.getParameter("curatedPid")
sourceId = self.request.getParameter("sourceIdentifier")
digitalObject = StorageUtils.getDigitalObject(self.storage, oid)
metadataJson = self.getTfPackage(digitalObject)
relationships = metadataJson.getArray("relationships")
found = False
for relationship in relationships:
if relationship.get("identifier") == sourceId:
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
found = True
if not found:
relationship = JsonObject()
relationship.put("isCurated",True)
relationship.put("curatedPid",curatedPid)
relationship.put("relationship",relationshipType)
relationship.put("identifier",sourceId)
relationships.add(relationship)
self.log.info(metadataJson.toString(True))
out.println(metadataJson.toString(True))
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
for pid in digitalObject.getPayloadIdList():
if pid.endswith(".tfpackage"):
StorageUtils.createOrUpdatePayload(digitalObject,pid,istream)
out.close()
finally:
self.sessionState.remove("username")
def __getJson(self):
rvtMap = JsonObject()
try:
oid = self.vc("formData").get("oid")
object = Services.storage.getObject(oid)
payload = object.getPayload("imsmanifest.xml")
try:
from xml.etree import ElementTree
xmlStr = IOUtils.toString(payload.open(), "UTF-8")
payload.close()
xml = ElementTree.XML(xmlStr.encode("UTF-8"))
ns = xml.tag[:xml.tag.find("}")+1]
resources = {}
for res in xml.findall(ns+"resources/"+ns+"resource"):
resources[res.attrib.get("identifier")] = res.attrib.get("href")
organizations = xml.find(ns+"organizations")
defaultName = organizations.attrib.get("default")
organizations = organizations.findall(ns+"organization")
organizations = [o for o in organizations if o.attrib.get("identifier")==defaultName]
organization = organizations[0]
title = organization.find(ns+"title").text
rvtMap.put("title", title)
items = organization.findall(ns+"item")
rvtMap.put("toc", self.__getJsonItems(ns, items, resources))
except Exception, e:
data["error"] = "Error - %s" % str(e)
print data["error"]
object.close()
def __getRvtManifest(self, manifest):
rvtMap = HashMap()
rvtMap.put("title", manifest.getTitle())
rvtMap.put("toc", self.__getRvtNodes(manifest.getTopNodes()))
json = JsonObject(rvtMap)
#print json.toString()
return json.toString()
def __getUserInfo(self, username):
"""
Query HibernateUser to get a user information
There are users managed by internal auth manager with no attributes other than password
There are users managed by external auth managers e.g. shibboleth who have attributes
Each user, at the time of writing: 20140904, each user has multiple identical attribute sets,
so, only the first one is used
We put all available attributes of a user in to return value
"""
username = username.strip()
authUserDao = ApplicationContextProvider.getApplicationContext(
).getBean("hibernateAuthUserDao")
parameters = HashMap()
parameters.put("username", username)
userObjectList = authUserDao.query("getUser", parameters)
userJson = JsonObject()
userJson.put("username", username)
try:
if userObjectList.size() > 0:
# One hit will be enough to get user object
userJson = self.__constructUserAttribs(userObjectList.get(0),
self.ATTRIB_FILTER)
else:
# This should not be reached with external sourced users
self.log.warn("Wrong username or internal user is queried")
except Exception, e:
self.log.error(
"%s: cannot construct user attribute JSON, detail = %s" %
(self.__class__.__name__, str(e)))
def sendMessage(self, oid):
message = JsonObject()
message.put("oid", oid)
message.put("task", "reharvest")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID,
message.toString())
def __getPackageTypes(self):
object = self.sysConfig.getObject(["portal", "packageTypes"])
packageTypes = JsonSimple.toJavaMap(object)
if packageTypes.isEmpty():
defaultPackage = JsonObject()
defaultPackage.put("jsonconfig", "packaging-config.json")
packageTypes.put("default", JsonSimple(defaultPackage))
return packageTypes
def sendMessage(self, oid):
# Fake a workflow reindex. ReDBox doesn't need to reindex,
# the VITAL subscriber just needs to think we did.
message = JsonObject()
message.put("oid", oid)
message.put("task", "publish")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def sendMessage(self, oid, eventType):
message = JsonObject()
message.put("oid", oid)
message.put("eventType", eventType)
message.put("username", self.vc("page").authentication.get_username())
message.put("context", "Workflow")
message.put("task", "workflow")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __csvToJson(self, fileName):
self.log.info("Converting '{}' to JSON...", fileName)
filePath = self.pBase(fileName)
timestamp = time.gmtime(os.path.getmtime(filePath))
## Parse our CSV file
f = open(filePath, "rb")
csvReader = csv.reader(f, dialect=self.csvDialect)
## We don't need the header row
try:
headerRow = csvReader.next()
except:
## File has no data??
self.log.error("File '{}' contains no rows of data!", fileName)
return []
## Process each row in turn
data = None
jsonList = []
for row in csvReader:
data = {
"viewId":
"default",
"title":
row[0].strip(),
"description":
row[1].strip(),
"workflow_source":
row[5].strip(),
"packageType":
"dataset",
"redbox:formVersion":
self.redboxVersion,
"redbox:newForm":
"true",
"redbox:submissionProcess.redbox:submitted":
"true",
"redbox:submissionProcess.dc:date":
time.strftime("%Y-%m-%d %H:%M:%S", timestamp),
"redbox:submissionProcess.dc:title":
row[0].strip(),
"redbox:submissionProcess.dc:description":
row[1].strip(),
"redbox:submissionProcess.locrel:prc.foaf:Person.foaf:name":
row[2].strip(),
"redbox:submissionProcess.locrel:prc.foaf:Person.foaf:phone":
row[3].strip(),
"redbox:submissionProcess.locrel:prc.foaf:Person.foaf:mbox":
row[4].strip(),
"redbox:submissionProcess.skos:note":
row[6].strip()
}
json = JsonSimple(JsonObject(data))
jsonList.append(json)
f.close()
return jsonList
def _addRelatedOid(self, tfPackageJson, relatedOid):
relatedOids = tfPackageJson.getArray("related.datasets")
if relatedOids is None:
relatedOids = JSONArray()
relatedOidJsonObject = JsonObject()
relatedOidJsonObject.put("oid",relatedOid)
relatedOids.add(relatedOidJsonObject)
jsonObject = tfPackageJson.getJsonObject()
jsonObject.put("related.datasets", relatedOids)
return jsonObject
def __sendMessage(self, oid, step):
message = JsonObject()
message.put("oid", oid)
if step is None:
message.put("eventType", "ReIndex")
else:
message.put("eventType", "NewStep : %s" % step)
message.put("newStep", step)
message.put("username", "admin")
message.put("context", "Workflow")
message.put("task", "workflow")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __constructUserAttribs(self, authUser, filter=["password"]):
"""
Construct a JSON object to store all attributes of a user object
Filter out do not wanted
"""
userJson = JsonObject()
# Get ID and username first
userJson.put("id", authUser.getId())
userJson.put("username", authUser.getUsername())
attrbs = authUser.getAttributes()
filtered = [
attribName for attribName in attrbs.keySet()
if attribName not in filter
]
for attribName in filtered:
attribValue = attrbs.get(attribName).getValStr()
userJson.put(attribName, attribValue)
return userJson
def getJson(self, state="open"):
title = "%s (%s)" % (self.getName(), self.getCount())
json = JsonSimple()
jsonObj = json.getJsonObject()
attributes = JsonObject()
attributes.put("id", self.getId())
attributes.put("fq", self.getFacetQuery())
attributes.put("title", title)
jsonObj.put("data", title)
jsonObj.put("attributes", attributes)
hasSubFacets = not self.getSubFacets().isEmpty()
if hasSubFacets:
jsonObj.put("state", state)
subFacetList = ArrayList()
for subFacet in self.getSubFacets():
subFacetList.add(subFacet.getJson("closed"))
children = JSONArray()
children.addAll(subFacetList)
jsonObj.put("children", children)
return json
def __activate__(self, context):
self.sysConfig = JsonSimpleConfig()
self.velocityContext = context
self.log = context["log"]
self.__meta = {}
formData = self.vc("formData")
self.isAjax = formData.get("ajax") != None
if self.isAjax:
ok = JsonObject()
ok.put("ok", "OK")
self.json = ok.toString()
else:
self.json = ""
self.__selectedPackageType = formData.get("packageType", "default")
self.log.debug("formData = %s" % self.vc("formData"))
self.log.debug("selectedPackageType = '%s'" %
self.__selectedPackageType)
self.__meta["packageType"] = formData.get("packageType", "default")
self.__meta["description"] = formData.get("description", "")
def __activate__(self, context):
try:
self.log = context["log"]
self.response = context["response"]
self.request = context["request"]
self.systemConfig = context["systemConfig"]
self.storage = context["Services"].getStorage()
self.indexer = context["Services"].getIndexer()
self.sessionState = context["sessionState"]
self.sessionState.set("username", "admin")
out = self.response.getPrintWriter("text/plain; charset=UTF-8")
identifier = self.request.getParameter("identifier")
oid = self.findOidByIdentifier(identifier)
responseObject = JsonObject()
responseObject.put("oid", oid)
out.println(JsonSimple(responseObject).toString(True))
out.close()
finally:
self.sessionState.remove("username")
def getCurationData(self, oid):
json = JsonObject()
try:
# Get the object from storage
storage = self.Services.getStorage()
object = storage.getObject(oid)
# Find the package payload
payload = None
pidList = object.getPayloadIdList()
for pid in pidList:
if (pid.endswith(".tfpackage")):
payload = object.getPayload(pid)
# Not found?
if payload is None:
self.log.error(" * detail.py => Can't find package data!")
json.put("error", True)
return json
# Parse the data
data = JsonSimple(payload.open())
payload.close()
# Some basic cosmetic fixes
relations = data.writeArray("relationships")
for relation in relations:
if not relation.containsKey("field"):
relation.put("field", "From Object " + relation.get("oid"))
# Return it
json.put("error", False)
json.put("relationships", relations)
return json
except StorageException, ex:
self.log.error(" * detail.py => Storage Error accessing data: ",
ex)
json.put("error", True)
return json
def getAttachedFiles(self, oid):
# Build a query
req = SearchRequest("attached_to:%s" % oid)
req.setParam("rows", "1000")
# Run a search
out = ByteArrayOutputStream()
self.Services.getIndexer().search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
# Process results
docs = JSONArray()
for doc in result.getResults():
attachmentType = self.escapeHtml(
WordUtils.capitalizeFully(
doc.getFirst("attachment_type").replace("-", " ")))
accessRights = self.escapeHtml(
WordUtils.capitalizeFully(doc.getFirst("access_rights")))
entry = JsonObject()
entry.put("filename", self.escapeHtml(doc.getFirst("filename")))
entry.put("attachment_type", attachmentType)
entry.put("access_rights", accessRights)
entry.put("id", self.escapeHtml(doc.getFirst("id")))
docs.add(entry)
return docs
def __createAttachmentJson(self,
oid,
attachment_type,
description,
filename,
access_right="public"):
"""
Create a JsonObject which contains basic information of an attachment
# An attachment is a JsonObject with keys:
# {
# "access_rights": "public",
# "attachment_type": "review-attachments",
# "description": "fileA",
# "filename": "boto.config",
# "oid": "ad8fa8238ce10813ef156a0a88262174"
# }
"""
attachment = JsonObject()
attachment.put("oid", oid)
attachment.put("attachment_type", attachment_type)
attachment.put("description", description)
attachment.put("filename", filename)
attachment.put("access_right", access_right)
return attachment
# Parse fails
except Exception, e:
## Move the XML to the 'failed' directory
shutil.move(filePath, self.pFail(fileName))
## And write our error data to disk beside it
self.writeError(fileName, e)
return None
# Close our file access objects
finally:
if reader is not None:
reader.close()
if inStream is not None:
inStream.close()
# Now go looking for all our data
json = JsonObject()
json.put("workflow_source", "XML Ingest") # Default
self.__mapXpathToFields(document, xmlMappings, xmlExceptions, json)
# Operational fields
json.put("viewId", "default")
json.put("packageType", "dataset")
json.put("redbox:formVersion", self.redboxVersion)
json.put("redbox:newForm", "true")
json.put("redbox:submissionProcess.redbox:submitted", "true")
json.put("redbox:submissionProcess.dc:date",
time.strftime("%Y-%m-%d %H:%M:%S", timestamp))
return JsonSimple(json)
## Used recursively
def __mapXpathToFields(self,
def getObjectMetaJson(self,objectMeta):
objMetaJson = JsonObject()
propertyNames = objectMeta.stringPropertyNames()
for propertyName in propertyNames:
objMetaJson.put(propertyName,objectMeta.get(propertyName))
return objMetaJson
def __toJson(self, dataDict):
return JsonSimple(JsonObject(dataDict))
def _reharvestPackage(self):
message = JsonObject()
message.put("oid", self.formData.get("toOid"))
message.put("task", "reharvest")
self.messaging.queueMessage(
TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
