def __activate__(self, context): self.auth = context["page"].authentication self.errorMsg = "" self.request = context["request"] self.response = context["response"] self.formData = context["formData"] self.storage = context["Services"].getStorage() self.log = context["log"] self.reportManager = context["Services"].getService("reportManager") fromOid = self.formData.get("fromOid") fromObject = self.storage.getObject(fromOid) if (self.auth.is_logged_in()): if (self.auth.is_admin() == True): pass elif (self.__isOwner(fromObject)): pass else: self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access." else: self.errorMsg = "Please login." if self.errorMsg == "": toOid = self.formData.get("toOid") toObject = self.storage.getObject(toOid) storeRelatedData = self.formData.get("relatedData") fromTFPackage = self._getTFPackage(fromObject) toTFPackage = self._getTFPackage(toObject) fromInputStream = fromTFPackage.open() try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream) except StorageException: print "error setting tfPackage" fromTFPackage.close() fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject() if storeRelatedData != "false" : # add relatedOid info fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid) inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream) except StorageException: print "error setting tfPackage" tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue") self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue) self._reharvestPackage() result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid , toOid) else: result = '{"status": "err", "message": "%s"}' % self.errorMsg writer = self.response.getPrintWriter("application/json; charset=UTF-8") writer.println(result) writer.close()
def __activate__(self, context): request = context["request"] storage = context["Services"].getStorage() auth = context["page"].authentication log = context["log"] username = auth.get_name() oid = request.getParameter("oid") approval = request.getParameter("approval") approval_comment = request.getParameter("approval_comment") storedObj = storage.getObject(oid) committeeResponses = None payloadList = storedObj.getPayloadIdList() if payloadList.contains("committee-responses.metadata"): committeeResponsePayload = storedObj.getPayload("committee-responses.metadata") committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject() else: committeeResponses = JsonObject() committeeResponse = JsonObject() committeeResponse.put("approval",approval) committeeResponse.put("approval_comment",approval_comment) committeeResponses.put(username,committeeResponse) log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment)) StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8")) context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def saveResponse(self, context): """ Save into object storage key to assessor's name It has four keys: status, recommendation, size-agreement and comments when status == "submitted", reviewer sees it """ oid = self.request.getParameter("oid") action = self.request.getParameter("action") if action and re.match("submit", action, re.I): status = "submitted" else: status = "draft" recommendation = self.request.getParameter("recommendation") sizeAgreement = self.request.getParameter("size-agreement") comments = self.request.getParameter("comments") storedObj, fileExisted = self.hasResponses(oid) if fileExisted: committeeResponses = self.getResponses(storedObj) else: committeeResponses = JsonObject() assessorResponse = JsonObject() assessorResponse.put("status", status) if status == 'submitted': assessorResponse.put("date",self._getToday()) assessorResponse.put("recommendation",recommendation) assessorResponse.put("size-agreement",sizeAgreement) assessorResponse.put("comments",comments) committeeResponses.put(self.assessor,assessorResponse) StorageUtils.createOrUpdatePayload(storedObj,self.PAYLOAD,IOUtils.toInputStream(committeeResponses.toString(), "UTF-8")) context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def __checkMetadataPayload(self): try: # Simple check for its existance self.object.getPayload("formData.tfpackage") self.firstHarvest = False except Exception: self.firstHarvest = True # We need to create it self.log.info("Creating 'formData.tfpackage' payload for object '{}'", self.oid) # Prep data data = { "viewId": "default", "workflow_source": "Edgar Import", "packageType": "dataset", "redbox:formVersion": self.redboxVersion, "redbox:newForm": "true" } package = JsonSimple(JsonObject(data)) # Store it inStream = IOUtils.toInputStream(package.toString(True), "UTF-8") try: self.object.createStoredPayload("formData.tfpackage", inStream) self.packagePid = "formData.tfpackage" except StorageException, e: self.log.error("Error creating 'formData.tfpackage' payload for object '{}'", self.oid, e) raise Exception("Error creating package payload: ", e)
def __checkMetadataPayload(self): try: # Simple check for its existance self.object.getPayload("formData.tfpackage") self.firstHarvest = False except Exception: self.firstHarvest = True # We need to create it self.log.info( "Creating 'formData.tfpackage' payload for object '{}'", self.oid) # Prep data data = { "viewId": "default", "workflow_source": "Edgar Import", "packageType": "dataset", "redbox:formVersion": self.redboxVersion, "redbox:newForm": "true" } package = JsonSimple(JsonObject(data)) # Store it inStream = IOUtils.toInputStream(package.toString(True), "UTF-8") try: self.object.createStoredPayload("formData.tfpackage", inStream) self.packagePid = "formData.tfpackage" except StorageException, e: self.log.error( "Error creating 'formData.tfpackage' payload for object '{}'", self.oid, e) raise Exception("Error creating package payload: ", e)
def __checkMetadataPayload(self, identifier): # We are just going to confirm the existance of # 'metadata.json', or create an empty one if it # doesn't exist. Makes curation function for this # option and removes some log errors on the details # screen. try: self.object.getPayload("metadata.json") # all is good, the above will throw an exception if it doesn't exist return except Exception: self.log.info("Creating 'metadata.json' payload for object '{}'", self.oid) # Prep data metadata = JsonSimple() metadata.getJsonObject().put("recordIDPrefix", "") metadata.writeObject("data") # The only real data we require is the ID for curation idHolder = metadata.writeObject("metadata") idHolder.put("dc.identifier", identifier) # Store it inStream = IOUtils.toInputStream(metadata.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "metadata.json", inStream) except StorageException, e: self.log.error("Error creating 'metadata.json' payload for object '{}'", self.oid, e) return
def updateWorkFlowMetadata(self, workflowMetadata, toWorkflowId, toWorkflowStage): workflowMetaDataJson = JsonSimple(workflowMetadata.open()).getJsonObject() workflowMetaDataJson.put("id", toWorkflowId) workflowMetaDataJson.put("step", toWorkflowStage) inStream = IOUtils.toInputStream(workflowMetaDataJson.toString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload"
def updatePackageType(self, tfPackage, toWorkflowId): tfPackageJson = JsonSimple(tfPackage.open()).getJsonObject() tfPackageJson.put("packageType", toWorkflowId) inStream = IOUtils.toInputStream(tfPackageJson.toString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, tfPackage.getId(), inStream) except StorageException: print " ERROR updating dataset payload"
def __createOrUpdateArrayPayload(self, oid, payloadName, newObject): """ Create or update a payload object in storage defined by oid The content of this digital object is a JsonArray of objects payloadName: name of the payload newObject: new object to be appended, e.g. a JsonObject """ objList = self.__getPayloadJsonArray(oid, payloadName) objList.add(newObject) storedObj = self.Services.getStorage().getObject(oid) StorageUtils.createOrUpdatePayload(storedObj, payloadName,IOUtils.toInputStream(objList.toString(), "UTF-8"))
def __createOrUpdateArrayPayload(self, oid, payloadName, newObject): """ Create or update a payload object in storage defined by oid The content of this digital object is a JsonArray of objects payloadName: name of the payload newObject: new object to be appended, e.g. a JsonObject """ objList = self.__getPayloadJsonArray(oid, payloadName) objList.add(newObject) storedObj = self.Services.getStorage().getObject(oid) StorageUtils.createOrUpdatePayload( storedObj, payloadName, IOUtils.toInputStream(objList.toString(), "UTF-8"))
def updateWorkFlowMetadata(self, workflowMetadata, toWorkflowId, toWorkflowStage): workflowMetaDataJson = JsonSimple( workflowMetadata.open()).getJsonObject() workflowMetaDataJson.put("id", toWorkflowId) workflowMetaDataJson.put("step", toWorkflowStage) inStream = IOUtils.toInputStream(workflowMetaDataJson.toString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload"
def __tidy(self, content): tidy = Tidy() tidy.setIndentAttributes(False) tidy.setIndentContent(False) tidy.setPrintBodyOnly(True) tidy.setSmartIndent(False) tidy.setWraplen(0) tidy.setXHTML(False) tidy.setNumEntities(True) out = ByteArrayOutputStream() doc = tidy.parseDOM(IOUtils.toInputStream(content, "UTF-8"), out) content = out.toString("UTF-8") return content, doc
def __updateMetadataPayload(self, data): # Get and parse payload = self.object.getPayload("formData.tfpackage") json = JsonSimple(payload.open()) payload.close() # Basic test for a mandatory field title = json.getString(None, ["dc:title"]) if title is not None: # We've done this before return # Merge json.getJsonObject().putAll(data) # Store it inStream = IOUtils.toInputStream(json.toString(True), "UTF-8") try: self.object.updatePayload("formData.tfpackage", inStream) except StorageException, e: self.log.error("Error updating 'formData.tfpackage' payload for object '{}'", self.oid, e)
def __updateMetadataPayload(self, data): # Get and parse payload = self.object.getPayload("formData.tfpackage") json = JsonSimple(payload.open()) payload.close() # Basic test for a mandatory field title = json.getString(None, ["dc:title"]) if title is not None: # We've done this before return # Merge json.getJsonObject().putAll(data) # Store it inStream = IOUtils.toInputStream(json.toString(True), "UTF-8") try: self.object.updatePayload("formData.tfpackage", inStream) except StorageException, e: self.log.error( "Error updating 'formData.tfpackage' payload for object '{}'", self.oid, e)
def __workflow(self): # Workflow data WORKFLOW_ID = "Parties_People" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) initialStep = 0 try: wfMeta = self.__getJsonPayload("workflow.metadata") # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = ["title", "description"] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self): # Workflow data WORKFLOW_ID = "dataset" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) if self.owner == "guest": pageTitle = "Submission Request" displayType = "submission-request" initialStep = 0 else: pageTitle = "Metadata Record" displayType = "package-dataset" initialStep = 1 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString( None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put( "label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = [ "title", "description", "manifest", "metaList", "relationships", "responses" ] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] #Used to make sure we have a created date createdDateFlag = False formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) self.log.debug("Set created date to :%s" % solrTime) createdDateFlag = True elif field == "redbox:embargo.dc:date": self.embargoedDate = value elif field == "create_timestamp": self.createTimeStamp = value # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) # check if this is an array field fnameparts = field.split(":") if fnameparts is not None and len(fnameparts) >= 3: if field.startswith("bibo") or field.startswith( "skos"): arrParts = fnameparts[1].split(".") else: arrParts = fnameparts[2].split(".") # we're not interested in: Relationship, Type and some redbox:origin if arrParts is not None and len( arrParts) >= 2 and field.find( ":Relationship.") == -1 and field.find( "dc:type") == -1 and field.find( "redbox:origin" ) == -1 and arrParts[1].isdigit(): # we've got an array field fldPart = ":%s" % arrParts[0] prefixEndIdx = field.find(fldPart) + len(fldPart) suffixStartIdx = prefixEndIdx + len( arrParts[1]) + 1 arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[ suffixStartIdx:] if field.endswith("Name"): arrFldName = self.reportingFieldPrefix + field[: prefixEndIdx] self.log.debug( "Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value)) if field.endswith("Name"): fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put( arrFldName, fullFieldMap) idx = arrParts[1] fullField = fullFieldMap.get(idx) if (fullField is None): fullField = "" if (field.endswith("givenName")): fullField = "%s, %s" % (fullField, value) if (field.endswith("familyName")): fullField = "%s%s" % (value, fullField) self.log.debug("fullname now is :%s" % fullField) fullFieldMap.put(idx, fullField) else: fieldlist = self.arrayBucket.get(arrFldName) if fieldlist is None: fieldlist = [] self.arrayBucket.put(arrFldName, fieldlist) fieldlist.append(value) for compfield in self.compFields: if field.startswith(compfield): arrFldName = self.reportingFieldPrefix + compfield fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put(arrFldName, fullFieldMap) fullField = fullFieldMap.get("1") if fullField is None: fullField = "" if field.endswith( self.compFieldsConfig[compfield]["end"]): fullField = "%s%s%s" % ( fullField, self.compFieldsConfig[compfield]["delim"], value) if field.endswith( self.compFieldsConfig[compfield]["start"]): fullField = "%s%s" % (value, fullField) self.log.debug("full field now is :%s" % fullField) fullFieldMap.put("1", fullField) self.utils.add(self.index, "display_type", displayType) # Make sure we have a creation date if not createdDateFlag: self.utils.add(self.index, "date_created", self.last_modified) self.log.debug( "Forced creation date to %s because it was not explicitly set." % self.last_modified) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def __activate__(self, context): self.auth = context["page"].authentication self.errorMsg = "" self.request = context["request"] self.response = context["response"] self.formData = context["formData"] self.storage = context["Services"].getStorage() self.log = context["log"] self.reportManager = context["Services"].getService("reportManager") fromOid = self.formData.get("fromOid") fromObject = self.storage.getObject(fromOid) if (self.auth.is_logged_in()): if (self.auth.is_admin() == True): pass elif (self.__isOwner(fromObject)): pass else: self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access." else: self.errorMsg = "Please login." if self.errorMsg == "": toOid = self.formData.get("toOid") toObject = self.storage.getObject(toOid) storeRelatedData = self.formData.get("relatedData") fromTFPackage = self._getTFPackage(fromObject) toTFPackage = self._getTFPackage(toObject) # get relevant dc:description from new object before you overwrite it with 'from' data, as this should exist from form data created in initial object packaging (packaging.py) toTFPackageJson = JsonSimple(toTFPackage.open()).getJsonObject() relevant_description = toTFPackageJson.get("dc:description") fromInputStream = fromTFPackage.open() try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream) except StorageException: print "error setting tfPackage" finally: fromTFPackage.close() tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue") if tfMetaPropertyValue == 'dmpToSelfSub': # fetch recently created new 'to' object from storage to get all data toTFPackage = self._getTFPackage(toObject) toTFPackageJson = JsonSimple(toTFPackage.open()).getJsonObject() self.setMultiDescription(toTFPackageJson, relevant_description) inStream = IOUtils.toInputStream(toTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), inStream) except StorageException: print "error setting description text in tfPackage" finally: inStream.close() self.log.info( "Completed migrating 'dc:description' to dc:decription.1.text' for oid: %s." % toTFPackage.getId()) self.log.debug("Result: %r" % toTFPackageJson) fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject() self.log.debug('from json is: %r' % fromTFPackageJson) if storeRelatedData != "false": # add relatedOid info fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid) self.log.debug('from tfPackage json is now: %r' % fromTFPackageJson) inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream) except StorageException: print "error setting tfPackage" finally: inStream.close() self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue) self._reharvestPackage() result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid, toOid) else: result = '{"status": "err", "message": "%s"}' % self.errorMsg writer = self.response.getPrintWriter("application/json; charset=UTF-8") writer.println(result) writer.close()
def __getDigitalItems(self, manifest): for itemHash in manifest.keySet(): payloadDict = {} item = manifest.get(itemHash) id = item.getString(None, "id") title = item.getString(None, "title") hidden = item.getBoolean(False, "hidden") if hidden: print "Skipping hidden item: %s (%s)" % (title, id) continue children = item.getJsonSimpleMap("children") isImage = False object = Services.storage.getObject(id) pid = object.getSourceId() htmlFileName = pid[:pid.rfind(".")] + ".htm" nodeHtm = "%s.htm" % itemHash #.replace("-", "_") sourcePayload = object.getPayload(pid) if sourcePayload and hidden != 'true': payloadType = sourcePayload.contentType htmlPayload = object.getPayload(htmlFileName) process = True if htmlPayload: #gather all the related payload payloadDict[nodeHtm] = htmlPayload, "application/xhtml+xml" payloadList = object.getPayloadIdList() for payloadid in payloadList: payload = object.getPayload(payloadid) if payloadid.find("_files") > -1: if payload.contentType.startswith("image"): #hash the name here.... filepath, filename = os.path.split(payload.id) filename, ext = os.path.splitext(filename) filename = hashlib.md5(filename).hexdigest() payloadid = os.path.join( "%s" % filepath.lower().replace(" ", "_"), "node-%s%s" % (filename, ext)) payloadDict[ payloadid] = payload, payload.contentType #elif sourcePayload: #for now only works for images elif payloadType.startswith("image") and sourcePayload: #hash the file name to avoid invalid id in epub... isImage = True #use thumbnail if exist ext = os.path.splitext(id)[1] filename = id[id.rfind("/") + 1:-len(ext)] #+ ".thumb.jpg" hashedFileName = hashlib.md5(filename).hexdigest() thumbNailPayload = object.getPayload("%s_thumbnail.jpg" % filename) htmlString = """<html xmlns="http://www.w3.org/1999/xhtml"><head><title>%s</title> <link rel="stylesheet" href="epub.css"/> </head><body><div><span style="display: block"><img src="%s" alt="%s"/></span></div></body></html>""" if thumbNailPayload: htmlString = htmlString % ( pid, "node-%s_thumbnail.jpg" % hashedFileName, pid) payloadDict[ "node-%s_thumbnail.jpg" % hashedFileName] = thumbNailPayload, "image/jpeg" else: htmlString = htmlString % (pid, pid.lower().replace( " ", "_"), pid) payloadDict[pid] = sourcePayload, payloadType payloadDict[nodeHtm] = IOUtils.toInputStream( htmlString, "UTF-8"), "application/xhtml+xml" else: process = False if process: self.__itemRefDict[ itemHash] = id, title, nodeHtm, payloadDict, isImage self.__orderedItem.append(itemHash) if children: self.__getDigitalItems(children) object.close()
def __copyString(self, s, out): IOUtils.copy(IOUtils.toInputStream(String(s), "UTF-8"), out)
def __getDigitalItems(self, manifest): for itemHash in manifest.keySet(): payloadDict = {} item = manifest[itemHash] id = item.get("id") title = item.get("title") hidden = item.get("hidden", "False") if hidden == "True": print "Skipping hidden item: %s (%s)" % (title, id) continue children = item.getJsonMap("children") isImage=False object = Services.storage.getObject(id) pid = object.getSourceId() htmlFileName = pid[:pid.rfind(".")] + ".htm" nodeHtm = "%s.htm" % itemHash #.replace("-", "_") sourcePayload = object.getPayload(pid) if sourcePayload and hidden != 'true': payloadType = sourcePayload.contentType htmlPayload = object.getPayload(htmlFileName) process = True if htmlPayload: #gather all the related payload payloadDict[nodeHtm] = htmlPayload, "application/xhtml+xml" payloadList = object.getPayloadIdList() for payloadid in payloadList: payload = object.getPayload(payloadid) if payloadid.find("_files") > -1: if payload.contentType.startswith("image"): #hash the name here.... filepath, filename = os.path.split(payload.id) filename, ext = os.path.splitext(filename) filename = hashlib.md5(filename).hexdigest() payloadid = os.path.join("%s" % filepath.lower().replace(" ", "_"), "node-%s%s" % (filename, ext)) payloadDict[payloadid] = payload, payload.contentType #elif sourcePayload: #for now only works for images elif payloadType.startswith("image") and sourcePayload: #hash the file name to avoid invalid id in epub... isImage=True #use thumbnail if exist ext = os.path.splitext(id)[1] filename = id[id.rfind("/")+1:-len(ext)] #+ ".thumb.jpg" hashedFileName = hashlib.md5(filename).hexdigest() thumbNailPayload = object.getPayload("%s_thumbnail.jpg" % filename) htmlString = """<html xmlns="http://www.w3.org/1999/xhtml"><head><title>%s</title> <link rel="stylesheet" href="epub.css"/> </head><body><div><span style="display: block"><img src="%s" alt="%s"/></span></div></body></html>""" if thumbNailPayload: htmlString = htmlString % (pid, "node-%s_thumbnail.jpg" % hashedFileName, pid) payloadDict["node-%s_thumbnail.jpg" % hashedFileName] = thumbNailPayload, "image/jpeg" else: htmlString = htmlString % (pid, pid.lower().replace(" ", "_"), pid) payloadDict[pid] = sourcePayload, payloadType payloadDict[nodeHtm] = IOUtils.toInputStream(htmlString, "UTF-8"), "application/xhtml+xml" else: process = False if process: self.__itemRefDict[itemHash] = id, title, nodeHtm, payloadDict, isImage self.__orderedItem.append(itemHash) if children: self.__getDigitalItems(children) object.close()
def __authorize_with_service_account(service_account_json): credential = GoogleCredential.fromStream(IOUtils.toInputStream(service_account_json, StandardCharsets.UTF_8)).createScoped(Collections.singleton(AndroidPublisherScopes.ANDROIDPUBLISHER)) return credential
def __workflow(self): # Workflow data WORKFLOW_ID = "dataset" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) if self.owner == "guest": pageTitle = "Submission Request" displayType = "submission-request" initialStep = 0 else: pageTitle = "Metadata Record" displayType = "package-dataset" initialStep = 1 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] #Used to make sure we have a created date createdDateFlag = False formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) self.log.debug("Set created date to :%s" % solrTime) createdDateFlag = True elif field == "redbox:embargo.dc:date": self.embargoedDate = value # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) # check if this is an array field fnameparts = field.split(":") if fnameparts is not None and len(fnameparts) >= 3: if field.startswith("bibo") or field.startswith("skos"): arrParts = fnameparts[1].split(".") else: arrParts = fnameparts[2].split(".") # we're not interested in: Relationship, Type and some redbox:origin if arrParts is not None and len(arrParts) >= 2 and field.find(":Relationship.") == -1 and field.find("dc:type") == -1 and field.find("redbox:origin") == -1 and arrParts[1].isdigit(): # we've got an array field fldPart = ":%s" % arrParts[0] prefixEndIdx = field.find(fldPart) + len(fldPart) suffixStartIdx = prefixEndIdx+len(arrParts[1])+1 arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[suffixStartIdx:] if field.endswith("Name"): arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] self.log.debug("Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value)) if field.endswith("Name"): fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put(arrFldName, fullFieldMap) idx = arrParts[1] fullField = fullFieldMap.get(idx) if (fullField is None): fullField = "" if (field.endswith("givenName")): fullField = "%s, %s" % (fullField, value) if (field.endswith("familyName")): fullField = "%s%s" % (value, fullField) self.log.debug("fullname now is :%s" % fullField) fullFieldMap.put(idx, fullField) else: fieldlist = self.arrayBucket.get(arrFldName) if fieldlist is None: fieldlist = [] self.arrayBucket.put(arrFldName, fieldlist) fieldlist.append(value) for compfield in self.compFields: if field.startswith(compfield): arrFldName = self.reportingFieldPrefix +compfield fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put(arrFldName, fullFieldMap) fullField = fullFieldMap.get("1") if fullField is None: fullField = "" if field.endswith(self.compFieldsConfig[compfield]["end"]): fullField = "%s%s%s" % (fullField, self.compFieldsConfig[compfield]["delim"] ,value) if field.endswith(self.compFieldsConfig[compfield]["start"]): fullField = "%s%s" % (value, fullField) self.log.debug("full field now is :%s" % fullField) fullFieldMap.put("1", fullField) self.utils.add(self.index, "display_type", displayType) # Make sure we have a creation date if not createdDateFlag: self.utils.add(self.index, "date_created", self.last_modified) self.log.debug("Forced creation date to %s because it was not explicitly set." % self.last_modified) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
class AnotarData: def __init__(self): pass def __activate__(self, context): self.velocityContext = context self.__auth = context["page"].authentication # This gets called a lot self.fd = self.vc("formData").get self.action = self.fd("action") self.rootUri = self.fd("rootUri") self.json = self.fd("json") self.type = self.fd("type") self.rootUriList = self.vc("formData").getValues("rootUriList") if self.rootUriList is None: self.rootUriList = self.vc("formData").getValues("rootUriList[]") self.portalPath = self.vc("portalPath") #print "action:'%s' formData:'%s'" % (self.action, formData) # used so that ajax requests don't cache if self.rootUri and self.rootUri.find("?ticks") > -1: self.rootUri = self.rootUri[:self.rootUri.find("?ticks")] # Portal path info portalPath = self.portalPath + "/" self.oid = self.rootUri if self.oid and self.oid.startswith(portalPath): self.oid = self.oid[len(portalPath):] # oid for packaged items if self.oid: hashIndex = self.oid.find("#") if hashIndex > -1: self.oid = self.oid[hashIndex + 1:] result = "" if self.action == "getList": # Response is a list of object (nested) #print "**** anotar.py : GET_SOLR : " + self.rootUri result = self.search_solr() elif self.action == "put": result = self.__authenticate() if result is None: # Response is an ID #print "**** anotar.py : PUT : " + self.rootUri result = self.put() elif self.action == "delete": result = self.__authenticate() if result is None: # Response is empty result = self.delete() if result != "": self.vc("response").setStatus(500) elif self.action == "get-image": # Response is the JSON format expected by image annotation plugin result = self.get_image() elif self.action == "save-image": result = self.__authenticate() if result is None: # Response is anotar JSON result = self.save_image() elif self.action == "delete-image": result = self.__authenticate() if result is None: result = self.delete_image() writer = self.vc("response").getPrintWriter( "text/plain; charset=UTF-8") writer.println(result) writer.close() def __authenticate(self): if not self.__auth.is_logged_in(): self.vc("response").setStatus(500) return "Only registered users can access this API" return None # Get from velocity context def vc(self, index): if self.velocityContext[index] is not None: return self.velocityContext[index] else: log.error("ERROR: Requested context entry '" + index + "' doesn't exist") return None def generate_id(self): counter = 0 fileName = "anotar." + str(counter) payloadList = self.obj.getPayloadIdList() while fileName in payloadList: counter = counter + 1 fileName = "anotar." + str(counter) self.pid = fileName print "New ID (" + self.pid + ")" def modify_json(self): #print "**** anotar.py : add_json() : adding json : " + json jsonSimple = JsonSimple(self.json) jsonObj = jsonSimple.getJsonObject() jsonObj.put("id", self.pid) rootUri = jsonSimple.getString(None, ["annotates", "rootUri"]) if rootUri is not None: baseUrl = "http://%s:%s/" % (self.vc("request").serverName, self.vc("serverPort")) myUri = baseUrl + rootUri + "#" + self.pid jsonObj.put("uri", myUri) jsonObj.put("schemaVersionUri", "http://www.purl.org/anotar/schema/0.1") self.json = jsonSimple.toString() def process_response(self, result): #print " ******** result =", result docs = [] rootDocs = [] docsDict = {} # Build a dictionary of the annotations for doc in result: #hack is done here to replace [] with null as json.py does not properly parse jsonStr = unicode(doc.get("jsonString").replace( "[]", "null")).encode("utf-8") doc = json.read(jsonStr) doc["replies"] = [] docs.append(doc) docsDict[doc["uri"]] = doc if doc["annotates"]["uri"] == doc["annotates"]["rootUri"]: rootDocs.append(doc) # Now process the dictionary for doc in docs: # If we are NOT a top level annotation if doc["annotates"]["uri"] != doc["annotates"]["rootUri"]: # Find what we are annotating try: d = docsDict[doc["annotates"]["uri"]] d["replies"].append(doc) # Add ourselves to its reply list except: # TODO KeyError pass return json.write(rootDocs) def process_tags(self, result): tags = [] tagsDict = {} # Build a dictionary of the tags for doc in result: # Get Anotar data from Solr data doc = JsonSimple(doc.get("jsonString")) # Get actual tag text tag = doc.getString(None, ["content", "literal"]) # Find out if they have locators locs = doc.getJsonSimpleList(["annotates", "locators"]).size() if locs == 0: # Basic tags, just aggregate counts if tag in tagsDict: # We've seen it before, just increment the counter existing = tagsDict[tag] count = existing.getInteger(0, ["tagCount"]) existing.getJsonObject().put("tagCount", str(count + 1)) else: # First time, store this object doc.getJsonObject().put("tagCount", str(1)) tagsDict[tag] = doc else: # Tags with a locator, special case for images etc. tags.append(doc.toString()) # Push all the 'basic' counts into the list to return for tag in tagsDict: tags.append(tagsDict[tag].toString()) return "[" + ",".join(tags) + "]" def put(self, pid=None): try: self.obj = Services.storage.getObject(self.oid) except StorageException, e: print " * anotar.py : Error creating object : ", e return e.getMessage() if pid: self.pid = pid else: self.generate_id() self.modify_json() try: p = StorageUtils.createOrUpdatePayload( self.obj, self.pid, IOUtils.toInputStream(self.json, "UTF-8")) except StorageException, e: print " * anotar.py : Error creating payload :", e return e.getMessage()
def __workflow(self): # Workflow data WORKFLOW_ID = "servicesUI2" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) pageTitle = "Services Record" displayType = "package-service" initialStep = 0 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) self.utils.add(self.index, "display_type", displayType) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def __workflow(self): # Workflow data WORKFLOW_ID = "dataset" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) #if self.owner == "guest": # pageTitle = "Submission Request" # displayType = "submission-request" # initialStep = 0 #else: # pageTitle = "Metadata Record" # displayType = "package-dataset" # initialStep = 1 ## Harvesting straight into the 'Published' stage pageTitle = "Metadata Record" displayType = "package-dataset" #initialStep = 4 initialStep = 3 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString( None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put( "label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print(" ERROR updating dataset payload") # Form processing coreFields = [ "title", "description", "manifest", "metaList", "relationships", "responses" ] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) self.utils.add(self.index, "display_type", displayType) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")