def __createFromSelected(self): self.vc("log").debug("Creating package from selected...") packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile() #self.vc("log").debug("packageType = '{}'", packageType) #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile) # if modifying existing manifest, we already have an identifier, # otherwise create a new one manifestId = self.__getActiveManifestId() if manifestId is None: manifestHash = "%s.tfpackage" % uuid.uuid4() else: manifestHash = self.__getActiveManifestPid() # store the manifest file for harvesting packageDir = FascinatorHome.getPathFile("packages") packageDir.mkdirs() manifestFile = File(packageDir, manifestHash) outStream = FileOutputStream(manifestFile) outWriter = OutputStreamWriter(outStream, "UTF-8") manifest = self.__getActiveManifest() oldType = manifest.getType() if oldType is None: manifest.setType(packageType) else: manifest.setType(oldType) #self.vc("log").debug("Manifest: {}", manifest) outWriter.write(manifest.toString(True)) outWriter.close() try: if manifestId is None: # harvest the package as an object username = self.vc("sessionState").get("username") if username is None: username = "******" # necessary? harvester = None # set up config files, and make sure they are both deployed workflowsDir = FascinatorHome.getPathFile("harvest/workflows") configFile = self.__getFile(workflowsDir, jsonConfigFile) rulesFile = self.__getFile(workflowsDir, "packaging-rules.py") # run the harvest client with our packaging workflow config harvester = HarvestClient(configFile, manifestFile, username) harvester.start() manifestId = harvester.getUploadOid() harvester.shutdown() else: # update existing object object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId) manifestStream = FileUtils.openInputStream(manifestFile) StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream) manifestStream.close() object.close() except Exception, ex: error = "Packager workflow failed: %s" % str(ex) self.vc("log").error(error, ex) if harvester is not None: harvester.shutdown() return '{ "status": "failed" }'
def __activate__(self, context): request = context["request"] storage = context["Services"].getStorage() auth = context["page"].authentication log = context["log"] username = auth.get_name() oid = request.getParameter("oid") approval = request.getParameter("approval") approval_comment = request.getParameter("approval_comment") storedObj = storage.getObject(oid) committeeResponses = None payloadList = storedObj.getPayloadIdList() if payloadList.contains("committee-responses.metadata"): committeeResponsePayload = storedObj.getPayload("committee-responses.metadata") committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject() else: committeeResponses = JsonObject() committeeResponse = JsonObject() committeeResponse.put("approval",approval) committeeResponse.put("approval_comment",approval_comment) committeeResponses.put(username,committeeResponse) log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment)) StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8")) context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def updateLocalRecordRelations(self, jobItems): oidIdentifierMap = HashMap() for jobItem in jobItems: oidIdentifierMap.put(jobItem.get("oid"),jobItem.get("required_identifiers")[0].get("identifier")) for jobItem in jobItems: type = jobItem.get("type"); targetSystem = self.systemConfig.getString(None, "curation", "supported-types", type); if targetSystem == "redbox": oid = jobItem.get("oid") digitalObject = StorageUtils.getDigitalObject(self.services.getStorage(), oid) tfPackagePid = self.getPackageData(digitalObject) metadataJsonPayload = digitalObject.getPayload(tfPackagePid) metadataJsonInstream = metadataJsonPayload.open() metadataJson = JsonSimple(metadataJsonInstream) metadataJsonPayload.close() relationships = metadataJson.getArray("relationships") if relationships is not None: for relationship in relationships: system = relationship.get("system") if system != "redbox" or system != None: url = self.systemConfig.getString("can't find it", "curation","external-system-urls","get-oid-for-identifier",system) client = BasicHttpClient(url+ "&identifier="+relationship.get("identifier")) get = GetMethod(url+ "&identifier="+relationship.get("identifier")) client.executeMethod(get) if get.getStatusCode() == 200: response = JsonSimple(get.getResponseBodyAsString()) relationship.put("curatedPid",oidIdentifierMap.get(response.getString(None,"oid"))) relationship.put("isCurated",True) #Now update the relationship on Mint's side break istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) StorageUtils.createOrUpdatePayload(digitalObject,tfPackagePid,istream)
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper") externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder") oid = self.request.getParameter("oid") if oid is None : identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) relationshipType = self.request.getParameter("relationship") curatedPid = self.request.getParameter("curatedPid") sourceId = self.request.getParameter("sourceIdentifier") digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJson = self.getTfPackage(digitalObject) relationships = metadataJson.getArray("relationships") found = False for relationship in relationships: if relationship.get("identifier") == sourceId: relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) found = True if not found: relationship = JsonObject() relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) relationship.put("relationship",relationshipType) relationship.put("identifier",sourceId) relationships.add(relationship) self.log.info(metadataJson.toString(True)) out.println(metadataJson.toString(True)) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) for pid in digitalObject.getPayloadIdList(): if pid.endswith(".tfpackage"): StorageUtils.createOrUpdatePayload(digitalObject,pid,istream) out.close() finally: self.sessionState.remove("username")
def __checkMetadataPayload(self, identifier): # We are just going to confirm the existance of # 'metadata.json', or create an empty one if it # doesn't exist. Makes curation function for this # option and removes some log errors on the details # screen. try: self.object.getPayload("metadata.json") # all is good, the above will throw an exception if it doesn't exist return except Exception: self.log.info("Creating 'metadata.json' payload for object '{}'", self.oid) # Prep data metadata = JsonSimple() metadata.getJsonObject().put("recordIDPrefix", "") metadata.writeObject("data") # The only real data we require is the ID for curation idHolder = metadata.writeObject("metadata") idHolder.put("dc.identifier", identifier) # Store it inStream = IOUtils.toInputStream(metadata.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "metadata.json", inStream) except StorageException, e: self.log.error("Error creating 'metadata.json' payload for object '{}'", self.oid, e) return
def updateRelationships(self, relationship,pid,identifier): oid = self.findOidByIdentifier(relationship.get("identifier")) self.writer.println(oid) digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJsonPayload = digitalObject.getPayload("metadata.json") metadataJsonInstream = metadataJsonPayload.open() metadataJson = JsonSimple(metadataJsonInstream) metadataJsonPayload.close() relationships = metadataJson.getArray("relationships") found = False if relationships is None: relationships = JSONArray() metadataJson.getJsonObject().put("relationships",relationships) for relationship1 in relationships: if relationship1.get("identifier") == identifier: relationship1.put("isCurated",True) relationship1.put("curatedPid",pid) found = True if not found: newRelationship = JsonObject() newRelationship.put("isCurated",True) newRelationship.put("curatedPid",pid) newRelationship.put("relationship",relationship.get("relationship")) newRelationship.put("identifier",identifier) relationships.add(newRelationship) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream)
def __activate__(self, context): self.auth = context["page"].authentication self.errorMsg = "" self.request = context["request"] self.response = context["response"] self.formData = context["formData"] self.storage = context["Services"].getStorage() self.log = context["log"] self.reportManager = context["Services"].getService("reportManager") fromOid = self.formData.get("fromOid") fromObject = self.storage.getObject(fromOid) if (self.auth.is_logged_in()): if (self.auth.is_admin() == True): pass elif (self.__isOwner(fromObject)): pass else: self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access." else: self.errorMsg = "Please login." if self.errorMsg == "": toOid = self.formData.get("toOid") toObject = self.storage.getObject(toOid) storeRelatedData = self.formData.get("relatedData") fromTFPackage = self._getTFPackage(fromObject) toTFPackage = self._getTFPackage(toObject) fromInputStream = fromTFPackage.open() try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream) except StorageException: print "error setting tfPackage" fromTFPackage.close() fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject() if storeRelatedData != "false" : # add relatedOid info fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid) inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream) except StorageException: print "error setting tfPackage" tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue") self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue) self._reharvestPackage() result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid , toOid) else: result = '{"status": "err", "message": "%s"}' % self.errorMsg writer = self.response.getPrintWriter("application/json; charset=UTF-8") writer.println(result) writer.close()
def saveResponse(self, context): """ Save into object storage key to assessor's name It has four keys: status, recommendation, size-agreement and comments when status == "submitted", reviewer sees it """ oid = self.request.getParameter("oid") action = self.request.getParameter("action") if action and re.match("submit", action, re.I): status = "submitted" else: status = "draft" recommendation = self.request.getParameter("recommendation") sizeAgreement = self.request.getParameter("size-agreement") comments = self.request.getParameter("comments") storedObj, fileExisted = self.hasResponses(oid) if fileExisted: committeeResponses = self.getResponses(storedObj) else: committeeResponses = JsonObject() assessorResponse = JsonObject() assessorResponse.put("status", status) if status == 'submitted': assessorResponse.put("date",self._getToday()) assessorResponse.put("recommendation",recommendation) assessorResponse.put("size-agreement",sizeAgreement) assessorResponse.put("comments",comments) committeeResponses.put(self.assessor,assessorResponse) StorageUtils.createOrUpdatePayload(storedObj,self.PAYLOAD,IOUtils.toInputStream(committeeResponses.toString(), "UTF-8")) context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def _addPropertyValueToTFMeta(self, object, tfMetaPropertyValue): objectMetadata = object.getMetadata() objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue) output = ByteArrayOutputStream(); objectMetadata.store(output, None); input = ByteArrayInputStream(output.toByteArray()); StorageUtils.createOrUpdatePayload(object,"TF-OBJ-META",input);
def _addPropertyValueToTFMeta(self, object, tfMetaPropertyValue): objectMetadata = object.getMetadata() objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue) output = ByteArrayOutputStream() objectMetadata.store(output, None) input = ByteArrayInputStream(output.toByteArray()) StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input)
def updateWorkFlowMetadata(self, workflowMetadata, toWorkflowId, toWorkflowStage): workflowMetaDataJson = JsonSimple(workflowMetadata.open()).getJsonObject() workflowMetaDataJson.put("id", toWorkflowId) workflowMetaDataJson.put("step", toWorkflowStage) inStream = IOUtils.toInputStream(workflowMetaDataJson.toString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload"
def _addPropertyValueToTFMeta(self, obj, tfMetaPropertyValue): objectMetadata = obj.getMetadata() objectMetadata.setProperty("copyTFPackage", tfMetaPropertyValue) objectMetadata.setProperty("render-pending", "true") output = ByteArrayOutputStream(); objectMetadata.store(output, None); input = ByteArrayInputStream(output.toByteArray()); StorageUtils.createOrUpdatePayload(obj,"TF-OBJ-META",input);
def updatePackageType(self, tfPackage, toWorkflowId): tfPackageJson = JsonSimple(tfPackage.open()).getJsonObject() tfPackageJson.put("packageType", toWorkflowId) inStream = IOUtils.toInputStream(tfPackageJson.toString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, tfPackage.getId(), inStream) except StorageException: print " ERROR updating dataset payload"
def __activate__(self, context): self.None = context["log"] self.systemConfig = context["systemConfig"] self.sessionState = context["sessionState"] self.response = context["response"] self.request = context["request"] self.indexer = context["Services"].getIndexer() self.storage = context["Services"].getStorage() self.log = context["log"] self.sessionState.set("username", "admin") self.writer = self.response.getPrintWriter("text/plain; charset=UTF-8") publishedRecords = self.findPublishedRecords() for publishedRecord in publishedRecords: digitalObject = StorageUtils.getDigitalObject( self.storage, publishedRecord.getString(None, "storage_id")) tfPackage = self.getTfPackage(digitalObject) metadata = digitalObject.getMetadata() configObject = StorageUtils.getDigitalObject( self.storage, metadata.getProperty("jsonConfigOid")) payload = configObject.getPayload( metadata.getProperty("jsonConfigPid")) inStream = payload.open() jsonConfig = JsonSimple(inStream) payload.close() requiredIdentifiers = jsonConfig.getArray("curation", "requiredIdentifiers") if requiredIdentifiers is not None: pidName = self.systemConfig.getString(None, "curation", "identifier-pids", requiredIdentifiers[0]) pid = metadata.getProperty(pidName) identifier = tfPackage.getString(pid, "metadata", "dc.identifier") relationships = tfPackage.getArray("relationships") if relationships is not None: for relationship in relationships: self.writer.println(relationship) if relationship.get("broker") is None: if relationship.get( "system" ) is not None and relationship.get( "system") != self.systemConfig.getString( None, "system"): self.writer.println( "notifyExternalRelationship") self.notifyExternalRelationship( relationship, pid, relationship.get("system"), identifier) else: self.updateRelationships(relationship, pid) self.writer.close() self.sessionState.remove("username")
def __createOrUpdateArrayPayload(self, oid, payloadName, newObject): """ Create or update a payload object in storage defined by oid The content of this digital object is a JsonArray of objects payloadName: name of the payload newObject: new object to be appended, e.g. a JsonObject """ objList = self.__getPayloadJsonArray(oid, payloadName) objList.add(newObject) storedObj = self.Services.getStorage().getObject(oid) StorageUtils.createOrUpdatePayload(storedObj, payloadName,IOUtils.toInputStream(objList.toString(), "UTF-8"))
def __change(self, context, oid, new_owner): storage = self.services.getStorage() object = storage.getObject(oid) objectMetadata = object.getMetadata() owner = objectMetadata.getProperty("owner") objectMetadata.setProperty("owner", new_owner) self.log.debug("grantAccess.py: Changed ownership from {} to {} ", owner, new_owner) output = ByteArrayOutputStream() objectMetadata.store(output, None) input = ByteArrayInputStream(output.toByteArray()) StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input) try: auth = context["page"].authentication source = context["formData"].get("source") self.log.debug("grantAccess.py: authentication plugin: = {}", source) auth.set_access_plugin(source) # special condition when setting admin as owner - revoke all viewers if new_owner == "admin": viewers = self.getViewers(oid, owner) self.log.debug( "grantAccess.py: New owner is admin, revoking all viewers") self.log.debug("grantAccess.py: Viewers: " + viewers.toString()) for viewer in viewers: self.log.debug("Revoking:%s" % viewer) auth.revoke_user_access(oid, viewer) # when there are viewers, the previous owner somehow joins the read-only list, revoke access to the previous owner as well. if viewers.size() > 0: auth.revoke_user_access(oid, owner) else: self.log.info( "Grant previous owner {} view access by adding them to security_execption.", owner) auth.grant_user_access( oid, owner) # give previous owner read access err = auth.get_error() if err is None or err == 'Duplicate! That user has already been applied to this record.': Services.indexer.index(oid) Services.indexer.commit() return '{"status":"ok", "new_owner": "' + new_owner + '"}' else: self.log.error( "grantAccess.py: Error raised during calling authentication for changing ownership. Exception: " + err) except Exception, e: self.log.error( "grantAccess.py: Unexpected error raised during changing ownership of data. Exception: " + str(e))
def updateWorkFlowMetadata(self, workflowMetadata, toWorkflowId, toWorkflowStage): workflowMetaDataJson = JsonSimple( workflowMetadata.open()).getJsonObject() workflowMetaDataJson.put("id", toWorkflowId) workflowMetaDataJson.put("step", toWorkflowStage) inStream = IOUtils.toInputStream(workflowMetaDataJson.toString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload"
def __createOrUpdateArrayPayload(self, oid, payloadName, newObject): """ Create or update a payload object in storage defined by oid The content of this digital object is a JsonArray of objects payloadName: name of the payload newObject: new object to be appended, e.g. a JsonObject """ objList = self.__getPayloadJsonArray(oid, payloadName) objList.add(newObject) storedObj = self.Services.getStorage().getObject(oid) StorageUtils.createOrUpdatePayload( storedObj, payloadName, IOUtils.toInputStream(objList.toString(), "UTF-8"))
def getTFPackagePid(self,oid): digitalObject = StorageUtils.getDigitalObject(self.storage,oid) for pid in digitalObject.getPayloadIdList(): pidString = String(pid) if pidString.endsWith("tfpackage"): return pid return None
def __activate__(self, context): self.None = context["log"] self.systemConfig = context["systemConfig"] self.sessionState = context["sessionState"] self.response = context["response"] self.request = context["request"] self.indexer = context["Services"].getIndexer() self.storage = context["Services"].getStorage() self.log = context["log"] self.sessionState.set("username","admin") self.writer = self.response.getPrintWriter("text/plain; charset=UTF-8") publishedRecords = self.findPublishedRecords() for publishedRecord in publishedRecords: self.log.info("processing record "+publishedRecord.getString(None,"storage_id")) digitalObject = StorageUtils.getDigitalObject(self.storage, publishedRecord.getString(None,"storage_id")) tfPackage = self.getTfPackage(digitalObject) metadata = digitalObject.getMetadata() configObject = StorageUtils.getDigitalObject(self.storage,metadata.getProperty("jsonConfigOid")) payload = configObject.getPayload(metadata.getProperty("jsonConfigPid")) inStream = payload.open() jsonConfig = JsonSimple(inStream) payload.close() requiredIdentifiers = jsonConfig.getArray("curation","requiredIdentifiers") if requiredIdentifiers is not None: pidName = self.systemConfig.getString(None,"curation","identifier-pids",requiredIdentifiers[0]) pid = metadata.getProperty(pidName) identifier = tfPackage.getString(pid,"metadata","dc.identifier") relationships = tfPackage.getArray("relationships") if relationships is not None: for relationship in relationships: if relationship.get("broker") is None: if relationship.get("system") is not None and relationship.get("system") != self.systemConfig.getString(None,"system"): self.notifyExternalRelationship(relationship,pid,relationship.get("system"),identifier) else: self.updateRelationships(relationship,pid,identifier) self.writer.close() self.sessionState.remove("username")
def __change(self, context, oid, new_owner): storage = self.services.getStorage() object = storage.getObject(oid) objectMetadata = object.getMetadata() owner = objectMetadata.getProperty("owner") objectMetadata.setProperty("owner", new_owner) self.log.debug("grantAccess.py: Changed ownership from {} to {} ", owner, new_owner) output = ByteArrayOutputStream() objectMetadata.store(output, None) input = ByteArrayInputStream(output.toByteArray()) StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input) try: auth = context["page"].authentication source = context["formData"].get("source") self.log.debug("grantAccess.py: authentication plugin: = {}", source) auth.set_access_plugin(source) # special condition when setting admin as owner - revoke all viewers if new_owner == "admin": viewers = self.getViewers(oid) self.log.debug("grantAccess.py: New owner is admin, revoking all viewers") self.log.debug("grantAccess.py: Viewers: " + viewers.toString()) for viewer in viewers: self.log.debug("Revoking:%s" % viewer) auth.revoke_user_access(oid, viewer) # when there are viewers, the previous owner somehow joins the read-only list, revoke access to the previous owner as well. if viewers.size() > 0: auth.revoke_user_access(oid, owner) else: self.log.info("Grant previous owner {} view access by adding them to security_execption.", owner) auth.grant_user_access(oid, owner) # give previous owner read access err = auth.get_error() if err is None or err == 'Duplicate! That user has already been applied to this record.': Services.indexer.index(oid) Services.indexer.commit() return '{"status":"ok", "new_owner": "' + new_owner + '"}' else: self.log.error("grantAccess.py: Error raised during calling authentication for changing ownership. Exception: " + err) except Exception, e: self.log.error("grantAccess.py: Unexpected error raised during changing ownership of data. Exception: " + str(e))
def __change(self, context, oid, new_owner): try: storage = context["Services"].getStorage() object = storage.getObject(oid) objectMetadata = object.getMetadata() owner = objectMetadata.getProperty("owner") objectMetadata.setProperty("owner", new_owner) self.log.debug("Changing ownership to : " + new_owner) output = ByteArrayOutputStream() objectMetadata.store(output, None) input = ByteArrayInputStream(output.toByteArray()) StorageUtils.createOrUpdatePayload(object, "TF-OBJ-META", input) auth = context["page"].authentication #special condition when setting admin as owner - revoke all viewers if new_owner == "admin": self.log.debug("New owner is admin, revoking all viewers") viewers = self.getViewers(oid) self.log.debug("Viewers: " + viewers.toString()) for viewer in viewers: self.log.debug("Revoking:%s" % viewer) auth.revoke_user_access(oid, viewer) auth.revoke_user_access(oid, owner) # don't know what's going on but this was originally set to "owner", changed it to new_owner. auth.grant_user_access(oid, new_owner) err = auth.get_error() if err is None or err == 'Duplicate! That user has already been applied to this record.': Services.indexer.index(oid) Services.indexer.commit() return '{"status":"ok", "new_owner": "' + new_owner + '"}' else: self.log.error( "Error during changing ownership of data. Exception: " + err) except Exception, e: self.log.error( "Error during changing ownership of data. Exception: " + str(e))
def updateObjectMetadata(self, objectMetaData, toWorkflowId): packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile( toWorkflowId) workflowsDir = FascinatorHome.getPathFile("harvest/workflows") configFile = File(workflowsDir, jsonConfigFile) configObject = StorageUtils.checkHarvestFile(self.storage, configFile) if configObject is None: oid = StorageUtils.generateOid(configFile) configObject = StorageUtils.getDigitalObject(self.storage, oid) objectMetaData.setProperty("jsonConfigPid", jsonConfigFile) objectMetaData.setProperty("jsonConfigOid", configObject.getId()) configJson = JsonSimple(configFile) rulesFileName = configJson.getString(None, "indexer", "script", "rules") rulesFile = File(workflowsDir, rulesFileName) rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile) if rulesObject is None: oid = StorageUtils.generateOid(rulesFile) rulesObject = StorageUtils.getDigitalObject(self.storage, oid) objectMetaData.setProperty("rulesPid", rulesFileName) objectMetaData.setProperty("rulesOid", rulesObject.getId()) objectMetaData.setProperty("workflowTransitioned", "true") objectMetaData.setProperty( "date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def updateObjectMetadata(self, objectMetaData, toWorkflowId): packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile(toWorkflowId) workflowsDir = FascinatorHome.getPathFile("harvest/workflows") configFile = File(workflowsDir, jsonConfigFile) configObject = StorageUtils.checkHarvestFile(self.storage, configFile); if configObject is None: oid = StorageUtils.generateOid(configFile); configObject = StorageUtils.getDigitalObject(self.storage, oid); objectMetaData.setProperty("jsonConfigPid", jsonConfigFile) objectMetaData.setProperty("jsonConfigOid", configObject.getId()) configJson = JsonSimple(configFile) rulesFileName = configJson.getString(None, "indexer","script","rules") rulesFile = File(workflowsDir,rulesFileName) rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile); if rulesObject is None: oid = StorageUtils.generateOid(rulesFile); rulesObject = StorageUtils.getDigitalObject(self.storage, oid); objectMetaData.setProperty("rulesPid", rulesFileName) objectMetaData.setProperty("rulesOid", rulesObject.getId()) objectMetaData.setProperty("workflowTransitioned", "true") objectMetaData.setProperty("date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def __change(self, context, oid, new_owner): try: storage = context["Services"].getStorage() object = storage.getObject(oid) objectMetadata = object.getMetadata() owner = objectMetadata.getProperty("owner") objectMetadata.setProperty("owner", new_owner) self.log.debug("Changing ownership to : " + new_owner) output = ByteArrayOutputStream() objectMetadata.store(output, None) input = ByteArrayInputStream(output.toByteArray()) StorageUtils.createOrUpdatePayload(object,"TF-OBJ-META",input) auth = context["page"].authentication #special condition when setting admin as owner - revoke all viewers if new_owner == "admin": self.log.debug("New owner is admin, revoking all viewers") viewers = self.getViewers(oid) self.log.debug("Viewers: " + viewers.toString()) for viewer in viewers: self.log.debug("Revoking:%s" % viewer) auth.revoke_user_access(oid, viewer) auth.revoke_user_access(oid, owner) # don't know what's going on but this was originally set to "owner", changed it to new_owner. auth.grant_user_access(oid, new_owner) err = auth.get_error() if err is None or err == 'Duplicate! That user has already been applied to this record.': Services.indexer.index(oid) Services.indexer.commit() return '{"status":"ok", "new_owner": "' + new_owner + '"}' else: self.log.error("Error during changing ownership of data. Exception: " + err) except Exception, e: self.log.error("Error during changing ownership of data. Exception: " + str(e))
def __workflow(self): # Workflow data WORKFLOW_ID = "Parties_People" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) initialStep = 0 try: wfMeta = self.__getJsonPayload("workflow.metadata") # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = ["title", "description"] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def getObjectMeta(self,oid): digitalObject = StorageUtils.getDigitalObject(self.storage, oid) return digitalObject.getMetadata()
def __createFromSelected(self): self.vc("log").debug("Creating package from selected...") packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile() #self.vc("log").debug("packageType = '{}'", packageType) #self.vc("log").debug("jsonConfigFile = '{}'", jsonConfigFile) # if modifying existing manifest, we already have an identifier, # otherwise create a new one manifestId = self.__getActiveManifestId() if manifestId is None: manifestHash = "%s.tfpackage" % uuid.uuid4() else: manifestHash = self.__getActiveManifestPid() # store the manifest file for harvesting packageDir = FascinatorHome.getPathFile("packages") packageDir.mkdirs() manifestFile = File(packageDir, manifestHash) outStream = FileOutputStream(manifestFile) outWriter = OutputStreamWriter(outStream, "UTF-8") manifest = self.__getActiveManifest() oldType = manifest.getType() if oldType is None: manifest.setType(packageType) else: manifest.setType(oldType) self.vc("log").debug("Manifest: %s" % manifest) outWriter.write(manifest.toString(True)) outWriter.close() try: if manifestId is None: # harvest the package as an object username = self.vc("sessionState").get("username") if username is None: username = "******" # necessary? harvester = None # set up config files, and make sure they are both deployed workflowsDir = FascinatorHome.getPathFile("harvest/workflows") configFile = self.__getFile(workflowsDir, jsonConfigFile) rulesFile = self.__getFile(workflowsDir, "packaging-rules.py") # run the harvest client with our packaging workflow config harvester = HarvestClient(configFile, manifestFile, username) harvester.start() manifestId = harvester.getUploadOid() harvester.shutdown() else: # update existing object object = StorageUtils.getDigitalObject(Services.getStorage(), manifestId) manifestStream = FileUtils.openInputStream(manifestFile) StorageUtils.createOrUpdatePayload(object, manifestHash, manifestStream) manifestStream.close() object.close() except Exception, ex: error = "Packager workflow failed: %s" % str(ex) self.vc("log").error(error, ex) if harvester is not None: harvester.shutdown() return '{ "status": "failed" }'
class AnotarData: def __init__(self): pass def __activate__(self, context): self.velocityContext = context self.__auth = context["page"].authentication # This gets called a lot self.fd = self.vc("formData").get self.action = self.fd("action") self.rootUri = self.fd("rootUri") self.json = self.fd("json") self.type = self.fd("type") self.rootUriList = self.vc("formData").getValues("rootUriList") if self.rootUriList is None: self.rootUriList = self.vc("formData").getValues("rootUriList[]") self.portalPath = self.vc("portalPath") #print "action:'%s' formData:'%s'" % (self.action, formData) # used so that ajax requests don't cache if self.rootUri and self.rootUri.find("?ticks") > -1: self.rootUri = self.rootUri[:self.rootUri.find("?ticks")] # Portal path info portalPath = self.portalPath + "/" self.oid = self.rootUri if self.oid and self.oid.startswith(portalPath): self.oid = self.oid[len(portalPath):] # oid for packaged items if self.oid: hashIndex = self.oid.find("#") if hashIndex > -1: self.oid = self.oid[hashIndex + 1:] result = "" if self.action == "getList": # Response is a list of object (nested) #print "**** anotar.py : GET_SOLR : " + self.rootUri result = self.search_solr() elif self.action == "put": result = self.__authenticate() if result is None: # Response is an ID #print "**** anotar.py : PUT : " + self.rootUri result = self.put() elif self.action == "delete": result = self.__authenticate() if result is None: # Response is empty result = self.delete() if result != "": self.vc("response").setStatus(500) elif self.action == "get-image": # Response is the JSON format expected by image annotation plugin result = self.get_image() elif self.action == "save-image": result = self.__authenticate() if result is None: # Response is anotar JSON result = self.save_image() elif self.action == "delete-image": result = self.__authenticate() if result is None: result = self.delete_image() writer = self.vc("response").getPrintWriter( "text/plain; charset=UTF-8") writer.println(result) writer.close() def __authenticate(self): if not self.__auth.is_logged_in(): self.vc("response").setStatus(500) return "Only registered users can access this API" return None # Get from velocity context def vc(self, index): if self.velocityContext[index] is not None: return self.velocityContext[index] else: log.error("ERROR: Requested context entry '" + index + "' doesn't exist") return None def generate_id(self): counter = 0 fileName = "anotar." + str(counter) payloadList = self.obj.getPayloadIdList() while fileName in payloadList: counter = counter + 1 fileName = "anotar." + str(counter) self.pid = fileName print "New ID (" + self.pid + ")" def modify_json(self): #print "**** anotar.py : add_json() : adding json : " + json jsonSimple = JsonSimple(self.json) jsonObj = jsonSimple.getJsonObject() jsonObj.put("id", self.pid) rootUri = jsonSimple.getString(None, ["annotates", "rootUri"]) if rootUri is not None: baseUrl = "http://%s:%s/" % (self.vc("request").serverName, self.vc("serverPort")) myUri = baseUrl + rootUri + "#" + self.pid jsonObj.put("uri", myUri) jsonObj.put("schemaVersionUri", "http://www.purl.org/anotar/schema/0.1") self.json = jsonSimple.toString() def process_response(self, result): #print " ******** result =", result docs = [] rootDocs = [] docsDict = {} # Build a dictionary of the annotations for doc in result: #hack is done here to replace [] with null as json.py does not properly parse jsonStr = unicode(doc.get("jsonString").replace( "[]", "null")).encode("utf-8") doc = json.read(jsonStr) doc["replies"] = [] docs.append(doc) docsDict[doc["uri"]] = doc if doc["annotates"]["uri"] == doc["annotates"]["rootUri"]: rootDocs.append(doc) # Now process the dictionary for doc in docs: # If we are NOT a top level annotation if doc["annotates"]["uri"] != doc["annotates"]["rootUri"]: # Find what we are annotating try: d = docsDict[doc["annotates"]["uri"]] d["replies"].append(doc) # Add ourselves to its reply list except: # TODO KeyError pass return json.write(rootDocs) def process_tags(self, result): tags = [] tagsDict = {} # Build a dictionary of the tags for doc in result: # Get Anotar data from Solr data doc = JsonSimple(doc.get("jsonString")) # Get actual tag text tag = doc.getString(None, ["content", "literal"]) # Find out if they have locators locs = doc.getJsonSimpleList(["annotates", "locators"]).size() if locs == 0: # Basic tags, just aggregate counts if tag in tagsDict: # We've seen it before, just increment the counter existing = tagsDict[tag] count = existing.getInteger(0, ["tagCount"]) existing.getJsonObject().put("tagCount", str(count + 1)) else: # First time, store this object doc.getJsonObject().put("tagCount", str(1)) tagsDict[tag] = doc else: # Tags with a locator, special case for images etc. tags.append(doc.toString()) # Push all the 'basic' counts into the list to return for tag in tagsDict: tags.append(tagsDict[tag].toString()) return "[" + ",".join(tags) + "]" def put(self, pid=None): try: self.obj = Services.storage.getObject(self.oid) except StorageException, e: print " * anotar.py : Error creating object : ", e return e.getMessage() if pid: self.pid = pid else: self.generate_id() self.modify_json() try: p = StorageUtils.createOrUpdatePayload( self.obj, self.pid, IOUtils.toInputStream(self.json, "UTF-8")) except StorageException, e: print " * anotar.py : Error creating payload :", e return e.getMessage()
wfMetaObj.put("step", "pending") wfMetaObj.put("pageTitle", "Uploaded Files - Management") stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == "pending": wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: jsonString = String(wfMeta.toString()) inStream = ByteArrayInputStream(jsonString.getBytes("UTF-8")) try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException, e: print " * workflow-harvester.py : Error updating workflow payload" self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfMeta.getString(None, ["step"])) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) def __filePath(self): baseFilePath = self.params["base.file.path"] filePath = self.object.getMetadata().getProperty("file.path") if baseFilePath:
def getTfPackage(self,oid, pid): digitalObject = StorageUtils.getDigitalObject(self.storage, oid) tfPackageInputStream = digitalObject.getPayload(pid).open() return JsonSimple(tfPackageInputStream)
def __workflow(self): # Workflow data WORKFLOW_ID = "dataset" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) if self.owner == "guest": pageTitle = "Submission Request" displayType = "submission-request" initialStep = 0 else: pageTitle = "Metadata Record" displayType = "package-dataset" initialStep = 1 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] #Used to make sure we have a created date createdDateFlag = False formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) self.log.debug("Set created date to :%s" % solrTime) createdDateFlag = True elif field == "redbox:embargo.dc:date": self.embargoedDate = value # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) # check if this is an array field fnameparts = field.split(":") if fnameparts is not None and len(fnameparts) >= 3: if field.startswith("bibo") or field.startswith("skos"): arrParts = fnameparts[1].split(".") else: arrParts = fnameparts[2].split(".") # we're not interested in: Relationship, Type and some redbox:origin if arrParts is not None and len(arrParts) >= 2 and field.find(":Relationship.") == -1 and field.find("dc:type") == -1 and field.find("redbox:origin") == -1 and arrParts[1].isdigit(): # we've got an array field fldPart = ":%s" % arrParts[0] prefixEndIdx = field.find(fldPart) + len(fldPart) suffixStartIdx = prefixEndIdx+len(arrParts[1])+1 arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[suffixStartIdx:] if field.endswith("Name"): arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] self.log.debug("Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value)) if field.endswith("Name"): fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put(arrFldName, fullFieldMap) idx = arrParts[1] fullField = fullFieldMap.get(idx) if (fullField is None): fullField = "" if (field.endswith("givenName")): fullField = "%s, %s" % (fullField, value) if (field.endswith("familyName")): fullField = "%s%s" % (value, fullField) self.log.debug("fullname now is :%s" % fullField) fullFieldMap.put(idx, fullField) else: fieldlist = self.arrayBucket.get(arrFldName) if fieldlist is None: fieldlist = [] self.arrayBucket.put(arrFldName, fieldlist) fieldlist.append(value) for compfield in self.compFields: if field.startswith(compfield): arrFldName = self.reportingFieldPrefix +compfield fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put(arrFldName, fullFieldMap) fullField = fullFieldMap.get("1") if fullField is None: fullField = "" if field.endswith(self.compFieldsConfig[compfield]["end"]): fullField = "%s%s%s" % (fullField, self.compFieldsConfig[compfield]["delim"] ,value) if field.endswith(self.compFieldsConfig[compfield]["start"]): fullField = "%s%s" % (value, fullField) self.log.debug("full field now is :%s" % fullField) fullFieldMap.put("1", fullField) self.utils.add(self.index, "display_type", displayType) # Make sure we have a creation date if not createdDateFlag: self.utils.add(self.index, "date_created", self.last_modified) self.log.debug("Forced creation date to %s because it was not explicitly set." % self.last_modified) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def getWorkflowMeta(self,oid): digitalObject = StorageUtils.getDigitalObject(self.storage, oid) workflowMetaInputStream = digitalObject.getPayload("workflow.metadata").open() return JsonSimple(workflowMetaInputStream)
def getPayloadJson(self,oid,payloadName): digitalObject = StorageUtils.getDigitalObject(self.storage, oid) workflowMetaInputStream = digitalObject.getPayload(payloadName).open() return JsonSimple(workflowMetaInputStream)
def __workflow(self): # Workflow data WORKFLOW_ID = "dataset" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) #if self.owner == "guest": # pageTitle = "Submission Request" # displayType = "submission-request" # initialStep = 0 #else: # pageTitle = "Metadata Record" # displayType = "package-dataset" # initialStep = 1 ## Harvesting straight into the 'Published' stage pageTitle = "Metadata Record" displayType = "package-dataset" #initialStep = 4 initialStep = 3 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString( None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put( "label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print(" ERROR updating dataset payload") # Form processing coreFields = [ "title", "description", "manifest", "metaList", "relationships", "responses" ] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) self.utils.add(self.index, "display_type", displayType) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
wfMetaObj.put("pageTitle", "Uploaded Files - Management") stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == "pending": wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: jsonString = String(wfMeta.toString()) inStream = ByteArrayInputStream(jsonString.getBytes("UTF-8")) try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException, e: print " * workflow-harvester.py : Error updating workflow payload" self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfMeta.getString(None, ["step"])) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) def __filePath(self): baseFilePath = self.params["base.file.path"]
def getPayloadJsonByExtension(self,oid,payloadExtension): digitalObject = StorageUtils.getDigitalObject(self.storage,oid) pid = self.findPidForExtenstion(digitalObject, payloadExtension) tfPackageInputStream = digitalObject.getPayload(pid).open() return JsonSimple(tfPackageInputStream)
def __attachFile(self): try: # WebKit/IE prefixes C:\fakepath\ with javascript manipulated file inputs uploadFile = self.formData.get("uploadFile") uploadFile = uploadFile.replace("C:\\fakepath\\", "") fileDetails = self.vc("sessionState").get(uploadFile) # Establish that we do have details on the uploaded file if fileDetails is None: uploadFile = uploadFile.rsplit("\\", 1)[-1] fileDetails = self.vc("sessionState").get(uploadFile) if fileDetails is None: self.log.error("**** fileDetails is None!!! ***") return self.__toJson({ "error": "fileDetails is None (no upload file!)" }) self.log.debug("Attach Upload: fileDetails: '{}'", fileDetails) errorDetails = fileDetails.get("error") if errorDetails: self.log.error("ERROR: %s" % errorDetails) return self.__toJson({"error": errorDetails}) # Look for the storage info we need jsonFormData = JsonSimple(self.formData.get("json")) oid = jsonFormData.getString(None, "oid") fname = fileDetails.get("name") foid = fileDetails.get("oid") self.log.debug("attach oid='{}', filename='{}', foid='{}'", [oid, fname, foid]) # Make sure it was actually stored try: attachObj = self.Services.getStorage().getObject(foid) except StorageException, e: return JsonSimple({"error": "Attached file - '%s'" % str(e)}) # Build up some metadata to store alongside the file attachFormData = JsonSimple(self.formData.get("json", "{}")) attachMetadata = { "type": "attachment", "created_by": "workflow.py", "formData": { "oid": foid, "attached_to": oid, "filename": fname, "access_rights": attachFormData.getString("private", ["accessRights"]), "attachment_type": attachFormData.getString("supporting-material", ["attachmentType"]) } } # We are going to send an update on all attachments back with our response attachedFiles = self.__getAttachedFiles(oid) attachedFiles.append(dict(attachMetadata["formData"])) # Now store our metadata for this file try: jsonMetadata = self.__toJson(attachMetadata) jsonIn = ByteArrayInputStream(jsonMetadata.toString()) StorageUtils.createOrUpdatePayload(attachObj, "workflow.metadata", jsonIn) jsonIn.close(); attachObj.close(); except StorageException, e: self.log.error("Failed to create attachment metadata!", e)
def __workflow(self): # Workflow data WORKFLOW_ID = "dataset" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) if self.owner == "guest": pageTitle = "Submission Request" displayType = "submission-request" initialStep = 0 else: pageTitle = "Metadata Record" displayType = "package-dataset" initialStep = 1 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString( None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put( "label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = [ "title", "description", "manifest", "metaList", "relationships", "responses" ] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] #Used to make sure we have a created date createdDateFlag = False formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) self.log.debug("Set created date to :%s" % solrTime) createdDateFlag = True elif field == "redbox:embargo.dc:date": self.embargoedDate = value elif field == "create_timestamp": self.createTimeStamp = value # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) # check if this is an array field fnameparts = field.split(":") if fnameparts is not None and len(fnameparts) >= 3: if field.startswith("bibo") or field.startswith( "skos"): arrParts = fnameparts[1].split(".") else: arrParts = fnameparts[2].split(".") # we're not interested in: Relationship, Type and some redbox:origin if arrParts is not None and len( arrParts) >= 2 and field.find( ":Relationship.") == -1 and field.find( "dc:type") == -1 and field.find( "redbox:origin" ) == -1 and arrParts[1].isdigit(): # we've got an array field fldPart = ":%s" % arrParts[0] prefixEndIdx = field.find(fldPart) + len(fldPart) suffixStartIdx = prefixEndIdx + len( arrParts[1]) + 1 arrFldName = self.reportingFieldPrefix + field[:prefixEndIdx] + field[ suffixStartIdx:] if field.endswith("Name"): arrFldName = self.reportingFieldPrefix + field[: prefixEndIdx] self.log.debug( "Array Field name is:%s from: %s, with value:%s" % (arrFldName, field, value)) if field.endswith("Name"): fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put( arrFldName, fullFieldMap) idx = arrParts[1] fullField = fullFieldMap.get(idx) if (fullField is None): fullField = "" if (field.endswith("givenName")): fullField = "%s, %s" % (fullField, value) if (field.endswith("familyName")): fullField = "%s%s" % (value, fullField) self.log.debug("fullname now is :%s" % fullField) fullFieldMap.put(idx, fullField) else: fieldlist = self.arrayBucket.get(arrFldName) if fieldlist is None: fieldlist = [] self.arrayBucket.put(arrFldName, fieldlist) fieldlist.append(value) for compfield in self.compFields: if field.startswith(compfield): arrFldName = self.reportingFieldPrefix + compfield fullFieldMap = self.arrayBucket.get(arrFldName) if fullFieldMap is None: fullFieldMap = HashMap() self.arrayBucket.put(arrFldName, fullFieldMap) fullField = fullFieldMap.get("1") if fullField is None: fullField = "" if field.endswith( self.compFieldsConfig[compfield]["end"]): fullField = "%s%s%s" % ( fullField, self.compFieldsConfig[compfield]["delim"], value) if field.endswith( self.compFieldsConfig[compfield]["start"]): fullField = "%s%s" % (value, fullField) self.log.debug("full field now is :%s" % fullField) fullFieldMap.put("1", fullField) self.utils.add(self.index, "display_type", displayType) # Make sure we have a creation date if not createdDateFlag: self.utils.add(self.index, "date_created", self.last_modified) self.log.debug( "Forced creation date to %s because it was not explicitly set." % self.last_modified) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def __activate__(self, context): self.auth = context["page"].authentication self.errorMsg = "" self.request = context["request"] self.response = context["response"] self.formData = context["formData"] self.storage = context["Services"].getStorage() self.log = context["log"] self.reportManager = context["Services"].getService("reportManager") fromOid = self.formData.get("fromOid") fromObject = self.storage.getObject(fromOid) if (self.auth.is_logged_in()): if (self.auth.is_admin() == True): pass elif (self.__isOwner(fromObject)): pass else: self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access." else: self.errorMsg = "Please login." if self.errorMsg == "": toOid = self.formData.get("toOid") toObject = self.storage.getObject(toOid) storeRelatedData = self.formData.get("relatedData") fromTFPackage = self._getTFPackage(fromObject) toTFPackage = self._getTFPackage(toObject) # get relevant dc:description from new object before you overwrite it with 'from' data, as this should exist from form data created in initial object packaging (packaging.py) toTFPackageJson = JsonSimple(toTFPackage.open()).getJsonObject() relevant_description = toTFPackageJson.get("dc:description") fromInputStream = fromTFPackage.open() try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream) except StorageException: print "error setting tfPackage" finally: fromTFPackage.close() tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue") if tfMetaPropertyValue == 'dmpToSelfSub': # fetch recently created new 'to' object from storage to get all data toTFPackage = self._getTFPackage(toObject) toTFPackageJson = JsonSimple(toTFPackage.open()).getJsonObject() self.setMultiDescription(toTFPackageJson, relevant_description) inStream = IOUtils.toInputStream(toTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), inStream) except StorageException: print "error setting description text in tfPackage" finally: inStream.close() self.log.info( "Completed migrating 'dc:description' to dc:decription.1.text' for oid: %s." % toTFPackage.getId()) self.log.debug("Result: %r" % toTFPackageJson) fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject() self.log.debug('from json is: %r' % fromTFPackageJson) if storeRelatedData != "false": # add relatedOid info fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid) self.log.debug('from tfPackage json is now: %r' % fromTFPackageJson) inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream) except StorageException: print "error setting tfPackage" finally: inStream.close() self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue) self._reharvestPackage() result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid, toOid) else: result = '{"status": "err", "message": "%s"}' % self.errorMsg writer = self.response.getPrintWriter("application/json; charset=UTF-8") writer.println(result) writer.close()
def __workflow(self): # Workflow data WORKFLOW_ID = "servicesUI2" wfChanged = False workflow_security = [] self.message_list = None stages = self.config.getJsonSimpleList(["stages"]) pageTitle = "Services Record" displayType = "package-service" initialStep = 0 try: wfMeta = self.__getJsonPayload("workflow.metadata") wfMeta.getJsonObject().put("pageTitle", pageTitle) # Are we indexing because of a workflow progression? targetStep = wfMeta.getString(None, ["targetStep"]) if targetStep is not None and targetStep != wfMeta.getString(None, ["step"]): wfChanged = True # Step change wfMeta.getJsonObject().put("step", targetStep) wfMeta.getJsonObject().remove("targetStep") # This must be a re-index then else: targetStep = wfMeta.getString(None, ["step"]) # Security change for stage in stages: if stage.getString(None, ["name"]) == targetStep: wfMeta.getJsonObject().put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) if wfChanged == True: self.message_list = stage.getStringList(["message"]) except StorageException: # No workflow payload, time to create initialStage = stages.get(initialStep).getString(None, ["name"]) wfChanged = True wfMeta = JsonSimple() wfMetaObj = wfMeta.getJsonObject() wfMetaObj.put("id", WORKFLOW_ID) wfMetaObj.put("step", initialStage) wfMetaObj.put("pageTitle", pageTitle) stages = self.config.getJsonSimpleList(["stages"]) for stage in stages: if stage.getString(None, ["name"]) == initialStage: wfMetaObj.put("label", stage.getString(None, ["label"])) self.item_security = stage.getStringList(["visibility"]) workflow_security = stage.getStringList(["security"]) self.message_list = stage.getStringList(["message"]) # Has the workflow metadata changed? if wfChanged == True: inStream = IOUtils.toInputStream(wfMeta.toString(True), "UTF-8") try: StorageUtils.createOrUpdatePayload(self.object, "workflow.metadata", inStream) except StorageException: print " ERROR updating dataset payload" # Form processing coreFields = ["title", "description", "manifest", "metaList", "relationships", "responses"] formData = wfMeta.getObject(["formData"]) if formData is not None: formData = JsonSimple(formData) # Core fields description = formData.getStringList(["description"]) if description: self.descriptionList = description # Non-core fields data = formData.getJsonObject() for field in data.keySet(): if field not in coreFields: self.customFields[field] = formData.getStringList([field]) # Manifest processing (formData not present in wfMeta) manifest = self.__getJsonPayload(self.packagePid) formTitles = manifest.getStringList(["title"]) if formTitles: for formTitle in formTitles: if self.title is None: self.title = formTitle self.descriptionList = [manifest.getString("", ["description"])] formData = manifest.getJsonObject() for field in formData.keySet(): if field not in coreFields: value = formData.get(field) if value is not None and value.strip() != "": self.utils.add(self.index, field, value) # We want to sort by date of creation, so it # needs to be indexed as a date (ie. 'date_*') if field == "dc:created": parsedTime = time.strptime(value, "%Y-%m-%d") solrTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsedTime) self.utils.add(self.index, "date_created", solrTime) # try to extract some common fields for faceting if field.startswith("dc:") and \ not (field.endswith(".dc:identifier.rdf:PlainLiteral") \ or field.endswith(".dc:identifier") \ or field.endswith(".rdf:resource")): # index dublin core fields for faceting basicField = field.replace("dc:", "dc_") dot = field.find(".") if dot > 0: facetField = basicField[:dot] else: facetField = basicField #print "Indexing DC field '%s':'%s'" % (field, facetField) if facetField == "dc_title": if self.title is None: self.title = value elif facetField == "dc_type": if self.dcType is None: self.dcType = value elif facetField == "dc_creator": if basicField.endswith("foaf_name"): self.utils.add(self.index, "dc_creator", value) else: self.utils.add(self.index, facetField, value) # index keywords for lookup if field.startswith("dc:subject.vivo:keyword."): self.utils.add(self.index, "keywords", value) self.utils.add(self.index, "display_type", displayType) # Workflow processing wfStep = wfMeta.getString(None, ["step"]) self.utils.add(self.index, "workflow_id", wfMeta.getString(None, ["id"])) self.utils.add(self.index, "workflow_step", wfStep) self.utils.add(self.index, "workflow_step_label", wfMeta.getString(None, ["label"])) for group in workflow_security: self.utils.add(self.index, "workflow_security", group) if self.owner is not None: self.utils.add(self.index, "workflow_security", self.owner) # set OAI-PMH status to deleted if wfStep == "retired": self.utils.add(self.index, "oai_deleted", "true")
def __attachFile(self): try: # WebKit/IE prefixes C:\fakepath\ with javascript manipulated file inputs uploadFile = self.formData.get("uploadFile") self.log.debug(uploadFile) self.log.debug(self.vc("sessionState").toString()) self.log.debug(self.formData.toString()) uploadFile = uploadFile.replace("C:\\fakepath\\", "") fileDetails = self.vc("sessionState").get(uploadFile) # Establish that we do have details on the uploaded file if fileDetails is None: #uploadFile = uploadFile.rsplit("\\", 1)[-1] fileDetails = self.vc("sessionState").get(uploadFile) if fileDetails is None: self.log.error("**** fileDetails is None!!! ***") return self.__toJson( {"error": "fileDetails is None (no upload file!)"}) self.log.debug("Attach Upload: fileDetails: '{}'", fileDetails) errorDetails = fileDetails.get("error") if errorDetails: self.log.error("ERROR: %s" % errorDetails) return self.__toJson({"error": errorDetails}) # Look for the storage info we need jsonFormData = JsonSimple(self.formData.get("json")) oid = jsonFormData.getString(None, "oid") fname = fileDetails.get("name") foid = fileDetails.get("oid") attachType = jsonFormData.getString("supporting-material", "attachmentType") self.log.debug("attach oid='{}', filename='{}', foid='{}'", [oid, fname, foid]) # Make sure it was actually stored try: attachObj = self.Services.getStorage().getObject(foid) except StorageException, e: return JsonSimple({"error": "Attached file - '%s'" % str(e)}) # Build up some metadata to store alongside the file attachFormData = JsonSimple(self.formData.get("json", "{}")) attachMetadata = { "type": "attachment", "created_by": "attachments.py", "formData": { "oid": foid, "attached_to": oid, "filename": fname, "access_rights": attachFormData.getString("private", ["accessRights"]), "attachment_type": attachType, "description": attachFormData.getString("", ["description"]), "owner": self.vc("page").authentication.get_username() } } # Record an attachment in attachments.metadata attachment = self.__createAttachmentJson( foid, attachMetadata["formData"]["attachment_type"], attachMetadata["formData"]["description"], attachMetadata["formData"]["filename"]) # print "Attaching %s" % attachment.toString().encode('utf-8') self.__createOrUpdateArrayPayload(oid, "attachments.metadata", attachment) # We are going to send an update on all attachments back with our response attachedFiles = self.__getAttachedFiles(oid, attachType) attachedFiles.append(dict(attachMetadata["formData"])) # Now store our metadata for this file try: jsonMetadata = self.__toJson(attachMetadata) jsonIn = ByteArrayInputStream(jsonMetadata.toString()) StorageUtils.createOrUpdatePayload(attachObj, "attachments.metadata", jsonIn) jsonIn.close() attachObj.close() except StorageException, e: self.log.error("Failed to create attachment metadata!", e)