def process(self): #We'll return a list with 1 JsonSimple object jsonList = [] data = None reader = None inStream = None document = None # Run the XML through our parser try: inStream = FileInputStream(File(self.file)) reader = InputStreamReader(inStream, "UTF-8") document = self.saxReader.read(reader) # Parse fails except: raise # Close our file access objects finally: if reader is not None: reader.close() if inStream is not None: inStream.close() # Now go looking for all our data data = JsonObject() data.put("workflow_source", "XML Alert") # Default self.__mapXpathToFields(document, self.map, data) if data is None: return None jsonList.append(JsonSimple(data)) return jsonList
def __getJson(self): rvtMap = JsonObject() try: oid = self.vc("formData").get("oid") object = Services.storage.getObject(oid) payload = object.getPayload("imsmanifest.xml") try: from xml.etree import ElementTree xmlStr = IOUtils.toString(payload.open(), "UTF-8") payload.close() xml = ElementTree.XML(xmlStr.encode("UTF-8")) ns = xml.tag[:xml.tag.find("}")+1] resources = {} for res in xml.findall(ns+"resources/"+ns+"resource"): resources[res.attrib.get("identifier")] = res.attrib.get("href") organizations = xml.find(ns+"organizations") defaultName = organizations.attrib.get("default") organizations = organizations.findall(ns+"organization") organizations = [o for o in organizations if o.attrib.get("identifier")==defaultName] organization = organizations[0] title = organization.find(ns+"title").text rvtMap.put("title", title) items = organization.findall(ns+"item") rvtMap.put("toc", self.__getJsonItems(ns, items, resources)) except Exception, e: data["error"] = "Error - %s" % str(e) print data["error"] object.close()
def send_message(self, oid): message = JsonObject() message.put("oid", oid) message.put("task", "curation-confirm") self.messaging.queueMessage( TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __getUserInfo(self, username): """ Query HibernateUser to get a user information There are users managed by internal auth manager with no attributes other than password There are users managed by external auth managers e.g. shibboleth who have attributes Each user, at the time of writing: 20140904, each user has multiple identical attribute sets, so, only the first one is used We put all available attributes of a user in to return value """ username = username.strip() authUserDao = ApplicationContextProvider.getApplicationContext().getBean("hibernateAuthUserDao") parameters = HashMap() parameters.put("username", username) userObjectList = authUserDao.query("getUser", parameters) userJson = JsonObject() userJson.put("username", username) try: if userObjectList.size() > 0: # One hit will be enough to get user object userJson = self.__constructUserAttribs(userObjectList.get(0), self.ATTRIB_FILTER) else: # This should not be reached with external sourced users self.log.warn("Wrong username or internal user is queried") except Exception, e: self.log.error("%s: cannot construct user attribute JSON, detail = %s" % (self.__class__.__name__ , str(e)))
def __activate__(self, context): request = context["request"] storage = context["Services"].getStorage() auth = context["page"].authentication log = context["log"] username = auth.get_name() oid = request.getParameter("oid") approval = request.getParameter("approval") approval_comment = request.getParameter("approval_comment") storedObj = storage.getObject(oid) committeeResponses = None payloadList = storedObj.getPayloadIdList() if payloadList.contains("committee-responses.metadata"): committeeResponsePayload = storedObj.getPayload("committee-responses.metadata") committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject() else: committeeResponses = JsonObject() committeeResponse = JsonObject() committeeResponse.put("approval",approval) committeeResponse.put("approval_comment",approval_comment) committeeResponses.put(username,committeeResponse) log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment)) StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8")) context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def sendMessage(self, oid): message = JsonObject() message.put("oid", oid) message.put("task", "reharvest") self.messaging.queueMessage( TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __getPackageTypes(self): object = self.sysConfig.getObject(["portal", "packageTypes"]) packageTypes = JsonSimple.toJavaMap(object) if packageTypes.isEmpty(): defaultPackage = JsonObject() defaultPackage.put("jsonconfig", "packaging-config.json") packageTypes.put("default", JsonSimple(defaultPackage)) return packageTypes
def sendMessage(self, oid, eventType): message = JsonObject() message.put("oid", oid) message.put("eventType", eventType) message.put("username", self.vc("page").authentication.get_username()) message.put("context", "Workflow") message.put("task", "workflow") self.messaging.queueMessage(TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def sendMessage(self, oid): # Fake a workflow reindex. ReDBox doesn't need to reindex, # the VITAL subscriber just needs to think we did. message = JsonObject() message.put("oid", oid) message.put("task", "publish") self.messaging.queueMessage( TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper") externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder") oid = self.request.getParameter("oid") if oid is None : identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) relationshipType = self.request.getParameter("relationship") curatedPid = self.request.getParameter("curatedPid") sourceId = self.request.getParameter("sourceIdentifier") system = self.request.getParameter("system") digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJsonPayload = digitalObject.getPayload("metadata.json") metadataJsonInstream = metadataJsonPayload.open() metadataJson = JsonSimple(metadataJsonInstream) metadataJsonPayload.close() relationships = metadataJson.getArray("relationships") found = False for relationship in relationships: if relationship.get("identifier") == sourceId: relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) found = True if not found: relationship = JsonObject() relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) relationship.put("relationship",relationshipType) relationship.put("identifier",sourceId) relationship.put("system",system) relationships.add(relationship) out.println(metadataJson.toString(True)) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream) out.close() finally: self.sessionState.remove("username")
def _addRelatedOid(self, tfPackageJson, relatedOid): relatedOids = tfPackageJson.getArray("related.datasets") if relatedOids is None: relatedOids = JSONArray() relatedOidJsonObject = JsonObject() relatedOidJsonObject.put("oid", relatedOid) relatedOids.add(relatedOidJsonObject) jsonObject = tfPackageJson.getJsonObject() jsonObject.put("related.datasets", relatedOids) return jsonObject
def _readReviewers(self, storedObj, tfpackage): """Read from TFPACKAGE for reviewer's recommendation and map to a json with short keys: reviewer-recommend-for : for reviewer-recommended-storage : storage """ reviewersPayload = storedObj.getPayload(tfpackage) reviewersRecommends = JsonSimple(reviewersPayload.open()).getJsonObject() reviewers = JsonObject() reviewers.put("for", reviewersRecommends.get("reviewer-recommend-for")) reviewers.put("storage", reviewersRecommends.get("reviewer-recommended-storage")) return reviewers
def transformLegacyArrays(self, originalObject): outputJsonObject = JsonObject() dataUtil = StorageDataUtil() outputJsonObject = JsonObject() jsonObject = originalObject.getJsonObject() for keyString in jsonObject.keySet(): if self.isAnArrayKey(keyString): prefix = self.getPrefix(keyString); outputJsonObject.put(prefix, dataUtil.getJavaList(json,prefix)); else: outputJsonObject.put(keyString, jsonObject.get(keyString)); return JsonSimple(outputJsonObject)
def __activate__(self, context): self.log = context["log"] self.services = context["Services"] auth = context["page"].authentication username = auth.get_username() # get system username try: result = self.__constructInfoJson(username) except Exception, e: self.log.error("%s: cannot get user attributes, detail = %s" % (self.__class__.__name__ , str(e))) result = JsonObject() result.put("realName", auth.get_name()) # default value: user's realName
def getCurationData(self, oid): json = JsonObject() try: # Get the object from storage storage = self.Services.getStorage() object = storage.getObject(oid) # Find the package payload payload = object.getPayload("metadata.json") # Not found? if payload is None: self.log.error(" * detail.py => Can't find package data!") json.put("error", True) return json # Parse the data data = JsonSimple(payload.open()) payload.close() # Return it json.put("error", False) json.put("relationships", data.writeArray("relationships")) return json except StorageException, ex: self.log.error(" * detail.py => Storage Error accessing data: ", ex) json.put("error", True) return json
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper") externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder") oid = self.request.getParameter("oid") if oid is None : identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) relationshipType = self.request.getParameter("relationship") curatedPid = self.request.getParameter("curatedPid") sourceId = self.request.getParameter("sourceIdentifier") digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJson = self.getTfPackage(digitalObject) relationships = metadataJson.getArray("relationships") found = False for relationship in relationships: if relationship.get("identifier") == sourceId: relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) found = True if not found: relationship = JsonObject() relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) relationship.put("relationship",relationshipType) relationship.put("identifier",sourceId) relationships.add(relationship) self.log.info(metadataJson.toString(True)) out.println(metadataJson.toString(True)) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) for pid in digitalObject.getPayloadIdList(): if pid.endswith(".tfpackage"): StorageUtils.createOrUpdatePayload(digitalObject,pid,istream) out.close() finally: self.sessionState.remove("username")
def __constructUserJson(self, username): """ There are users managed by internal auth manager with no attributes There are users managed by external auth manages e.g. shibboleth who have attributes These users username is not necessarily the same as there normal display name This function currently solves this issue by checking commonName attribute for shibboleth users """ username = username.strip() userJson = JsonObject() userJson.put("userName", username) parameters = HashMap() # print "Checking user info for %s" % username parameters.put("username", username) userObjectList = self.authUserDao.query("getUser", parameters) # print "Returned size = %d" % userObjectList.size() if userObjectList.size() > 0: userObject = userObjectList.get(0) #Check if this is a user with attributes? attrb = userObject.getAttributes().get("commonName") if attrb is None: # print "We cannot find so called commonName, use %s instead" % username userJson.put("realName", username) else: # print "We found so called commonName, use %s instead of %s" % (attrb.getValStr(), username) userJson.put("realName", attrb.getValStr().strip()) else: # This should not be reached self.log.warn("What is going on here? why ends up here?") userJson.put("realName", username) return userJson
def regenerate_key(self): clientArray = self.getKeysArray() name = self.formData.get("name") clientToBeReplaced = None index = 0 for client in clientArray: if client.get("name") == name: clientToBeReplaced = client break index = index + 1 if clientToBeReplaced is not None: clientObject = JsonObject() clientObject.put("name", self.formData.get("name")) clientObject.put("key", self.get_random_key()) clientArray.set(index,clientObject) self.apiKeyService.updateAndSaveKeys(clientArray)
def getFullHistory(self, oids): """Query the history and save the latest full set of workflow steps to the return JsonObject""" req = SearchRequest('context:"Workflow" AND newStep:[* TO *] AND oid:' + oids + '') req.setParam("fl",'eventTime,newStep,oid') req.setParam("rows", Dashboard.MAX_ROWS) req.setParam("sort", "oid asc, eventTime asc") events = self._packageResults(req, "eventLog") latest = JsonObject() for e in events: oid = e.get("oid") if oid not in latest: jObj = JsonObject() latest.put(oid,jObj) else: jObj = latest.get(oid) jObj.put('%s_eventTime' % e.get("newStep"), self.formatDate(e.get("eventTime"))) return latest
def __createAttachmentJson(self, oid, attachment_type, description, filename, access_right="public"): """ Create a JsonObject which contains basic information of an attachment # An attachment is a JsonObject with keys: # { # "access_rights": "public", # "attachment_type": "review-attachments", # "description": "fileA", # "filename": "boto.config", # "oid": "ad8fa8238ce10813ef156a0a88262174" # } """ attachment = JsonObject() attachment.put("oid", oid) attachment.put("attachment_type", attachment_type) attachment.put("description", description) attachment.put("filename", filename) attachment.put("access_right", access_right) return attachment
def __activate__(self, context): self.sysConfig = JsonSimpleConfig() self.velocityContext = context self.log = context["log"] self.__meta = {} formData = self.vc("formData") self.isAjax = formData.get("ajax") != None if self.isAjax: ok = JsonObject() ok.put("ok", "OK") self.json = ok.toString() else: self.json = "" self.__selectedPackageType = formData.get("packageType", "default") self.log.debug("formData = %s" % self.vc("formData")) self.log.debug("selectedPackageType = '%s'" % self.__selectedPackageType) self.__meta["packageType"] = formData.get("packageType", "default") self.__meta["description"] = formData.get("description", "")
def updateRelationships(self, relationship,pid,identifier): oid = self.findOidByIdentifier(relationship.get("identifier")) self.writer.println(oid) digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJsonPayload = digitalObject.getPayload("metadata.json") metadataJsonInstream = metadataJsonPayload.open() metadataJson = JsonSimple(metadataJsonInstream) metadataJsonPayload.close() relationships = metadataJson.getArray("relationships") found = False if relationships is None: relationships = JSONArray() metadataJson.getJsonObject().put("relationships",relationships) for relationship1 in relationships: if relationship1.get("identifier") == identifier: relationship1.put("isCurated",True) relationship1.put("curatedPid",pid) found = True if not found: newRelationship = JsonObject() newRelationship.put("isCurated",True) newRelationship.put("curatedPid",pid) newRelationship.put("relationship",relationship.get("relationship")) newRelationship.put("identifier",identifier) relationships.add(newRelationship) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream)
def __constructInfoJson(self, username): """ There are users managed by internal auth manager with no attributes There are users managed by external auth managers e.g. shibboleth who have attributes We put all available attributes of a user in to return value """ # print "Query username = %s" % username username = username.strip() authUserDao = ApplicationContextProvider.getApplicationContext().getBean("hibernateAuthUserDao") parameters = HashMap() parameters.put("username", username) userObjectList = authUserDao.query("getUser", parameters) # print "Returned object = %s" % str(userObjectList) # print "Returned size = %d" % userObjectList.size() userJson = JsonObject() try: if userObjectList.size() > 0: # One hit will be enough to get user object userObj = userObjectList.get(0) attrbs = userObj.getAttributes() for attrb in attrbs.keySet(): # print "Attribute %s = %s) % (attrb, attrbs.get(attrb).getValStr()) userJson.put(attrb, attrbs.get(attrb).getValStr()) else: # This should not be reached with external sourced users self.log.warn("Wrong username? Every user should have a record") userJson.put("userName", username) except Exception, e: self.log.error("%s: cannot construct user attribute JSON, detail = %s" % (self.__class__.__name__ , str(e))) userJson.put("userName", username)
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) responseObject = JsonObject() responseObject.put("oid", oid) out.println(JsonSimple(responseObject).toString(True)) out.close() finally: self.sessionState.remove("username")
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) responseObject = JsonObject() responseObject.put("oid",oid) out.println(JsonSimple(responseObject).toString(True)) out.close() finally: self.sessionState.remove("username")
def __activate__(self, context): self.request = context['request'] self.response = context['response'] self.services = context['Services'] self.sessionState = context["sessionState"] #Assumption for time being is that users have admin access try: self.sessionState.set("username", "admin") self.storage = self.services.storage oid = self.request.getParameter("oid") tfPackage = self.getTfPackage(oid, self.getTFPackagePid(oid)) objectMeta = self.getObjectMeta(oid) workflowMeta = self.getWorkflowMeta(oid) jsonObject = JsonObject() jsonObject.put("tfPackage", tfPackage.getJsonObject()) jsonObject.put("workflow.metadata", workflowMeta.getJsonObject()) jsonObject.put("TF-OBJ-META", self.getObjectMetaJson(objectMeta)) writer = self.response.getPrintWriter("application/json; charset=UTF-8") writer.println(JsonSimple(jsonObject).toString(True)) writer.close() finally: self.sessionState.remove("username")
def __activate__(self, context): self.log = context["log"] self.services = context["Services"] auth = context["page"].authentication if not auth.is_admin(): result = JsonObject() # only in this case, returning JSON has status to get attention result.put("status", "403") result.put("message","Error: Only administrative users can access this feature") self.__respond(context["response"], result) return qterm = context["formData"].get("qt") qvalue = context["formData"].get("qv") if not qterm: result = self.listUserAttributes() else: try: result = self.getUsers(qterm, qvalue) except Exception, e: self.log.error("%s: cannot get user attributes, detail = %s" % (self.__class__.__name__ , str(e))) result = JsonObject()
def getAttachedFiles(self, oid): # Build a query req = SearchRequest("attached_to:%s" % oid) req.setParam("rows", "1000") # Run a search out = ByteArrayOutputStream() self.Services.getIndexer().search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Process results docs = JSONArray() for doc in result.getResults(): attachmentType = self.escapeHtml(WordUtils.capitalizeFully(doc.getFirst("attachment_type").replace("-", " "))) accessRights = self.escapeHtml(WordUtils.capitalizeFully(doc.getFirst("access_rights"))) entry = JsonObject() entry.put("filename", self.escapeHtml(doc.getFirst("filename"))) entry.put("attachment_type", attachmentType) entry.put("access_rights", accessRights) entry.put("id", self.escapeHtml(doc.getFirst("id"))) docs.add(entry) return docs
def getCurationData(self, oid): json = JsonObject() try: # Get the object from storage storage = self.Services.getStorage() object = storage.getObject(oid) # Find the package payload payload = None pidList = object.getPayloadIdList() for pid in pidList: if (pid.endswith(".tfpackage")): payload = object.getPayload(pid) # Not found? if payload is None: self.log.error(" * detail.py => Can't find package data!") json.put("error", True) return json # Parse the data data = JsonSimple(payload.open()) payload.close() # Some basic cosmetic fixes relations = data.writeArray("relationships") for relation in relations: if not relation.containsKey("field"): relation.put("field", "From Object " + relation.get("oid")) # Return it json.put("error", False) json.put("relationships", relations) return json except StorageException, ex: self.log.error(" * detail.py => Storage Error accessing data: ", ex) json.put("error", True) return json
def getAttachedFiles(self, oid): # Build a query req = SearchRequest("attached_to:%s" % oid) req.setParam("rows", "1000") # Run a search out = ByteArrayOutputStream() self.Services.getIndexer().search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Process results docs = JSONArray() for doc in result.getResults(): attachmentType = self.escapeHtml( WordUtils.capitalizeFully( doc.getFirst("attachment_type").replace("-", " "))) accessRights = self.escapeHtml( WordUtils.capitalizeFully(doc.getFirst("access_rights"))) entry = JsonObject() entry.put("filename", self.escapeHtml(doc.getFirst("filename"))) entry.put("attachment_type", attachmentType) entry.put("access_rights", accessRights) entry.put("id", self.escapeHtml(doc.getFirst("id"))) docs.add(entry) return docs
def add_key(self): clientArray = self.getKeysArray() name = self.formData.get("name") for client in clientArray: if client.get("name") == name: return clientObject = JsonObject() clientObject.put("username", self.formData.get("name")) if(self.formData.get("generateKey") == "true"): clientObject.put("apiKey", self.get_random_key()) else: clientObject.put("apiKey", self.formData.get("key")) clientArray.add(clientObject) self.apiKeyService.updateAndSaveKeys(clientArray)
def getCurationData(self, oid): json = JsonObject() try: # Get the object from storage storage = self.Services.getStorage() object = storage.getObject(oid) # Find the package payload payload = None pidList = object.getPayloadIdList() for pid in pidList: if (pid.endswith(".tfpackage")): payload = object.getPayload(pid) # Not found? if payload is None: self.log.error(" * detail.py => Can't find package data!") json.put("error", True) return json # Parse the data data = JsonSimple(payload.open()) payload.close() # Some basic cosmetic fixes relations = data.writeArray("relationships") for relation in relations: if not relation.containsKey("field"): relation.put("field", "From Object "+relation.get("oid")) # Return it json.put("error", False) json.put("relationships", relations) return json except StorageException, ex: self.log.error(" * detail.py => Storage Error accessing data: ", ex) json.put("error", True) return json
def __constructUserAttribs(self, authUser, filter=["password"]): """ Construct a JSON object to store all attributes of a user object Filter out do not wanted """ userJson = JsonObject() # Get ID and username first userJson.put("id", authUser.getId()) userJson.put("username", authUser.getUsername()) attrbs = authUser.getAttributes() filtered = [attribName for attribName in attrbs.keySet() if attribName not in filter] for attribName in filtered: attribValue = attrbs.get(attribName).getValStr() userJson.put(attribName, attribValue) return userJson
def __constructUserAttribs(self, authUser, filter=["password"]): """ Construct a JSON object to store all attributes of a user object Filter out do not wanted """ userJson = JsonObject() # Get ID and username first userJson.put("id", authUser.getId()) userJson.put("username", authUser.getUsername()) attrbs = authUser.getAttributes() filtered = [ attribName for attribName in attrbs.keySet() if attribName not in filter ] for attribName in filtered: attribValue = attrbs.get(attribName).getValStr() userJson.put(attribName, attribValue) return userJson
def getJson(self, state="open"): title = "%s (%s)" % (self.getName(), self.getCount()) json = JsonSimple() jsonObj = json.getJsonObject() attributes = JsonObject() attributes.put("id", self.getId()) attributes.put("fq", self.getFacetQuery()) attributes.put("title", title) jsonObj.put("data", title) jsonObj.put("attributes", attributes) hasSubFacets = not self.getSubFacets().isEmpty() if hasSubFacets: jsonObj.put("state", state) subFacetList = ArrayList() for subFacet in self.getSubFacets(): subFacetList.add(subFacet.getJson("closed")) children = JSONArray() children.addAll(subFacetList) jsonObj.put("children", children) return json
def searchNla(self): # Start row start = self.formData.get("start") if start is None: start = "1" # Row limit rows = self.formData.get("rows") if rows is None: rows = "10" # Query query = "" # Test searches #query = "pa.surname=\"Smith\"" #query = "cql.anywhere=\"monash\"" if query == "": # Surname surname = self.formData.get("surname") if surname is not None: query = "pa.surname=\"%s\"" % (surname) # Firstname firstName = self.formData.get("firstName") if firstName is not None: if query != "": query += " AND " query += "pa.firstname=\"%s\"" % (firstName) if query != "": query += " AND " query += "pa.type=\"person\"" # Some basic metadata we already know self.metadata.put("searchString", query) self.metadata.put("startRecord", start) self.metadata.put("rowsRequested", rows) # Search NLA self.log.debug("Submitting query to NLA: '{}', Start: '{}', Rows: '{}'", [query, start, rows]) sru = SRUClient() # If using the NLA's test server, comment out the line above and uncomment the line below # sru = SRUClient("http://www-test.nla.gov.au/apps/srw/search/peopleaustralia") response = sru.nlaGetResponseBySearch(query, start, rows) if response is None: raise Exception("Error searching NLA. Please check system logs for details.") self.metadata.put("rowsReturned", response.getRows()) self.metadata.put("totalHits", response.getTotalResults()) identities = NLAIdentity.convertNodesToIdentities(response.getResults()) for id in identities: idEntry = JsonObject() idEntry.put("nlaId", id.getId()) idEntry.put("displayName", id.getDisplayName()) idEntry.put("firstName", id.getFirstName()) idEntry.put("surname", id.getSurame()) idEntry.put("institution", id.getInstitution()) knownIds = JSONArray() idEntry.put("knownIdentities", knownIds) for knownId in id.getKnownIdentities(): thisId = JsonObject() thisId.put("displayName", knownId.get("displayName")) thisId.put("institution", knownId.get("institution")) knownIds.add(thisId) self.results.add(idEntry) return True
except Exception, e: ## Move the XML to the 'failed' directory shutil.move(filePath, self.pFail(fileName)) ## And write our error data to disk beside it self.writeError(fileName, e) return None # Close our file access objects finally: if reader is not None: reader.close() if inStream is not None: inStream.close() # Now go looking for all our data json = JsonObject() json.put("workflow_source", "XML Ingest") # Default self.__mapXpathToFields(document, xmlMappings, xmlExceptions, json) # Operational fields json.put("viewId", "default") json.put("packageType", "dataset") json.put("redbox:formVersion", self.redboxVersion) json.put("redbox:newForm", "true") json.put("redbox:submissionProcess.redbox:submitted", "true") json.put("redbox:submissionProcess.dc:date", time.strftime("%Y-%m-%d %H:%M:%S", timestamp)) return JsonSimple(json) ## Used recursively def __mapXpathToFields(self, sourceData,
def __activate__(self, context): response = context["response"] json = JsonObject() auth = context["page"].authentication if auth.is_logged_in(): formData = context["formData"] oid = formData.get("oid") if oid: # TODO check security on object json.put("oid", oid) try: metaNode = JsonObject() json.put("meta", metaNode) object = context["Services"].storage.getObject(oid) metadata = object.getMetadata() for key in metadata.keySet(): metaNode.put(key, metadata.get(key)) except StorageException: response.setStatus(500) json.put("error", "Object '%s' not found" % oid) else: response.setStatus(500) json.put("error", "An object identifier is required") else: response.setStatus(500) json.put("error", "Only registered users can access this API") writer = response.getPrintWriter("text/plain; charset=UTF-8") writer.println(json.toString()) writer.close()
def getObjectMetaJson(self,objectMeta): objMetaJson = JsonObject() propertyNames = objectMeta.stringPropertyNames() for propertyName in propertyNames: objMetaJson.put(propertyName,objectMeta.get(propertyName)) return objMetaJson
def __sendMessage(self, oid, step): message = JsonObject() message.put("oid", oid) if step is None: message.put("eventType", "ReIndex") else: message.put("eventType", "NewStep : %s" % step) message.put("newStep", step) message.put("username", "admin") message.put("context", "Workflow") message.put("task", "workflow") self.messaging.queueMessage( TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def __activate__(self, context): self.log = context["log"] response = context["response"] writer = response.getPrintWriter("text/plain; charset=UTF-8") auth = context["page"].authentication sessionState = context["sessionState"] result = JsonObject() result.put("status", "error") result.put("message", "An unknown error has occurred") if auth.is_admin(): services = context["Services"] formData = context["formData"] func = formData.get("func") oid = formData.get("oid") portalId = formData.get("portalId") portalManager = services.portalManager if func == "reharvest": if oid: self.log.debug("Reharvesting object '{}'", oid) self.sendMessage(oid) result.put("status", "ok") result.put("message", "Object '%s' queued for reharvest") elif portalId: self.log.debug("Reharvesting view '{}'", portalId) sessionState.set("reharvest/running/" + portalId, "true") # TODO security filter - not necessary because this requires admin anyway? portal = portalManager.get(portalId) query = "*:*" if portal.query != "": query = portal.query if portal.searchQuery != "": if query == "*:*": query = portal.searchQuery else: query = query + " AND " + portal.searchQuery # query solr to get the objects to reharvest rows = 25 req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') req.setParam("rows", str(rows)) req.setParam("fl", "id") done = False count = 0 while not done: req.setParam("start", str(count)) out = ByteArrayOutputStream() services.indexer.search(req, out) json = SolrResult( ByteArrayInputStream(out.toByteArray())) objectIds = HashSet(json.getFieldList("id")) if not objectIds.isEmpty(): for oid in objectIds: self.sendMessage(oid) count = count + rows total = json.getNumFound() self.log.debug("Queued {} of {}...", min(count, total), total) done = (count >= total) sessionState.remove("reharvest/running/" + portalId) result.put("status", "ok") result.put( "message", "Objects in '%s' queued for reharvest" % portalId) else: response.setStatus(500) result.put("message", "No object or view specified for reharvest") elif func == "reindex": if oid: self.log.debug("Reindexing object '{}'", oid) self.sendMessage(oid) result.put("status", "ok") result.put("message", "Object '%s' queued for reindex" % oid) else: response.setStatus(500) result.put("message", "No object specified to reindex") else: response.setStatus(500) result.put("message", "Unknown action '%s'" % func) else: response.setStatus(500) result.put("message", "Only administrative users can access this API") writer.println(result.toString()) writer.close()
def _reharvestPackage(self): message = JsonObject() message.put("oid", self.formData.get("toOid")) message.put("task", "reharvest") self.messaging.queueMessage( TransactionManagerQueueConsumer.LISTENER_ID, message.toString())
def process(self): self.log.debug(" * settings.py: formData={}", self.vc("formData")) valid = self.vc("page").csrfSecurePage() if not valid: self.throw_error("Invalid request") return result = "{}" portalManager = self.vc("Services").getPortalManager() portal = portalManager.get(self.vc("portalId")) func = self.vc("formData").get("func") if func == "view-update": portal.setDescription(self.vc("formData").get("view-description")) portal.setQuery(self.vc("formData").get("view-query")) portal.setSearchQuery(self.vc("formData").get("view-search-query")) portal.setRecordsPerPage( int(self.vc("formData").get("view-records-per-page"))) portal.setFacetCount( int(self.vc("formData").get("view-facet-count"))) portal.setFacetDisplay( int(self.vc("formData").get("view-facet-display"))) portal.setFacetSort( self.vc("formData").get("view-facet-sort") is not None) portalManager.save(portal) elif func == "general-update": config = JsonSimpleConfig() email = StringUtils.trimToEmpty( self.vc("formData").get("general-email")) systemEmail = StringUtils.trimToEmpty( config.getString(None, ["email"])) if systemEmail != email: obj = config.writableSystemConfig() obj.put("email", self.vc("formData").get("general-email")) obj.put("configured", "true") config.storeSystemConfig() # mark restart self.vc("Services").getHouseKeepingManager( ).requestUrgentRestart() else: self.log.debug( " * settings.py: email not updated: did not change") self.throw_error("Email address is the same! No change saved.") elif func == "facets-update": portal.getObject(["portal"]).remove("facet-fields") fields = self.vc("formData").getValues("field") labels = self.vc("formData").getValues("label") displays = self.vc("formData").getValues("display") deletes = self.vc("formData").getValues("delete") for i in range(0, len(fields)): field = fields[i] if deletes[i] == "false": node = portal.writeObject( ["portal", "facet-fields", field]) node.put("label", labels[i]) node.put("display", displays[i]) portalManager.save(portal) elif func == "sort-update": portal.getObject(["portal"]).remove("sort-fields") default = self.vc("formData").get("default") if default: portal.setSortFieldDefault(default) order = self.vc("formData").get("order") if order: portal.setSortFieldDefaultOrder(order) fields = self.vc("formData").getValues("field") labels = self.vc("formData").getValues("label") deletes = self.vc("formData").getValues("delete") for i in range(0, len(fields)): field = fields[i] if deletes[i] == "false": node = portal.writeObject(["portal", "sort-fields"]) node.put(field, labels[i]) portalManager.save(portal) elif func == "watcher-update": configFile = self.getWatcherFile() if configFile is not None: json = JsonSimpleConfig(configFile) pathIds = self.vc("formData").get("pathIds").split(",") actives = self.vc("formData").getValues("watcher-active") if actives is None: actives = [] deletes = self.vc("formData").getValues("watcher-delete") if deletes is None: deletes = [] for pathId in pathIds: if pathId not in deletes: path = self.vc("formData").get("%s-path" % pathId) stopped = str(pathId not in actives).lower() watchDir = json.writeObject( ["watcher", "watchDirs", path]) watchDir.put( "ignoreFileFilter", self.vc("formData").get("%s-file" % pathId)) watchDir.put( "ignoreDirectories", self.vc("formData").get("%s-dir" % pathId)) watchDir.put("stopped", stopped) json.writeArray( ["watcher", "watchDirs", path, "cxtTags"]) writer = FileWriter(configFile) writer.write(json.toString(True)) writer.close() else: result = "The Watcher is not installed properly." elif func == "restore-default-config": # backup the file JsonSimpleConfig.backupSystemFile() # delete the file JsonSimpleConfig.getSystemFile().delete() # restore default JsonSimpleConfig.getSystemFile() # mark restart self.vc("Services").getHouseKeepingManager().requestUrgentRestart() elif func == "housekeeping-update": config = JsonSimpleConfig() freq = StringUtils.trimToEmpty( self.vc("formData").get("housekeeping-timeout")) systemFreq = StringUtils.trimToEmpty( config.getString( None, ["portal", "houseKeeping", "config", "frequency"])) result = "House Keeper refreshed" if systemFreq != freq: # Get writeable access to underlying system sysConfig = JsonSimple(config.writableSystemConfig()) # Modify the house keeping node hkConfig = sysConfig.writeObject( ["portal", "houseKeeping", "config"]) hkConfig.put("frequency", freq) # Write the underlying config back to disk config.storeSystemConfig() result = "Frequency updated, refreshing House Keeper" # Refresh the HouseKeeper message = JsonObject() message.put("type", "refresh") self.vc("Services").getHouseKeepingManager().sendMessage( message.toString()) self.writer.println(result) self.writer.close()