def __activate__(self, context): self.request = context["request"] self.response = context["response"] self.formData = context["formData"] self.log = context["log"] # Basic response text message = JsonSimple() self.metadata = message.writeObject(["metadata"]) self.results = message.writeArray(["results"]) # Prepare response Object format = self.formData.get("format") if format == "json": out = self.response.getPrintWriter("application/json; charset=UTF-8") else: out = self.response.getPrintWriter("text/plain; charset=UTF-8") # Success Response try: self.searchNla() out.println(message.toString(True)) out.close() except Exception, ex: self.log.error("Error during search: ", ex) self.response.setStatus(500) message = JsonSimple() message.getJsonObject().put("error", ex.getMessage()) out.println(message.toString(True)) out.close()
def __activate__(self, context): self.auth = context["page"].authentication self.errorMsg = "" self.request = context["request"] self.response = context["response"] self.formData = context["formData"] self.storage = context["Services"].getStorage() self.log = context["log"] self.reportManager = context["Services"].getService("reportManager") fromOid = self.formData.get("fromOid") fromObject = self.storage.getObject(fromOid) if (self.auth.is_logged_in()): if (self.auth.is_admin() == True): pass elif (self.__isOwner(fromObject)): pass else: self.errorMsg = "Requires Admin / Librarian / Reviewer / owner access." else: self.errorMsg = "Please login." if self.errorMsg == "": toOid = self.formData.get("toOid") toObject = self.storage.getObject(toOid) storeRelatedData = self.formData.get("relatedData") fromTFPackage = self._getTFPackage(fromObject) toTFPackage = self._getTFPackage(toObject) fromInputStream = fromTFPackage.open() try: StorageUtils.createOrUpdatePayload(toObject, toTFPackage.getId(), fromInputStream) except StorageException: print "error setting tfPackage" fromTFPackage.close() fromTFPackageJson = JsonSimple(fromTFPackage.open()).getJsonObject() if storeRelatedData != "false" : # add relatedOid info fromTFPackageJson = self._addRelatedOid(JsonSimple(fromTFPackage.open()), toOid) inStream = IOUtils.toInputStream(fromTFPackageJson.toJSONString(), "UTF-8") try: StorageUtils.createOrUpdatePayload(fromObject, fromTFPackage.getId(), inStream) except StorageException: print "error setting tfPackage" tfMetaPropertyValue = self.formData.get("tfMetaPropertyValue") self._addPropertyValueToTFMeta(toObject, tfMetaPropertyValue) self._reharvestPackage() result = '{"status": "ok", "url": "%s/workflow/%s", "oid": "%s" }' % (context["portalPath"], toOid , toOid) else: result = '{"status": "err", "message": "%s"}' % self.errorMsg writer = self.response.getPrintWriter("application/json; charset=UTF-8") writer.println(result) writer.close()
def __activate__(self, context): self.log = context["log"] self.request = context["request"] self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") processingSet = self.request.getParameter("processingSet") self.procMsg = None # read configuration and trigger processing stream sets # storing the return object on the map configFilePath = FascinatorHome.getPath( "process") + "/processConfig.json" procConfigFile = File(configFilePath) if procConfigFile.exists() == True: self.dataMap = HashMap() self.dataMap.put("indexer", context['Services'].getIndexer()) self.procConfigJson = JsonSimple(procConfigFile) for configObj in self.procConfigJson.getJsonArray(): configJson = JsonSimple(configObj) procId = configJson.getString("", "id") if processingSet is not None: if procId == processingSet: self.execProcSet(procId, configJson) else: self.execProcSet(procId, configJson) if self.procMsg is None: self.procMsg = "Processing complete!" else: self.procMsg = "Configuration file does not exist: " + configFilePath
def getLabel(self, jsonFile, key): value = self.metadata.get(key) jsonLabelFile = System.getProperty("fascinator.home") + jsonFile entries = JsonSimple(File(jsonLabelFile)).getJsonArray() for entry in entries: entryJson = JsonSimple(entry) if value == entryJson.getString("", "value"): return entryJson.getString("", "label") return None
def __activate__(self, context): self.velocityContext = context formData = self.vc("formData") # build the URL and query parameters to retrieve proxyUrls = JsonSimple(self.vc("systemConfig").getObject("proxy-urls")) url = "" key = formData.get("ns", "") if proxyUrls.getJsonObject().containsKey(key): url = proxyUrls.getString("", [key]) queryStr = formData.get("qs") if queryStr == "searchTerms={searchTerms}": queryStr = None if queryStr: if formData.get("jaffa2autocomplete", "false") == "true": url += "?searchTerms=%s" % queryStr.lower() else: url += "?%s" % queryStr self.vc("log").debug("Proxy URL = '{}'", url) data = None try: data = self.__wget(url) except Exception, e: data = '{"error":"%s"}' % str(e) self.vc("log").error("ERROR accessing URL:", e)
def process(self): '''Read the XML file and map xpath items to metadata Return a list with 1 JsonSimple object (at most) ''' jsonList = [] data = None reader = None inStream = None document = None # Run the XML through our parser try: inStream = FileInputStream(File(self.file)) reader = InputStreamReader(inStream, "UTF-8") document = self.saxReader.read(reader) # Parse fails except: raise # Close our file access objects finally: if reader is not None: reader.close() if inStream is not None: inStream.close() # Now go looking for all our data data = self.getNewJsonObject() self.__mapXpathToFields(document, self.map, data) if data is None: return None jsonList.append(JsonSimple(data)) return jsonList
def process(self): #We'll return a list with 1 JsonSimple object jsonList = [] data = None reader = None inStream = None document = None # Run the XML through our parser try: inStream = FileInputStream(File(self.file)) reader = InputStreamReader(inStream, "UTF-8") document = self.saxReader.read(reader) # Parse fails except: raise # Close our file access objects finally: if reader is not None: reader.close() if inStream is not None: inStream.close() # Now go looking for all our data data = JsonObject() data.put("workflow_source", "XML Alert") # Default self.__mapXpathToFields(document, self.map, data) if data is None: return None jsonList.append(JsonSimple(data)) return jsonList
def parseFFmpeg(self, parent): if parent is not None: object = parent.getObject() if object is not None: payload = None try: payload = object.getPayload("ffmpeg.info") # Stream the content out to string out = ByteArrayOutputStream() IOUtils.copy(payload.open(), out) payload.close() self.__ffmpegRaw = out.toString("UTF-8") out.close() payload.close() # And parse it self.__ffmpegData = JsonSimple(self.__ffmpegRaw) if self.__ffmpegData is None: return False else: self.__ffmpegOutputs = self.__ffmpegData.getJsonSimpleMap( ["outputs"]) return True except: if payload is not None: payload.close() return False
def __hotfix(self, formData): oldType = formData.getString(None, ["dc:type"]) newType = formData.getString(None, ["dc:type.rdf:PlainLiteral"]) if oldType != newType or newType is None: self.log.debug("Bugged Type?: v1.4: '{}', OLD: '{}'", newType, oldType) else: ## No fix required return None ## Get Backup data ## NOTE: The only known production system affected by this bug ## was caught during a v1.4 upgrade. Alter this line if required. pid = "1.4.workflow.backup" oldData = None try: payload = self.object.getPayload(pid) try: oldData = JsonSimple(payload.open()) except Exception: self.log.error("Error parsing JSON '{}'", pid) finally: payload.close() except StorageException: self.log.error("Error accessing '{}'", pid) return None oldType = oldData.getString(None, ["dc:type"]) self.log.debug("Old Type: '{}' => 'dc:type.rdf:PlainLiteral'", oldType) formData.getJsonObject().put("dc:type.rdf:PlainLiteral", oldType) return formData
def __activate__(self, context): self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] out = self.response.getPrintWriter("text/plain; charset=UTF-8") callType = self.request.getParameter("callType") apiClass = self.systemConfig.getObject("api").get(callType) className = apiClass.get("className") apiCallClass = Class.forName(className) apiCallObject = apiCallClass.newInstance() setScriptingServiceMethod = apiCallClass.getMethod( "setScriptingServices", self.get_class( "com.googlecode.fascinator.portal.services.ScriptingServices")) setScriptingServiceMethod.invoke(apiCallObject, context['Services']) if callType == "mint-stats": setScriptingServiceMethod = apiCallClass.getMethod( "setConfig", self.get_class("com.googlecode.fascinator.common.JsonSimple")) setScriptingServiceMethod.invoke( apiCallObject, JsonSimple(self.systemConfig.getObject("api", "mint-stats"))) handleRequestMethod = apiCallClass.getMethod( "handleRequest", self.get_class("org.apache.tapestry5.services.Request")) responseString = handleRequestMethod.invoke(apiCallObject, context["request"]) out.println(responseString) out.close()
def process_tags(self, result): tags = [] tagsDict = {} # Build a dictionary of the tags for doc in result: # Get Anotar data from Solr data doc = JsonSimple(doc.get("jsonString")) # Get actual tag text tag = doc.getString(None, ["content", "literal"]) # Find out if they have locators locs = doc.getJsonSimpleList(["annotates", "locators"]).size() if locs == 0: # Basic tags, just aggregate counts if tag in tagsDict: # We've seen it before, just increment the counter existing = tagsDict[tag] count = existing.getInteger(0, ["tagCount"]) existing.getJsonObject().put("tagCount", str(count + 1)) else: # First time, store this object doc.getJsonObject().put("tagCount", str(1)) tagsDict[tag] = doc else: # Tags with a locator, special case for images etc. tags.append(doc.toString()) # Push all the 'basic' counts into the list to return for tag in tagsDict: tags.append(tagsDict[tag].toString()) return "[" + ",".join(tags) + "]"
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") relationshipMapper = ApplicationContextProvider.getApplicationContext( ).getBean("relationshipMapper") externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext( ).getBean("externalCurationMessageBuilder") builder = StringBuilder() aux = "" reader = self.httpRequest.getReader() aux = reader.readLine() while aux is not None: builder.append(aux) aux = reader.readLine() requestJsonString = builder.toString() requestJson = JsonSimple(requestJsonString) # out.println(relationshipMapJsonObject.toString(True)) out.close() finally: self.sessionState.remove("username")
def __checkMetadataPayload(self): try: # Simple check for its existance self.object.getPayload("formData.tfpackage") self.firstHarvest = False except Exception: self.firstHarvest = True # We need to create it self.log.info( "Creating 'formData.tfpackage' payload for object '{}'", self.oid) # Prep data data = { "viewId": "default", "workflow_source": "Edgar Import", "packageType": "dataset", "redbox:formVersion": self.redboxVersion, "redbox:newForm": "true" } package = JsonSimple(JsonObject(data)) # Store it inStream = IOUtils.toInputStream(package.toString(True), "UTF-8") try: self.object.createStoredPayload("formData.tfpackage", inStream) self.packagePid = "formData.tfpackage" except StorageException, e: self.log.error( "Error creating 'formData.tfpackage' payload for object '{}'", self.oid, e) raise Exception("Error creating package payload: ", e)
def __activate__(self, context): request = context["request"] storage = context["Services"].getStorage() auth = context["page"].authentication log = context["log"] username = auth.get_name() oid = request.getParameter("oid") approval = request.getParameter("approval") approval_comment = request.getParameter("approval_comment") storedObj = storage.getObject(oid) committeeResponses = None payloadList = storedObj.getPayloadIdList() if payloadList.contains("committee-responses.metadata"): committeeResponsePayload = storedObj.getPayload("committee-responses.metadata") committeeResponses = JsonSimple(committeeResponsePayload.open()).getJsonObject() else: committeeResponses = JsonObject() committeeResponse = JsonObject() committeeResponse.put("approval",approval) committeeResponse.put("approval_comment",approval_comment) committeeResponses.put(username,committeeResponse) log.debug(" %s: Committee %s, approval = %s, comment = %s" % ( oid, username, approval, approval_comment)) StorageUtils.createOrUpdatePayload(storedObj,"committee-responses.metadata",IOUtils.toInputStream(committeeResponses.toString(), "UTF-8")) context["response"].sendRedirect(context["portalPath"] +"/detail/"+oid)
def __getJsonData(self, pid): data = None object = self.__getObject() payload = object.getPayload(pid) data = JsonSimple(payload.open()) payload.close() return data
def getCurationData(self, oid): json = JsonObject() try: # Get the object from storage storage = self.Services.getStorage() object = storage.getObject(oid) # Find the package payload payload = object.getPayload("metadata.json") # Not found? if payload is None: self.log.error(" * detail.py => Can't find package data!") json.put("error", True) return json # Parse the data data = JsonSimple(payload.open()) payload.close() # Return it json.put("error", False) json.put("relationships", data.writeArray("relationships")) return json except StorageException, ex: self.log.error(" * detail.py => Storage Error accessing data: ", ex) json.put("error", True) return json
def __activate__(self, context): self.velocityContext = context oid = self.vc("formData").get("oid") print "--- Creating ePub for: %s ---" % oid try: self.__epubMimetypeStream = None self.__epubContainerStream = None self.__epubcss = None self.__orderedItem = [] self.__itemRefDict = {} # get the package manifest object = Services.getStorage().getObject(oid) sourceId = object.getSourceId() payload = object.getPayload(sourceId) self.__manifest = JsonSimple(payload.open()) payload.close() object.close() # create the epub self.__getDigitalItems( self.__manifest.getJsonSimpleMap("manifest")) self.__createEpub() except Exception, e: log.error("Failed to create epub", e) self.vc("response").setStatus(500) writer = self.vc("response").getPrintWriter( "text/plain; charset=UTF-8") writer.println(str(e)) writer.close()
def getLabel(self, jsonFile, key, listKey, valKey, labelKey): value = self.metadata.get(key) jsonLabelFile = System.getProperty("fascinator.home") + jsonFile jsonLabel = JsonSimple(File(jsonLabelFile)) entries = jsonLabel.getJsonArray() # the structure of the json file is fun and complicated if entries is None: entries = jsonLabel.getArray(listKey) else: valKey = "value" labelKey = "label" for entry in entries: entryJson = JsonSimple(entry) if value == entryJson.getString("", valKey): return entryJson.getString("", labelKey) return None
def updateObjectMetadata(self, objectMetaData, toWorkflowId): packageType, jsonConfigFile = self.__getPackageTypeAndJsonConfigFile( toWorkflowId) workflowsDir = FascinatorHome.getPathFile("harvest/workflows") configFile = File(workflowsDir, jsonConfigFile) configObject = StorageUtils.checkHarvestFile(self.storage, configFile) if configObject is None: oid = StorageUtils.generateOid(configFile) configObject = StorageUtils.getDigitalObject(self.storage, oid) objectMetaData.setProperty("jsonConfigPid", jsonConfigFile) objectMetaData.setProperty("jsonConfigOid", configObject.getId()) configJson = JsonSimple(configFile) rulesFileName = configJson.getString(None, "indexer", "script", "rules") rulesFile = File(workflowsDir, rulesFileName) rulesObject = StorageUtils.checkHarvestFile(self.storage, rulesFile) if rulesObject is None: oid = StorageUtils.generateOid(rulesFile) rulesObject = StorageUtils.getDigitalObject(self.storage, oid) objectMetaData.setProperty("rulesPid", rulesFileName) objectMetaData.setProperty("rulesOid", rulesObject.getId()) objectMetaData.setProperty("workflowTransitioned", "true") objectMetaData.setProperty( "date_transitioned", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
def __messages(self): if self.message_list is not None and len(self.message_list) > 0: msg = JsonSimple() msg.getJsonObject().put("oid", self.oid) message = msg.toString() for target in self.message_list: self.utils.sendMessage(target, message)
def __activate__(self, context): response = context["response"] writer = response.getPrintWriter("text/plain; charset=UTF-8") auth = context["page"].authentication result = JsonSimple() obj = result.getJsonObject() obj.put("status", "error") obj.put("message", "An unknown error has occurred") if auth.is_logged_in(): services = context["Services"] formData = context["formData"] sessionState = context["sessionState"] urlBase = context["urlBase"] if urlBase.endswith("/"): urlBase = urlBase[:-1] func = formData.get("func") portalManager = services.portalManager if func == "create-view": try: fq = [q for q in sessionState.get("fq") if q != 'item_type:"object"'] id = formData.get("id") description = formData.get("description") print "Creating view '%s': '%s'" % (id, description) portal = Portal(id) portal.setDescription(formData.get("description")) portal.setQuery(" OR ".join(fq)) portal.setSearchQuery(sessionState.get("searchQuery")) portal.setFacetFields(portalManager.default.facetFields) portalManager.add(portal) portalManager.save(portal) obj.put("status", "ok") obj.put("message", "View '%s' successfully created" % id) obj.put("url", "%s/%s/home" % (urlBase, id)) except Exception, e: response.setStatus(500) obj.put("message", str(e)) elif func == "delete-view": defaultPortal = context["defaultPortal"] portalId = formData.get("view") if auth.is_admin(): if not portalId: response.setStatus(500) obj.put("message", "No view specified to be deleted") elif portalId != defaultPortal: # sanity check: don't delete default portal print "Deleting view '%s'" % portalId try: portalManager.remove(portalId) obj.put("status", "ok") obj.put("message", "View '%s' successfully removed" % portalId) obj.put("url", "%s/%s/home" % (urlBase, defaultPortal)) except Exception, e: obj.put("message", str(e)) else: response.setStatus(500) obj.put("message", "The default view cannot be deleted") else: response.setStatus(403) obj.put("message", "Only administrative users can access this API")
class ProxyGetData: def __activate__(self, context): self.velocityContext = context formData = self.vc("formData") # build the URL and query parameters to retrieve proxyUrls = JsonSimple(self.vc("systemConfig").getObject("proxy-urls")) url = "" key = formData.get("ns", "") if proxyUrls.getJsonObject().containsKey(key): url = proxyUrls.getString("", [key]) queryStr = formData.get("qs") if queryStr == "searchTerms={searchTerms}": queryStr = None if queryStr: if formData.get("jaffa2autocomplete", "false") == "true": url += "?searchTerms=%s" % queryStr.lower() else: url += "?%s" % queryStr self.vc("log").debug("Proxy URL = '{}'", url) data = None try: data = self.__wget(url) except Exception, e: data = '{"error":"%s"}' % str(e) self.vc("log").error("ERROR accessing URL:", e) try: # parse json to check well-formedness json = JsonSimple(data).getJsonObject() # format for jquery.autocomplete if formData.get("autocomplete", "false") == "true": rows = [] fields = formData.get("fields", "").split(",") results = json["results"] for result in results: row = "" for field in fields: if row != "": row += "::" value = result.get(field) if value is None: value = result["result-metadata"]["all"].get(field) if isinstance(value, JSONArray): value = ",".join(value) #self.vc("log").debug(" *** value from all: {}", value) if value: row += value else: row += "*" rows.append(row) if len(rows) > 0: data = "\n".join(rows) else: data = "" except Exception, e: data = '{"error":"%s"}' % str(e) self.vc("log").error("ERROR invalid JSON:", e)
def __getPackageTypes(self): object = self.sysConfig.getObject(["portal", "packageTypes"]) packageTypes = JsonSimple.toJavaMap(object) if packageTypes.isEmpty(): defaultPackage = JsonObject() defaultPackage.put("jsonconfig", "packaging-config.json") packageTypes.put("default", JsonSimple(defaultPackage)) return packageTypes
def get_file_list_as_json(self, rootPath, pattern): json = JsonSimple().getJsonObject() for root, dirs, files in os.walk(rootPath): for filename in fnmatch.filter(files, pattern): match = os.path.join(root, filename) self.log.debug("appending %s to file list" % match) json.put(match, filename) return json
def __getPackageManifest(self, oid): object = Services.getStorage().getObject(oid) sourceId = object.getSourceId() payload = object.getPayload(sourceId) manifest = JsonSimple(payload.open()) payload.close() object.close() return manifest
def __activate__(self, context): try: self.log = context["log"] self.response = context["response"] self.request = context["request"] self.systemConfig = context["systemConfig"] self.storage = context["Services"].getStorage() self.indexer = context["Services"].getIndexer() self.sessionState = context["sessionState"] self.sessionState.set("username", "admin") out = self.response.getPrintWriter("text/plain; charset=UTF-8") relationshipMapper = ApplicationContextProvider.getApplicationContext().getBean("relationshipMapper") externalCurationMessageBuilder = ApplicationContextProvider.getApplicationContext().getBean("externalCurationMessageBuilder") oid = self.request.getParameter("oid") if oid is None : identifier = self.request.getParameter("identifier") oid = self.findOidByIdentifier(identifier) relationshipType = self.request.getParameter("relationship") curatedPid = self.request.getParameter("curatedPid") sourceId = self.request.getParameter("sourceIdentifier") system = self.request.getParameter("system") digitalObject = StorageUtils.getDigitalObject(self.storage, oid) metadataJsonPayload = digitalObject.getPayload("metadata.json") metadataJsonInstream = metadataJsonPayload.open() metadataJson = JsonSimple(metadataJsonInstream) metadataJsonPayload.close() relationships = metadataJson.getArray("relationships") found = False for relationship in relationships: if relationship.get("identifier") == sourceId: relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) found = True if not found: relationship = JsonObject() relationship.put("isCurated",True) relationship.put("curatedPid",curatedPid) relationship.put("relationship",relationshipType) relationship.put("identifier",sourceId) relationship.put("system",system) relationships.add(relationship) out.println(metadataJson.toString(True)) istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) StorageUtils.createOrUpdatePayload(digitalObject,"metadata.json",istream) out.close() finally: self.sessionState.remove("username")
def parseJson(self, jsonString): ## Parse JSON Metadata try: return JsonSimple(jsonString) except Exception, e: self.log.error("ERROR invalid JSON: ", e) self.log.error("Invalid JSON was:\n{}", jsonString) self.throwError("Parsing JSON failed : " + e.getMessage()) return None
def process(self): '''Parse our CSV file Return a list of JsonSimple instances ''' f = None reader = None try: f = open(self.file, "rb") reader = csv.DictReader(f, dialect=self.dialect) if reader is None: raise AlertException("The requested file [%s] didn't return values" % self.file) jsonList = [] data = None for row in reader: data = {} print row for col in self.fieldMap: if col not in row: raise AlertException("The requested field [%s] was not present in [%s]" % (col, self.file)) if self.multiValue and col in self.multiValueFields: val = row[col].strip() values = val.split(self.multiValueDelimiter) i = 1 for value in values: #List fields look like dc:subject.vivo:keyword.0.rdf:PlainLiteral #Plus the config may provide a list of target keys tmpkey = self.fieldMap[col] if type(tmpkey) is list: for k in tmpkey: key = k.replace(".0.", ".%s." % i) data[key] = value.strip() else: key = tmpkey.replace(".0.", ".%s." % i) data[key] = value.strip() i += 1 else: key = self.fieldMap[col] if type(key) is list: for k in key: data[k] = row[col].strip() else: data[key] = row[col].strip() objdict = self.getNewJsonObjectDict(data) json = JsonSimple(objdict) jsonList.append(json) except: raise finally: if f is not None: f.close() return jsonList
def __loadConfig(self, file): f = None try: f = open(os.path.join("config/", file)) config = JsonSimple(f) finally: if not f is None: f.close() return config
def __getWorkflow(self): if self.__getObject() and self.__workflow is None: try: wfPayload = self.__object.getPayload("workflow.metadata") self.__workflow = JsonSimple(wfPayload.open()) wfPayload.close() except StorageException, e: self.__errorMessage = "Failed to retrieve workflow metadata: " + e.getMessage( )