def __search(self): indexer = Services.getIndexer() portalQuery = Services.getPortalManager().get(portalId).getQuery() req = SearchRequest("last_modified:[NOW-1MONTH TO *]") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); out = ByteArrayOutputStream() indexer.search(req, out) self.__latest = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) req = SearchRequest("*:*") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) req.addParam("fq", "") req.setParam("rows", "0") out = ByteArrayOutputStream() indexer.search(req, out) sessionState.set("fq", 'item_type:"object"') #sessionState.set("query", portalQuery.replace("\"", "'")) self.__result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
def __messages(self): if self.message_list is not None and len(self.message_list) > 0: msg = JsonConfigHelper() msg.set("oid", self.oid) message = msg.toString() for target in self.message_list: self.utils.sendMessage(target, message)
def __init__(self): action = formData.get("verb") portalName = formData.get("value") sessionState.remove("fq") if action == "delete-portal": print " * home.py: delete portal %s" % portalName Services.portalManager.remove(portalName) if action == "backup-portal": print "Backing up: ", portalName backupPath = "" email = "" portal = Services.portalManager.get(portalName) if portal: email = portal.email if portal.backupPaths: for key in portal.backupPaths: if portal.backupPaths[key]=="default": backupPath = key portalQuery = portal.getQuery() #print " ***** portalQuery: ", portalQuery #print " ***** backupPath: ", backupPath #print " ***** email: ", email #print " ***** description: ", description if backupPath is None: " ** Default backup path configured in system-config.json will be used " Services.portalManager.backup(email, backupPath, portalQuery) self.__latest = JsonConfigHelper() self.__result = JsonConfigHelper() self.__search()
def __isLinked(self, ids, map): query = 'package_node_id:("' + '" OR "'.join(ids) + '")' req = SearchRequest(query) req.setParam("fq", 'recordtype:"master"') req.addParam("fq", 'item_type:"object"') req.setParam("rows", "9999") out = ByteArrayOutputStream() self.__indexer.search(req, out) result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) currentList = [] for doc in result.getJsonList("response/docs"): currentList.extend(doc.getList("package_node_id")) if type(map).__name__ == "LinkedHashMap": for author in map.keySet(): authorDocs = map.get(author) for doc in authorDocs: if doc.get("id") in currentList: doc.set("linked", "true") else: for author in map.keys(): authorList = map[author] for count in authorList: doc = authorList[count] if doc.get("id") in currentList: doc.set("linked", "true")
def parseFFmpeg(self, parent): if parent is not None: object = parent.getObject() if object is not None: payload = None try: payload = object.getPayload("ffmpeg.info") # Stream the content out to string out = ByteArrayOutputStream() IOUtils.copy(payload.open(), out) payload.close() self.__ffmpegRaw = out.toString("UTF-8") out.close() payload.close() # And parse it jsonData = JsonConfigHelper(self.__ffmpegRaw) if jsonData is None: return False else: self.__ffmpegData = jsonData.getJsonMap("/outputs") return True except: if payload is not None: payload.close() return False
def search_solr(self): query = "(rootUri:" if self.rootUriList: query += "(" + " OR ".join(self.rootUriList) + ")" else: query += "\"" + self.rootUri + "\"" if self.type: query += " AND type:\"" + self.type + "\"" query += ")" #print "**********", query req = SearchRequest(query) req.setParam("facet", "false") req.setParam("rows", str(99999)) req.setParam("sort", "dateCreated asc") req.setParam("start", str(0)) #security_roles = page.authentication.get_roles_list(); #security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' #req.addParam("fq", security_query) out = ByteArrayOutputStream() Services.indexer.annotateSearch(req, out) result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) result = result.getJsonList("response/docs") # Every annotation for this URI if self.type == "http://www.purl.org/anotar/ns/type/0.1#Tag": return self.process_tags(result) else: return self.process_response(result)
def __getPackageTypes(self): json = JsonConfigHelper(JsonConfig.getSystemFile()) packageTypes = json.getMap("portal/packageTypes") packageTypes = dict(packageTypes) if packageTypes=={}: packageTypes["default"] = {"jsonconfig":"packaging-config.json"} return packageTypes
def __getStorageId(self, oid): req = SearchRequest('id:"%s"' % oid) req.addParam("fl", "storage_id") out = ByteArrayOutputStream() Services.indexer.search(req, out) json = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) return json.getList("response/docs").get(0).get("storage_id")
def __activate__(self, context): self.velocityContext = context self.vc("sessionState").remove("fq") self.__latest = JsonConfigHelper() self.__mine = JsonConfigHelper() self.__workflows = JsonConfigHelper() self.__result = JsonConfigHelper() self.__search()
def __getActiveManifest(self): activeManifest = sessionState.get("package/active") if not activeManifest: activeManifest = JsonConfigHelper() activeManifest.set("title", "New package") activeManifest.set("viewId", portalId) sessionState.set("package/active", activeManifest) return activeManifest
def getSuggestedNames(self): # search common forms lookupNames = [] surname = self.__metadata.getList("surname").get(0) firstName = self.__metadata.getList("firstName").get(0) firstInitial = firstName[0].upper() secondName = self.__metadata.getList("secondName") if not secondName.isEmpty(): secondName = secondName.get(0) if secondName and secondName != "": secondInitial = secondName[0].upper() lookupNames.append("%s, %s. %s." % (surname, firstInitial, secondInitial)) lookupNames.append("%s, %s %s." % (surname, firstName, secondInitial)) lookupNames.append("%s, %s %s" % (surname, firstName, secondName)) lookupNames.append("%s %s %s" % (firstName, secondName, surname)) lookupNames.append("%s, %s." % (surname, firstInitial)) lookupNames.append("%s, %s" % (surname, firstName)) lookupNames.append("%s %s" % (firstName, surname)) query = '" OR dc_title:"'.join(lookupNames) # general word search from each part of the name parts = [p for p in self.getPackageTitle().split(" ") if len(p) > 0] query2 = " OR dc_title:".join(parts) req = SearchRequest('(dc_title:"%s")^2.5 OR (dc_title:%s)^0.5' % (query, query2)) self.log.info("suggestedNames query={}", req.query) req.setParam("fq", 'recordtype:"author"') req.addParam("fq", 'item_type:"object"') req.setParam("rows", "9999") req.setParam("fl", "score") req.setParam("sort", "score desc") # Make sure 'fq' has already been set in the session ##security_roles = self.authentication.get_roles_list(); ##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' ##req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer = self.services.getIndexer() indexer.search(req, out) result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) # self.log.info("result={}", result.toString()) docs = result.getJsonList("response/docs") map = LinkedHashMap() for doc in docs: authorName = doc.getList("dc_title").get(0) if map.containsKey(authorName): authorDocs = map.get(authorName) else: authorDocs = ArrayList() map.put(authorName, authorDocs) authorDocs.add(doc) self.__maxScore = max(1.0, float(result.get("response/maxScore"))) return map
def __search(self): self.__result = JsonConfigHelper() portal = self.services.getPortalManager().get(self.vc("portalId")) recordsPerPage = portal.recordsPerPage query = self.vc("formData").get("query") if query is None or query == "": query = "*:*" req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", portal.facetFieldList) req.setParam("facet.limit", str(portal.facetCount)) req.setParam("sort", "f_dc_title asc") portalQuery = portal.query print " * portalQuery=%s" % portalQuery if portalQuery: portalQuery += "&item_type:object" else: portalQuery += "item_type:object" req.addParam("fq", portalQuery) #Check if there's resumption token exist in the formData if self.__currentToken: start = self.__currentToken.getStart() totalFound = self.__currentToken.getTotalFound() nextTokenStart = start+recordsPerPage if nextTokenStart < totalFound: self.__token = ResumptionToken(start=nextTokenStart, metadataPrefix=self.__metadataPrefix, sessionExpiry=self.__sessionExpiry) else: start = 0 metadataPrefix = self.vc("formData").get("metadataPrefix") self.__token = ResumptionToken(start=recordsPerPage, metadataPrefix=self.__metadataPrefix, sessionExpiry=self.__sessionExpiry) req.setParam("start", str(start)) print " * oai.py:", req.toString() out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) totalFound = int(self.__result.get("response/numFound")) if totalFound == 0: self.__token = None elif self.__token: if self.__token.getStart() < totalFound: self.__token.setTotalFound(totalFound) else: self.__token = None #Storing the resumptionToken to session if self.__token: self.__resumptionTokenList[self.__token.getToken()] = self.__token #(totalFound, self.__token.getConstructedToken()) #Need to know how long the server need to store this token self.sessionState.set("resumptionTokenList", self.__resumptionTokenList)
def test(self): s = "data " json = JsonConfigHelper() try: l = json.getList("test") s += str(l.size()) s += " '%s' " % json.get("test1") except Exception, e: s += "Error '%s'" % str(e)
def __metadata(self): ISOCode = self.params.getProperty("ISOcode") print "*** Processing: ", ISOCode #Index the country metadata metadataPayload = self.object.getPayload("%s.json" % ISOCode) json = JsonConfigHelper(metadataPayload.open()) allMeta = json.getMap(".") self.utils.add(self.index, "recordType", "country") for key in allMeta.keySet(): self.utils.add(self.index, key, allMeta.get(key)); metadataPayload.close() #Index the country detail geoNamePayload = self.object.getPayload("%s.txt" % ISOCode) countryName = self.params.getProperty("countryName") countryAreaStream = geoNamePayload.open() reader = BufferedReader(InputStreamReader(countryAreaStream, "UTF-8")); line = reader.readLine() headerList = ["geonameid", "name", "asciiname", "alternatenames", "latitude", "longitude", \ "feature class", "feature code", "country code", "cc2", "admin1 code", "admin2 code", \ "admin3 code", "admin4 code", "population", "elevation", "gtopo30", "timezone", "modification date"] while (line != None): arraySplit = line.split("\t") geonamesId = arraySplit[0] oid = DigestUtils.md5Hex("http://geonames.org/" + geonamesId) if oid == self.oid: extraIndex = self.index else: extraIndex = HashMap() self.utils.add(extraIndex, "recordType", "area") self.utils.add(extraIndex, "item_type", self.itemType) self.utils.add(extraIndex, "dc_identifier", oid) self.utils.add(extraIndex, "id", oid) self.utils.add(extraIndex, "storage_id", self.oid) #Use parent object self.utils.add(extraIndex, "last_modified", time.strftime("%Y-%m-%dT%H:%M:%SZ")) self.utils.add(extraIndex, "display_type", "geonames") self.utils.add(extraIndex, "countryName", countryName) self.utils.add(extraIndex, "repository_name", self.params["repository.name"]) self.utils.add(extraIndex, "repository_type", self.params["repository.type"]) self.utils.add(extraIndex, "security_filter", "guest") self.utils.add(extraIndex, "dc_title", arraySplit[1]) # The rest of the metadata count = 0 for array in arraySplit: if headerList[count] !="alternatenames" and array: self.utils.add(extraIndex, headerList[count], array) count +=1 self.indexer.sendIndexToBuffer(oid, extraIndex) line = reader.readLine() geoNamePayload.close()
def __getManifest(self, oid): object = self.services.getStorage().getObject(oid) sourceId = object.getSourceId() payload = object.getPayload(sourceId) payloadReader = InputStreamReader(payload.open(), "UTF-8") manifest = JsonConfigHelper(payloadReader) payloadReader.close() payload.close() object.close() return manifest.getJsonMap("manifest")
def __init__(self): action = formData.get("verb") portalName = formData.get("value") sessionState.remove("fq") if action == "delete-portal": print " * home.py: delete portal %s" % portalName Services.portalManager.remove(portalName) self.__latest = JsonConfigHelper() self.__mine = JsonConfigHelper() self.__workflows = JsonConfigHelper() self.__result = JsonConfigHelper() self.__search()
def __getPackageTypeAndJsonConfigFile(self): try: packageType = self.vc("formData").get("packageType", "default") if packageType == "": packageType = "default" json = JsonConfigHelper(JsonConfig.getSystemFile()) pt = json.getMap("portal/packageTypes/" + packageType) jsonConfigFile = pt.get("jsonconfig") if jsonConfigFile is None: jsonConfigFile = "packaging-config.json" except Exception, e: jsonConfigFile = "packaging-config.json"
def __getMetadata(self, oid): req = SearchRequest('id:%s' % oid) req.setParam("fq", 'item_type:"object"') # Make sure 'fq' has already been set in the session ##security_roles = self.authentication.get_roles_list(); ##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' ##req.addParam("fq", security_query) out = ByteArrayOutputStream() self.__indexer.search(req, out) result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) #self.log.info("result={}", result.toString()) return result.getJsonList("response/docs").get(0)
def __init__(self): responseType = "text/html; charset=UTF-8" responseMsg = "" func = formData.get("func") if func == "url-history": responseType = "text/plain; charset=UTF-8" responseMsg = "\n".join(self.getUrls()) else: try: url = formData.get("url") title = formData.get("title") username = formData.get("username") password = formData.get("password") try: auth = ProxyBasicAuthStrategy(username, password, url) self.__service = AtomClientFactory.getAtomService(url, auth) oid = formData.get("oid") if oid is not None: self.__object = Services.getStorage().getObject(oid) sourceId = self.__object.getSourceId() payload = self.__object.getPayload(sourceId) print "payload=%s,%s" % (payload, payload.getContentType()) # FIXME see https://fascinator.usq.edu.au/trac/ticket/647 if payload and sourceId.endswith(".tfpackage"): #payload.getContentType() == "application/x-fascinator-package": jsonManifest = JsonConfigHelper(payload.open()) print jsonManifest.toString() content = self.__getManifestContent(jsonManifest) payload.close() else: content = self.__getContent(oid) self.__object.close() else: content = "<div>Object not found!</div>" success, value = self.__post(title, content) except Exception, e: e.printStackTrace() success = False value = e.getMessage() if success: altLinks = value.getAlternateLinks() if altLinks is not None: self.saveUrl(url) responseMsg = "<p>Success! Visit the <a href='%s' target='_blank'>blog post</a>.</p>" % altLinks[0].href else: responseMsg = "<p class='warning'>The server did not return a valid link!</p>" else: responseMsg = "<p class='error'>%s</p>" % value except Exception, e: print "Failed to post: %s" % e.getMessage() responseMsg = "<p class='error'>%s</p>" % e.getMessage()
def __activate__(self, context): self.velocityContext = context action = self.vc("formData").get("verb") portalName = self.vc("formData").get("value") self.vc("sessionState").remove("fq") if action == "delete-portal": print " * home.py: delete portal %s" % portalName Services.portalManager.remove(portalName) self.__latest = JsonConfigHelper() self.__mine = JsonConfigHelper() self.__workflows = JsonConfigHelper() self.__result = JsonConfigHelper() self.__search()
def __search(self): indexer = Services.getIndexer() portalQuery = Services.getPortalManager().get(self.vc("portalId")).getQuery() portalSearchQuery = Services.getPortalManager().get(self.vc("portalId")).getSearchQuery() # Security prep work current_user = self.vc("page").authentication.get_username() security_roles = self.vc("page").authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" isAdmin = self.vc("page").authentication.is_admin() req = SearchRequest("last_modified:[NOW-1MONTH TO *]") req.setParam("fq", 'item_type:"object"') req.addParam("fq", "workflow_modified:false") if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__latest = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) req = SearchRequest("*:*") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("facet", "true") req.setParam("facet.field", "workflow_step") req.setParam("fl", "id") req.setParam("rows", "1") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) # self.vc("sessionState").set("fq", 'item_type:"object"') # sessionState.set("query", portalQuery.replace("\"", "'")) self.__result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
def __getAuthorDetails(self, citationIds): query = " OR id:".join(citationIds) req = SearchRequest('id:%s' % query) req.setParam("fq", 'recordtype:"author"') req.addParam("fq", 'item_type:"object"') req.setParam("rows", "9999") # Make sure 'fq' has already been set in the session ##security_roles = self.authentication.get_roles_list(); ##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' ##req.addParam("fq", security_query) out = ByteArrayOutputStream() self.__indexer.search(req, out) result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) return result.getJsonList("response/docs")
def __activate__(self, context): self.services = context["Services"] self.page = context["page"] self.formData = context["formData"] self.sessionState = context["sessionState"] self.request = context["request"] self.pageName = context["pageName"] self.__portal = context["page"].getPortal() sessionNav = self.__portal.get("portal/use-session-navigation", "true") self.__useSessionNavigation = Boolean.parseBoolean(sessionNav) self.__result = JsonConfigHelper() if self.__useSessionNavigation: self.__pageNum = self.sessionState.get("pageNum", 1) else: self.__pageNum = 1 self.__selected = ArrayList() self.__fqParts = [] self.__searchField = self.formData.get("searchField", "full_text") if self.__portal.getName() != self.lastPortalId: self.sessionState.remove("fq") self.sessionState.remove("pageNum") self.__pageNum = 1 self.lastPortalId = self.__portal.getName() self.__search()
def numberOfModifiedRecord(self): indexer = self.services.getIndexer() portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery() portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery() # Security prep work current_user = self.page.authentication.get_username() security_roles = self.page.authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" req = SearchRequest("modified:true") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.addParam("fq", "") req.setParam("rows", "0") if not self.page.authentication.is_admin(): req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) return self.__result.get("response/numFound")
def __init__(self): oid = formData.get("oid") print "--- Creating ePub for: %s ---" % oid try: self.__epubMimetypeStream = None self.__epubContainerStream = None self.__epubcss = None self.__orderedItem = [] self.__itemRefDict = {} # get the package manifest object = Services.getStorage().getObject(oid) sourceId = object.getSourceId() payload = object.getPayload(sourceId) self.__manifest = JsonConfigHelper(payload.open()) payload.close() object.close() # create the epub self.__getDigitalItems(self.__manifest.getJsonMap("manifest")) self.__createEpub() except Exception, e: log.error("Failed to create epub", e) response.setStatus(500) writer = response.getPrintWriter("text/plain") writer.println(str(e)) writer.close()
class QueuesData: def __init__(self): pass def __activate__(self, context): self.request = context["request"] self.response = context["response"] self.formData = context["formData"] if self.request.isXHR(): print " **** formData: %s" % self.formData queue = self.formData.get("queueName") msg = self.formData.get("queueMessage") self.queueMessage(queue, msg); out = self.response.getPrintWriter("text/plain") out.println(self.formData) out.close() self.config = JsonConfigHelper(JsonConfig.getSystemFile()) self.threads = self.config.getJsonList("messaging/threads") def getDescription(self, queue): for thread in self.threads: name = thread.get("config/name") if name == queue: return thread.get("description") def queueMessage(self, queue, msg): ms = MessagingServices.getInstance() ms.queueMessage(queue, msg); ms.release()
def __getTableOfContents(self, package, oid): try: # get the package manifest object = Services.getStorage().getObject(package) sourceId = object.getSourceId() payload = object.getPayload(sourceId) payloadReader = InputStreamReader(payload.open(), "UTF-8") manifest = JsonConfigHelper(payloadReader) payloadReader.close() payload.close() object.close() # generate toc result = self.__toc(manifest.getJsonMap("manifest/" + oid.replace("blank-", "node-") + "/children")) except Exception, e: print "Failed to load manifest '%s': '%s'" % (package, str(e)) result = '<div class="error">Failed to generate table of contents!</div><pre>%s</pre>' % str(e)
def modify_json(self): #print "**** anotar.py : add_json() : adding json : " + json jsonObj = JsonConfigHelper(self.json) jsonObj.set("id", self.pid) rootUri = jsonObj.get("annotates/rootUri") if rootUri is not None: baseUrl = "http://%s:%s/" % (self.vc("request").serverName, self.vc("serverPort")) myUri = baseUrl + rootUri + "#" + self.pid jsonObj.set("uri", myUri) jsonObj.set("schemaVersionUri", "http://www.purl.org/anotar/schema/0.1") self.json = jsonObj.toString(False)
def __search(self, searchField): indexer = self.services.getIndexer() portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery() portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery() # Security prep work current_user = self.page.authentication.get_username() security_roles = self.page.authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" startRow = 0 numPerPage = 25 numFound = 0 req = SearchRequest(searchField) if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) if not self.page.authentication.is_admin(): req.addParam("fq", security_query) objectIdList = [] while True: req.addParam("fq", 'item_type:"object"') req.addParam("rows", str(numPerPage)) req.addParam("start", str(startRow)) out = ByteArrayOutputStream() indexer.search(req, out) result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray())) docs = result.getList("response/docs/storage_id") objectIdList.extend(docs) startRow += numPerPage numFound = int(result.get("response/numFound")) if startRow > numFound: break return objectIdList
def __init__(self): self.__result = JsonConfigHelper() self.__portal = Services.getPortalManager().get(portalId) pageNum = sessionState.get("pageNum") if pageNum is None: self.__pageNum = 1 else: self.__pageNum = int(pageNum) self.__search()