def checkEventLogForEdits(self, oid): req = SearchRequest("oid:" + oid + " AND context:Workflow AND eventType:Save") out = ByteArrayOutputStream() self.indexer.searchByIndex(req, out, "eventLog") solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getRows() > 0
def checkAprovedRequests(self, provisioned=0, startPage=1): """ A customised query for package type of arms at workflow_step of arms-approved Get a list of requests filtered by provisioning_checklist """ workflowStep = "arms-approved" if self.packageType: req = SearchRequest(self.packageType) else: req = SearchRequest("packageType:arms") req.addParam("fq", 'workflow_step:' + workflowStep) if provisioned: req.addParam("fq", '-provisioning_checklist.4:null') else: req.addParam("fq", 'provisioning_checklist.4:null') req.setParam("sort", "date_object_modified desc, f_dc_title asc") req.setParam("fl","id,dc_title,date-provisioned") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResults = SolrResult(ByteArrayInputStream(out.toByteArray())) if solrResults: results = solrResults.getResults() if results: results = self.mergeEvents(results, ["arms_draft","arms_redraft","arms_review","arms_approved","arms_rejected"]) self._setPaging(results.size()) return results else: return ArrayList()
def handleWorkflowStep(self): out = ByteArrayOutputStream() req = SearchRequest("workflow_step_label:[* TO *]" ) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): recordTypeResults = res.getResults() for recordTypeResult in recordTypeResults: recordTypeList = recordTypeResult.getList("workflow_step_label") if (recordTypeList.isEmpty()==False): for hit in recordTypeList: hits.add(hit) self.writer.println("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.println(",{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit)) else: self.writer.println("{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit)) hitnum += 1 self.writer.println("]") else: self.writer.println("[\"\"]") self.writer.close()
def export(self, exportType): exportQuery = "%s:%s" % (self.facetField, self.facetFieldValue) outputType = "text/%s; charset=UTF-8" % type responseHeader = "attachment; filename=%s.%s" % (self.facetFieldValue, exportType) try: out = ByteArrayOutputStream() recnumreq = SearchRequest(exportQuery) recnumreq.setParam("fl","create_timestamp") recnumreq.setParam("rows", "0") self.indexer.search(recnumreq, out) recnumres = SolrResult(ByteArrayInputStream(out.toByteArray())) self.__rowsFoundSolr = "%s" % recnumres.getNumFound() except: self.errorMsg = "Export query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Export query threw an exception (package type was %s): %s - %s" % (self.facetFieldValue, sys.exc_info()[0], sys.exc_info()[1])) return out = ByteArrayOutputStream() req = SearchRequest(exportQuery) req.setParam("wt", exportType) req.setParam("rows", self.__rowsFoundSolr) self.indexer.search(req, out) self.response.setHeader("Content-Disposition", responseHeader) writer = self.response.getPrintWriter(outputType) writer.println(out.toString("UTF-8")) writer.close()
def handleQuery(self, query, fieldName, formatStr): out = ByteArrayOutputStream() req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): results = res.getResults() for searchRes in results: searchResList = searchRes.getList(fieldName) if (searchResList.isEmpty()==False): for hit in searchResList: if self.term is not None: if hit.find(self.term) != -1: hits.add(hit) else: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print(","+formatStr % {"hit":hit}) else: self.writer.print(formatStr % {"hit":hit}) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def handleQuery(self, query, fieldName, formatStr): out = ByteArrayOutputStream() req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): results = res.getResults() for searchRes in results: searchResList = searchRes.getList(fieldName) if (searchResList.isEmpty() == False): for hit in searchResList: if self.term is not None: if hit.find(self.term) != -1: hits.add(hit) else: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print("," + formatStr % {"hit": hit}) else: self.writer.print(formatStr % {"hit": hit}) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def __getUsers(self, oid): indexer = Services.getIndexer() req = SearchRequest("id:" + oid) req.setParam("fl", "security_exception,owner") out = ByteArrayOutputStream() indexer.search(req, out) rtJson = "" try: qresult = SolrResult(ByteArrayInputStream( out.toByteArray())).getResults().get(0) owner = qresult.getString(None, 'owner') secException = qresult.getArray('security_exception') if secException is None: secException = JSONArray() self.log.debug("Owner of object: " + owner) self.log.debug("Viewer(s) of object: " + secException.toString()) if secException.contains(owner): secException.remove(owner) return '{"owner":"' + owner + '", "viewers": ' + secException.toString( ) + '}' except Exception, e: self.log.error("Error during query/package ownership data" + str(e))
def findPublishedRecords(self): #req = SearchRequest("published:\"true\"") req = SearchRequest("storage_id:\"c6a214670dc644e5ebdaede4a2243f67\"") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def handleGrantNumber(self): out = ByteArrayOutputStream() req = SearchRequest("grant_numbers:%s*" % self.term) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): creatorResults = res.getResults() for creatorRes in creatorResults: creatorList = creatorRes.getList("grant_numbers") if (creatorList.isEmpty()==False): for hit in creatorList: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print(",\"%s\"" % hit) else: self.writer.print("\"%s\"" % hit) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def findPackagesToTransition(self, fromWorkflowId, fromWorkflowStage): req = SearchRequest("workflow_id:"+fromWorkflowId+" AND _query_:\"workflow_step:"+fromWorkflowStage+"\"") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def checkRequests(self, checklist_filter=['1'], role_filter='reviewer', exclusive=True, startPage=1): """ A customised query for arms at arms-review Get a list of requests filtered by provisioning_checklist For assessor, it is user based and queries against committee-responses.metadata """ workflowStep = "arms-review" if self.packageType: req = SearchRequest(self.packageType) else: req = SearchRequest("packageType:arms") req.addParam("fq", 'workflow_step:' + workflowStep) for item in ['1','2','3','4']: if item in checklist_filter: req.addParam("fq", '-provisioning_checklist.' + item + ':null' + ' AND provisioning_checklist.' + item + ':[* TO *]') else: if exclusive: # ensure that brand new submissions (not yet saved by reviewer) are also returned req.addParam("fq", 'provisioning_checklist.' + item + ':null' + ' OR (*:* -provisioning_checklist.' + item + ':[* TO *])') req.setParam("sort", "date_object_modified desc, f_dc_title asc") req.setParam("fl",self.returnFields) out = ByteArrayOutputStream() self.indexer.search(req, out) solrResults = SolrResult(ByteArrayInputStream(out.toByteArray())) if solrResults: results = solrResults.getResults() if results: packageResults = results results = self.mergeEvents(packageResults, ["arms_draft","arms_redraft","arms_review","arms_approved","arms_rejected"]) returnArray = JSONArray() if role_filter.startswith('assessor'): if role_filter == 'assessor': query_status = ['new', 'draft'] elif role_filter == 'assessor-assessed': query_status = ['submitted'] x = Assessment() x.activate(self.velocityContext) i = 0 rows = self.recordsPerPage start = (startPage - 1) * self.recordsPerPage for r in results: status = x.queryStatus(r.get("id")) if status in query_status: if i >= start and i - start < rows: if status == 'submitted': assessment_submitted_date = x.queryMyAttr('date') if assessment_submitted_date: r.getJsonObject().put('date', assessment_submitted_date) returnArray.add(r) i = i + 1 else: returnArray = results self._setPaging(returnArray.size()) return returnArray else: return ArrayList()
def findPackagesToPurge(self,packageType): req = SearchRequest("display_type:"+packageType +" AND date_object_created:[* TO NOW-7DAY]") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") req.setParam("fl", "storage_id,date_object_created,date_object_modified") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def getAttachments(self): attachmentType = "review-attachments" req = SearchRequest("attached_to:%s AND attachment_type:%s" % (self.oid, attachmentType)) req.setParam("rows", "1000") out = ByteArrayOutputStream() self.Services.indexer.search(req, out) response = SolrResult(ByteArrayInputStream(out.toByteArray())) return response.getResults()
def findPackagesToTransition(self, fromWorkflowId, fromWorkflowStage): req = SearchRequest("workflow_id:" + fromWorkflowId + " AND _query_:\"workflow_step:" + fromWorkflowStage + "\"") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def findPackagesToPurge(self, packageType): req = SearchRequest("display_type:" + packageType + " AND date_object_created:[* TO NOW-7DAY]") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") req.setParam("fl", "storage_id,date_object_created,date_object_modified") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def _packageResults(self, req, solrLog=None): out = ByteArrayOutputStream() if solrLog: self.indexer.searchByIndex(req, out, solrLog) else: self.indexer.searchByIndex(req, out) solrResults = SolrResult(ByteArrayInputStream(out.toByteArray())) if solrResults: return solrResults.getResults() else: return ArrayList()
def __search(self): indexer = self.services.getIndexer() # Security prep work isAdmin = self.vc("page").authentication.is_admin() if not isAdmin: print "ERROR: User is not an admin '" return None req = SearchRequest('eventType:harvestStart') req.setParam("rows", "100") out = ByteArrayOutputStream() indexer.searchByIndex(req, out, "eventLog") self.__harvestList = SolrResult(ByteArrayInputStream( out.toByteArray()))
def search_solr(self): query = "(rootUri:" if self.rootUriList: query += "(" + " OR ".join(self.rootUriList) + ")" else: query += "\"" + self.rootUri + "\"" if self.type: query += " AND type:\"" + self.type + "\"" query += ")" #print "**********", query req = SearchRequest(query) req.setParam("facet", "false") req.setParam("rows", str(99999)) req.setParam("sort", "dateCreated asc") req.setParam("start", str(0)) #security_roles = page.authentication.get_roles_list(); #security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' #req.addParam("fq", security_query) out = ByteArrayOutputStream() Services.indexer.annotateSearch(req, out) result = SolrResult(ByteArrayInputStream( out.toByteArray())).getResults() # Every annotation for this URI if self.type == "http://www.purl.org/anotar/ns/type/0.1#Tag": return self.process_tags(result) else: return self.process_response(result)
def __feed(self): self.portal = self.services.getPortalManager().get(self.vc("portalId")) recordsPerPage = self.portal.recordsPerPage pageNum = self.vc("sessionState").get("pageNum", 1) query = "*:*" if self.vc("formData").get("query"): query = self.vc("formData").get("query") query = self.__escapeQuery(query) req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", self.portal.facetFieldList) req.setParam("facet.sort", "true") req.setParam("facet.limit", str(self.portal.facetCount)) req.setParam("sort", "f_dc_title asc") portalQuery = self.portal.query if portalQuery: req.addParam("fq", portalQuery) else: fq = self.vc("sessionState").get("fq") if fq is not None: req.setParam("fq", fq) req.setParam("start", str((pageNum - 1) * recordsPerPage)) self.log.debug(" * Query: '{}'", query) self.log.debug(" * portalQuery: '{}'", portalQuery) self.log.debug(" * feed.py: '{}'", req) out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __search(self): query = self.formData.get("query") searchQuery = self.sessionState.get("searchQuery") if query is None or query == "": query = "*:*" if searchQuery and query == "*:*": query = searchQuery elif searchQuery: query += " AND " + searchQuery facetField = self.formData.get("facet.field") req = SearchRequest(query) req.setParam("facet", "true") req.setParam("fl", "id") req.setParam("rows", "0") req.setParam("facet.limit", "-1") req.setParam("facet.field", facetField) fq = self.sessionState.get("fq") if fq is not None: req.setParam("fq", fq) req.addParam("fq", 'item_type:"object"') # Make sure 'fq' has already been set in the session security_roles = self.auth.get_roles_list() security_query = 'security_filter:("' + '" OR "'.join( security_roles) + '")' req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer = self.services.indexer indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) return FacetList(facetField, result)
def __searchExecute(self, search, count): try: search.setParam("start", str(count)) out = ByteArrayOutputStream() self.services.indexer.search(search, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Error during search: ", e) return None
class ReportsData: def __init__(self): pass def __activate__(self, context): #import pydevd;pydevd.settrace() self.velocityContext = context self.vc("sessionState").remove("fq") self.services = self.vc("Services") self.log = context["log"] self.__harvestList = None self.__search() # Get from velocity context def vc(self, index): if self.velocityContext[index] is not None: return self.velocityContext[index] else: print "ERROR: Requested context entry '" + index + "' doesn't exist" return None def __search(self): indexer = self.services.getIndexer() # Security prep work isAdmin = self.vc("page").authentication.is_admin() if not isAdmin: print "ERROR: User is not an admin '" return None req = SearchRequest('eventType:harvestStart') req.setParam("rows", "100") out = ByteArrayOutputStream() indexer.searchByIndex(req, out, "eventLog") self.__harvestList = SolrResult(ByteArrayInputStream(out.toByteArray())) def getHarvestlist(self): return self.__harvestList.getResults() def getItemCount(self): return self.__harvestList.getNumFound()
def __activate__(self, context): formData = context["formData"] services = context["Services"] response = context["response"] # Prepare a query q = formData.get("q") if q is not None and q != "": query = "(item_type:object AND " + self.titleTokens( q.strip()) + ")" else: query = "item_type:object" # Can't link to yourself AND we're not interested in attachments oid = formData.get("qs") query += " AND -storage_id:\"" + oid + "\"" # And we're not interested in attachments query += " AND display_type:\"package-dataset\"" req = SearchRequest(query) req.setParam("fl", "dc_title,storage_id,pidProperty") limit = formData.get("limit") if limit is None: limit = 10 req.setParam("rows", limit) # Search Solr indexer = services.getIndexer() out = ByteArrayOutputStream() indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Build a response list = [] for doc in result.getResults(): title = doc.getFirst("dc_title") #oid = doc.getFirst("storage_id") oid = doc.getFirst("pidProperty") list.append("%s::%s" % (oid, title)) result = "\n".join(list) writer = response.getPrintWriter("text/plain; charset=UTF-8") writer.println(result) writer.close()
class ReportsData: def __init__(self): pass def __activate__(self, context): #import pydevd;pydevd.settrace() self.velocityContext = context self.vc("sessionState").remove("fq") self.services = self.vc("Services") self.log = context["log"] self.__harvestList = None self.__search() # Get from velocity context def vc(self, index): if self.velocityContext[index] is not None: return self.velocityContext[index] else: print "ERROR: Requested context entry '" + index + "' doesn't exist" return None def __search(self): indexer = self.services.getIndexer() # Security prep work isAdmin = self.vc("page").authentication.is_admin() if not isAdmin: print "ERROR: User is not an admin '" return None req = SearchRequest('eventType:harvestStart') req.setParam("rows", "100") out = ByteArrayOutputStream() indexer.searchByIndex(req, out, "eventLog") self.__harvestList = SolrResult(ByteArrayInputStream( out.toByteArray())) def getHarvestlist(self): return self.__harvestList.getResults() def getItemCount(self): return self.__harvestList.getNumFound()
def __loadSolrData(self, oid): portal = self.vc("page").getPortal() query = 'id:"%s"' % oid if portal.getSearchQuery(): query += " AND " + portal.getSearchQuery() req = SearchRequest(query) req.addParam("fq", 'item_type:"object"') req.addParam("fq", portal.getQuery()) out = ByteArrayOutputStream() self.vc("Services").getIndexer().search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def getAttachedFiles(self, oid): # Build a query req = SearchRequest("attached_to:%s" % oid) req.setParam("rows", "1000") # Run a search out = ByteArrayOutputStream() self.Services.getIndexer().search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Process results docs = JSONArray() for doc in result.getResults(): attachmentType = self.escapeHtml(WordUtils.capitalizeFully(doc.getFirst("attachment_type").replace("-", " "))) accessRights = self.escapeHtml(WordUtils.capitalizeFully(doc.getFirst("access_rights"))) entry = JsonObject() entry.put("filename", self.escapeHtml(doc.getFirst("filename"))) entry.put("attachment_type", attachmentType) entry.put("access_rights", accessRights) entry.put("id", self.escapeHtml(doc.getFirst("id"))) docs.add(entry) return docs
def getFacetFields(self): try: out = ByteArrayOutputStream() req = SearchRequest("*:*") req.setParam("fl","facet_counts") req.setParam("facet", "on") req.setParam("facet.field", self.facetField) req.setParam("wt", "json") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) facets = res.getFacets() facet = facets.get(self.facetField) if facet is not None and facet.values().size() > 0: self.facetFields = facet.values() else: self.errorMsg = "No facet field values to export. Please enter/harvest some data first." except: self.errorMsg = "Get facet field query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Get facet field threw an exception : %s - %s" % (sys.exc_info()[0], sys.exc_info()[1])) return
def __activate__(self, context): formData = context["formData"] services = context["Services"] response = context["response"] # Prepare a query q = formData.get("q") if q is not None and q != "": query = "(item_type:object AND " + self.titleTokens(q.strip()) + ")" else: query = "item_type:object" # Can't link to yourself AND we're not interested in attachments oid = formData.get("qs") query += " AND -storage_id:\""+oid+"\"" # And we're not interested in attachments query += " AND display_type:\"package-dataset\"" req = SearchRequest(query) req.setParam("fl", "dc_title,storage_id,pidProperty") limit = formData.get("limit") if limit is None: limit = 10 req.setParam("rows", limit) # Search Solr indexer = services.getIndexer() out = ByteArrayOutputStream() indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Build a response list = [] for doc in result.getResults(): title = doc.getFirst("dc_title") #oid = doc.getFirst("storage_id") oid = doc.getFirst("pidProperty") list.append("%s::%s" % (oid, title)) result = "\n".join(list) writer = response.getPrintWriter("text/plain; charset=UTF-8") writer.println(result) writer.close()
def getViewers(self, oid): indexer = self.services.getIndexer() req = SearchRequest("id:" + oid) req.setParam("fl", "security_exception,owner") out = ByteArrayOutputStream() indexer.search(req, out) try: qresult = SolrResult(ByteArrayInputStream(out.toByteArray())).getResults().get(0) owner = qresult.getString(None, 'owner') secException = qresult.getArray('security_exception') if secException is None: secException = JSONArray() self.log.debug("Owner of object: " + owner) self.log.debug("Viewer(s) of object: " + secException.toString()) if secException.contains(owner): secException.remove(owner) return secException except Exception, e: self.log.error("Error during query/package ownership data" + str(e))
def __search(self): indexer = self.services.getIndexer() # Security prep work isAdmin = self.vc("page").authentication.is_admin() if not isAdmin: print "ERROR: User is not an admin '" return None req = SearchRequest('eventType:harvestStart') req.setParam("rows", "100") out = ByteArrayOutputStream() indexer.searchByIndex(req, out, "eventLog") self.__harvestList = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __activate__(self, context): formData = context["formData"] services = context["Services"] response = context["response"] query = "keywords:[* TO *]" q = formData.get("q") if q: query += " AND keywords:(%(q)s OR %(q)s*)" % { "q": q } req = SearchRequest(query) req.setParam("fl", "keywords") req.setParam("rows", "50") keywords = TreeSet() indexer = services.getIndexer() out = ByteArrayOutputStream() indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) for doc in result.getResults(): for keyword in doc.getList("keywords"): if keyword.startswith(q): keywords.add(keyword) writer = response.getPrintWriter("text/plain; charset=UTF-8") writer.println("\n".join(keywords)) writer.close()
def getAttachedFiles(self, oid): # Build a query req = SearchRequest("attached_to:%s" % oid) req.setParam("rows", "1000") # Run a search out = ByteArrayOutputStream() self.Services.getIndexer().search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Process results docs = JSONArray() for doc in result.getResults(): attachmentType = self.escapeHtml( WordUtils.capitalizeFully( doc.getFirst("attachment_type").replace("-", " "))) accessRights = self.escapeHtml( WordUtils.capitalizeFully(doc.getFirst("access_rights"))) entry = JsonObject() entry.put("filename", self.escapeHtml(doc.getFirst("filename"))) entry.put("attachment_type", attachmentType) entry.put("access_rights", accessRights) entry.put("id", self.escapeHtml(doc.getFirst("id"))) docs.add(entry) return docs
def getFilteredAssessments(self, packageType, stageName, filterType, startPage=1): """ A customised query to use filter to get certain assessment with desired status """ ## reference /redbox-rdsi-arms/src/main/config/home/lib/jython/util/Assessment.py for methods filters = {'assessment-draft': ['new','draft'], 'assessment-submitted':['submitted']} statusFilter = filters[filterType] req = SearchRequest("packageType:" + packageType) req.addParam("fq", 'workflow_step:' + stageName) req.setParam("sort", "date_object_modified desc, f_dc_title asc") req.setParam("fl",self.returnFields) out = ByteArrayOutputStream() self.indexer.search(req, out) solrResults = SolrResult(ByteArrayInputStream(out.toByteArray())) if solrResults: results = solrResults.getResults() returnArray = JSONArray() x = Assessment() x.activate(self.velocityContext) i = 0 rows = self.recordsPerPage start = (startPage - 1) * self.recordsPerPage for r in results: status = x.queryStatus(r.get("id")) if status in statusFilter: if i >= start and i - start < rows: assessment_submitted_date = x.queryAttr(r.get("id"), 'date') if assessment_submitted_date: r.getJsonObject().put('date', assessment_submitted_date) returnArray.add(r) i = i + 1 self._setPaging(returnArray.size()) return returnArray else: return ArrayList()
def _searchSets(self, indexer, searchType, isAdmin=True, security_query=''): req = SearchRequest("packageType:" + searchType) req.setParam("fq", 'item_type:"object"') req.addParam("fq", "") req.setParam("sort", "last_modified desc, f_dc_title asc") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __searchSolr(self): query = "handle:http* AND item_type:object" req = SearchRequest(query) req.setParam("rows", "99999") req.setParam("fl", "id,dc_title,handle,repository_type,repository_name") req.setParam("sort", "handle asc") req.setParam("facet", "true") req.setParam("facet.field", "repository_type,repository_name") out = ByteArrayOutputStream() self.services.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def findOidByIdentifier(self, identifier): query = "known_ids:\"" + identifier + "\"" request = SearchRequest(query) out = ByteArrayOutputStream() # Now search and parse response result = None try: self.indexer.search(request, out) inputStream = ByteArrayInputStream(out.toByteArray()) result = SolrResult(inputStream) except Exception, ex: self.log.error("Error searching Solr: ", ex) raise ex return None
def __getSolrData(self): level = self.getFormData("level", None) if level: if level == "top": query = 'rdf_type:"http://purl.org/asc/1297.0/2008/seo/SEO2"' else: query = 'skos_broader:"%s"' % level else: prefix = self.getSearchTerms() if prefix != "": terms = prefix.split(" ") if len(terms) > 1: termsQuery = " OR %s" ' OR '.join(terms) else: termsQuery = "" queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % { "prefix": prefix, "terms": termsQuery } query = 'dc_title:(%(qv)s)^2 OR dc_identifier:(%(qv)s)^0.5' % { "qv": queryValue } else: query = "*:*" portal = self.services.portalManager.get(self.portalId) sq = portal.searchQuery if sq not in ["", "*:*"]: query = query + " AND " + portal.searchQuery req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') if portal.query: req.addParam("fq", portal.query) req.setParam("fl", "score") req.setParam("sort", "score desc, f_dc_title asc") req.setParam("start", self.getStartIndex()) req.setParam("rows", self.getItemsPerPage()) try: out = ByteArrayOutputStream() indexer = self.services.getIndexer() indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def search_solr(self): # Build our solr query readyForNla = "ready_for_nla:ready" nlaPidExists = "nlaId:http*" query = readyForNla + " AND NOT " + nlaPidExists # Prepare the query req = SearchRequest(query) req.setParam("facet", "false") req.setParam("rows", "20") # Run the query try: out = ByteArrayOutputStream() self.services.getIndexer().search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Error searching solr: ", e) self.throw_error("failure searching solr: " + e.getMessage()) return None
def _searchSets(self, startPage=1): req = SearchRequest(self.getQuery()) req.setParam("fq", 'item_type:"object"') req.setParam("rows", str(self.getRecordsPerPage())) req.setParam("start", str((startPage - 1) * self.getRecordsPerPage())) req.addParam("fq", self.getFilterQuery()) req.setParam("fl", self.getReturnFields()) req.setParam("sort", "date_object_modified desc, f_dc_title asc") if not self.isAdmin(): req.addParam("fq", self.getSecurityQuery()) out = ByteArrayOutputStream() self.indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) self._setPaging(result.getNumFound()) result.getJsonObject().put("lastPage", str(self.paging.getLastPage())) result.getJsonObject().put("curPage", str(startPage)) return result
def _searchStage(self, stage, startPage=1): req = SearchRequest("packageType:arms") req.setParam("rows", str(self.recordsPerPage)) req.setParam("start", str((startPage - 1) * self.recordsPerPage)) if ',' in stage: stages = stage.split(',') for s in stages: s = "workflow_step:" + s req.addParam("fq", " OR ".join(stages)) # print "Searching stages %s" % " OR ".join(stages) else: req.addParam("fq", 'workflow_step:' + stage) req.setParam("sort", "last_modified desc, f_dc_title asc") req.setParam("fl", self.returnFields) out = ByteArrayOutputStream() self.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def _searchSets(self, searchType, isAdmin=True, security_query='', startPage=1): req = SearchRequest("packageType:" + searchType) req.setParam("rows", str(self.recordsPerPage)) req.setParam("start", str((startPage - 1) * self.recordsPerPage)) req.setParam("fq", 'item_type:"object"') req.addParam("fq", "") req.setParam("sort", "last_modified desc, f_dc_title asc") req.setParam("fl", self.returnFields) if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() self.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __getSolrData(self): prefix = self.getSearchTerms() if prefix != "": terms = prefix.split(" ") if len(terms) > 1: termsQuery = " OR %s" ' OR '.join(terms) else: termsQuery = "" queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % { "prefix": prefix, "terms": termsQuery } query = 'dc_title:(%(qv)s)^0.5 OR grant_number:(%(qvu)s)^2' % { "qv": queryValue, "qvu": queryValue.upper() } else: query = "*:*" portal = self.services.portalManager.get(self.portalId) sq = portal.searchQuery if sq not in ["", "*:*"]: query = query + " AND " + portal.searchQuery req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') if portal.query: req.addParam("fq", portal.query) req.setParam("fl", "score") req.setParam("sort", "score desc, f_dc_title asc") req.setParam("start", self.getStartIndex()) req.setParam("rows", self.getItemsPerPage()) try: out = ByteArrayOutputStream() indexer = self.services.getIndexer() indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def __search(self): indexer = self.services.getIndexer() portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery() portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery() # Security prep work current_user = self.vc("page").authentication.get_username() security_roles = self.vc("page").authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" isAdmin = self.vc("page").authentication.is_admin() req = SearchRequest("last_modified:[NOW-1MONTH TO *]") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest(owner_query) req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest('workflow_security:"' + current_user + '"') req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest("*:*") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.addParam("fq", "") req.setParam("rows", "0") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.vc("sessionState").set("fq", 'item_type:"object"') # sessionState.set("query", portalQuery.replace("\"", "'")) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
class SearchData: def __activate__(self, context): self.services = context["Services"] self.page = context["page"] self.formData = context["formData"] self.portalId = context["portalId"] self.sessionState = context["sessionState"] self.request = context["request"] self.pageName = context["pageName"] self.log = context["log"] self.__portal = context["page"].getPortal() self.__useSessionNavigation = self.__portal.getBoolean(True, ["portal", "use-session-navigation"]) self.__result = None if self.__useSessionNavigation: self.__pageNum = self.sessionState.get("pageNum", 1) else: self.__pageNum = 1 self.__selected = ArrayList() self.__fqParts = [] self.__searchField = self.formData.get("searchField", "full_text") self.__sortField = self.formData.get("sort-field") self.__sortOrder = self.formData.get("sort-order") if not (self.__sortField or self.__sortOrder): # use form data not specified, check session sortField = self.__portal.sortFieldDefault or "score" sortOrder = self.__portal.sortFieldDefaultOrder or "desc" #print "f:%s,o:%s" % (sortField, sortOrder) self.__sortField = self.sessionState.get("sortField", sortField) self.__sortOrder = self.sessionState.get("sortOrder", sortOrder) self.sessionState.set("sortField", self.__sortField) self.sessionState.set("sortOrder", self.__sortOrder) self.__sortBy = "%s %s" % (self.__sortField, self.__sortOrder) # reset the query and facet selections when changing views lastPortalId = self.sessionState.get("lastPortalId") if lastPortalId != self.portalId: self.sessionState.remove("fq") self.sessionState.remove("pageNum") self.sessionState.remove("sortField") self.sessionState.remove("sortOrder") self.__pageNum = 1 self.sessionState.set("lastPortalId", self.portalId) self.__search() def usingSessionNavigation(self): return self.__restful def getPortalName(self): return self.__portal.getDescription() def getSearchField(self): return self.__searchField def __search(self): requireEscape = False recordsPerPage = self.__portal.recordsPerPage uri = URLDecoder.decode(self.request.getAttribute("RequestURI")) query = None pagePath = self.__portal.getName() + "/" + self.pageName if query is None or query == "": query = self.formData.get("query") requireEscape = True if query is None or query == "": query = "*:*" if query == "*:*": self.__query = "" else: self.__query = query if requireEscape: query = self.__escapeQuery(query) query = "%s:%s" % (self.__searchField, query) self.sessionState.set("query", self.__query) # find objects with annotations matching the query if query != "*:*": anotarQuery = self.__query if requireEscape: anotarQuery = self.__escapeQuery(anotarQuery) annoReq = SearchRequest(anotarQuery) annoReq.setParam("facet", "false") annoReq.setParam("rows", str(99999)) annoReq.setParam("sort", "dateCreated asc") annoReq.setParam("start", str(0)) anotarOut = ByteArrayOutputStream() self.services.indexer.annotateSearch(annoReq, anotarOut) resultForAnotar = SolrResult(ByteArrayInputStream(anotarOut.toByteArray())) resultForAnotar = resultForAnotar.getResults() ids = HashSet() for annoDoc in resultForAnotar: annotatesUri = annoDoc.getFirst("annotatesUri") ids.add(annotatesUri) self.log.debug("Found annotation for %s" % annotatesUri) # add annotation ids to query query += ' OR id:("' + '" OR "'.join(ids) + '")' portalSearchQuery = self.__portal.searchQuery if portalSearchQuery == "": portalSearchQuery = query else: if query != "*:*": query += " AND " + portalSearchQuery else: query = portalSearchQuery req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", self.__portal.facetFieldList) req.setParam("facet.sort", Boolean.toString(self.__portal.getFacetSort())) req.setParam("facet.limit", str(self.__portal.facetCount)) req.setParam("sort", self.__sortBy) # setup facets if self.__useSessionNavigation: action = self.formData.get("verb") value = self.formData.get("value") fq = self.sessionState.get("fq") if fq is not None: self.__pageNum = 1 req.setParam("fq", fq) if action == "add_fq": self.__pageNum = 1 req.addParam("fq", URLDecoder.decode(value, "UTF-8")) elif action == "remove_fq": self.__pageNum = 1 req.removeParam("fq", URLDecoder.decode(value, "UTF-8")) elif action == "clear_fq": self.__pageNum = 1 req.removeParam("fq") elif action == "select-page": self.__pageNum = int(value) else: navUri = uri[len(pagePath):] self.__pageNum, fq, self.__fqParts = self.__parseUri(navUri) savedfq = self.sessionState.get("savedfq") limits = [] if savedfq: limits.extend(savedfq) if fq: limits.extend(fq) self.sessionState.set("savedfq", limits) for q in fq: req.addParam("fq", URLDecoder.decode(q, "UTF-8")) portalQuery = self.__portal.query if portalQuery: req.addParam("fq", portalQuery) req.addParam("fq", 'item_type:"object"') if req.getParams("fq"): self.__selected = ArrayList(req.getParams("fq")) if self.__useSessionNavigation: self.sessionState.set("fq", self.__selected) self.sessionState.set("searchQuery", portalSearchQuery) self.sessionState.set("pageNum", self.__pageNum) # Make sure 'fq' has already been set in the session if not self.page.authentication.is_admin(): current_user = self.page.authentication.get_username() security_roles = self.page.authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" req.addParam("fq", security_query) req.setParam("start", str((self.__pageNum - 1) * recordsPerPage)) #print " * search.py:", req.toString(), self.__pageNum out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray())) if self.__result is not None: self.__paging = Pagination(self.__pageNum, self.__result.getNumFound(), self.__portal.recordsPerPage) def __escapeQuery(self, q): temp = "" chars = "+-&|!(){}[]^\"~*?:\\" for c in q: if c in chars: temp += "\%s" % c else: temp += c return temp # eq = q # # escape all solr/lucene special chars # # from http://lucene.apache.org/java/2_4_0/queryparsersyntax.html#Escaping%20Special%20Characters # for c in "+-&|!(){}[]^\"~*?:\\": # eq = eq.replace(c, "\\%s" % c) # ## Escape UTF8 # try: # return URLEncoder.encode(eq, "UTF-8") # except UnsupportedEncodingException, e: # print "Error during UTF8 escape! ", repr(eq) # return eq def getQueryTime(self): return int(self.__result.getQueryTime()) / 1000.0; def getPaging(self): return self.__paging def getResult(self): return self.__result def getFacetField(self, key): return self.__portal.facetFields.get(key) def getFacetName(self, key): return self.__portal.facetFields.get(key).getString(None, ["label"]) def getFacetCounts(self, key): if self.__useSessionNavigation: facetData = self.__result.getFacets() if facetData is None: return LinkedHashMap() if not facetData.containsKey(key): return LinkedHashMap() return facetData.get(key).values() else: return LinkedHashMap() # TODO : What were these doing? Hiding file path facets unless some facets are selected? #if name.find("/") == -1 or self.hasSelectedFacets(): # values.put(name, count) def getFacetDisplay(self): return self.__portal.facetDisplay def hasSelectedFacets(self): return (self.__selected is not None and len(self.__selected) > 1) and \ not (self.__portal.query in self.__selected and len(self.__selected) == 2) def getSelectedFacets(self): return self.__selected def isPortalQueryFacet(self, fq): return fq == self.__portal.query def isSelected(self, fq): return fq in self.__selected def getSelectedFacetIds(self): return [md5.new(fq).hexdigest() for fq in self.__selected] def getFileName(self, path): return os.path.splitext(os.path.basename(path))[0] def getFacetQuery(self, name, value): return '%s:"%s"' % (name, value) # Packaging support def getActiveManifestTitle(self): return self.__getActiveManifest().getTitle() def getActiveManifestId(self): return self.sessionState.get("package/active/id") def getSelectedItemsCount(self): return self.__getActiveManifest().size() def isSelectedForPackage(self, oid): result = self.__getActiveManifest().getNode("node-%s" % oid) return (result is not None) def getManifestItemTitle(self, oid, defaultValue): result = self.__getActiveManifest().getNode("node-%s" % oid) if result is None: return defaultValue return result.getTitle() def __getActiveManifest(self): activeManifest = self.sessionState.get("package/active") if not activeManifest: activeManifest = Manifest(None) activeManifest.setTitle("New package") activeManifest.setViewId(self.__portal.getName()) self.sessionState.set("package/active", activeManifest) return activeManifest def isSelectableForPackage(self, oid): return oid != self.getActiveManifestId() def getSortFields(self): return self.__portal.sortFields def getSortField(self): return self.__sortField def getSortOrder(self): return self.__sortOrder # RESTful style URL support methods def getPageQuery(self, page): prefix = "" if self.__fqParts: prefix = "/" + "/".join(self.__fqParts) suffix = "" if page > 1: suffix = "/page/%s" % page return prefix + suffix def getFacetQueryUri(self, name, value): return "%s/%s" % (name, value) def getFacetValue(self, facetValue): return facetValue.split("/")[-1] def getLimitQueryWith(self, fq): limits = ArrayList(self.__fqParts) limits.add("category/" + fq) return "/".join(limits) def getFacetIndent(self, facetValue): return len(facetValue.split("/")) def getLimitQueryWithout(self, fq): limits = ArrayList(self.__fqParts) limits.remove("category/" + fq) if limits.isEmpty(): return "" return "/".join(limits) def __parseUri(self, uri): page = 1 fq = [] fqParts = [] if uri != "": parts = uri.split("/") partType = None facetKey = None facetValues = None for part in parts: if partType == "page": facetKey = None page = int(part) elif partType == "category": partType = "category-value" facetValues = None facetKey = part elif partType == "category-value": if facetValues is None: facetValues = [] if part in ["page", "category"]: partType = part facetQuery = '%s:"%s"' % (facetKey, "/".join(facetValues)) fq.append(facetQuery) fqParts.append("category/%s/%s" % (facetKey, "/".join(facetValues))) facetKey = None facetValues = None else: facetValues.append(URLDecoder.decode(part)) else: partType = part if partType == "category-value": facetQuery = '%s:"%s"' % (facetKey, "/".join(facetValues)) fq.append(facetQuery) fqParts.append("category/%s/%s" % (facetKey, "/".join(facetValues))) return page, fq, fqParts
def findPublishedRecords(self): req = SearchRequest("published:\"true\"") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
class OaiData: def __init__(self): self.tokensDB = None def __activate__(self, context): if self.tokensDB is None: self.tokensDB = TokensDatabase(context) # Set up configuration self.systemConfig = JsonSimpleConfig() self.oaiConfig = None self.getMetadataFormats() self.velocityContext = context self.services = context["Services"] self.log = context["log"] self.sessionState = context["sessionState"] self.portalDir = context["portalDir"] self.__result = None self.lastPage = False # Check if the OAI request has an overriding portal ('set') to the URL paramSet = self.vc("formData").get("set") self.__portalName = context["page"].getPortal().getName() illegalSet = False if paramSet is not None: portals = self.vc("page").getPortals().keySet() if portals.contains(paramSet): self.__portalName = paramSet else: illegalSet = True self.__metadataPrefix = "" self.__sessionExpiry = self.systemConfig.getInteger(None, ["portal", "oai-pmh", "sessionExpiry"]) # Check if there's a resumption token in the formData self.__currentToken = None resumptionToken = self.vc("formData").get("resumptionToken") if resumptionToken is not None: # Split out the start component from the actual resumption token (resumptionTokenPart, start) = resumptionToken.strip().split(":") # This could still be be null self.__currentToken = self.tokensDB.getToken(resumptionTokenPart) # Code to handle null token is handled later on if self.__currentToken is not None: self.__currentToken.setStart(start) # Process/parse the request we've received for validity self.vc("request").setAttribute("Content-Type", "text/xml") self.__request = OaiPmhVerb(context, self.tokensDB, self.__currentToken) if self.getError() is None and illegalSet: self.__request.setError("badArgument", "Set '%s' is not valid!" % paramSet) # If there are no errors... and the request requires some additional # data (like a search result) do so now. Everything else can be # handled in the templates. if self.getError() is None and self.getVerb() in ["GetRecord", "ListIdentifiers", "ListRecords"]: # Find the metadata prefix requested self.__metadataPrefix = self.vc("formData").get("metadataPrefix") if self.__metadataPrefix is None: self.__metadataPrefix = self.__currentToken.getMetadataPrefix() # Only list records if the metadata format is enabled in this view if self.isInView(self.__metadataPrefix): self.__search() # Get from velocity context def vc(self, index): if self.velocityContext[index] is not None: return self.velocityContext[index] else: self.log.error("ERROR: Requested context entry '" + index + "' doesn't exist") return None def isInView(self, format, view=None): # Sanity check if format is None or format == "": return False # Default to current poral if view is None: view = self.__portalName # Make sure there is some config for this format formatConfig = self.getMetadataFormats().get(format) if formatConfig is None: return False # Is it visible everywhere? allViews = formatConfig.getBoolean(False, ["enabledInAllViews"]) if allViews: return True # Check if it is visible in this view else: allowedViews = formatConfig.getStringList(["enabledViews"]) if view in allowedViews: return True # Rejection return False def getID(self, item): identifier = item.getFirst("oai_identifier") # Fallback to the default if identifier is None or identifier == "": return "oai:fascinator.usq.edu.au:" + item.getFirst("id") # Use the indexed value return identifier def isDeleted(self, item): return bool(item.getFirst("oai_deleted")) def getSet(self, item): set = item.getFirst("oai_set") # Fallback to the portal name if set is None or set == "": return self.__portalName # Use the required set return set def getVerb(self): return self.getRequest().getVerb() def getError(self): return self.getRequest().getError() def getResponseDate(self): return time.strftime("%Y-%m-%dT%H:%M:%SZ") def getRequest(self): return self.__request def getResult(self): return self.__result def getElement(self, elementName, values): elementStr = "" if values: for value in values: elementStr += "<%s>%s</%s>" % (elementName, value, elementName) return elementStr def __search(self): self.__result = SolrResult(None) portal = self.services.getPortalManager().get(self.__portalName) recordsPerPage = portal.recordsPerPage # Resolve our identifier id = self.vc("formData").get("identifier") query = "*:*" if id is not None and id != "": # A default TF2 OID if id.startswith("oai:fascinator.usq.edu.au:"): idString = id.replace("oai:fascinator.usq.edu.au:", "") idString = self.__escapeQuery(idString) query = "id:" + idString # Or a custom OAI ID else: idString = self.__escapeQuery(id) query = "oai_identifier:" + idString req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", portal.facetFieldList) req.setParam("facet.limit", str(portal.facetCount)) req.setParam("sort", "f_dc_title asc") portalQuery = portal.query if portalQuery: req.addParam("fq", portalQuery) req.addParam("fq", "item_type:object") # Date data... is supplied fromDate = self.__request.getFromDate() untilDate = self.__request.getUntilDate() if fromDate is not None: fromStr = fromDate.isoformat() + "Z" self.log.debug("From Date: '{}'", fromStr) if untilDate is not None: untilStr = untilDate.isoformat() + "Z" self.log.debug("Until Date: '{}'", untilStr) queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr) else: queryStr = "last_modified:[%s TO *]" % (fromStr) self.log.debug("Date query: '{}'", queryStr) req.addParam("fq", queryStr) else: if untilDate is not None: untilStr = untilDate.isoformat() + "Z" self.log.debug("Until Date: '{}'", untilDate.isoformat()) queryStr = "last_modified:[* TO %s]" % (untilStr) self.log.debug("Date query: '{}'", queryStr) req.addParam("fq", queryStr) # Check if there's resumption token exist in the formData newToken = None if self.__currentToken is not None: start = int(self.__currentToken.getStart()) totalFound = int(self.__currentToken.getTotalFound()) nextTokenStart = start + recordsPerPage if nextTokenStart < totalFound: newToken = self.__currentToken newToken.resetExpiry(self.__sessionExpiry) newToken.setStart(nextTokenStart) # or start a new resumption token else: start = 0 newToken = ResumptionToken(None, recordsPerPage, self.__metadataPrefix, self.__sessionExpiry) req.setParam("start", str(start)) out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray())) totalFound = self.__result.getNumFound() if totalFound == 0: newToken = None # If an ID was requested, and not found, this is an error if id is not None and id != "": self.__request.setError("idDoesNotExist", "ID: '%s' not found" % id) else: self.__request.setError("noRecordsMatch", "No records match this request") # We need to store this for NEW tokens elif self.__currentToken is None: # Assuming there are enough results to even keep the token if newToken.getStart() < totalFound: newToken.setTotalFound(totalFound) else: newToken = None # Check if we need to remove the resumption token else: if (start + recordsPerPage) >= totalFound: self.tokensDB.removeToken(self.__currentToken) self.lastPage = True # Store/update the resumption token if newToken is not None: # Brand new token if self.__currentToken is None: self.tokensDB.storeToken(newToken) # Or update an old token else: self.tokensDB.updateToken(newToken) self.__currentToken = newToken def getToken(self): if self.isInView(self.__metadataPrefix) and not self.lastPage: return self.__currentToken return None def getMetadataFormats(self): if self.oaiConfig is None: self.oaiConfig = self.systemConfig.getJsonSimpleMap(["portal", "oai-pmh", "metadataFormats"]) return self.oaiConfig def encodeXml(self, string): return StringEscapeUtils.escapeXml(string) def getPayload(self, oid, metadataFileName): # First get the Object from storage object = None try: object = self.services.getStorage().getObject(oid) except StorageException, e: return None # Check whether the payload exists try: return object.getPayload(metadataFileName) except StorageException, e: return None
def __search(self): requireEscape = False recordsPerPage = self.__portal.recordsPerPage uri = URLDecoder.decode(self.request.getAttribute("RequestURI")) query = None pagePath = self.__portal.getName() + "/" + self.pageName if query is None or query == "": query = self.formData.get("query") requireEscape = True if query is None or query == "": query = "*:*" if query == "*:*": self.__query = "" else: self.__query = query if requireEscape: query = self.__escapeQuery(query) query = "%s:%s" % (self.__searchField, query) self.sessionState.set("query", self.__query) # find objects with annotations matching the query if query != "*:*": anotarQuery = self.__query if requireEscape: anotarQuery = self.__escapeQuery(anotarQuery) annoReq = SearchRequest(anotarQuery) annoReq.setParam("facet", "false") annoReq.setParam("rows", str(99999)) annoReq.setParam("sort", "dateCreated asc") annoReq.setParam("start", str(0)) anotarOut = ByteArrayOutputStream() self.services.indexer.annotateSearch(annoReq, anotarOut) resultForAnotar = SolrResult(ByteArrayInputStream(anotarOut.toByteArray())) resultForAnotar = resultForAnotar.getResults() ids = HashSet() for annoDoc in resultForAnotar: annotatesUri = annoDoc.getFirst("annotatesUri") ids.add(annotatesUri) self.log.debug("Found annotation for %s" % annotatesUri) # add annotation ids to query query += ' OR id:("' + '" OR "'.join(ids) + '")' portalSearchQuery = self.__portal.searchQuery if portalSearchQuery == "": portalSearchQuery = query else: if query != "*:*": query += " AND " + portalSearchQuery else: query = portalSearchQuery req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", self.__portal.facetFieldList) req.setParam("facet.sort", Boolean.toString(self.__portal.getFacetSort())) req.setParam("facet.limit", str(self.__portal.facetCount)) req.setParam("sort", self.__sortBy) # setup facets if self.__useSessionNavigation: action = self.formData.get("verb") value = self.formData.get("value") fq = self.sessionState.get("fq") if fq is not None: self.__pageNum = 1 req.setParam("fq", fq) if action == "add_fq": self.__pageNum = 1 req.addParam("fq", URLDecoder.decode(value, "UTF-8")) elif action == "remove_fq": self.__pageNum = 1 req.removeParam("fq", URLDecoder.decode(value, "UTF-8")) elif action == "clear_fq": self.__pageNum = 1 req.removeParam("fq") elif action == "select-page": self.__pageNum = int(value) else: navUri = uri[len(pagePath):] self.__pageNum, fq, self.__fqParts = self.__parseUri(navUri) savedfq = self.sessionState.get("savedfq") limits = [] if savedfq: limits.extend(savedfq) if fq: limits.extend(fq) self.sessionState.set("savedfq", limits) for q in fq: req.addParam("fq", URLDecoder.decode(q, "UTF-8")) portalQuery = self.__portal.query if portalQuery: req.addParam("fq", portalQuery) req.addParam("fq", 'item_type:"object"') if req.getParams("fq"): self.__selected = ArrayList(req.getParams("fq")) if self.__useSessionNavigation: self.sessionState.set("fq", self.__selected) self.sessionState.set("searchQuery", portalSearchQuery) self.sessionState.set("pageNum", self.__pageNum) # Make sure 'fq' has already been set in the session if not self.page.authentication.is_admin(): current_user = self.page.authentication.get_username() security_roles = self.page.authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" req.addParam("fq", security_query) req.setParam("start", str((self.__pageNum - 1) * recordsPerPage)) #print " * search.py:", req.toString(), self.__pageNum out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray())) if self.__result is not None: self.__paging = Pagination(self.__pageNum, self.__result.getNumFound(), self.__portal.recordsPerPage)
def __search(self): self.__result = SolrResult(None) portal = self.services.getPortalManager().get(self.__portalName) recordsPerPage = portal.recordsPerPage # Resolve our identifier id = self.vc("formData").get("identifier") query = "*:*" if id is not None and id != "": # A default TF2 OID if id.startswith("oai:fascinator.usq.edu.au:"): idString = id.replace("oai:fascinator.usq.edu.au:", "") idString = self.__escapeQuery(idString) query = "id:" + idString # Or a custom OAI ID else: idString = self.__escapeQuery(id) query = "oai_identifier:" + idString req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", portal.facetFieldList) req.setParam("facet.limit", str(portal.facetCount)) req.setParam("sort", "f_dc_title asc") portalQuery = portal.query if portalQuery: req.addParam("fq", portalQuery) req.addParam("fq", "item_type:object") # Date data... is supplied fromDate = self.__request.getFromDate() untilDate = self.__request.getUntilDate() if fromDate is not None: fromStr = fromDate.isoformat() + "Z" self.log.debug("From Date: '{}'", fromStr) if untilDate is not None: untilStr = untilDate.isoformat() + "Z" self.log.debug("Until Date: '{}'", untilStr) queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr) else: queryStr = "last_modified:[%s TO *]" % (fromStr) self.log.debug("Date query: '{}'", queryStr) req.addParam("fq", queryStr) else: if untilDate is not None: untilStr = untilDate.isoformat() + "Z" self.log.debug("Until Date: '{}'", untilDate.isoformat()) queryStr = "last_modified:[* TO %s]" % (untilStr) self.log.debug("Date query: '{}'", queryStr) req.addParam("fq", queryStr) # Check if there's resumption token exist in the formData newToken = None if self.__currentToken is not None: start = int(self.__currentToken.getStart()) totalFound = int(self.__currentToken.getTotalFound()) nextTokenStart = start + recordsPerPage if nextTokenStart < totalFound: newToken = self.__currentToken newToken.resetExpiry(self.__sessionExpiry) newToken.setStart(nextTokenStart) # or start a new resumption token else: start = 0 newToken = ResumptionToken(None, recordsPerPage, self.__metadataPrefix, self.__sessionExpiry) req.setParam("start", str(start)) out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray())) totalFound = self.__result.getNumFound() if totalFound == 0: newToken = None # If an ID was requested, and not found, this is an error if id is not None and id != "": self.__request.setError("idDoesNotExist", "ID: '%s' not found" % id) else: self.__request.setError("noRecordsMatch", "No records match this request") # We need to store this for NEW tokens elif self.__currentToken is None: # Assuming there are enough results to even keep the token if newToken.getStart() < totalFound: newToken.setTotalFound(totalFound) else: newToken = None # Check if we need to remove the resumption token else: if (start + recordsPerPage) >= totalFound: self.tokensDB.removeToken(self.__currentToken) self.lastPage = True # Store/update the resumption token if newToken is not None: # Brand new token if self.__currentToken is None: self.tokensDB.storeToken(newToken) # Or update an old token else: self.tokensDB.updateToken(newToken) self.__currentToken = newToken
def __activate__(self, context): response = context["response"] log = context["log"] writer = response.getPrintWriter("text/plain; charset=UTF-8") auth = context["page"].authentication sessionState = context["sessionState"] result = JsonObject() result.put("status", "error") result.put("message", "An unknown error has occurred") if auth.is_admin(): services = context["Services"] formData = context["formData"] func = formData.get("func") oid = formData.get("oid") portalId = formData.get("portalId") portalManager = services.portalManager if func == "reharvest": # One object if oid: log.info(" * Reharvesting object '{}'", oid) portalManager.reharvest(oid) result.put("status", "ok") result.put("message", "Object '%s' queued for reharvest") # The whole portal elif portalId: log.info(" * Reharvesting view '{}'", portalId) sessionState.set("reharvest/running/" + portalId, "true") # TODO security filter - not necessary because this requires admin anyway? portal = portalManager.get(portalId) query = "*:*" if portal.query != "": query = portal.query if portal.searchQuery != "": if query == "*:*": query = portal.searchQuery else: query = query + " AND " + portal.searchQuery # query solr to get the objects to reharvest rows = 25 req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') req.setParam("rows", str(rows)) req.setParam("fl", "id") done = False count = 0 while not done: req.setParam("start", str(count)) out = ByteArrayOutputStream() services.indexer.search(req, out) json = SolrResult(ByteArrayInputStream(out.toByteArray())) objectIds = HashSet(json.getFieldList("id")) if not objectIds.isEmpty(): portalManager.reharvest(objectIds) count = count + rows total = json.getNumFound() log.info(" * Queued {} of {}...", (min(count, total), total)) done = (count >= total) sessionState.remove("reharvest/running/" + portalId) result.put("status", "ok") result.put("message", "Objects in '%s' queued for reharvest" % portalId) else: response.setStatus(500) result.put("message", "No object or view specified for reharvest") elif func == "reindex": if oid: log.info(" * Reindexing object '{}'", oid) services.indexer.index(oid) services.indexer.commit() result.put("status", "ok") result.put("message", "Object '%s' queued for reindex" % portalId) else: response.setStatus(500) result.put("message", "No object specified to reindex") else: response.setStatus(500) result.put("message", "Unknown action '%s'" % func) else: response.setStatus(500) result.put("message", "Only administrative users can access this API") writer.println(result.toString()) writer.close()
def checkEventLogForEdits(self,oid): req = SearchRequest("oid:"+oid +" AND context:Workflow AND eventType:Save") out = ByteArrayOutputStream() self.indexer.searchByIndex(req, out, "eventLog") solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getRows() > 0
class HomeData: def __init__(self): pass def __activate__(self, context): self.velocityContext = context self.vc("sessionState").remove("fq") self.services = self.vc("Services") self.__latest = None self.__mine = None self.__workflows = None self.__result = None self.__steps = None self.__selfservicesStages = None self.__search() # Get from velocity context Mint version def vc(self, index): if self.velocityContext[index] is not None: return self.velocityContext[index] else: print "ERROR: Requested context entry '" + index + "' doesn't exist" return None def __search(self): indexer = self.services.getIndexer() portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery() portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery() # Security prep work current_user = self.vc("page").authentication.get_username() security_roles = self.vc("page").authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" isAdmin = self.vc("page").authentication.is_admin() req = SearchRequest("last_modified:[NOW-1MONTH TO *]") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest(owner_query) req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest('workflow_security:"' + current_user + '"') req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest("*:*") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.addParam("fq", "") req.setParam("rows", "0") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.vc("sessionState").set("fq", 'item_type:"object"') #sessionState.set("query", portalQuery.replace("\"", "'")) # Load in the services UI workflow selfSubmitWfConfig = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/servicesUI.json")) selfSubmitJsonStageList = selfSubmitWfConfig.getJsonSimpleList(["stages"]) servicesStages = [] for jsonStage in selfSubmitJsonStageList: wfStage = WorkflowStage(jsonStage, self.__steps) servicesStages.append(wfStage) self.__selfservicesStages = servicesStages self.__result = SolrResult(ByteArrayInputStream(out.toByteArray())) def getLatest(self): return self.__latest.getResults() def getMine(self): return self.__mine.getResults() def getWorkflows(self): return self.__workflows.getResults() def getItemCount(self): return self.__result.getNumFound() def getServicesStages(self): return self.__servicesStages
def __search(self): indexer = self.services.getIndexer() portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery() portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery() # Security prep work current_user = self.vc("page").authentication.get_username() security_roles = self.vc("page").authentication.get_roles_list() security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")' security_exceptions = 'security_exception:"' + current_user + '"' owner_query = 'owner:"' + current_user + '"' security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")" isAdmin = self.vc("page").authentication.is_admin() req = SearchRequest("last_modified:[NOW-1MONTH TO *]") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest(owner_query) req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest('workflow_security:"' + current_user + '"') req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.setParam("rows", "10") req.setParam("sort", "last_modified desc, f_dc_title asc"); if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray())) req = SearchRequest("*:*") req.setParam("fq", 'item_type:"object"') if portalQuery: req.addParam("fq", portalQuery) if portalSearchQuery: req.addParam("fq", portalSearchQuery) req.addParam("fq", "") req.setParam("rows", "0") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) self.vc("sessionState").set("fq", 'item_type:"object"') #sessionState.set("query", portalQuery.replace("\"", "'")) # Load in the services UI workflow selfSubmitWfConfig = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/servicesUI.json")) selfSubmitJsonStageList = selfSubmitWfConfig.getJsonSimpleList(["stages"]) servicesStages = [] for jsonStage in selfSubmitJsonStageList: wfStage = WorkflowStage(jsonStage, self.__steps) servicesStages.append(wfStage) self.__selfservicesStages = servicesStages self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))