def findPublishedRecords(self): #req = SearchRequest("published:\"true\"") req = SearchRequest("storage_id:\"c6a214670dc644e5ebdaede4a2243f67\"") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def handleGrantNumber(self): out = ByteArrayOutputStream() req = SearchRequest("grant_numbers:%s*" % self.term) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): creatorResults = res.getResults() for creatorRes in creatorResults: creatorList = creatorRes.getList("grant_numbers") if (creatorList.isEmpty()==False): for hit in creatorList: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print(",\"%s\"" % hit) else: self.writer.print("\"%s\"" % hit) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def handleWorkflowStep(self): out = ByteArrayOutputStream() req = SearchRequest("workflow_step_label:[* TO *]" ) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): recordTypeResults = res.getResults() for recordTypeResult in recordTypeResults: recordTypeList = recordTypeResult.getList("workflow_step_label") if (recordTypeList.isEmpty()==False): for hit in recordTypeList: hits.add(hit) self.writer.println("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.println(",{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit)) else: self.writer.println("{\"value\": \"%s\",\n\"label\": \"%s\"}" % (hit,hit)) hitnum += 1 self.writer.println("]") else: self.writer.println("[\"\"]") self.writer.close()
def checkEventLogForEdits(self, oid): req = SearchRequest("oid:" + oid + " AND context:Workflow AND eventType:Save") out = ByteArrayOutputStream() self.indexer.searchByIndex(req, out, "eventLog") solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getRows() > 0
def __feed(self): self.portal = self.services.getPortalManager().get(self.vc("portalId")) recordsPerPage = self.portal.recordsPerPage pageNum = self.vc("sessionState").get("pageNum", 1) query = "*:*" if self.vc("formData").get("query"): query = self.vc("formData").get("query") query = self.__escapeQuery(query) req = SearchRequest(query) req.setParam("facet", "true") req.setParam("rows", str(recordsPerPage)) req.setParam("facet.field", self.portal.facetFieldList) req.setParam("facet.sort", "true") req.setParam("facet.limit", str(self.portal.facetCount)) req.setParam("sort", "f_dc_title asc") portalQuery = self.portal.query if portalQuery: req.addParam("fq", portalQuery) else: fq = self.vc("sessionState").get("fq") if fq is not None: req.setParam("fq", fq) req.setParam("start", str((pageNum - 1) * recordsPerPage)) self.log.debug(" * Query: '{}'", query) self.log.debug(" * portalQuery: '{}'", portalQuery) self.log.debug(" * feed.py: '{}'", req) out = ByteArrayOutputStream() self.services.indexer.search(req, out) self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __search(self): query = self.formData.get("query") searchQuery = self.sessionState.get("searchQuery") if query is None or query == "": query = "*:*" if searchQuery and query == "*:*": query = searchQuery elif searchQuery: query += " AND " + searchQuery facetField = self.formData.get("facet.field") req = SearchRequest(query) req.setParam("facet", "true") req.setParam("fl", "id") req.setParam("rows", "0") req.setParam("facet.limit", "-1") req.setParam("facet.field", facetField) fq = self.sessionState.get("fq") if fq is not None: req.setParam("fq", fq) req.addParam("fq", 'item_type:"object"') # Make sure 'fq' has already been set in the session security_roles = self.auth.get_roles_list() security_query = 'security_filter:("' + '" OR "'.join( security_roles) + '")' req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer = self.services.indexer indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) return FacetList(facetField, result)
def export(self, exportType): exportQuery = "%s:%s" % (self.facetField, self.facetFieldValue) outputType = "text/%s; charset=UTF-8" % type responseHeader = "attachment; filename=%s.%s" % (self.facetFieldValue, exportType) try: out = ByteArrayOutputStream() recnumreq = SearchRequest(exportQuery) recnumreq.setParam("fl","create_timestamp") recnumreq.setParam("rows", "0") self.indexer.search(recnumreq, out) recnumres = SolrResult(ByteArrayInputStream(out.toByteArray())) self.__rowsFoundSolr = "%s" % recnumres.getNumFound() except: self.errorMsg = "Export query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Export query threw an exception (package type was %s): %s - %s" % (self.facetFieldValue, sys.exc_info()[0], sys.exc_info()[1])) return out = ByteArrayOutputStream() req = SearchRequest(exportQuery) req.setParam("wt", exportType) req.setParam("rows", self.__rowsFoundSolr) self.indexer.search(req, out) self.response.setHeader("Content-Disposition", responseHeader) writer = self.response.getPrintWriter(outputType) writer.println(out.toString("UTF-8")) writer.close()
def search_solr(self): query = "(rootUri:" if self.rootUriList: query += "(" + " OR ".join(self.rootUriList) + ")" else: query += "\"" + self.rootUri + "\"" if self.type: query += " AND type:\"" + self.type + "\"" query += ")" #print "**********", query req = SearchRequest(query) req.setParam("facet", "false") req.setParam("rows", str(99999)) req.setParam("sort", "dateCreated asc") req.setParam("start", str(0)) #security_roles = page.authentication.get_roles_list(); #security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")' #req.addParam("fq", security_query) out = ByteArrayOutputStream() Services.indexer.annotateSearch(req, out) result = SolrResult(ByteArrayInputStream( out.toByteArray())).getResults() # Every annotation for this URI if self.type == "http://www.purl.org/anotar/ns/type/0.1#Tag": return self.process_tags(result) else: return self.process_response(result)
def handleQuery(self, query, fieldName, formatStr): out = ByteArrayOutputStream() req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') req.setParam("fq", 'workflow_id:"dataset"') req.setParam("rows", "1000") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) hits = HashSet() if (res.getNumFound() > 0): results = res.getResults() for searchRes in results: searchResList = searchRes.getList(fieldName) if (searchResList.isEmpty() == False): for hit in searchResList: if self.term is not None: if hit.find(self.term) != -1: hits.add(hit) else: hits.add(hit) self.writer.print("[") hitnum = 0 for hit in hits: if (hitnum > 0): self.writer.print("," + formatStr % {"hit": hit}) else: self.writer.print(formatStr % {"hit": hit}) hitnum += 1 self.writer.print("]") else: self.writer.println("[\"\"]") self.writer.close()
def __getUsers(self, oid): indexer = Services.getIndexer() req = SearchRequest("id:" + oid) req.setParam("fl", "security_exception,owner") out = ByteArrayOutputStream() indexer.search(req, out) rtJson = "" try: qresult = SolrResult(ByteArrayInputStream( out.toByteArray())).getResults().get(0) owner = qresult.getString(None, 'owner') secException = qresult.getArray('security_exception') if secException is None: secException = JSONArray() self.log.debug("Owner of object: " + owner) self.log.debug("Viewer(s) of object: " + secException.toString()) if secException.contains(owner): secException.remove(owner) return '{"owner":"' + owner + '", "viewers": ' + secException.toString( ) + '}' except Exception, e: self.log.error("Error during query/package ownership data" + str(e))
def __searchExecute(self, search, count): try: search.setParam("start", str(count)) out = ByteArrayOutputStream() self.services.indexer.search(search, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Error during search: ", e) return None
def getAttachments(self): attachmentType = "review-attachments" req = SearchRequest("attached_to:%s AND attachment_type:%s" % (self.oid, attachmentType)) req.setParam("rows", "1000") out = ByteArrayOutputStream() self.Services.indexer.search(req, out) response = SolrResult(ByteArrayInputStream(out.toByteArray())) return response.getResults()
def findPackagesToTransition(self, fromWorkflowId, fromWorkflowStage): req = SearchRequest("workflow_id:" + fromWorkflowId + " AND _query_:\"workflow_step:" + fromWorkflowStage + "\"") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def findPackagesToPurge(self, packageType): req = SearchRequest("display_type:" + packageType + " AND date_object_created:[* TO NOW-7DAY]") req.setParam("fq", "owner:[* TO *]") req.setParam("fq", "security_filter:[* TO *]") req.setParam("fl", "storage_id,date_object_created,date_object_modified") out = ByteArrayOutputStream() self.indexer.search(req, out) solrResult = SolrResult(ByteArrayInputStream(out.toByteArray())) return solrResult.getResults()
def __loadSolrData(self, oid): portal = self.vc("page").getPortal() query = 'id:"%s"' % oid if portal.getSearchQuery(): query += " AND " + portal.getSearchQuery() req = SearchRequest(query) req.addParam("fq", 'item_type:"object"') req.addParam("fq", portal.getQuery()) out = ByteArrayOutputStream() self.vc("Services").getIndexer().search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def _searchSets(self, indexer, searchType, isAdmin=True, security_query=''): req = SearchRequest("packageType:" + searchType) req.setParam("fq", 'item_type:"object"') req.addParam("fq", "") req.setParam("sort", "last_modified desc, f_dc_title asc") if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __search(self): indexer = self.services.getIndexer() # Security prep work isAdmin = self.vc("page").authentication.is_admin() if not isAdmin: print "ERROR: User is not an admin '" return None req = SearchRequest('eventType:harvestStart') req.setParam("rows", "100") out = ByteArrayOutputStream() indexer.searchByIndex(req, out, "eventLog") self.__harvestList = SolrResult(ByteArrayInputStream( out.toByteArray()))
def __searchSolr(self): query = "handle:http* AND item_type:object" req = SearchRequest(query) req.setParam("rows", "99999") req.setParam("fl", "id,dc_title,handle,repository_type,repository_name") req.setParam("sort", "handle asc") req.setParam("facet", "true") req.setParam("facet.field", "repository_type,repository_name") out = ByteArrayOutputStream() self.services.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def findOidByIdentifier(self, identifier): query = "known_ids:\"" + identifier + "\"" request = SearchRequest(query) out = ByteArrayOutputStream() # Now search and parse response result = None try: self.indexer.search(request, out) inputStream = ByteArrayInputStream(out.toByteArray()) result = SolrResult(inputStream) except Exception, ex: self.log.error("Error searching Solr: ", ex) raise ex return None
def _searchSets(self, startPage=1): req = SearchRequest(self.getQuery()) req.setParam("fq", 'item_type:"object"') req.setParam("rows", str(self.getRecordsPerPage())) req.setParam("start", str((startPage - 1) * self.getRecordsPerPage())) req.addParam("fq", self.getFilterQuery()) req.setParam("fl", self.getReturnFields()) req.setParam("sort", "date_object_modified desc, f_dc_title asc") if not self.isAdmin(): req.addParam("fq", self.getSecurityQuery()) out = ByteArrayOutputStream() self.indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) self._setPaging(result.getNumFound()) result.getJsonObject().put("lastPage", str(self.paging.getLastPage())) result.getJsonObject().put("curPage", str(startPage)) return result
def __getSolrData(self): level = self.getFormData("level", None) if level: if level == "top": query = 'rdf_type:"http://purl.org/asc/1297.0/2008/seo/SEO2"' else: query = 'skos_broader:"%s"' % level else: prefix = self.getSearchTerms() if prefix != "": terms = prefix.split(" ") if len(terms) > 1: termsQuery = " OR %s" ' OR '.join(terms) else: termsQuery = "" queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % { "prefix": prefix, "terms": termsQuery } query = 'dc_title:(%(qv)s)^2 OR dc_identifier:(%(qv)s)^0.5' % { "qv": queryValue } else: query = "*:*" portal = self.services.portalManager.get(self.portalId) sq = portal.searchQuery if sq not in ["", "*:*"]: query = query + " AND " + portal.searchQuery req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') if portal.query: req.addParam("fq", portal.query) req.setParam("fl", "score") req.setParam("sort", "score desc, f_dc_title asc") req.setParam("start", self.getStartIndex()) req.setParam("rows", self.getItemsPerPage()) try: out = ByteArrayOutputStream() indexer = self.services.getIndexer() indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def search_solr(self): # Build our solr query readyForNla = "ready_for_nla:ready" nlaPidExists = "nlaId:http*" query = readyForNla + " AND NOT " + nlaPidExists # Prepare the query req = SearchRequest(query) req.setParam("facet", "false") req.setParam("rows", "20") # Run the query try: out = ByteArrayOutputStream() self.services.getIndexer().search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Error searching solr: ", e) self.throw_error("failure searching solr: " + e.getMessage()) return None
def _searchStage(self, stage, startPage=1): req = SearchRequest("packageType:arms") req.setParam("rows", str(self.recordsPerPage)) req.setParam("start", str((startPage - 1) * self.recordsPerPage)) if ',' in stage: stages = stage.split(',') for s in stages: s = "workflow_step:" + s req.addParam("fq", " OR ".join(stages)) # print "Searching stages %s" % " OR ".join(stages) else: req.addParam("fq", 'workflow_step:' + stage) req.setParam("sort", "last_modified desc, f_dc_title asc") req.setParam("fl", self.returnFields) out = ByteArrayOutputStream() self.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def _searchSets(self, searchType, isAdmin=True, security_query='', startPage=1): req = SearchRequest("packageType:" + searchType) req.setParam("rows", str(self.recordsPerPage)) req.setParam("start", str((startPage - 1) * self.recordsPerPage)) req.setParam("fq", 'item_type:"object"') req.addParam("fq", "") req.setParam("sort", "last_modified desc, f_dc_title asc") req.setParam("fl", self.returnFields) if not isAdmin: req.addParam("fq", security_query) out = ByteArrayOutputStream() self.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __activate__(self, context): formData = context["formData"] services = context["Services"] response = context["response"] # Prepare a query q = formData.get("q") if q is not None and q != "": query = "(item_type:object AND " + self.titleTokens( q.strip()) + ")" else: query = "item_type:object" # Can't link to yourself AND we're not interested in attachments oid = formData.get("qs") query += " AND -storage_id:\"" + oid + "\"" # And we're not interested in attachments query += " AND display_type:\"package-dataset\"" req = SearchRequest(query) req.setParam("fl", "dc_title,storage_id,pidProperty") limit = formData.get("limit") if limit is None: limit = 10 req.setParam("rows", limit) # Search Solr indexer = services.getIndexer() out = ByteArrayOutputStream() indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Build a response list = [] for doc in result.getResults(): title = doc.getFirst("dc_title") #oid = doc.getFirst("storage_id") oid = doc.getFirst("pidProperty") list.append("%s::%s" % (oid, title)) result = "\n".join(list) writer = response.getPrintWriter("text/plain; charset=UTF-8") writer.println(result) writer.close()
def getFacetFields(self): try: out = ByteArrayOutputStream() req = SearchRequest("*:*") req.setParam("fl","facet_counts") req.setParam("facet", "on") req.setParam("facet.field", self.facetField) req.setParam("wt", "json") self.indexer.search(req, out) res = SolrResult(ByteArrayInputStream(out.toByteArray())) facets = res.getFacets() facet = facets.get(self.facetField) if facet is not None and facet.values().size() > 0: self.facetFields = facet.values() else: self.errorMsg = "No facet field values to export. Please enter/harvest some data first." except: self.errorMsg = "Get facet field query failure. The issue has been logged (%s - %s)." % (sys.exc_info()[0], sys.exc_info()[1]) self.log.error("Get facet field threw an exception : %s - %s" % (sys.exc_info()[0], sys.exc_info()[1])) return
def __getSolrData(self): prefix = self.getSearchTerms() if prefix != "": terms = prefix.split(" ") if len(terms) > 1: termsQuery = " OR %s" ' OR '.join(terms) else: termsQuery = "" queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % { "prefix": prefix, "terms": termsQuery } query = 'dc_title:(%(qv)s)^0.5 OR grant_number:(%(qvu)s)^2' % { "qv": queryValue, "qvu": queryValue.upper() } else: query = "*:*" portal = self.services.portalManager.get(self.portalId) sq = portal.searchQuery if sq not in ["", "*:*"]: query = query + " AND " + portal.searchQuery req = SearchRequest(query) req.setParam("fq", 'item_type:"object"') if portal.query: req.addParam("fq", portal.query) req.setParam("fl", "score") req.setParam("sort", "score desc, f_dc_title asc") req.setParam("start", self.getStartIndex()) req.setParam("rows", self.getItemsPerPage()) try: out = ByteArrayOutputStream() indexer = self.services.getIndexer() indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except Exception, e: self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def __activate__(self, context): formData = context["formData"] services = context["Services"] response = context["response"] query = "keywords:[* TO *]" q = formData.get("q") if q: query += " AND keywords:(%(q)s OR %(q)s*)" % { "q": q } req = SearchRequest(query) req.setParam("fl", "keywords") req.setParam("rows", "50") keywords = TreeSet() indexer = services.getIndexer() out = ByteArrayOutputStream() indexer.search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) for doc in result.getResults(): for keyword in doc.getList("keywords"): if keyword.startswith(q): keywords.add(keyword) writer = response.getPrintWriter("text/plain; charset=UTF-8") writer.println("\n".join(keywords)) writer.close()
def __searchDataSetOids(self, oids): query_ids = "storage_id:" try: if (len(oids) > 1): query_ids += "(" + oids[0].get('oid') for oid in oids[1:]: query_ids += " OR " + oid.get('oid') query_ids += ")" else: query_ids = oids[0].get('oid') self.log.debug("related.datasets: query_ids = {}", query_ids) req = SearchRequest(query_ids) req.setParam("fq", 'item_type:"object"') req.addParam("fq", "") req.setParam("sort", "last_modified desc, f_dc_title asc") # FIXME: security? out = ByteArrayOutputStream() self.indexer.search(req, out) return SolrResult(ByteArrayInputStream(out.toByteArray())) except: return None
def getAttachedFiles(self, oid): # Build a query req = SearchRequest("attached_to:%s" % oid) req.setParam("rows", "1000") # Run a search out = ByteArrayOutputStream() self.Services.getIndexer().search(req, out) result = SolrResult(ByteArrayInputStream(out.toByteArray())) # Process results docs = JSONArray() for doc in result.getResults(): attachmentType = self.escapeHtml( WordUtils.capitalizeFully( doc.getFirst("attachment_type").replace("-", " "))) accessRights = self.escapeHtml( WordUtils.capitalizeFully(doc.getFirst("access_rights"))) entry = JsonObject() entry.put("filename", self.escapeHtml(doc.getFirst("filename"))) entry.put("attachment_type", attachmentType) entry.put("access_rights", accessRights) entry.put("id", self.escapeHtml(doc.getFirst("id"))) docs.add(entry) return docs