def __search(self):
query = self.formData.get("query")
searchQuery = self.sessionState.get("searchQuery")
if query is None or query == "":
query = "*:*"
if searchQuery and query == "*:*":
query = searchQuery
elif searchQuery:
query += " AND " + searchQuery
facetField = self.formData.get("facet.field")
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("fl", "id")
req.setParam("rows", "0")
req.setParam("facet.limit", "-1")
req.setParam("facet.field", facetField)
fq = self.sessionState.get("fq")
if fq is not None:
req.setParam("fq", fq)
req.addParam("fq", 'item_type:"object"')
# Make sure 'fq' has already been set in the session
security_roles = self.auth.get_roles_list()
security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.indexer
indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
return FacetList(facetField, result)
def __getSolrData(self):
prefix = self.getSearchTerms()
if prefix != "":
terms = prefix.split(" ")
if len(terms)>1:
termsQuery = " OR %s" ' OR '.join(terms)
else:
termsQuery = ""
queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % { "prefix": prefix, "terms": termsQuery }
query = 'dc_title:(%(qv)s)^2 OR dc_identifier:(%(qv)s)^0.5' % { "qv": queryValue }
else:
query = "*:*"
portal = self.services.portalManager.get(self.portalId)
sq = portal.searchQuery
if sq not in ["", "*:*"]:
query = query + " AND " + portal.searchQuery
req = SearchRequest(query)
req.setParam("fq", 'item_type:"object"')
if portal.query:
req.addParam("fq", portal.query)
req.setParam("fl", "score")
req.setParam("sort", "score desc, f_dc_title asc")
req.setParam("start", self.getStartIndex())
req.setParam("rows", self.getItemsPerPage())
try:
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except Exception, e:
self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def __feed(self):
portal = Services.getPortalManager().get(self.vc("portalId"))
recordsPerPage = portal.recordsPerPage
pageNum = self.vc("sessionState").get("pageNum", 1)
query = "*:*"
if self.vc("formData").get("query"):
query = self.vc("formData").get("query")
query = self.__escapeQuery(query)
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.sort", "true")
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = portal.query
if portalQuery:
req.addParam("fq", portalQuery)
else:
fq = sessionState.get("fq")
if fq is not None:
req.setParam("fq", fq)
req.setParam("start", str((pageNum - 1) * recordsPerPage))
print " * query: ", query
print " * portalQuery='%s'" % portalQuery
print " * feed.py:", req.toString()
out = ByteArrayOutputStream()
Services.indexer.search(req, out)
self.__result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
def __feed(self):
self.portal = self.services.getPortalManager().get(self.vc("portalId"))
recordsPerPage = self.portal.recordsPerPage
pageNum = self.vc("sessionState").get("pageNum", 1)
query = "*:*"
if self.vc("formData").get("query"):
query = self.vc("formData").get("query")
query = self.__escapeQuery(query)
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", self.portal.facetFieldList)
req.setParam("facet.sort", "true")
req.setParam("facet.limit", str(self.portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = self.portal.query
if portalQuery:
req.addParam("fq", portalQuery)
else:
fq = self.vc("sessionState").get("fq")
if fq is not None:
req.setParam("fq", fq)
req.setParam("start", str((pageNum - 1) * recordsPerPage))
self.log.debug(" * Query: '{}'", query)
self.log.debug(" * portalQuery: '{}'", portalQuery)
self.log.debug(" * feed.py: '{}'", req)
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __search(self):
query = self.formData.get("query")
searchQuery = self.sessionState.get("searchQuery")
if query is None or query == "":
query = "*:*"
if searchQuery and query == "*:*":
query = searchQuery
elif searchQuery:
query += " AND " + searchQuery
facetField = self.formData.get("facet.field")
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("fl", "id")
req.setParam("rows", "0")
req.setParam("facet.limit", "-1")
req.setParam("facet.field", facetField)
fq = self.sessionState.get("fq")
if fq is not None:
req.setParam("fq", fq)
req.addParam("fq", 'item_type:"object"')
# Make sure 'fq' has already been set in the session
security_roles = self.auth.get_roles_list()
security_query = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.indexer
indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
return FacetList(facetField, result)
def getSuggestedNames(self):
# search common forms
lookupNames = []
surname = self.__metadata.getList("surname").get(0)
firstName = self.__metadata.getList("firstName").get(0)
firstInitial = firstName[0].upper()
secondName = self.__metadata.getList("secondName")
if not secondName.isEmpty():
secondName = secondName.get(0)
if secondName and secondName != "":
secondInitial = secondName[0].upper()
lookupNames.append("%s, %s. %s." % (surname, firstInitial, secondInitial))
lookupNames.append("%s, %s %s." % (surname, firstName, secondInitial))
lookupNames.append("%s, %s %s" % (surname, firstName, secondName))
lookupNames.append("%s %s %s" % (firstName, secondName, surname))
lookupNames.append("%s, %s." % (surname, firstInitial))
lookupNames.append("%s, %s" % (surname, firstName))
lookupNames.append("%s %s" % (firstName, surname))
query = '" OR dc_title:"'.join(lookupNames)
# general word search from each part of the name
parts = [p for p in self.getPackageTitle().split(" ") if len(p) > 0]
query2 = " OR dc_title:".join(parts)
req = SearchRequest('(dc_title:"%s")^2.5 OR (dc_title:%s)^0.5' % (query, query2))
self.log.info("suggestedNames query={}", req.query)
req.setParam("fq", 'recordtype:"author"')
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "9999")
req.setParam("fl", "score")
req.setParam("sort", "score desc")
# Make sure 'fq' has already been set in the session
##security_roles = self.authentication.get_roles_list();
##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
##req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
#self.log.info("result={}", result.toString())
docs = result.getJsonList("response/docs")
map = LinkedHashMap()
for doc in docs:
authorName = doc.getList("dc_title").get(0)
if map.containsKey(authorName):
authorDocs = map.get(authorName)
else:
authorDocs = ArrayList()
map.put(authorName, authorDocs)
authorDocs.add(doc)
self.__maxScore = max(1.0, float(result.get("response/maxScore")))
return map
def _searchSets(self, indexer, searchType, isAdmin=True, security_query=''):
req = SearchRequest("packageType:"+searchType)
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", "")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __loadSolrData(self, oid):
portal = self.vc("page").getPortal()
query = 'id:"%s"' % oid
if portal.getSearchQuery():
query += " AND " + portal.getSearchQuery()
req = SearchRequest(query)
req.addParam("fq", 'item_type:"object"')
req.addParam("fq", portal.getQuery())
out = ByteArrayOutputStream()
self.vc("Services").getIndexer().search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __loadSolrData(self, oid):
portal = self.vc("page").getPortal()
query = 'id:"%s"' % oid
if portal.getSearchQuery():
query += " AND " + portal.getSearchQuery()
req = SearchRequest(query)
req.addParam("fq", 'item_type:"object"')
req.addParam("fq", portal.getQuery())
out = ByteArrayOutputStream()
self.vc("Services").getIndexer().search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def _searchEmbargoes(self):
req = SearchRequest("item_type:object")
req.setParam("fq", 'redbox\:embargo.redbox\:isEmbargoed:on')
req.addParam("fq", 'workflow_step:final-review')
req.addParam("fq", "")
req.setParam("fl","id,date_embargoed,dc_title")
req.setParam("rows", "25")
req.setParam("sort", "date_embargoed asc, dc_title asc");
out = ByteArrayOutputStream()
indexer = Services.getIndexer()
indexer.search(req, out)
self.__embargoes = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.velocityContext["log"].info("searchEmbargoes call ended" + str(self.__embargoes))
def numberOfModifiedRecord(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
req = SearchRequest("modified:true")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
return self.__result.getString(None, "response", "numFound")
def _searchSets(self,
indexer,
searchType,
isAdmin=True,
security_query=''):
req = SearchRequest("packageType:" + searchType)
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", "")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def _searchSets(self, packageType, isAdmin=True, security_query='', startPage=1):
req = SearchRequest("packageType:"+packageType)
req.setParam("rows", str(self.recordsPerPage))
req.setParam("start", str((startPage - 1) * self.recordsPerPage))
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", "")
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
req.setParam("fl",self.returnFields)
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
self.indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def __getFeed(self):
portal = self.services.getPortalManager().get(self.portalId)
req = SearchRequest("*:*")
req.setParam("facet", "true")
req.setParam("rows", "1000")
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.sort", "true")
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
req.setParam("fq", 'item_type:"object"')
portalQuery = portal.getQuery()
if portalQuery:
req.addParam("fq", portalQuery)
out = ByteArrayOutputStream()
self.services.getIndexer().search(req, out)
return JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
def __getAuthorDetails(self, authorIds):
query = " OR id:".join(authorIds)
req = SearchRequest('id:%s' % query)
req.setParam("fq", 'recordtype:"author"')
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "9999")
# Make sure 'fq' has already been set in the session
##security_roles = self.authentication.get_roles_list();
##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
##req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
return result.getJsonList("response/docs")
def __getAuthorDetails(self, authorIds):
query = " OR id:".join(authorIds)
req = SearchRequest('id:%s' % query)
req.setParam("fq", 'recordtype:"author"')
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "9999")
# Make sure 'fq' has already been set in the session
##security_roles = self.authentication.get_roles_list();
##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
##req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
return result.getJsonList("response/docs")
def _searchSets(self, startPage=1):
req = SearchRequest(self.getQuery())
req.setParam("fq", 'item_type:"object"')
req.setParam("rows", str(self.getRecordsPerPage()))
req.setParam("start", str((startPage - 1) * self.getRecordsPerPage()))
req.addParam("fq", self.getFilterQuery())
req.setParam("fl", self.getReturnFields())
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
if not self.isAdmin():
req.addParam("fq", self.getSecurityQuery())
out = ByteArrayOutputStream()
self.indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
self._setPaging(result.getNumFound())
result.getJsonObject().put("lastPage", str(self.paging.getLastPage()))
result.getJsonObject().put("curPage", str(startPage))
return result
def _searchSets(self, startPage=1):
req = SearchRequest(self.getQuery())
req.setParam("fq", 'item_type:"object"')
req.setParam("rows", str(self.getRecordsPerPage()))
req.setParam("start", str((startPage - 1) * self.getRecordsPerPage()))
req.addParam("fq", self.getFilterQuery())
req.setParam("fl", self.getReturnFields())
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
if not self.isAdmin():
req.addParam("fq", self.getSecurityQuery())
out = ByteArrayOutputStream()
self.indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
self._setPaging(result.getNumFound())
result.getJsonObject().put("lastPage", str(self.paging.getLastPage()))
result.getJsonObject().put("curPage", str(startPage))
return result
def __getSolrData(self):
level = self.getFormData("level", None)
if level:
if level == "top":
query = 'rdf_type:"http://purl.org/asc/1297.0/2008/seo/SEO2"'
else:
query = 'skos_broader:"%s"' % level
else:
prefix = self.getSearchTerms()
if prefix != "":
terms = prefix.split(" ")
if len(terms) > 1:
termsQuery = " OR %s" ' OR '.join(terms)
else:
termsQuery = ""
queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % {
"prefix": prefix,
"terms": termsQuery
}
query = 'dc_title:(%(qv)s)^2 OR dc_identifier:(%(qv)s)^0.5' % {
"qv": queryValue
}
else:
query = "*:*"
portal = self.services.portalManager.get(self.portalId)
sq = portal.searchQuery
if sq not in ["", "*:*"]:
query = query + " AND " + portal.searchQuery
req = SearchRequest(query)
req.setParam("fq", 'item_type:"object"')
if portal.query:
req.addParam("fq", portal.query)
req.setParam("fl", "score")
req.setParam("sort", "score desc, f_dc_title asc")
req.setParam("start", self.getStartIndex())
req.setParam("rows", self.getItemsPerPage())
try:
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except Exception, e:
self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def getAuthorities(self):
req = SearchRequest("package_node_id:%s" % self.metadata.get("id"))
req.setParam("fq", 'recordtype:"master"')
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "9999")
# Make sure 'fq' has already been set in the session
##security_roles = self.authentication.get_roles_list();
##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
##req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response/docs")
return docs
def _searchSets(self,
searchType,
isAdmin=True,
security_query='',
startPage=1):
req = SearchRequest("packageType:" + searchType)
req.setParam("rows", str(self.recordsPerPage))
req.setParam("start", str((startPage - 1) * self.recordsPerPage))
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", "")
req.setParam("sort", "last_modified desc, f_dc_title asc")
req.setParam("fl", self.returnFields)
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
self.indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def _searchStage(self, stage, startPage=1):
req = SearchRequest("packageType:arms")
req.setParam("rows", str(self.recordsPerPage))
req.setParam("start", str((startPage - 1) * self.recordsPerPage))
if ',' in stage:
stages = stage.split(',')
for s in stages:
s = "workflow_step:" + s
req.addParam("fq", " OR ".join(stages))
# print "Searching stages %s" % " OR ".join(stages)
else:
req.addParam("fq", 'workflow_step:' + stage)
req.setParam("sort", "last_modified desc, f_dc_title asc")
req.setParam("fl", self.returnFields)
out = ByteArrayOutputStream()
self.indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def getAuthorities(self):
req = SearchRequest('package_node_id:%s' % self.metadata.get("id"))
req.setParam("fq", 'recordtype:"master"')
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "9999")
# Make sure 'fq' has already been set in the session
##security_roles = self.authentication.get_roles_list();
##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
##req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response/docs")
return docs
def _searchStage(self, packageType, stage, startPage=1):
req = SearchRequest("packageType:"+packageType)
req.setParam("rows", str(self.recordsPerPage))
req.setParam("start", str((startPage - 1) * self.recordsPerPage))
if ',' in stage:
stages = stage.split(',')
for s in stages:
s = "workflow_step:"+s
req.addParam("fq", " OR ".join(stages))
# print "Searching stages %s" % " OR ".join(stages)
else:
req.addParam("fq", 'workflow_step:' + stage)
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
req.setParam("fl",self.returnFields)
out = ByteArrayOutputStream()
self.indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
def checkAprovedRequests(self, provisioned=0, startPage=1):
""" A customised query for package type of arms at workflow_step of arms-approved
Get a list of requests filtered by provisioning_checklist
"""
workflowStep = "arms-approved"
if self.packageType:
req = SearchRequest(self.packageType)
else:
req = SearchRequest("packageType:arms")
req.addParam("fq", 'workflow_step:' + workflowStep)
if provisioned:
req.addParam("fq", '-provisioning_checklist.4:null')
else:
req.addParam("fq", 'provisioning_checklist.4:null')
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
req.setParam("fl","id,dc_title,date-provisioned")
out = ByteArrayOutputStream()
self.indexer.search(req, out)
solrResults = SolrResult(ByteArrayInputStream(out.toByteArray()))
if solrResults:
results = solrResults.getResults()
if results:
results = self.mergeEvents(results, ["arms_draft","arms_redraft","arms_review","arms_approved","arms_rejected"])
self._setPaging(results.size())
return results
else:
return ArrayList()
def __getSolrData(self):
prefix = self.getSearchTerms()
if prefix != "":
terms = prefix.split(" ")
if len(terms) > 1:
termsQuery = " OR %s" ' OR '.join(terms)
else:
termsQuery = ""
queryValue = "%(prefix)s OR %(prefix)s*%(terms)s" % {
"prefix": prefix,
"terms": termsQuery
}
query = 'dc_title:(%(qv)s)^0.5 OR grant_number:(%(qvu)s)^2' % {
"qv": queryValue,
"qvu": queryValue.upper()
}
else:
query = "*:*"
portal = self.services.portalManager.get(self.portalId)
sq = portal.searchQuery
if sq not in ["", "*:*"]:
query = query + " AND " + portal.searchQuery
req = SearchRequest(query)
req.setParam("fq", 'item_type:"object"')
if portal.query:
req.addParam("fq", portal.query)
req.setParam("fl", "score")
req.setParam("sort", "score desc, f_dc_title asc")
req.setParam("start", self.getStartIndex())
req.setParam("rows", self.getItemsPerPage())
try:
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except Exception, e:
self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def checkRequests(self, checklist_filter=['1'], role_filter='reviewer', exclusive=True, startPage=1):
""" A customised query for arms at arms-review
Get a list of requests filtered by provisioning_checklist
For assessor, it is user based and queries against committee-responses.metadata
"""
workflowStep = "arms-review"
if self.packageType:
req = SearchRequest(self.packageType)
else:
req = SearchRequest("packageType:arms")
req.addParam("fq", 'workflow_step:' + workflowStep)
for item in ['1','2','3','4']:
if item in checklist_filter:
req.addParam("fq", '-provisioning_checklist.' + item + ':null' + ' AND provisioning_checklist.' + item + ':[* TO *]')
else:
if exclusive:
# ensure that brand new submissions (not yet saved by reviewer) are also returned
req.addParam("fq", 'provisioning_checklist.' + item + ':null' + ' OR (*:* -provisioning_checklist.' + item + ':[* TO *])')
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
req.setParam("fl",self.returnFields)
out = ByteArrayOutputStream()
self.indexer.search(req, out)
solrResults = SolrResult(ByteArrayInputStream(out.toByteArray()))
if solrResults:
results = solrResults.getResults()
if results:
packageResults = results
results = self.mergeEvents(packageResults, ["arms_draft","arms_redraft","arms_review","arms_approved","arms_rejected"])
returnArray = JSONArray()
if role_filter.startswith('assessor'):
if role_filter == 'assessor':
query_status = ['new', 'draft']
elif role_filter == 'assessor-assessed':
query_status = ['submitted']
x = Assessment()
x.activate(self.velocityContext)
i = 0
rows = self.recordsPerPage
start = (startPage - 1) * self.recordsPerPage
for r in results:
status = x.queryStatus(r.get("id"))
if status in query_status:
if i >= start and i - start < rows:
if status == 'submitted':
assessment_submitted_date = x.queryMyAttr('date')
if assessment_submitted_date:
r.getJsonObject().put('date', assessment_submitted_date)
returnArray.add(r)
i = i + 1
else:
returnArray = results
self._setPaging(returnArray.size())
return returnArray
else:
return ArrayList()
def __searchDataSetOids(self, oids):
query_ids = "storage_id:"
try:
if (len(oids) > 1):
query_ids += "(" + oids[0].get('oid')
for oid in oids[1:]:
query_ids += " OR " + oid.get('oid')
query_ids += ")"
else:
query_ids = oids[0].get('oid')
self.log.debug("related.datasets: query_ids = {}", query_ids)
req = SearchRequest(query_ids)
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", "")
req.setParam("sort", "last_modified desc, f_dc_title asc")
# FIXME: security?
out = ByteArrayOutputStream()
self.indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except:
return None
def getSearchTerms(self):
searchTerms = []
prefix = self.getSuggestionPrefix()
query = '%(prefix)s OR %(prefix)s*' % { "prefix" : prefix }
req = SearchRequest(query)
req.addParam("fq", self.page.getPortal().getQuery())
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "50")
req.setParam("fl", "score,id,dc_title")
req.setParam("sort", "score desc")
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response/docs")
for doc in docs:
dc_title = doc.getList("dc_title").get(0)
searchTerms.append(dc_title)
return '", "'.join(searchTerms)
def __searchDataSetOids(self, oids):
query_ids = "storage_id:"
try:
if (len(oids) > 1):
query_ids += "(" + oids[0].get('oid')
for oid in oids[1:]:
query_ids += " OR " + oid.get('oid')
query_ids += ")"
else:
query_ids = oids[0].get('oid')
self.log.debug("related.datasets: query_ids = {}", query_ids)
req = SearchRequest(query_ids)
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", "")
req.setParam("sort", "last_modified desc, f_dc_title asc");
# FIXME: security?
out = ByteArrayOutputStream()
self.indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except:
return None
def getSearchTerms(self):
searchTerms = []
prefix = self.getSuggestionPrefix()
query = '%(prefix)s OR %(prefix)s*' % {"prefix": prefix}
req = SearchRequest(query)
req.addParam("fq", self.page.getPortal().getQuery())
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "50")
req.setParam("fl", "score,id,dc_title")
req.setParam("sort", "score desc")
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response/docs")
for doc in docs:
dc_title = doc.getList("dc_title").get(0)
searchTerms.append(dc_title)
return '", "'.join(searchTerms)
def getFilteredAssessments(self, packageType, stageName, filterType, startPage=1):
""" A customised query to use filter to get certain assessment with desired status """
## reference /redbox-rdsi-arms/src/main/config/home/lib/jython/util/Assessment.py for methods
filters = {'assessment-draft': ['new','draft'], 'assessment-submitted':['submitted']}
statusFilter = filters[filterType]
req = SearchRequest("packageType:" + packageType)
req.addParam("fq", 'workflow_step:' + stageName)
req.setParam("sort", "date_object_modified desc, f_dc_title asc")
req.setParam("fl",self.returnFields)
out = ByteArrayOutputStream()
self.indexer.search(req, out)
solrResults = SolrResult(ByteArrayInputStream(out.toByteArray()))
if solrResults:
results = solrResults.getResults()
returnArray = JSONArray()
x = Assessment()
x.activate(self.velocityContext)
i = 0
rows = self.recordsPerPage
start = (startPage - 1) * self.recordsPerPage
for r in results:
status = x.queryStatus(r.get("id"))
if status in statusFilter:
if i >= start and i - start < rows:
assessment_submitted_date = x.queryAttr(r.get("id"), 'date')
if assessment_submitted_date:
r.getJsonObject().put('date', assessment_submitted_date)
returnArray.add(r)
i = i + 1
self._setPaging(returnArray.size())
return returnArray
else:
return ArrayList()
def __getSolrData(self):
query = "*:*"
req = SearchRequest(query)
req.setParam("fq", 'item_type:"object"')
req.addParam("fq", 'repository_type:"Parties"')
req.addParam("fq", 'repository_name:"People"')
req.addParam("fq", 'ready_for_nla:"ready"')
req.addParam("fq", "-nlaId:[* TO *]")
req.setParam("fl", "score")
req.setParam("sort", "score desc, f_dc_title asc")
req.setParam("start", "0")
req.setParam("rows", "99999")
try:
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
return SolrResult(ByteArrayInputStream(out.toByteArray()))
except Exception, e:
self.log.error("Failed to lookup '{}': {}", prefix, e.getMessage())
def __search(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("last_modified:[NOW-1MONTH TO *]")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest(owner_query)
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest('workflow_security:"' + current_user + '"')
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.vc("sessionState").set("fq", 'item_type:"object"')
#sessionState.set("query", portalQuery.replace("\"", "'"))
# Load in the services UI workflow
selfSubmitWfConfig = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/servicesUI.json"))
selfSubmitJsonStageList = selfSubmitWfConfig.getJsonSimpleList(["stages"])
servicesStages = []
for jsonStage in selfSubmitJsonStageList:
wfStage = WorkflowStage(jsonStage, self.__steps)
servicesStages.append(wfStage)
self.__selfservicesStages = servicesStages
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def getSuggestedNames(self):
# search common forms
lookupNames = []
surname = self.__metadata.getList("surname").get(0)
firstName = self.__metadata.getList("firstName").get(0)
firstInitial = firstName[0].upper()
secondName = self.__metadata.getList("secondName")
if not secondName.isEmpty():
secondName = secondName.get(0)
if secondName and secondName != "":
secondInitial = secondName[0].upper()
lookupNames.append("%s, %s. %s." %
(surname, firstInitial, secondInitial))
lookupNames.append("%s, %s %s." %
(surname, firstName, secondInitial))
lookupNames.append("%s, %s %s" % (surname, firstName, secondName))
lookupNames.append("%s %s %s" % (firstName, secondName, surname))
lookupNames.append("%s, %s." % (surname, firstInitial))
lookupNames.append("%s, %s" % (surname, firstName))
lookupNames.append("%s %s" % (firstName, surname))
query = '" OR dc_title:"'.join(lookupNames)
# general word search from each part of the name
parts = [p for p in self.getPackageTitle().split(" ") if len(p) > 0]
query2 = " OR dc_title:".join(parts)
req = SearchRequest('(dc_title:"%s")^2.5 OR (dc_title:%s)^0.5' %
(query, query2))
self.log.info("suggestedNames query={}", req.query)
req.setParam("fq", 'recordtype:"author"')
req.addParam("fq", 'item_type:"object"')
req.setParam("rows", "9999")
req.setParam("fl", "score")
req.setParam("sort", "score desc")
# Make sure 'fq' has already been set in the session
##security_roles = self.authentication.get_roles_list();
##security_query = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
##req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer = self.services.getIndexer()
indexer.search(req, out)
result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
#self.log.info("result={}", result.toString())
docs = result.getJsonList("response/docs")
map = LinkedHashMap()
for doc in docs:
authorName = doc.getList("dc_title").get(0)
if map.containsKey(authorName):
authorDocs = map.get(authorName)
else:
authorDocs = ArrayList()
map.put(authorName, authorDocs)
authorDocs.add(doc)
self.__maxScore = max(1.0, float(result.get("response/maxScore")))
return map
def __search(self, searchField):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(
self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(
self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
startRow = 0
numPerPage = 25
numFound = 0
req = SearchRequest(searchField)
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
objectIdList = []
while True:
req.addParam("fq", 'item_type:"object"')
req.addParam("rows", str(numPerPage))
req.addParam("start", str(startRow))
out = ByteArrayOutputStream()
indexer.search(req, out)
result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response", "docs")
docIds = []
for doc in docs:
docId = doc.getString(None, "storage_id")
if docId is not None:
docIds.append(docId)
objectIdList.extend(docs)
startRow += numPerPage
numFound = int(result.getString(None, "response", "numFound"))
if (startRow > numFound):
break
return objectIdList
def __search(self, searchField):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.portal.getName()).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.portal.getName()).getSearchQuery()
# Security prep work
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
startRow = 0
numPerPage = 25
numFound = 0
req = SearchRequest(searchField)
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
if not self.page.authentication.is_admin():
req.addParam("fq", security_query)
objectIdList = []
while True:
req.addParam("fq", 'item_type:"object"')
req.addParam("rows", str(numPerPage))
req.addParam("start", str(startRow))
out = ByteArrayOutputStream()
indexer.search(req, out)
result = JsonSimpleConfig(ByteArrayInputStream(out.toByteArray()))
docs = result.getJsonList("response", "docs")
docIds = []
for doc in docs:
docId = doc.getString(None, "storage_id")
if docId is not None:
docIds.append(docId)
objectIdList.extend(docs)
startRow += numPerPage
numFound = int(result.getString(None, "response", "numFound"))
if (startRow > numFound):
break
return objectIdList
def __buildResumptionTokenSets(self):
self.__result = SolrResult(None)
portal = self.services.getPortalManager().get(self.__portalName)
recordsPerPage = portal.recordsPerPage
# Resolve our identifier
id = self.vc("formData").get("identifier")
query = "*:*"
if id is not None and id != "":
# A default TF2 OID
if id.startswith("oai:fascinator.usq.edu.au:"):
idString = id.replace("oai:fascinator.usq.edu.au:", "")
idString = self.__escapeQuery(idString)
query = "id:" + idString
# Or a custom OAI ID
else:
idString = self.__escapeQuery(id)
query = "oai_identifier:" + idString
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", "item_type:object")
# Date data... is supplied
fromDate = self.__request.getFromDate()
untilDate = self.__request.getUntilDate()
if fromDate is not None:
fromStr = fromDate.isoformat() + "Z"
self.log.debug("From Date: '{}'", fromStr)
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilStr)
queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr)
else:
queryStr = "last_modified:[%s TO *]" % (fromStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
else:
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilDate.isoformat())
queryStr = "last_modified:[* TO %s]" % (untilStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
# Check if there's resumption token exist in the formData
start = 0
req.setParam("start", str(start))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
totalFound = self.__result.getNumFound()
self.log.debug("Total found:" + str(totalFound))
if totalFound > recordsPerPage:
startRow = 0
random.seed()
resumptionToken = "%016x" % random.getrandbits(128)
nextResumptionToken = resumptionToken
firstLoop = True
while True:
self.log.debug("Current Resumption Token: " + resumptionToken)
req.setParam("start", str(startRow))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
tokenObject = ResumptionToken(resumptionToken,self.__metadataPrefix,nextResumptionToken,result.toString())
if firstLoop:
self.__currentToken = ResumptionToken(None,self.__metadataPrefix,resumptionToken,None)
tokenObject = None
firstLoop = False
startRow = startRow + recordsPerPage
if startRow > totalFound:
tokenObject = ResumptionToken(resumptionToken,self.__metadataPrefix,"",result.toString())
self.tokensDB.storeToken(tokenObject)
break;
if tokenObject is not None:
self.tokensDB.storeToken(tokenObject)
resumptionToken = nextResumptionToken
nextResumptionToken = "%016x" % random.getrandbits(128)
self.log.debug("Resumption Token: " + resumptionToken + " next resumption token:" + nextResumptionToken)
def __search(self):
self.__result = SolrResult(None)
portal = self.services.getPortalManager().get(self.__portalName)
recordsPerPage = portal.recordsPerPage
# Resolve our identifier
id = self.vc("formData").get("identifier")
query = "*:*"
if id is not None and id != "":
# A default TF2 OID
if id.startswith("oai:fascinator.usq.edu.au:"):
idString = id.replace("oai:fascinator.usq.edu.au:", "")
idString = self.__escapeQuery(idString)
query = "id:" + idString
# Or a custom OAI ID
else:
idString = self.__escapeQuery(id)
query = "oai_identifier:" + idString
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", "item_type:object")
# Date data... is supplied
fromDate = self.__request.getFromDate()
untilDate = self.__request.getUntilDate()
if fromDate is not None:
fromStr = fromDate.isoformat() + "Z"
self.log.debug("From Date: '{}'", fromStr)
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilStr)
queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr)
else:
queryStr = "last_modified:[%s TO *]" % (fromStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
else:
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilDate.isoformat())
queryStr = "last_modified:[* TO %s]" % (untilStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
# Check if there's resumption token exist in the formData
newToken = None
if self.__currentToken is not None:
start = int(self.__currentToken.getStart())
totalFound = int(self.__currentToken.getTotalFound())
nextTokenStart = start + recordsPerPage
if nextTokenStart < totalFound:
newToken = self.__currentToken
newToken.resetExpiry(self.__sessionExpiry)
newToken.setStart(nextTokenStart)
# or start a new resumption token
else:
start = 0
newToken = ResumptionToken(None, recordsPerPage, self.__metadataPrefix, self.__sessionExpiry)
req.setParam("start", str(start))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
totalFound = self.__result.getNumFound()
if totalFound == 0:
newToken = None
# If an ID was requested, and not found, this is an error
if id is not None and id != "":
self.__request.setError("idDoesNotExist", "ID: '%s' not found" % id)
else:
self.__request.setError("noRecordsMatch", "No records match this request")
# We need to store this for NEW tokens
elif self.__currentToken is None:
# Assuming there are enough results to even keep the token
if newToken.getStart() < totalFound:
newToken.setTotalFound(totalFound)
else:
newToken = None
# Check if we need to remove the resumption token
else:
if (start + recordsPerPage) >= totalFound:
self.tokensDB.removeToken(self.__currentToken)
self.lastPage = True
# Store/update the resumption token
if newToken is not None:
# Brand new token
if self.__currentToken is None:
self.tokensDB.storeToken(newToken)
# Or update an old token
else:
self.tokensDB.updateToken(newToken)
self.__currentToken = newToken
def __search(self):
requireEscape = False
recordsPerPage = self.__portal.recordsPerPage
uri = URLDecoder.decode(self.request.getAttribute("RequestURI"))
query = None
pagePath = self.__portal.getName() + "/" + self.pageName
if query is None or query == "":
query = self.formData.get("query")
requireEscape = True
if query is None or query == "":
query = "*:*"
if query == "*:*":
self.__query = ""
else:
self.__query = query
if requireEscape:
query = self.__escapeQuery(query)
query = "%s:%s" % (self.__searchField, query)
self.sessionState.set("query", self.__query)
# find objects with annotations matching the query
if query != "*:*":
anotarQuery = self.__query
if requireEscape:
anotarQuery = self.__escapeQuery(anotarQuery)
annoReq = SearchRequest(anotarQuery)
annoReq.setParam("facet", "false")
annoReq.setParam("rows", str(99999))
annoReq.setParam("sort", "dateCreated asc")
annoReq.setParam("start", str(0))
anotarOut = ByteArrayOutputStream()
self.services.indexer.annotateSearch(annoReq, anotarOut)
resultForAnotar = SolrResult(ByteArrayInputStream(anotarOut.toByteArray()))
resultForAnotar = resultForAnotar.getResults()
ids = HashSet()
for annoDoc in resultForAnotar:
annotatesUri = annoDoc.getFirst("annotatesUri")
ids.add(annotatesUri)
self.log.debug("Found annotation for %s" % annotatesUri)
# add annotation ids to query
query += ' OR id:("' + '" OR "'.join(ids) + '")'
portalSearchQuery = self.__portal.searchQuery
if portalSearchQuery == "":
portalSearchQuery = query
else:
if query != "*:*":
query += " AND " + portalSearchQuery
else:
query = portalSearchQuery
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", self.__portal.facetFieldList)
req.setParam("facet.sort", Boolean.toString(self.__portal.getFacetSort()))
req.setParam("facet.limit", str(self.__portal.facetCount))
req.setParam("sort", self.__sortBy)
# setup facets
if self.__useSessionNavigation:
action = self.formData.get("verb")
value = self.formData.get("value")
fq = self.sessionState.get("fq")
if fq is not None:
self.__pageNum = 1
req.setParam("fq", fq)
if action == "add_fq":
self.__pageNum = 1
req.addParam("fq", URLDecoder.decode(value, "UTF-8"))
elif action == "remove_fq":
self.__pageNum = 1
req.removeParam("fq", URLDecoder.decode(value, "UTF-8"))
elif action == "clear_fq":
self.__pageNum = 1
req.removeParam("fq")
elif action == "select-page":
self.__pageNum = int(value)
else:
navUri = uri[len(pagePath):]
self.__pageNum, fq, self.__fqParts = self.__parseUri(navUri)
savedfq = self.sessionState.get("savedfq")
limits = []
if savedfq:
limits.extend(savedfq)
if fq:
limits.extend(fq)
self.sessionState.set("savedfq", limits)
for q in fq:
req.addParam("fq", URLDecoder.decode(q, "UTF-8"))
portalQuery = self.__portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", 'item_type:"object"')
if req.getParams("fq"):
self.__selected = ArrayList(req.getParams("fq"))
if self.__useSessionNavigation:
self.sessionState.set("fq", self.__selected)
self.sessionState.set("searchQuery", portalSearchQuery)
self.sessionState.set("pageNum", self.__pageNum)
# Make sure 'fq' has already been set in the session
if not self.page.authentication.is_admin():
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
req.addParam("fq", security_query)
req.setParam("start", str((self.__pageNum - 1) * recordsPerPage))
#print " * search.py:", req.toString(), self.__pageNum
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
if self.__result is not None:
self.__paging = Pagination(self.__pageNum,
self.__result.getNumFound(),
self.__portal.recordsPerPage)
def __search(self):
indexer = Services.getIndexer()
portalQuery = Services.getPortalManager().get(
self.vc("portalId")).getQuery()
portalSearchQuery = Services.getPortalManager().get(
self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
req.setParam("facet", "true")
req.setParam("facet.field", "workflow_step")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
steps = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__steps = steps.getFacets().get("workflow_step")
wfConfig = JsonSimple(
FascinatorHome.getPathFile("harvest/workflows/dataset.json"))
jsonStageList = wfConfig.getJsonSimpleList(["stages"])
stages = []
for jsonStage in jsonStageList:
wfStage = WorkflowStage(jsonStage, self.__steps)
stages.append(wfStage)
self.__stages = stages
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "25")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
req.addParam("fq", "workflow_step:%s" % stages[0].getName())
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__alerts = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest(
"last_modified:[NOW-1MONTH TO *] AND workflow_step:live")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.vc("sessionState").set("fq", 'item_type:"object"')
def __search(self):
requireEscape = False
recordsPerPage = self.__portal.recordsPerPage
uri = URLDecoder.decode(self.request.getAttribute("RequestURI"))
query = None
pagePath = self.__portal.getName() + "/" + self.pageName
if query is None or query == "":
query = self.formData.get("query")
requireEscape = True
if query is None or query == "":
query = "*:*"
if query == "*:*":
self.__query = ""
else:
self.__query = query
if requireEscape:
query = self.__escapeQuery(query)
query = "%s:%s" % (self.__searchField, query)
self.sessionState.set("query", self.__query)
# find objects with annotations matching the query
if query != "*:*":
anotarQuery = self.__query
if requireEscape:
anotarQuery = self.__escapeQuery(anotarQuery)
annoReq = SearchRequest(anotarQuery)
annoReq.setParam("facet", "false")
annoReq.setParam("rows", str(99999))
annoReq.setParam("sort", "dateCreated asc")
annoReq.setParam("start", str(0))
anotarOut = ByteArrayOutputStream()
self.services.indexer.annotateSearch(annoReq, anotarOut)
resultForAnotar = SolrResult(
ByteArrayInputStream(anotarOut.toByteArray()))
resultForAnotar = resultForAnotar.getResults()
ids = HashSet()
for annoDoc in resultForAnotar:
annotatesUri = annoDoc.getFirst("annotatesUri")
ids.add(annotatesUri)
self.log.debug("Found annotation for %s" % annotatesUri)
# add annotation ids to query
query += ' OR id:("' + '" OR "'.join(ids) + '")'
portalSearchQuery = self.__portal.searchQuery
if portalSearchQuery == "":
portalSearchQuery = query
else:
if query != "*:*":
query += " AND " + portalSearchQuery
else:
query = portalSearchQuery
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", self.__portal.facetFieldList)
req.setParam("facet.sort",
Boolean.toString(self.__portal.getFacetSort()))
req.setParam("facet.limit", str(self.__portal.facetCount))
req.setParam("sort", self.__sortBy)
# setup facets
if self.__useSessionNavigation:
action = self.formData.get("verb")
value = self.formData.get("value")
fq = self.sessionState.get("fq")
if fq is not None:
self.__pageNum = 1
req.setParam("fq", fq)
if action == "add_fq":
self.__pageNum = 1
req.addParam("fq", URLDecoder.decode(value, "UTF-8"))
elif action == "remove_fq":
self.__pageNum = 1
req.removeParam("fq", URLDecoder.decode(value, "UTF-8"))
elif action == "clear_fq":
self.__pageNum = 1
req.removeParam("fq")
elif action == "select-page":
self.__pageNum = int(value)
else:
navUri = uri[len(pagePath):]
self.__pageNum, fq, self.__fqParts = self.__parseUri(navUri)
savedfq = self.sessionState.get("savedfq")
limits = []
if savedfq:
limits.extend(savedfq)
if fq:
limits.extend(fq)
self.sessionState.set("savedfq", limits)
for q in fq:
req.addParam("fq", URLDecoder.decode(q, "UTF-8"))
portalQuery = self.__portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", 'item_type:"object"')
if req.getParams("fq"):
self.__selected = ArrayList(req.getParams("fq"))
if self.__useSessionNavigation:
self.sessionState.set("fq", self.__selected)
self.sessionState.set("searchQuery", portalSearchQuery)
self.sessionState.set("pageNum", self.__pageNum)
# Make sure 'fq' has already been set in the session
if not self.page.authentication.is_admin():
current_user = self.page.authentication.get_username()
security_roles = self.page.authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(
security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
req.addParam("fq", security_query)
## uncomment to ensure guest users not logged in cannot see alerts in browse page
# self.filterOutWorkflowStepForUnAuth(req, "inbox")
req.setParam("start", str((self.__pageNum - 1) * recordsPerPage))
self.log.debug(" * search.py: %s, page: %s" %
(req.toString(), self.__pageNum))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
if self.__result is not None:
self.__paging = Pagination(self.__pageNum,
self.__result.getNumFound(),
self.__portal.recordsPerPage)
def __search(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("last_modified:[NOW-1MONTH TO *]")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest(owner_query)
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest('workflow_security:"' + current_user + '"')
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.vc("sessionState").set("fq", 'item_type:"object"')
# sessionState.set("query", portalQuery.replace("\"", "'"))
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __search(self):
indexer = self.services.getIndexer()
portalQuery = self.services.getPortalManager().get(self.vc("portalId")).getQuery()
portalSearchQuery = self.services.getPortalManager().get(self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("last_modified:[NOW-1MONTH TO *]")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest(owner_query)
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__mine = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest('workflow_security:"' + current_user + '"')
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__workflows = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.vc("sessionState").set("fq", 'item_type:"object"')
#sessionState.set("query", portalQuery.replace("\"", "'"))
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
def __search(self):
indexer = Services.getIndexer()
portalQuery = Services.getPortalManager().get(self.vc("portalId")).getQuery()
portalSearchQuery = Services.getPortalManager().get(self.vc("portalId")).getSearchQuery()
# Security prep work
current_user = self.vc("page").authentication.get_username()
security_roles = self.vc("page").authentication.get_roles_list()
security_filter = 'security_filter:("' + '" OR "'.join(security_roles) + '")'
security_exceptions = 'security_exception:"' + current_user + '"'
owner_query = 'owner:"' + current_user + '"'
security_query = "(" + security_filter + ") OR (" + security_exceptions + ") OR (" + owner_query + ")"
isAdmin = self.vc("page").authentication.is_admin()
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "0")
req.setParam("facet", "true")
req.setParam("facet.field", "workflow_step")
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
steps = SolrResult(ByteArrayInputStream(out.toByteArray()))
self.__steps = steps.getFacets().get("workflow_step")
wfConfig = JsonSimple(FascinatorHome.getPathFile("harvest/workflows/dataset.json"))
jsonStageList = wfConfig.getJsonSimpleList(["stages"])
stages = []
for jsonStage in jsonStageList:
wfStage = WorkflowStage(jsonStage, self.__steps)
stages.append(wfStage)
self.__stages = stages
req = SearchRequest("*:*")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.addParam("fq", "")
req.setParam("rows", "25")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
req.addParam("fq", "workflow_step:%s" % stages[0].getName())
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__alerts = SolrResult(ByteArrayInputStream(out.toByteArray()))
req = SearchRequest("last_modified:[NOW-1MONTH TO *] AND workflow_step:live")
req.setParam("fq", 'item_type:"object"')
if portalQuery:
req.addParam("fq", portalQuery)
if portalSearchQuery:
req.addParam("fq", portalSearchQuery)
req.setParam("rows", "10")
req.setParam("sort", "last_modified desc, f_dc_title asc");
if not isAdmin:
req.addParam("fq", security_query)
out = ByteArrayOutputStream()
indexer.search(req, out)
self.__latest = SolrResult(ByteArrayInputStream(out.toByteArray()))
self._searchEmbargoes()
self.vc("sessionState").set("fq", 'item_type:"object"')
def __buildResumptionTokenSets(self):
self.__result = SolrResult(None)
portal = self.services.getPortalManager().get(self.__portalName)
recordsPerPage = portal.recordsPerPage
# Resolve our identifier
id = self.vc("formData").get("identifier")
query = "*:*"
if id is not None and id != "":
# A default TF2 OID
if id.startswith("oai:fascinator.usq.edu.au:"):
idString = id.replace("oai:fascinator.usq.edu.au:", "")
idString = self.__escapeQuery(idString)
query = "id:" + idString
# Or a custom OAI ID
else:
idString = self.__escapeQuery(id)
query = "oai_identifier:" + idString
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", str(recordsPerPage))
req.setParam("facet.field", portal.facetFieldList)
req.setParam("facet.limit", str(portal.facetCount))
req.setParam("sort", "f_dc_title asc")
portalQuery = portal.query
if portalQuery:
req.addParam("fq", portalQuery)
req.addParam("fq", "item_type:object")
# Date data... is supplied
fromDate = self.__request.getFromDate()
untilDate = self.__request.getUntilDate()
if fromDate is not None:
fromStr = fromDate.isoformat() + "Z"
self.log.debug("From Date: '{}'", fromStr)
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilStr)
queryStr = "last_modified:[%s TO %s]" % (fromStr, untilStr)
else:
queryStr = "last_modified:[%s TO *]" % (fromStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
else:
if untilDate is not None:
untilStr = untilDate.isoformat() + "Z"
self.log.debug("Until Date: '{}'", untilDate.isoformat())
queryStr = "last_modified:[* TO %s]" % (untilStr)
self.log.debug("Date query: '{}'", queryStr)
req.addParam("fq", queryStr)
# Check if there's resumption token exist in the formData
start = 0
req.setParam("start", str(start))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
self.__result = SolrResult(ByteArrayInputStream(out.toByteArray()))
totalFound = self.__result.getNumFound()
self.log.debug("Total found:" + str(totalFound))
if totalFound > recordsPerPage:
startRow = 0
random.seed()
resumptionToken = "%016x" % random.getrandbits(128)
nextResumptionToken = resumptionToken
firstLoop = True
while True:
self.log.debug("Current Resumption Token: " + resumptionToken)
req.setParam("start", str(startRow))
out = ByteArrayOutputStream()
self.services.indexer.search(req, out)
result = SolrResult(ByteArrayInputStream(out.toByteArray()))
tokenObject = ResumptionToken(resumptionToken,
self.__metadataPrefix,
nextResumptionToken,
result.toString())
if firstLoop:
self.__currentToken = ResumptionToken(
None, self.__metadataPrefix, resumptionToken, None)
tokenObject = None
firstLoop = False
startRow = startRow + recordsPerPage
if startRow > totalFound:
tokenObject = ResumptionToken(resumptionToken,
self.__metadataPrefix, "",
result.toString())
self.tokensDB.storeToken(tokenObject)
break
if tokenObject is not None:
self.tokensDB.storeToken(tokenObject)
resumptionToken = nextResumptionToken
nextResumptionToken = "%016x" % random.getrandbits(128)
self.log.debug("Resumption Token: " + resumptionToken +
" next resumption token:" + nextResumptionToken)
def __search(self):
recordsPerPage = self.__portal.recordsPerPage
query = None
if query is None or query == "":
query = formData.get("query")
if query is None or query == "":
query = "*:*"
req = SearchRequest(query)
req.setParam("facet", "true")
req.setParam("rows", "1000")
req.setParam("facet.field", self.__portal.facetFieldList)
req.setParam("facet.sort", "true")
req.setParam("facet.limit", str(self.__portal.facetCount))
req.setParam("sort", "f_dc_title asc")
# setup facets
action = formData.get("verb")
value = formData.get("value")
fq = sessionState.get("fq")
if fq is not None:
self.__pageNum = 1
req.setParam("fq", fq)
if action == "add_fq":
self.__pageNum = 1
name = formData.get("name")
print " * add_fq: %s" % value
req.addParam("fq", URLDecoder.decode(value, "UTF-8"))
elif action == "remove_fq":
self.__pageNum = 1
req.removeParam("fq", URLDecoder.decode(value, "UTF-8"))
elif action == "clear_fq":
self.__pageNum = 1
req.removeParam("fq")
elif action == "select-page":
self.__pageNum = int(value)
req.addParam("fq", 'item_type:"object"')
portalQuery = self.__portal.query
print " * portalQuery=%s" % portalQuery
if portalQuery:
req.addParam("fq", portalQuery)
self.__selected = req.getParams("fq")
sessionState.set("fq", self.__selected)
sessionState.set("pageNum", self.__pageNum)
req.setParam("start", str((self.__pageNum - 1) * recordsPerPage))
print " * combined.py:", req.toString(), self.__pageNum
out = ByteArrayOutputStream()
Services.indexer.search(req, out)
self.__result = JsonConfigHelper(ByteArrayInputStream(out.toByteArray()))
if self.__result is not None:
self.__paging = Pagination(self.__pageNum,
int(self.__result.get("response/numFound")),
self.__portal.recordsPerPage)
print " * combined.py: updating manifest..."
portal = self.getPortal()
manifest = portal.getJsonMap("manifest")
#add new items from search
for doc in self.__result.getList("response/docs"):
hashId = md5.new(doc.get("id")).hexdigest()
node = portal.get("manifest//node-%s" % hashId)
print " ********node=", node
if node is None:
portal.set("manifest/node-%s/title" % hashId, doc.get("dc_title").get(0))
portal.set("manifest/node-%s/id" % hashId, doc.get("id"))
#remove manifest items missing from search result
print manifest
for key in manifest.keySet():
item = manifest.get(key)
id = item.get("id")
doc = self.__result.getList('response/docs[@id="%s"]' % id)
if len(doc) == 0:
portal.removePath("manifest//%s" % key)
Services.getPortalManager().save(portal)
