def testQuery(self):
wq = WebQuery('cats', antiUnaryClause='antiunary exact true')
self.assertFalse(wq.isBooleanQuery())
self.assertFalse(wq.isPlusMinusQuery())
self.assertTrue(wq.isDefaultQuery())
self.assertFalse(wq.needsBooleanHelp())
self.assertEquals(parseCql('cats'), wq.ast)
def testFilterX4(self):
wq = WebQuery('fiets')
wq.addFilter('field1', 'value1')
wq.addFilter('field2', 'value2')
wq.addFilter('field3', 'value3')
wq.addFilter('field4', 'value4')
self.assertCql(parseCql('(fiets) AND field1 exact value1 AND field2 exact value2 AND field3 exact value3 AND field4 exact value4'), wq.ast)
def testReplaceTermsWithFilters(self):
wq = WebQuery('fiets kaart')
wq.addFilter('label', 'value')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEquals('fiets kaart', wq.original)
self.assertEquals('bike AND kaart', newWq.original)
self.assertCql(parseCql('(bike AND kaart) AND label exact value'), newWq.ast)
def handleRequest(self, RequestURI='', **kwargs):
yield httputils.okRss
yield """<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"><channel>"""
try:
Scheme, Netloc, Path, Query, Fragment = urlsplit(RequestURI)
arguments = parse_qs(Query)
sortKeys = arguments.get('sortKeys', [self._sortKeys])[0]
sortBy, sortDescending = None, None
if sortKeys:
sortBy, ignored, sortDescending = sortKeys.split(',')
sortDescending = sortDescending == '1'
maximumRecords = int(arguments.get('maximumRecords', [self._maximumRecords])[0])
query = arguments.get('query', [''])[0]
filters = arguments.get('filter', [])
startRecord = 1
if not query and not self._antiUnaryClause:
raise SruMandatoryParameterNotSuppliedException("query")
webquery = WebQuery(query, antiUnaryClause=self._antiUnaryClause)
for filter in filters:
if not ':' in filter:
raise BadRequestException('Invalid filter: %s' % filter)
field,term = filter.split(':', 1)
webquery.addFilter(field, term)
ast = webquery.ast
except (SruMandatoryParameterNotSuppliedException, BadRequestException, CQLParseException), e:
yield '<title>ERROR %s</title>' % xmlEscape(self._title)
yield '<link>%s</link>' % xmlEscape(self._link)
yield "<description>An error occurred '%s'</description>" % xmlEscape(str(e))
yield """</channel></rss>"""
raise StopIteration()
def testFilterFilter(self):
wq = WebQuery('fiets')
wq.addFilter('field1', 'value1')
wq.addFilter('field2', 'value2')
self.assertCql(
parseCql(
'(fiets) AND field1 exact value1 AND field2 exact value2'),
wq.ast)
def testReplaceTermsWithFilters(self):
wq = WebQuery('fiets kaart')
wq.addFilter('label', 'value')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEqual('fiets kaart', wq.original)
self.assertEqual('bike AND kaart', newWq.original)
self.assertCql(parseCql('(bike AND kaart) AND label exact value'),
newWq.ast)
def _assertQuery(self, expected, input, boolean=False, plusminus=False, default=False, needsBooleanHelp=False, asString=None):
input = expected if input == None else input
asString = expected if asString == None else asString
wq = WebQuery(input, antiUnaryClause='antiunary exact true')
self.assertEquals((boolean, plusminus, default, needsBooleanHelp), (wq.isBooleanQuery(), wq.isPlusMinusQuery(), wq.isDefaultQuery(), wq.needsBooleanHelp()))
self.assertEquals(parseCql(expected), wq.ast)
self.assertEqual(cqlToExpression(expected), wq.query)
self.assertEquals(asString, wq.asString())
self.assertEquals(input, wq.original)
def testHasFilters(self):
wq = WebQuery('fiets kaart')
self.assertFalse(wq.hasFilters())
wq.addFilter('label', 'value')
self.assertTrue(wq.hasFilters())
wq = WebQuery('fiets kaart')
self.assertFalse(wq.hasFilters())
wq.addTermFilter("water")
self.assertTrue(wq.hasFilters())
def testError(self):
wq = WebQuery('\'"a')
self.assertCql(parseCql('"\'" AND "a"'), wq.ast)
wq = WebQuery(
'<?xml version="1.0" encoding="ISO-8859-1"?>\n<!DOCTYPE foo [\n<!ELEMENT foo ANY >\n<!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo>'
)
self.assertCql(
parseCql(
'"<?xml version=\\"1.0\\" encoding=\\"ISO-8859-1\\"?>\n<!DOCTYPE foo [\n<!ELEMENT foo ANY >\n<!ENTITY xxe SYSTEM \\"file:///etc/passwd\\" >]><foo>&xxe;</foo>"'
), wq.ast)
def handleRequest(self, RequestURI='', **kwargs):
yield httputils.okRss
yield """<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel>"""
try:
Scheme, Netloc, Path, Query, Fragment = urlsplit(RequestURI)
arguments = parse_qs(Query)
sortKeys = arguments.get('sortKeys', [self._sortKeys])[0]
sortBy, sortDescending = None, None
if sortKeys:
sortBy, ignored, sortDescending = sortKeys.split(',')
sortDescending = sortDescending == '0'
# Set Language:
prefLanguage = arguments.get('preflang', [self._supportedLanguages[0]])[0]
# This might be German:-( Not anymore :-)
prefLanguage = prefLanguage if prefLanguage.lower() in self._supportedLanguages else self._supportedLanguages[0]
# Get max records, from request or get default
maximumRecords = int(arguments.get('maximumRecords', [self._maximumRecords])[0])
maximumRecords = maximumRecords if maximumRecords <= self._max_maximumRecords else int(self._max_maximumRecords) # Check if requested max does not exceed our max...
#set startrecord: we support rss paging...
startRecord = int(arguments.get('startRecord', [self._startRecord])[0])
#set querylabel:
querylabel = arguments.get('querylabel', [self._title.get(prefLanguage)])[0]
#get userquery:
query = arguments.get('query', [''])[0]
filters = arguments.get('filter', [])
# startRecord = 1
if not query and not self._antiUnaryClause:
raise SruMandatoryParameterNotSuppliedException("query")
webquery = WebQuery(query, antiUnaryClause=self._antiUnaryClause)
for filter in filters:
if not ':' in filter:
raise BadRequestException('Invalid filter: %s' % filter)
field,term = filter.split(':', 1)
webquery.addFilter(field, term)
ast = webquery.ast
except (SruMandatoryParameterNotSuppliedException, BadRequestException, CQLParseException), e:
yield '<title>ERROR %s</title>' % xmlEscape(self._title)
# yield '<link>%s</link>' % xmlEscape(self._link)
yield '<link>%s</link>' % xmlEscape(self._link.get(prefLanguage))
yield "<description>An error occurred '%s'</description>" % xmlEscape(str(e))
yield """</channel></rss>"""
raise StopIteration()
def handleRequest(self, RequestURI='', **kwargs):
yield httputils.okRss
yield """<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"><channel>"""
try:
Scheme, Netloc, Path, Query, Fragment = urlsplit(RequestURI)
arguments = parse_qs(Query)
sortKeys = arguments.get('sortKeys', [self._sortKeys])[0]
sortBy, sortDescending = None, None
if sortKeys:
sortBy, ignored, sortDescending = sortKeys.split(',')
sortDescending = sortDescending == '1'
maximumRecords = int(arguments.get('maximumRecords', [self._maximumRecords])[0])
query = arguments.get('query', [''])[0]
filters = arguments.get('filter', [])
startRecord = 1
if not query and not self._antiUnaryClause:
raise SruMandatoryParameterNotSuppliedException("query")
webquery = WebQuery(query, antiUnaryClause=self._antiUnaryClause)
for filter in filters:
if not ':' in filter:
raise BadRequestException('Invalid filter: %s' % filter)
field,term = filter.split(':', 1)
webquery.addFilter(field, term)
ast = webquery.ast
except (SruMandatoryParameterNotSuppliedException, BadRequestException, CQLParseException) as e:
yield '<title>ERROR %s</title>' % xmlEscape(self._title)
yield '<link>%s</link>' % xmlEscape(self._link)
yield "<description>An error occurred '%s'</description>" % xmlEscape(str(e))
yield """</channel></rss>"""
return
yield '<title>%s</title>' % xmlEscape(self._title)
yield '<description>%s</description>' % xmlEscape(self._description)
yield '<link>%s</link>' % xmlEscape(self._link)
SRU_IS_ONE_BASED = 1 #And our RSS plugin is closely based on SRU
yield self._yieldResults(
query=cqlToExpression(ast),
start=startRecord - SRU_IS_ONE_BASED,
stop=startRecord - SRU_IS_ONE_BASED+maximumRecords,
sortBy=sortBy,
sortDescending=sortDescending)
yield """</channel>"""
yield """</rss>"""
def testQuery(self):
wq = WebQuery('cats', antiUnaryClause='antiunary exact true')
self.assertFalse(wq.isBooleanQuery())
self.assertFalse(wq.isPlusMinusQuery())
self.assertTrue(wq.isDefaultQuery())
self.assertFalse(wq.needsBooleanHelp())
self.assertEqual(parseCql('cats'), wq.ast)
def testCqlQuery(self):
self.assertEqual(cqlToExpression('field = value'),
WebQuery('field=value').query)
self.assertTrue(WebQuery('field = value').isDefaultQuery())
self.assertFalse(WebQuery('field = value').isBooleanQuery())
self.assertFalse(WebQuery('field = value').isPlusMinusQuery())
self.assertEqual(cqlToExpression('field = value'),
WebQuery('field = value').query)
self.assertEqual(cqlToExpression('field exact value'),
WebQuery('field exact value').query)
def _assertQuery(self,
expected,
input,
boolean=False,
plusminus=False,
default=False,
needsBooleanHelp=False,
asString=None):
input = expected if input == None else input
asString = expected if asString == None else asString
wq = WebQuery(input, antiUnaryClause='antiunary exact true')
self.assertEqual((boolean, plusminus, default, needsBooleanHelp),
(wq.isBooleanQuery(), wq.isPlusMinusQuery(),
wq.isDefaultQuery(), wq.needsBooleanHelp()))
self.assertEqual(parseCql(expected), wq.ast)
self.assertEqual(cqlToExpression(expected), wq.query)
self.assertEqual(asString, wq.asString())
self.assertEqual(input, wq.original)
def testFilter(self):
wq = WebQuery('fiets')
wq.addFilter('field', 'value')
self.assertCql(parseCql('(fiets) AND field exact value'), wq.ast)
def testHasFilters(self):
wq = WebQuery('fiets kaart')
self.assertFalse(wq.hasFilters())
wq.addFilter('label', 'value')
self.assertTrue(wq.hasFilters())
wq = WebQuery('fiets kaart')
self.assertFalse(wq.hasFilters())
wq.addTermFilter("water")
self.assertTrue(wq.hasFilters())
def testTermFilter(self):
wq = WebQuery('fiets')
wq.addTermFilter("water")
#self.assertCql(parseCql('water AND (fiets)'), wq.ast)
self.assertCql(parseCql('(fiets) AND water'), wq.ast)
def testReplaceTerms(self):
wq = WebQuery('fiets kaart')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEqual('fiets kaart', wq.original)
self.assertEqual('bike AND kaart', newWq.original)
def testReplaceIndex(self):
wq = WebQuery('transport=fiets')
newWq = wq.replaceIndex(dict(transport="vervoer"))
self.assertEqual('vervoer=fiets', newWq.original)
def testReplaceTermOnLabelQuery(self):
wq = WebQuery('transport=fiets')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEqual('transport=bike', newWq.original)
def testFilterWithSpaces(self):
wq = WebQuery('fiets')
wq.addFilter('field', 'value with spaces')
self.assertCql(parseCql('(fiets) AND field exact "value with spaces"'), wq.ast)
def testFilterWithSpaces(self):
wq = WebQuery('fiets')
wq.addFilter('field', 'value with spaces')
self.assertCql(parseCql('(fiets) AND field exact "value with spaces"'),
wq.ast)
def testReplaceTermOnLabelQuery(self):
wq = WebQuery('transport=fiets')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEquals('transport=bike', newWq.original)
def testReplaceTerm(self):
wq = WebQuery('fiets')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEquals('fiets', wq.original)
self.assertEquals('bike', newWq.original)
def testFilter(self):
wq = WebQuery('fiets')
wq.addFilter('field', 'value')
self.assertCql(parseCql('(fiets) AND field exact value'), wq.ast)
def __init__(self, arguments, default_facet_terms_count,
maximum_record_number, page_size, default_record_schema):
query = arguments.pop('query', [None])[0]
if query is None:
raise MissingArgument('query')
self._request = dict(query=query)
self._next_request = dict(query=query)
self.query_expression = WebQuery(query, antiUnaryClause="*").query
page = getInt(arguments, 'page', 1)
if page <= 0:
raise InvalidArgument('page', 'expected value > 0')
if 'page' in arguments:
self._request['page'] = page
self._next_request['page'] = page
arguments.pop('page')
sortKeys = None
if 'sort' in arguments:
sort = [
s.strip() for sortOption in arguments.pop('sort')
for s in sortOption.split(',') if s.strip()
]
sortKeys = []
for key in sort:
ascending = True
if key.startswith('-'):
ascending = False
key = key[1:]
if key:
sortKeys.append(
dict(sortBy=key, sortDescending=not ascending))
self._request.setdefault(
'sort', []).append(('' if ascending else '-') + key)
self._next_request.setdefault(
'sort', []).append(('' if ascending else '-') + key)
self.recordSchema = arguments.pop('recordSchema',
[default_record_schema])[0]
pageSize = getInt(arguments, 'page-size', page_size)
if pageSize < 0:
raise InvalidArgument('page-size', 'expected value >= 0')
if 'page-size' in arguments:
self._request['page-size'] = pageSize
self._next_request['page-size'] = pageSize
arguments.pop('page-size')
self.start = (page - 1) * pageSize
if self.start > maximum_record_number:
raise InvalidArgument(
'page', 'expected value <= {}'.format(
(maximum_record_number + pageSize - 1) // pageSize))
self.stop = min(self.start + pageSize, maximum_record_number)
facets = arguments.pop('facet', [])
queryFacets = []
for facet in facets:
fieldname = facet
maxTerms = default_facet_terms_count
splitted = facet.rsplit(':', 1)
try:
maxTerms = int(splitted[1]) if len(
splitted) > 1 else default_facet_terms_count
fieldname = splitted[0]
except ValueError:
pass
queryFacets.append(
dict(fieldname=fieldname,
maxTerms=maxTerms,
sortBy=DRILLDOWN_SORTBY_COUNT))
self._request.setdefault('facet', []).append({
'index': fieldname,
'max-terms': maxTerms
})
facet_term_count = ':{0}'.format(
maxTerms) if maxTerms != default_facet_terms_count else ''
self._next_request.setdefault('facet', []).append(fieldname +
facet_term_count)
facetFilters = arguments.pop('facet-filter', [])
if facetFilters:
self._request['facet-filter'] = []
self._next_request['facet-filter'] = []
q = QueryExpression.nested('AND')
q.operands.append(self.query_expression)
for facetFilter in facetFilters:
if '=' not in facetFilter:
raise InvalidArgument(
'facet-filter', "expected <field>=<value> as a filter")
index, term = facetFilter.split('=', 1)
q.operands.append(
QueryExpression.searchterm(index=index,
relation='exact',
term=term))
self._request['facet-filter'].append({
'index': index,
'term': term
})
self._next_request['facet-filter'].append('{}={}'.format(
index, term))
self.query_expression = q
self.queryKwargs = dict(
start=self.start,
stop=self.stop,
query=self.query_expression,
facets=queryFacets or None,
sortKeys=sortKeys or None,
)
extra_arguments = {}
for k, v in arguments.items():
if k.startswith('x-'):
extra_arguments[k] = v
else:
raise BadArgument(k)
if extra_arguments:
self._request.update(extra_arguments)
self._next_request.update(extra_arguments)
self.queryKwargs['extraArguments'] = extra_arguments
def testTermFilter(self):
wq = WebQuery('fiets')
wq.addTermFilter("water")
#self.assertCql(parseCql('water AND (fiets)'), wq.ast)
self.assertCql(parseCql('(fiets) AND water'), wq.ast)
def testReplaceTerms(self):
wq = WebQuery('fiets kaart')
newWq = wq.replaceTerm('fiets', 'bike')
self.assertEquals('fiets kaart', wq.original)
self.assertEquals('bike AND kaart', newWq.original)