def test_dict():
d = MultiDict({'a': 1})
assert list(d.items()) == [('a', 1)]
d['b'] = 2
d['c'] = 3
assert list(d.items()) == [('a', 1), ('b', 2), ('c', 3)]
d['b'] = 4
assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4)]
d.add('b', 5)
assert_raises(KeyError, d.getone, "b")
assert d.getall('b') == [4, 5]
assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4), ('b', 5)]
del d['b']
assert list(d.items()) == [('a', 1), ('c', 3)]
assert d.pop('xxx', 5) == 5
assert d.getone('a') == 1
assert d.popitem() == ('c', 3)
assert list(d.items()) == [('a', 1)]
item = []
assert d.setdefault('z', item) is item
assert list(d.items()) == [('a', 1), ('z', item)]
assert d.setdefault('y', 6) == 6
assert d.mixed() == {'a': 1, 'y': 6, 'z': item}
assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]}
assert 'a' in d
dcopy = d.copy()
assert dcopy is not d
assert dcopy == d
d['x'] = 'x test'
assert dcopy != d
d[(1, None)] = (None, 1)
assert list(d.items()) == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'),
((1, None), (None, 1))]
def test_dict():
d = MultiDict({'a': 1})
assert d.items() == [('a', 1)]
d['b'] = 2
d['c'] = 3
assert d.items() == [('a', 1), ('b', 2), ('c', 3)]
d['b'] = 4
assert d.items() == [('a', 1), ('c', 3), ('b', 4)]
d.add('b', 5)
raises(KeyError, 'd.getone("b")')
assert d.getall('b') == [4, 5]
assert d.items() == [('a', 1), ('c', 3), ('b', 4), ('b', 5)]
del d['b']
assert d.items() == [('a', 1), ('c', 3)]
assert d.pop('xxx', 5) == 5
assert d.getone('a') == 1
assert d.popitem() == ('c', 3)
assert d.items() == [('a', 1)]
item = []
assert d.setdefault('z', item) is item
assert d.items() == [('a', 1), ('z', item)]
assert d.setdefault('y', 6) == 6
assert d.mixed() == {'a': 1, 'y': 6, 'z': item}
assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]}
assert 'a' in d
dcopy = d.copy()
assert dcopy is not d
assert dcopy == d
d['x'] = 'x test'
assert dcopy != d
d[(1, None)] = (None, 1)
assert d.items() == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'),
((1, None), (None, 1))]
def search(self, ver=None, register=None):
log.debug('search %s params: %r', register, request.params)
if register == 'revision':
since_time = None
if 'since_id' in request.params:
id = request.params['since_id']
if not id:
return self._finish_bad_request(
_(u'No revision specified'))
rev = model.Session.query(model.Revision).get(id)
if rev is None:
return self._finish_not_found(
_(u'There is no revision with id: %s') % id)
since_time = rev.timestamp
elif 'since_time' in request.params:
since_time_str = request.params['since_time']
try:
since_time = h.date_str_to_datetime(since_time_str)
except ValueError as inst:
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
_("Missing search term ('since_id=UUID' or " +
" 'since_time=TIMESTAMP')"))
revs = model.Session.query(model.Revision) \
.filter(model.Revision.timestamp > since_time) \
.order_by(model.Revision.timestamp) \
.limit(50) # reasonable enough for a page
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
params = MultiDict(self._get_search_params(request.params))
except ValueError as e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(
query=params.get('q'),
fields=query_fields,
options=options
)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.\
convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
# Remove any existing fq param and set the capacity to
# public
if 'fq' in params:
del params['fq']
params['fq'] = '+capacity:public'
# if callback is specified we do not want to send that to
# the search
if 'callback' in params:
del params['callback']
results = query.run(params)
return self._finish_ok(results)
except search.SearchError as e:
log.exception(e)
return self._finish_bad_request(
_('Bad search option: %s') % e)
else:
return self._finish_not_found(
_('Unknown register: %s') % register)
params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(_("Could not read parameters: %r" % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ["dataset", "package"] and not params.get("fl"):
params["fl"] = "id" if ver == 2 else "name"
try:
if register == "resource":
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if k in search.DEFAULT_OPTIONS.keys():
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
except ValueError, e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
def search(self, ver=None, register=None):
log.debug('search %s params: %r', register, request.params)
if register == 'revision':
since_time = None
if 'since_id' in request.params:
id = request.params['since_id']
if not id:
return self._finish_bad_request(
_(u'No revision specified'))
rev = model.Session.query(model.Revision).get(id)
if rev is None:
return self._finish_not_found(
_(u'There is no revision with id: %s') % id)
since_time = rev.timestamp
elif 'since_time' in request.params:
since_time_str = request.params['since_time']
try:
since_time = h.date_str_to_datetime(since_time_str)
except ValueError as inst:
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
_("Missing search term ('since_id=UUID' or " +
" 'since_time=TIMESTAMP')"))
revs = model.Session.query(model.Revision) \
.filter(model.Revision.timestamp > since_time) \
.order_by(model.Revision.timestamp) \
.limit(50) # reasonable enough for a page
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
params = MultiDict(self._get_search_params(request.params))
except ValueError as e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(query=params.get('q'),
fields=query_fields,
options=options)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.\
convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
# Remove any existing fq param and set the capacity to
# public
if 'fq' in params:
del params['fq']
params['fq'] = '+capacity:public'
# if callback is specified we do not want to send that to
# the search
if 'callback' in params:
del params['callback']
results = query.run(params)
return self._finish_ok(results)
except search.SearchError as e:
log.exception(e)
return self._finish_bad_request(_('Bad search option: %s') % e)
else:
return self._finish_not_found(_('Unknown register: %s') % register)
class QueryParser(object):
"""
The query parser will take any incoming query specifications and turn
them into field-specific and general query parts.
"""
def __init__(self, query, terms, fields):
self._query = query
self._terms = terms
self._fields = MultiDict(fields)
@property
def query(self):
if not hasattr(self, '_combined_query'):
parts = [self._query if self._query is not None else '']
for term in self._terms:
if term.find(u' ') != -1:
term = u"\"%s\"" % term
parts.append(term.strip())
for field, value in self._fields.items():
if value.find(' ') != -1:
value = u"\"%s\"" % value
parts.append(u"%s:%s" % (field.strip(), value.strip()))
self._combined_query = u' '.join(parts)
return self._combined_query
def _query_tokens(self):
""" Split the query string, leaving quoted strings intact. """
if self._query:
inside_quote = False
buf = u''
for ch in self._query:
if ch == u' ' and not inside_quote:
if len(buf):
yield buf.strip()
buf = u''
elif ch == inside_quote:
inside_quote = False
elif ch in [u"\"", u"'"]:
inside_quote = ch
else:
buf += ch
if len(buf):
yield buf.strip()
def _parse_query(self):
""" Decompose the query string into fields and terms. """
self._combined_fields = MultiDict(self._fields)
self._combined_terms = list(self._terms)
for token in self._query_tokens():
colon_pos = token.find(u':')
if colon_pos != -1:
field = token[:colon_pos]
value = token[colon_pos + 1:]
value = value.strip('"').strip("'").strip()
self._combined_fields.add(field, value)
else:
self._combined_terms.append(token)
@property
def fields(self):
if not hasattr(self, '_combined_fields'):
self._parse_query()
return self._combined_fields
@property
def terms(self):
if not hasattr(self, '_combined_terms'):
self._parse_query()
return self._combined_terms
def validate(self):
""" Check that this is a valid query. """
pass
def __str__(self):
return self.query
def __repr__(self):
return "Query(%r)" % self.query
class QueryParser(object):
"""
The query parser will take any incoming query specifications and turn
them into field-specific and general query parts.
"""
def __init__(self, query, terms, fields):
self._query = query
self._terms = terms
self._fields = MultiDict(fields)
@property
def query(self):
if not hasattr(self, '_combined_query'):
parts = [self._query if self._query is not None else '']
for term in self._terms:
if term.find(u' ') != -1:
term = u"\"%s\"" % term
parts.append(term.strip())
for field, value in self._fields.items():
if value.find(' ') != -1:
value = u"\"%s\"" % value
parts.append(u"%s:%s" % (field.strip(), value.strip()))
self._combined_query = u' '.join(parts)
return self._combined_query
def _query_tokens(self):
""" Split the query string, leaving quoted strings intact. """
if self._query:
inside_quote = False
buf = u''
for ch in self._query:
if ch == u' ' and not inside_quote:
if len(buf):
yield buf.strip()
buf = u''
elif ch == inside_quote:
inside_quote = False
elif ch in [u"\"", u"'"]:
inside_quote = ch
else:
buf += ch
if len(buf):
yield buf.strip()
def _parse_query(self):
""" Decompose the query string into fields and terms. """
self._combined_fields = MultiDict(self._fields)
self._combined_terms = list(self._terms)
for token in self._query_tokens():
colon_pos = token.find(u':')
if colon_pos != -1:
field = token[:colon_pos]
value = token[colon_pos+1:]
value = value.strip('"').strip("'").strip()
self._combined_fields.add(field, value)
else:
self._combined_terms.append(token)
@property
def fields(self):
if not hasattr(self, '_combined_fields'):
self._parse_query()
return self._combined_fields
@property
def terms(self):
if not hasattr(self, '_combined_terms'):
self._parse_query()
return self._combined_terms
def validate(self):
""" Check that this is a valid query. """
if not len(self.query):
raise SearchError("No query has been specified")
def __str__(self):
return self.query
def __repr__(self):
return "Query(%s)" % self
