def test_any_value(self):
clause = lucene_to_sqlalchemy(u'user:*',
{'user': User.user_name}, [User.user_name])
self.assert_clause_equals(clause, User.user_name != None)
clause = lucene_to_sqlalchemy(u'-user:*',
{'user': User.user_name}, [User.user_name])
self.assert_clause_equals(clause, User.user_name == None)
def test_unterminated_quote(self):
clause = lucene_to_sqlalchemy(u'"what?',
{'user_name': User.user_name}, [User.user_name])
self.assert_clause_equals(clause, User.user_name == u'"what?')
clause = lucene_to_sqlalchemy(u'user_name:"what?',
{'user_name': User.user_name}, [User.user_name])
self.assert_clause_equals(clause, User.user_name == u'"what?')
def test_datetime_range(self):
clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO 2014-08-31]',
{'date_added': System.date_added},
[System.date_added])
self.assert_clause_equals(
clause,
and_(
System.date_added >= datetime.datetime(2014, 8, 1, 0, 0),
System.date_added <= datetime.datetime(2014, 8, 31, 23, 59,
59)))
clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO *]',
{'date_added': System.date_added},
[System.date_added])
self.assert_clause_equals(
clause,
and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0),
true()))
clause = lucene_to_sqlalchemy(u'date_added:[* TO 2014-08-31]',
{'date_added': System.date_added},
[System.date_added])
self.assert_clause_equals(
clause,
and_(
true(), System.date_added <= datetime.datetime(
2014, 8, 31, 23, 59, 59)))
clause = lucene_to_sqlalchemy(u'date_added:[* TO *]',
{'date_added': System.date_added},
[System.date_added])
self.assert_clause_equals(clause, and_(true(), true()))
clause = lucene_to_sqlalchemy(u'date_added:[fnord TO blorch]',
{'date_added': System.date_added},
[System.date_added])
self.assert_clause_equals(clause, and_(false(), false()))
def test_integer_column(self):
clause = lucene_to_sqlalchemy(u'memory:1024',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause, System.memory == 1024)
# searching invalid numbers against a numeric column is just False
clause = lucene_to_sqlalchemy(u'memory:much',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause, and_(false()))
def test_string_exclusive_range(self):
clause = lucene_to_sqlalchemy(u'fqdn:{aaa TO zzz]',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause,
and_(System.fqdn > u'aaa', System.fqdn <= u'zzz'))
clause = lucene_to_sqlalchemy(u'fqdn:[aaa TO zzz}',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause,
and_(System.fqdn >= u'aaa', System.fqdn < u'zzz'))
def test_integer_exclusive_range(self):
clause = lucene_to_sqlalchemy(u'memory:{1024 TO 2048]',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause,
and_(System.memory > 1024, System.memory <= 2048))
clause = lucene_to_sqlalchemy(u'memory:[1024 TO 2048}',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause,
and_(System.memory >= 1024, System.memory < 2048))
def test_numeric_column(self):
clause = lucene_to_sqlalchemy(u'weight:1.2',
{'weight': LabInfo.weight},
[LabInfo.weight])
self.assert_clause_equals(clause, LabInfo.weight == Decimal('1.2'))
# searching invalid numbers against a numeric column is just False
clause = lucene_to_sqlalchemy(u'weight:heavy',
{'weight': LabInfo.weight},
[LabInfo.weight])
self.assert_clause_equals(clause, and_(false()))
def test_datetime_column(self):
clause = lucene_to_sqlalchemy(u'date_added:2014-09-08',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause,
and_(System.date_added >= datetime.datetime(2014, 9, 8, 0, 0),
System.date_added <= datetime.datetime(2014, 9, 8, 23, 59, 59)))
# searching invalid dates against a datetime column is just False
clause = lucene_to_sqlalchemy(u'date_added:fnord',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause, and_(false()))
def test_malformed_range(self):
# missing the "TO" separator
clause = lucene_to_sqlalchemy(u'[what]',
{'fqdn': System.fqdn, 'memory': System.memory},
[System.fqdn, System.memory])
self.assert_clause_equals(clause,
or_(System.fqdn == u'[what]', false()))
clause = lucene_to_sqlalchemy(u'memory:[1024, 2048]',
{'fqdn': System.fqdn, 'memory': System.memory},
[System.fqdn, System.memory])
self.assert_clause_equals(clause, and_(false()))
def test_datetime_exclusive_range(self):
clause = lucene_to_sqlalchemy(u'date_added:{2014-08-01 TO 2014-08-31]',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause,
and_(System.date_added > datetime.datetime(2014, 8, 1, 0, 0),
System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59)))
clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO 2014-08-31}',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause,
and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0),
System.date_added < datetime.datetime(2014, 8, 31, 23, 59, 59)))
def test_string_range(self):
clause = lucene_to_sqlalchemy(u'fqdn:[aaa TO zzz]',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause,
and_(System.fqdn >= u'aaa', System.fqdn <= u'zzz'))
clause = lucene_to_sqlalchemy(u'fqdn:[aaa TO *]',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause, and_(System.fqdn >= u'aaa', true()))
clause = lucene_to_sqlalchemy(u'fqdn:[* TO zzz]',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause, and_(true(), System.fqdn <= u'zzz'))
clause = lucene_to_sqlalchemy(u'fqdn:[* TO *]',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause, and_(true(), true()))
def test_collection_relationship(self):
clause = lucene_to_sqlalchemy(u'device.driver:e1000',
{'fqdn': System.fqdn,
'device.driver': (System.devices, Device.driver)},
[System.fqdn])
self.assert_clause_equals(clause,
System.devices.any(Device.driver == u'e1000'))
def test_default_field(self):
clause = lucene_to_sqlalchemy(u'rmancy*',
{'user_name': User.user_name, 'email_address': User.email_address},
[User.user_name, User.email_address])
self.assert_clause_equals(clause,
or_(User.user_name.like(u'rmancy%'),
User.email_address.like(u'rmancy%')))
def test_multiple_terms(self):
clause = lucene_to_sqlalchemy(
u'user_name:rmancy email_address:[email protected]',
{'user_name': User.user_name, 'email_address': User.email_address},
[User.user_name, User.email_address])
self.assert_clause_equals(clause,
and_(User.user_name == u'rmancy',
User.email_address == u'*****@*****.**'))
def test_integer_range(self):
clause = lucene_to_sqlalchemy(u'memory:[1024 TO 2048]',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause,
and_(System.memory >= 1024, System.memory <= 2048))
clause = lucene_to_sqlalchemy(u'memory:[1024 TO *]',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause,
and_(System.memory >= 1024, true()))
clause = lucene_to_sqlalchemy(u'memory:[* TO 2048]',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause,
and_(true(), System.memory <= 2048))
clause = lucene_to_sqlalchemy(u'memory:[* TO *]',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause, and_(true(), true()))
clause = lucene_to_sqlalchemy(u'memory:[fnord TO blorch]',
{'memory': System.memory}, [System.memory])
self.assert_clause_equals(clause, and_(false(), false()))
def test_datetime_range(self):
clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO 2014-08-31]',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause,
and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0),
System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59)))
clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO *]',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause,
and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0),
true()))
clause = lucene_to_sqlalchemy(u'date_added:[* TO 2014-08-31]',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause,
and_(true(),
System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59)))
clause = lucene_to_sqlalchemy(u'date_added:[* TO *]',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause, and_(true(), true()))
clause = lucene_to_sqlalchemy(u'date_added:[fnord TO blorch]',
{'date_added': System.date_added}, [System.date_added])
self.assert_clause_equals(clause, and_(false(), false()))
def _json_collection_decorated(*args, **kwargs):
result = {}
query = func(*args, **kwargs)
if request.args.get('q') and columns:
query = query.filter(
lucene_to_sqlalchemy(request.args['q'],
search_columns=columns,
default_columns=set(
columns.values())))
total_count = query.count()
result['count'] = total_count
if request.args.get('sort_by') in columns:
result['sort_by'] = request.args['sort_by']
sort_column = columns[request.args['sort_by']]
if request.args.get('order') == 'desc':
result['order'] = 'desc'
sort_criterion = sort_column.desc()
else:
result['order'] = 'asc'
sort_criterion = sort_column
query = query.order_by(None).order_by(sort_criterion)
with convert_internal_errors():
if 'page_size' in request.args:
page_size = int(request.args['page_size'])
page_size = min(max(page_size, min_page_size),
max_page_size)
query = query.limit(page_size)
page = int(request.args.get('page', 1))
if page > 1:
query = query.offset((page - 1) * page_size)
result['page'] = page
result['page_size'] = page_size
elif total_count > force_paging_for_count:
url = request.base_url
if '?' not in url:
url += '?page_size=%d' % default_page_size
else:
url += '&page_size=%s' % default_page_size
return redirect(url)
result['entries'] = query.all()
return jsonify(result)
def _json_collection_decorated(*args, **kwargs):
result = {}
query = func(*args, **kwargs)
if request.args.get('q') and columns:
query = query.filter(lucene_to_sqlalchemy(request.args['q'],
search_columns=columns,
default_columns=set(columns.values())))
total_count = query.count()
result['count'] = total_count
if request.args.get('sort_by') in columns:
result['sort_by'] = request.args['sort_by']
sort_column = columns[request.args['sort_by']]
if request.args.get('order') == 'desc':
result['order'] = 'desc'
sort_criterion = sort_column.desc()
else:
result['order'] = 'asc'
sort_criterion = sort_column
query = query.order_by(None).order_by(sort_criterion)
with convert_internal_errors():
if 'page_size' in request.args:
page_size = int(request.args['page_size'])
page_size = min(max(page_size, min_page_size), max_page_size)
query = query.limit(page_size)
page = int(request.args.get('page', 1))
if page > 1:
query = query.offset((page - 1) * page_size)
result['page'] = page
result['page_size'] = page_size
elif total_count > force_paging_for_count:
url = request.base_url
if '?' not in url:
url += '?page_size=%d' % default_page_size
else:
url += '&page_size=%s' % default_page_size
return redirect(url)
result['entries'] = query.all()
return jsonify(result)
def test_single_term(self):
clause = lucene_to_sqlalchemy(u'user_name:rmancy',
{'user_name': User.user_name},
[User.user_name])
self.assert_clause_equals(clause, User.user_name == u'rmancy')
def json_collection(query, columns=None, extra_sort_columns=None, min_page_size=20,
max_page_size=500, default_page_size=20, force_paging_for_count=500,
skip_count=False):
"""
Helper function for Flask request handlers which want to return
a collection of resources as JSON.
The return value is a Python dict suitable for serialization into JSON, in
a format corresponding to Beaker's API for "pageable JSON collections"
(defined in documentation/server-api/http.rst). The caller can either
return a JSON response directly by passing the return value to
flask.jsonify(), or serialize it and embed it in an HTML response.
"""
if columns is None:
columns = {}
if extra_sort_columns is None:
extra_sort_columns = {}
result = {}
if request.args.get('q') and columns:
query = query.filter(lucene_to_sqlalchemy(request.args['q'],
search_columns=columns,
default_columns=set(columns.values())))
result['q'] = request.args['q']
if not skip_count:
total_count = query.order_by(None).count()
result['count'] = total_count
force_paging = (total_count > force_paging_for_count)
else:
force_paging = True
total_columns = columns.copy()
total_columns.update(extra_sort_columns)
if request.args.get('sort_by') in total_columns:
result['sort_by'] = request.args['sort_by']
sort_columns = total_columns[request.args['sort_by']]
if not isinstance(sort_columns, tuple):
sort_columns = (sort_columns,)
if request.args.get('order') == 'desc':
sort_order = 'desc'
else:
sort_order = 'asc'
result['order'] = sort_order
query = query.order_by(None)
for sort_column in sort_columns:
if sort_order == 'desc':
query = query.order_by(sort_column.desc())
else:
query = query.order_by(sort_column)
with convert_internal_errors():
if 'page_size' in request.args:
page_size = int(request.args['page_size'])
page_size = min(max(page_size, min_page_size), max_page_size)
elif not request_wants_json():
# For the web UI, we always do paging even if the query params
# didn't request it.
page_size = default_page_size
elif force_paging and request_wants_json():
# If an API user didn't request paging but we are forcing it
# because of the result size, then we redirect them to the URL with
# a page_size= param added, as a way of indicating that they should
# be using paging. There's no point doing this for the web UI though.
if request.query_string:
url = request.url + ('&page_size=%s' % default_page_size)
else:
url = request.base_url + ('?page_size=%s' % default_page_size)
raise PaginationRequiredException(response=redirect(url))
elif force_paging:
page_size = default_page_size
else:
page_size = None
if page_size:
query = query.limit(page_size)
page = int(request.args.get('page', 1))
if page > 1:
query = query.offset((page - 1) * page_size)
result['page'] = page
result['page_size'] = page_size
result['entries'] = query.all()
if len(result['entries']) < page_size and 'count' not in result:
# Even if we're not counting rows for performance reason, we know
# we have reached the end of the rows if we returned fewer than the
# page size. In this case we can infer the total count and return
# it to the client, so that they know they are at the last page.
result['count'] = (page - 1) * page_size + len(result['entries'])
return result
def json_collection(query,
columns=None,
extra_sort_columns=None,
min_page_size=20,
max_page_size=500,
default_page_size=20,
force_paging_for_count=500,
skip_count=False):
"""
Helper function for Flask request handlers which want to return
a collection of resources as JSON.
The return value is a Python dict suitable for serialization into JSON, in
a format corresponding to Beaker's API for "pageable JSON collections"
(defined in documentation/server-api/http.rst). The caller can either
return a JSON response directly by passing the return value to
flask.jsonify(), or serialize it and embed it in an HTML response.
"""
if columns is None:
columns = {}
if extra_sort_columns is None:
extra_sort_columns = {}
result = {}
if request.args.get('q') and columns:
query = query.filter(
lucene_to_sqlalchemy(request.args['q'],
search_columns=columns,
default_columns=set(columns.values())))
result['q'] = request.args['q']
if not skip_count:
total_count = query.order_by(None).count()
result['count'] = total_count
force_paging = (total_count > force_paging_for_count)
else:
force_paging = True
total_columns = columns.copy()
total_columns.update(extra_sort_columns)
if request.args.get('sort_by') in total_columns:
result['sort_by'] = request.args['sort_by']
sort_columns = total_columns[request.args['sort_by']]
if not isinstance(sort_columns, tuple):
sort_columns = (sort_columns, )
if request.args.get('order') == 'desc':
sort_order = 'desc'
else:
sort_order = 'asc'
result['order'] = sort_order
query = query.order_by(None)
for sort_column in sort_columns:
if sort_order == 'desc':
query = query.order_by(sort_column.desc())
else:
query = query.order_by(sort_column)
with convert_internal_errors():
if 'page_size' in request.args:
page_size = int(request.args['page_size'])
page_size = min(max(page_size, min_page_size), max_page_size)
elif not request_wants_json():
# For the web UI, we always do paging even if the query params
# didn't request it.
page_size = default_page_size
elif force_paging and request_wants_json():
# If an API user didn't request paging but we are forcing it
# because of the result size, then we redirect them to the URL with
# a page_size= param added, as a way of indicating that they should
# be using paging. There's no point doing this for the web UI though.
if request.query_string:
url = request.url + ('&page_size=%s' % default_page_size)
else:
url = request.base_url + ('?page_size=%s' % default_page_size)
raise PaginationRequiredException(response=redirect(url))
elif force_paging:
page_size = default_page_size
else:
page_size = None
if page_size:
query = query.limit(page_size)
page = int(request.args.get('page', 1))
if page > 1:
query = query.offset((page - 1) * page_size)
result['page'] = page
result['page_size'] = page_size
result['entries'] = query.all()
if len(result['entries']) < page_size and 'count' not in result:
# Even if we're not counting rows for performance reason, we know
# we have reached the end of the rows if we returned fewer than the
# page size. In this case we can infer the total count and return
# it to the client, so that they know they are at the last page.
result['count'] = (page - 1) * page_size + len(result['entries'])
return result
def test_nonexistent_field(self):
clause = lucene_to_sqlalchemy(u'favourite_color:green',
{'user_name': User.user_name, 'email_address': User.email_address},
[User.user_name, User.email_address])
self.assert_clause_equals(clause, and_(false()))
def test_empty_term(self):
clause = lucene_to_sqlalchemy(u'fqdn:""',
{'fqdn': System.fqdn}, [System.fqdn])
self.assert_clause_equals(clause, System.fqdn == u'')
def test_quoted_term(self):
clause = lucene_to_sqlalchemy(u'display_name:"Raymond Mancy"',
{'display_name': User.display_name},
[User.display_name])
self.assert_clause_equals(clause, User.display_name == u'Raymond Mancy')
def test_negation(self):
clause = lucene_to_sqlalchemy(u'-user_name:rmancy',
{'user_name': User.user_name},
[User.user_name])
self.assert_clause_equals(clause, User.user_name != u'rmancy')
def test_wildcards(self):
clause = lucene_to_sqlalchemy(
u'email_address:*rmancy*',
{'user_name': User.user_name, 'email_address': User.email_address},
[User.user_name, User.email_address])
self.assert_clause_equals(clause, User.email_address.like('%rmancy%'))