def test_any_value(self): clause = lucene_to_sqlalchemy(u'user:*', {'user': User.user_name}, [User.user_name]) self.assert_clause_equals(clause, User.user_name != None) clause = lucene_to_sqlalchemy(u'-user:*', {'user': User.user_name}, [User.user_name]) self.assert_clause_equals(clause, User.user_name == None)
def test_unterminated_quote(self): clause = lucene_to_sqlalchemy(u'"what?', {'user_name': User.user_name}, [User.user_name]) self.assert_clause_equals(clause, User.user_name == u'"what?') clause = lucene_to_sqlalchemy(u'user_name:"what?', {'user_name': User.user_name}, [User.user_name]) self.assert_clause_equals(clause, User.user_name == u'"what?')
def test_datetime_range(self): clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO 2014-08-31]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals( clause, and_( System.date_added >= datetime.datetime(2014, 8, 1, 0, 0), System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59))) clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO *]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals( clause, and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0), true())) clause = lucene_to_sqlalchemy(u'date_added:[* TO 2014-08-31]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals( clause, and_( true(), System.date_added <= datetime.datetime( 2014, 8, 31, 23, 59, 59))) clause = lucene_to_sqlalchemy(u'date_added:[* TO *]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(true(), true())) clause = lucene_to_sqlalchemy(u'date_added:[fnord TO blorch]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(false(), false()))
def test_integer_column(self): clause = lucene_to_sqlalchemy(u'memory:1024', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, System.memory == 1024) # searching invalid numbers against a numeric column is just False clause = lucene_to_sqlalchemy(u'memory:much', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(false()))
def test_string_exclusive_range(self): clause = lucene_to_sqlalchemy(u'fqdn:{aaa TO zzz]', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, and_(System.fqdn > u'aaa', System.fqdn <= u'zzz')) clause = lucene_to_sqlalchemy(u'fqdn:[aaa TO zzz}', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, and_(System.fqdn >= u'aaa', System.fqdn < u'zzz'))
def test_integer_exclusive_range(self): clause = lucene_to_sqlalchemy(u'memory:{1024 TO 2048]', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(System.memory > 1024, System.memory <= 2048)) clause = lucene_to_sqlalchemy(u'memory:[1024 TO 2048}', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(System.memory >= 1024, System.memory < 2048))
def test_numeric_column(self): clause = lucene_to_sqlalchemy(u'weight:1.2', {'weight': LabInfo.weight}, [LabInfo.weight]) self.assert_clause_equals(clause, LabInfo.weight == Decimal('1.2')) # searching invalid numbers against a numeric column is just False clause = lucene_to_sqlalchemy(u'weight:heavy', {'weight': LabInfo.weight}, [LabInfo.weight]) self.assert_clause_equals(clause, and_(false()))
def test_datetime_column(self): clause = lucene_to_sqlalchemy(u'date_added:2014-09-08', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(System.date_added >= datetime.datetime(2014, 9, 8, 0, 0), System.date_added <= datetime.datetime(2014, 9, 8, 23, 59, 59))) # searching invalid dates against a datetime column is just False clause = lucene_to_sqlalchemy(u'date_added:fnord', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(false()))
def test_malformed_range(self): # missing the "TO" separator clause = lucene_to_sqlalchemy(u'[what]', {'fqdn': System.fqdn, 'memory': System.memory}, [System.fqdn, System.memory]) self.assert_clause_equals(clause, or_(System.fqdn == u'[what]', false())) clause = lucene_to_sqlalchemy(u'memory:[1024, 2048]', {'fqdn': System.fqdn, 'memory': System.memory}, [System.fqdn, System.memory]) self.assert_clause_equals(clause, and_(false()))
def test_datetime_exclusive_range(self): clause = lucene_to_sqlalchemy(u'date_added:{2014-08-01 TO 2014-08-31]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(System.date_added > datetime.datetime(2014, 8, 1, 0, 0), System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59))) clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO 2014-08-31}', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0), System.date_added < datetime.datetime(2014, 8, 31, 23, 59, 59)))
def test_string_range(self): clause = lucene_to_sqlalchemy(u'fqdn:[aaa TO zzz]', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, and_(System.fqdn >= u'aaa', System.fqdn <= u'zzz')) clause = lucene_to_sqlalchemy(u'fqdn:[aaa TO *]', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, and_(System.fqdn >= u'aaa', true())) clause = lucene_to_sqlalchemy(u'fqdn:[* TO zzz]', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, and_(true(), System.fqdn <= u'zzz')) clause = lucene_to_sqlalchemy(u'fqdn:[* TO *]', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, and_(true(), true()))
def test_collection_relationship(self): clause = lucene_to_sqlalchemy(u'device.driver:e1000', {'fqdn': System.fqdn, 'device.driver': (System.devices, Device.driver)}, [System.fqdn]) self.assert_clause_equals(clause, System.devices.any(Device.driver == u'e1000'))
def test_default_field(self): clause = lucene_to_sqlalchemy(u'rmancy*', {'user_name': User.user_name, 'email_address': User.email_address}, [User.user_name, User.email_address]) self.assert_clause_equals(clause, or_(User.user_name.like(u'rmancy%'), User.email_address.like(u'rmancy%')))
def test_multiple_terms(self): clause = lucene_to_sqlalchemy( u'user_name:rmancy email_address:[email protected]', {'user_name': User.user_name, 'email_address': User.email_address}, [User.user_name, User.email_address]) self.assert_clause_equals(clause, and_(User.user_name == u'rmancy', User.email_address == u'*****@*****.**'))
def test_integer_range(self): clause = lucene_to_sqlalchemy(u'memory:[1024 TO 2048]', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(System.memory >= 1024, System.memory <= 2048)) clause = lucene_to_sqlalchemy(u'memory:[1024 TO *]', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(System.memory >= 1024, true())) clause = lucene_to_sqlalchemy(u'memory:[* TO 2048]', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(true(), System.memory <= 2048)) clause = lucene_to_sqlalchemy(u'memory:[* TO *]', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(true(), true())) clause = lucene_to_sqlalchemy(u'memory:[fnord TO blorch]', {'memory': System.memory}, [System.memory]) self.assert_clause_equals(clause, and_(false(), false()))
def test_datetime_range(self): clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO 2014-08-31]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0), System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59))) clause = lucene_to_sqlalchemy(u'date_added:[2014-08-01 TO *]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(System.date_added >= datetime.datetime(2014, 8, 1, 0, 0), true())) clause = lucene_to_sqlalchemy(u'date_added:[* TO 2014-08-31]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(true(), System.date_added <= datetime.datetime(2014, 8, 31, 23, 59, 59))) clause = lucene_to_sqlalchemy(u'date_added:[* TO *]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(true(), true())) clause = lucene_to_sqlalchemy(u'date_added:[fnord TO blorch]', {'date_added': System.date_added}, [System.date_added]) self.assert_clause_equals(clause, and_(false(), false()))
def _json_collection_decorated(*args, **kwargs): result = {} query = func(*args, **kwargs) if request.args.get('q') and columns: query = query.filter( lucene_to_sqlalchemy(request.args['q'], search_columns=columns, default_columns=set( columns.values()))) total_count = query.count() result['count'] = total_count if request.args.get('sort_by') in columns: result['sort_by'] = request.args['sort_by'] sort_column = columns[request.args['sort_by']] if request.args.get('order') == 'desc': result['order'] = 'desc' sort_criterion = sort_column.desc() else: result['order'] = 'asc' sort_criterion = sort_column query = query.order_by(None).order_by(sort_criterion) with convert_internal_errors(): if 'page_size' in request.args: page_size = int(request.args['page_size']) page_size = min(max(page_size, min_page_size), max_page_size) query = query.limit(page_size) page = int(request.args.get('page', 1)) if page > 1: query = query.offset((page - 1) * page_size) result['page'] = page result['page_size'] = page_size elif total_count > force_paging_for_count: url = request.base_url if '?' not in url: url += '?page_size=%d' % default_page_size else: url += '&page_size=%s' % default_page_size return redirect(url) result['entries'] = query.all() return jsonify(result)
def _json_collection_decorated(*args, **kwargs): result = {} query = func(*args, **kwargs) if request.args.get('q') and columns: query = query.filter(lucene_to_sqlalchemy(request.args['q'], search_columns=columns, default_columns=set(columns.values()))) total_count = query.count() result['count'] = total_count if request.args.get('sort_by') in columns: result['sort_by'] = request.args['sort_by'] sort_column = columns[request.args['sort_by']] if request.args.get('order') == 'desc': result['order'] = 'desc' sort_criterion = sort_column.desc() else: result['order'] = 'asc' sort_criterion = sort_column query = query.order_by(None).order_by(sort_criterion) with convert_internal_errors(): if 'page_size' in request.args: page_size = int(request.args['page_size']) page_size = min(max(page_size, min_page_size), max_page_size) query = query.limit(page_size) page = int(request.args.get('page', 1)) if page > 1: query = query.offset((page - 1) * page_size) result['page'] = page result['page_size'] = page_size elif total_count > force_paging_for_count: url = request.base_url if '?' not in url: url += '?page_size=%d' % default_page_size else: url += '&page_size=%s' % default_page_size return redirect(url) result['entries'] = query.all() return jsonify(result)
def test_single_term(self): clause = lucene_to_sqlalchemy(u'user_name:rmancy', {'user_name': User.user_name}, [User.user_name]) self.assert_clause_equals(clause, User.user_name == u'rmancy')
def json_collection(query, columns=None, extra_sort_columns=None, min_page_size=20, max_page_size=500, default_page_size=20, force_paging_for_count=500, skip_count=False): """ Helper function for Flask request handlers which want to return a collection of resources as JSON. The return value is a Python dict suitable for serialization into JSON, in a format corresponding to Beaker's API for "pageable JSON collections" (defined in documentation/server-api/http.rst). The caller can either return a JSON response directly by passing the return value to flask.jsonify(), or serialize it and embed it in an HTML response. """ if columns is None: columns = {} if extra_sort_columns is None: extra_sort_columns = {} result = {} if request.args.get('q') and columns: query = query.filter(lucene_to_sqlalchemy(request.args['q'], search_columns=columns, default_columns=set(columns.values()))) result['q'] = request.args['q'] if not skip_count: total_count = query.order_by(None).count() result['count'] = total_count force_paging = (total_count > force_paging_for_count) else: force_paging = True total_columns = columns.copy() total_columns.update(extra_sort_columns) if request.args.get('sort_by') in total_columns: result['sort_by'] = request.args['sort_by'] sort_columns = total_columns[request.args['sort_by']] if not isinstance(sort_columns, tuple): sort_columns = (sort_columns,) if request.args.get('order') == 'desc': sort_order = 'desc' else: sort_order = 'asc' result['order'] = sort_order query = query.order_by(None) for sort_column in sort_columns: if sort_order == 'desc': query = query.order_by(sort_column.desc()) else: query = query.order_by(sort_column) with convert_internal_errors(): if 'page_size' in request.args: page_size = int(request.args['page_size']) page_size = min(max(page_size, min_page_size), max_page_size) elif not request_wants_json(): # For the web UI, we always do paging even if the query params # didn't request it. page_size = default_page_size elif force_paging and request_wants_json(): # If an API user didn't request paging but we are forcing it # because of the result size, then we redirect them to the URL with # a page_size= param added, as a way of indicating that they should # be using paging. There's no point doing this for the web UI though. if request.query_string: url = request.url + ('&page_size=%s' % default_page_size) else: url = request.base_url + ('?page_size=%s' % default_page_size) raise PaginationRequiredException(response=redirect(url)) elif force_paging: page_size = default_page_size else: page_size = None if page_size: query = query.limit(page_size) page = int(request.args.get('page', 1)) if page > 1: query = query.offset((page - 1) * page_size) result['page'] = page result['page_size'] = page_size result['entries'] = query.all() if len(result['entries']) < page_size and 'count' not in result: # Even if we're not counting rows for performance reason, we know # we have reached the end of the rows if we returned fewer than the # page size. In this case we can infer the total count and return # it to the client, so that they know they are at the last page. result['count'] = (page - 1) * page_size + len(result['entries']) return result
def json_collection(query, columns=None, extra_sort_columns=None, min_page_size=20, max_page_size=500, default_page_size=20, force_paging_for_count=500, skip_count=False): """ Helper function for Flask request handlers which want to return a collection of resources as JSON. The return value is a Python dict suitable for serialization into JSON, in a format corresponding to Beaker's API for "pageable JSON collections" (defined in documentation/server-api/http.rst). The caller can either return a JSON response directly by passing the return value to flask.jsonify(), or serialize it and embed it in an HTML response. """ if columns is None: columns = {} if extra_sort_columns is None: extra_sort_columns = {} result = {} if request.args.get('q') and columns: query = query.filter( lucene_to_sqlalchemy(request.args['q'], search_columns=columns, default_columns=set(columns.values()))) result['q'] = request.args['q'] if not skip_count: total_count = query.order_by(None).count() result['count'] = total_count force_paging = (total_count > force_paging_for_count) else: force_paging = True total_columns = columns.copy() total_columns.update(extra_sort_columns) if request.args.get('sort_by') in total_columns: result['sort_by'] = request.args['sort_by'] sort_columns = total_columns[request.args['sort_by']] if not isinstance(sort_columns, tuple): sort_columns = (sort_columns, ) if request.args.get('order') == 'desc': sort_order = 'desc' else: sort_order = 'asc' result['order'] = sort_order query = query.order_by(None) for sort_column in sort_columns: if sort_order == 'desc': query = query.order_by(sort_column.desc()) else: query = query.order_by(sort_column) with convert_internal_errors(): if 'page_size' in request.args: page_size = int(request.args['page_size']) page_size = min(max(page_size, min_page_size), max_page_size) elif not request_wants_json(): # For the web UI, we always do paging even if the query params # didn't request it. page_size = default_page_size elif force_paging and request_wants_json(): # If an API user didn't request paging but we are forcing it # because of the result size, then we redirect them to the URL with # a page_size= param added, as a way of indicating that they should # be using paging. There's no point doing this for the web UI though. if request.query_string: url = request.url + ('&page_size=%s' % default_page_size) else: url = request.base_url + ('?page_size=%s' % default_page_size) raise PaginationRequiredException(response=redirect(url)) elif force_paging: page_size = default_page_size else: page_size = None if page_size: query = query.limit(page_size) page = int(request.args.get('page', 1)) if page > 1: query = query.offset((page - 1) * page_size) result['page'] = page result['page_size'] = page_size result['entries'] = query.all() if len(result['entries']) < page_size and 'count' not in result: # Even if we're not counting rows for performance reason, we know # we have reached the end of the rows if we returned fewer than the # page size. In this case we can infer the total count and return # it to the client, so that they know they are at the last page. result['count'] = (page - 1) * page_size + len(result['entries']) return result
def test_nonexistent_field(self): clause = lucene_to_sqlalchemy(u'favourite_color:green', {'user_name': User.user_name, 'email_address': User.email_address}, [User.user_name, User.email_address]) self.assert_clause_equals(clause, and_(false()))
def test_empty_term(self): clause = lucene_to_sqlalchemy(u'fqdn:""', {'fqdn': System.fqdn}, [System.fqdn]) self.assert_clause_equals(clause, System.fqdn == u'')
def test_quoted_term(self): clause = lucene_to_sqlalchemy(u'display_name:"Raymond Mancy"', {'display_name': User.display_name}, [User.display_name]) self.assert_clause_equals(clause, User.display_name == u'Raymond Mancy')
def test_negation(self): clause = lucene_to_sqlalchemy(u'-user_name:rmancy', {'user_name': User.user_name}, [User.user_name]) self.assert_clause_equals(clause, User.user_name != u'rmancy')
def test_wildcards(self): clause = lucene_to_sqlalchemy( u'email_address:*rmancy*', {'user_name': User.user_name, 'email_address': User.email_address}, [User.user_name, User.email_address]) self.assert_clause_equals(clause, User.email_address.like('%rmancy%'))