def index(self, dataset, format='html'):
# Get the dataset into the context variable 'c'
self._get_dataset(dataset)
# If the format is either json or csv we direct the user to the search
# API instead
if format in ['json', 'csv']:
return redirect(h.url_for(controller='api/version2',
action='search',
format=format, dataset=dataset,
**request.params))
# Get the default view
handle_request(request, c, c.dataset)
# Parse the parameters using the SearchParamParser (used by the API)
parser = EntryIndexParamParser(request.params)
params, errors = parser.parse()
# We have to remove page from the parameters because that's also
# used in the Solr browser (which fetches the queries)
params.pop('page')
# We limit ourselve to only our dataset
params['filter']['dataset'] = [c.dataset.name]
facet_dimensions = {field.name: field
for field in c.dataset.dimensions
if field.facet}
params['facet_field'] = facet_dimensions.keys()
# Create a Solr browser and execute it
b = Browser(**params)
try:
b.execute()
except SolrException as e:
return {'errors': [unicode(e)]}
# Get the entries, each item is a tuple of the dataset and entry
solr_entries = b.get_entries()
entries = [entry for (dataset, entry) in solr_entries]
# Get expanded facets for this dataset,
c.facets = b.get_expanded_facets(c.dataset)
# Create a pager for the entries
c.entries = templating.Page(entries, **request.params)
# Set the search word and default to empty string
c.search = params.get('q', '')
# Set filters (but remove the dataset as we don't need it)
c.filters = params['filter']
del c.filters['dataset']
# We also make the facet dimensions and dimension names available
c.facet_dimensions = facet_dimensions
c.dimensions = [dimension.name for dimension in c.dataset.dimensions]
# Render the entries page
return templating.render('entry/index.html')
def test_entries_order(self):
self.conn.raw_query.return_value = make_response([1, 2, 3])
self.dataset.entries.return_value = make_entries([3, 1, 2])
b = Browser()
b.execute()
entries = b.get_entries()
h.assert_equal(map(lambda (a, b): b, entries), make_entries([1, 2, 3]))
def test_entries_order(self):
self.conn.raw_query.return_value = make_response([1, 2, 3])
self.dataset.entries.return_value = make_entries([3, 1, 2])
b = Browser()
b.execute()
entries = b.get_entries()
assert map(lambda a_b: a_b[1], entries) == make_entries([1, 2, 3])
def test_entries_order(self):
self.conn.raw_query.return_value = make_response([1, 2, 3])
self.dataset.entries.return_value = make_entries([3, 1, 2])
b = Browser()
b.execute()
entries = b.get_entries()
assert map(lambda a_b: a_b[1], entries) == make_entries([1, 2, 3])
def test_entries_order(self):
self.conn.raw_query.return_value = make_response([1, 2, 3])
self.dataset.entries.return_value = make_entries([3, 1, 2])
b = Browser()
b.execute()
entries = b.get_entries()
h.assert_equal(map(lambda (a, b): b, entries), make_entries([1, 2, 3]))
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return to_jsonp({'errors': errors})
expand_facets = params.pop('expand_facet_dimensions')
format = params.pop('format')
if format == 'csv':
params['stats'] = False
params['facet_field'] = None
datasets = params.pop('dataset', None)
if datasets is None or not datasets:
q = Dataset.all_by_account(c.account)
if params.get('category'):
q = q.filter_by(category=params.pop('category'))
datasets = q.all()
expand_facets = False
if not datasets:
return {'errors': ["No dataset available."]}
params['filter']['dataset'] = []
for dataset in datasets:
require.dataset.read(dataset)
params['filter']['dataset'].append(dataset.name)
response.last_modified = max([d.updated_at for d in datasets])
etag_cache_keygen(parser.key(), response.last_modified)
if params['pagesize'] > parser.defaults['pagesize']:
# http://wiki.nginx.org/X-accel#X-Accel-Buffering
response.headers['X-Accel-Buffering'] = 'no'
if format == 'csv':
csv_headers(response, 'entries.csv')
streamer = CSVStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize']
)
return streamer.response()
else:
json_headers(filename='entries.json')
streamer = JSONStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize'],
expand_facets=util.expand_facets
if expand_facets else None,
callback=request.params.get('callback')
)
return streamer.response()
solr_browser = Browser(**params)
try:
solr_browser.execute()
except SolrException as e:
return {'errors': [unicode(e)]}
entries = []
for dataset, entry in solr_browser.get_entries():
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
entries.append(entry)
if format == 'csv':
return write_csv(entries, response,
filename='entries.csv')
if expand_facets and len(datasets) == 1:
facets = solr_browser.get_expanded_facets(datasets[0])
else:
facets = solr_browser.get_facets()
return to_jsonp({
'stats': solr_browser.get_stats(),
'facets': facets,
'results': entries
})
def index(self, dataset, format='html'):
# Get the dataset into the context variable 'c'
self._get_dataset(dataset)
# If the format is either json or csv we direct the user to the search
# API instead
if format in ['json', 'csv']:
return redirect(
h.url_for(controller='api/version2',
action='search',
format=format,
dataset=dataset,
**request.params))
# Get the default view
handle_request(request, c, c.dataset)
# Parse the parameters using the SearchParamParser (used by the API)
parser = EntryIndexParamParser(request.params)
params, errors = parser.parse()
# We have to remove page from the parameters because that's also
# used in the Solr browser (which fetches the queries)
params.pop('page')
# We limit ourselve to only our dataset
params['filter']['dataset'] = [c.dataset.name]
facet_dimensions = {
field.name: field
for field in c.dataset.dimensions if field.facet
}
params['facet_field'] = facet_dimensions.keys()
# Create a Solr browser and execute it
b = Browser(**params)
try:
b.execute()
except SolrException as e:
return {'errors': [unicode(e)]}
# Get the entries, each item is a tuple of (dataset, entry)
solr_entries = b.get_entries()
# We are only interested in the entry in the tuple since we know
# the dataset
entries = [entry[1] for entry in solr_entries]
# Get expanded facets for this dataset,
c.facets = b.get_expanded_facets(c.dataset)
# Create a pager for the entries
c.entries = templating.Page(entries, **request.params)
# Set the search word and default to empty string
c.search = params.get('q', '')
# Set filters (but remove the dataset as we don't need it)
c.filters = params['filter']
del c.filters['dataset']
# We also make the facet dimensions and dimension names available
c.facet_dimensions = facet_dimensions
c.dimensions = [dimension.name for dimension in c.dataset.dimensions]
# Render the entries page
return templating.render('entry/index.html')
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return to_jsonp({'errors': errors})
expand_facets = params.pop('expand_facet_dimensions')
format = params.pop('format')
if format == 'csv':
params['stats'] = False
params['facet_field'] = None
datasets = params.pop('dataset', None)
if datasets is None or not datasets:
q = Dataset.all_by_account(c.account)
if params.get('category'):
q = q.filter_by(category=params.pop('category'))
datasets = q.all()
expand_facets = False
if not datasets:
return {'errors': ["No dataset available."]}
params['filter']['dataset'] = []
for dataset in datasets:
require.dataset.read(dataset)
params['filter']['dataset'].append(dataset.name)
response.last_modified = max([d.updated_at for d in datasets])
etag_cache_keygen(parser.key(), response.last_modified)
if params['pagesize'] > parser.defaults['pagesize']:
# http://wiki.nginx.org/X-accel#X-Accel-Buffering
response.headers['X-Accel-Buffering'] = 'no'
if format == 'csv':
csv_headers(response, 'entries.csv')
streamer = CSVStreamingResponse(
datasets, params, pagesize=parser.defaults['pagesize'])
return streamer.response()
else:
json_headers(filename='entries.json')
streamer = JSONStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize'],
expand_facets=util.expand_facets
if expand_facets else None,
callback=request.params.get('callback'))
return streamer.response()
solr_browser = Browser(**params)
try:
solr_browser.execute()
except SolrException as e:
return {'errors': [unicode(e)]}
entries = []
for dataset, entry in solr_browser.get_entries():
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
entries.append(entry)
if format == 'csv':
return write_csv(entries, response, filename='entries.csv')
if expand_facets and len(datasets) == 1:
facets = solr_browser.get_expanded_facets(datasets[0])
else:
facets = solr_browser.get_facets()
return to_jsonp({
'stats': solr_browser.get_stats(),
'facets': facets,
'results': entries
})
