def view(self, dataset, id, format='html'):
self._get_dataset(dataset)
entries = list(c.dataset.entries(c.dataset.alias.c.id == id))
if not len(entries) == 1:
abort(404, _('Sorry, there is no entry %r') % id)
c.entry = entry_apply_links(dataset, entries.pop())
c.id = c.entry.get('id')
c.from_ = c.entry.get('from')
c.to = c.entry.get('to')
c.currency = c.entry.get('currency', c.dataset.currency).upper()
c.amount = c.entry.get('amount')
c.time = c.entry.get('time')
c.custom_html = h.render_entry_custom_html(c.dataset,
c.entry)
excluded_keys = ('time', 'amount', 'currency', 'from',
'to', 'dataset', 'id', 'name', 'description')
c.extras = {}
if c.dataset:
c.desc = dict([(d.name, d) for d in c.dataset.dimensions])
for key in c.entry:
if key in c.desc and \
not key in excluded_keys:
c.extras[key] = c.entry[key]
if format == 'json':
return to_jsonp(c.entry)
elif format == 'csv':
return write_csv([c.entry], response)
else:
return render('entry/view.html')
def view(self, dataset, id, format="html"):
self._get_dataset(dataset)
entries = list(c.dataset.entries(c.dataset.alias.c.id == id))
if not len(entries) == 1:
abort(404, _("Sorry, there is no entry %r") % id)
c.entry = entry_apply_links(dataset, entries.pop())
c.id = c.entry.get("id")
c.from_ = c.entry.get("from")
c.to = c.entry.get("to")
c.currency = c.entry.get("currency", c.dataset.currency).upper()
c.amount = c.entry.get("amount")
c.time = c.entry.get("time")
c.custom_html = h.render_entry_custom_html(c.dataset, c.entry)
excluded_keys = ("time", "amount", "currency", "from", "to", "dataset", "id", "name", "description")
c.extras = {}
if c.dataset:
c.desc = dict([(d.name, d) for d in c.dataset.dimensions])
for key in c.entry:
if key in c.desc and not key in excluded_keys:
c.extras[key] = c.entry[key]
if format == "json":
return to_jsonp(c.entry)
elif format == "csv":
return write_csv([c.entry], response)
else:
return render("entry/view.html")
def view(self, dataset, id, format='html'):
self._get_dataset(dataset)
entries = list(c.dataset.entries(c.dataset.alias.c.id == id))
if not len(entries) == 1:
abort(404, _('Sorry, there is no entry %r') % id)
c.entry = entry_apply_links(dataset, entries.pop())
c.id = c.entry.get('id')
c.from_ = c.entry.get('from')
c.to = c.entry.get('to')
c.currency = c.entry.get('currency', c.dataset.currency).upper()
c.amount = c.entry.get('amount')
c.time = c.entry.get('time')
c.custom_html = h.render_entry_custom_html(c.dataset, c.entry)
excluded_keys = ('time', 'amount', 'currency', 'from', 'to', 'dataset',
'id', 'name', 'description')
c.extras = {}
if c.dataset:
c.desc = dict([(d.name, d) for d in c.dataset.dimensions])
for key in c.entry:
if key in c.desc and \
not key in excluded_keys:
c.extras[key] = c.entry[key]
if format == 'json':
return to_jsonp(c.entry)
elif format == 'csv':
return write_csv([c.entry], response)
else:
return templating.render('entry/view.html')
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return {'errors': errors}
expand_facets = params.pop('expand_facet_dimensions')
datasets = params.pop('dataset', None)
if datasets is None:
datasets = model.Dataset.all_by_account(c.account)
expand_facets = False
for dataset in datasets:
require.dataset.read(dataset)
b = Browser(**params)
stats, facets, entries = b.execute()
entries = [entry_apply_links(d.name, e) for d, e in entries]
if expand_facets and len(datasets) == 1:
_expand_facets(facets, datasets[0])
return {
'stats': stats,
'facets': facets,
'results': entries
}
def entries(self, dataset, dimension, name, format='html'):
self._get_member(dataset, dimension, name)
if format in ['json', 'csv']:
return redirect(url_for(controller='api2', action='search',
format=format, dataset=dataset,
filter='%s.name:%s' % (dimension, name),
**request.params))
handle_request(request, c, c.member, c.dimension.name)
entries = c.dataset.entries(c.dimension.alias.c.name == c.member['name'])
entries = (entry_apply_links(dataset, e) for e in entries)
return render('dimension/entries.html')
def entries(self, dataset, dimension, name, format="html"):
self._get_member(dataset, dimension, name)
handle_request(request, c, c.member, c.dimension.name)
entries = c.dataset.entries(c.dimension.alias.c.name == c.member["name"])
entries = (entry_apply_links(dataset, e) for e in entries)
attachment_name = "__".join([dataset, dimension, name])
if format == "json":
return write_json(entries, response, filename=attachment_name + ".json")
elif format == "csv":
return write_csv(entries, response, filename=attachment_name + ".csv")
else:
return render("dimension/entries.html")
def entries(self, dataset, dimension, name, format='html'):
self._get_member(dataset, dimension, name)
handle_request(request, c, c.member, c.dimension.name)
entries = c.dataset.entries(c.dimension.alias.c.name == c.member['name'])
entries = (entry_apply_links(dataset, e) for e in entries)
attachment_name = '__'.join([dataset, dimension, name])
if format == 'json':
return write_json(entries, response, filename=attachment_name + '.json')
elif format == 'csv':
return write_csv(entries, response, filename=attachment_name + '.csv')
else:
return render('dimension/entries.html')
def entries(self, dataset, dimension, name, format='html'):
self._get_member(dataset, dimension, name)
if format in ['json', 'csv']:
return redirect(
url_for(controller='api/version2',
action='search',
format=format,
dataset=dataset,
filter='%s.name:%s' % (dimension, name),
**request.params))
handle_request(request, c, c.member, c.dimension.name)
entries = c.dataset.entries(
c.dimension.alias.c.name == c.member['name'])
entries = (entry_apply_links(dataset, e) for e in entries)
return templating.render('dimension/entries.html')
def entries(self, dataset, dimension, name, format="html"):
self._get_member(dataset, dimension, name)
if format in ["json", "csv"]:
return redirect(
url_for(
controller="api2",
action="search",
format=format,
dataset=dataset,
filter="%s.name:%s" % (dimension, name),
**request.params
)
)
handle_request(request, c, c.member, c.dimension.name)
entries = c.dataset.entries(c.dimension.alias.c.name == c.member["name"])
entries = (entry_apply_links(dataset, e) for e in entries)
return templating.render("dimension/entries.html")
def entries(self, dataset, dimension, name, format='html'):
self._get_member(dataset, dimension, name)
handle_request(request, c, c.member, c.dimension.name)
entries = c.dataset.entries(
c.dimension.alias.c.name == c.member['name'])
entries = (entry_apply_links(dataset, e) for e in entries)
attachment_name = '__'.join([dataset, dimension, name])
if format == 'json':
return write_json(entries,
response,
filename=attachment_name + '.json')
elif format == 'csv':
return write_csv(entries,
response,
filename=attachment_name + '.csv')
else:
return render('dimension/entries.html')
class Api2Controller(BaseController):
def _response_params(self, params):
for k, v in params.items():
k = k.replace('_', ' ').replace('-', ' ').split()
k = '-'.join(['X'] + [l.capitalize() for l in k])
response.headers[k] = unicode(v).encode('ascii', 'ignore')
def aggregate(self):
parser = AggregateParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return to_jsonp({'errors': errors})
params['cuts'] = params.pop('cut')
params['drilldowns'] = params.pop('drilldown')
dataset = params.pop('dataset')
format = params.pop('format')
require.dataset.read(dataset)
self._response_params(params)
try:
cache = AggregationCache(dataset)
result = cache.aggregate(**params)
if 'drilldown' in result:
result['drilldown'] = drilldowns_apply_links(
dataset.name, result['drilldown'])
response.last_modified = dataset.updated_at
if cache.cache_enabled and 'cache_key' in result['summary']:
etag_cache(result['summary']['cache_key'])
except (KeyError, ValueError) as ve:
log.exception(ve)
response.status = 400
return to_jsonp({'errors': [unicode(ve)]})
if format == 'csv':
return write_csv(result['drilldown'],
response,
filename=dataset.name + '.csv')
return to_jsonp(result)
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return to_jsonp({'errors': errors})
expand_facets = params.pop('expand_facet_dimensions')
format = params.pop('format')
if format == 'csv':
params['stats'] = False
params['facet_field'] = None
datasets = params.pop('dataset', None)
if datasets is None or not datasets:
q = model.Dataset.all_by_account(c.account)
if params.get('category'):
q = q.filter_by(category=params.pop('category'))
datasets = q.all()
expand_facets = False
if not datasets:
return {'errors': ["No dataset available."]}
params['filter']['dataset'] = []
for dataset in datasets:
require.dataset.read(dataset)
params['filter']['dataset'].append(dataset.name)
response.last_modified = max([d.updated_at for d in datasets])
etag_cache_keygen(parser.key(), response.last_modified)
self._response_params(params)
if params['pagesize'] > parser.defaults['pagesize']:
# http://wiki.nginx.org/X-accel#X-Accel-Buffering
response.headers['X-Accel-Buffering'] = 'no'
if format == 'csv':
csv_headers(response, 'entries.csv')
streamer = CSVStreamingResponse(
datasets, params, pagesize=parser.defaults['pagesize'])
return streamer.response()
else:
json_headers(filename='entries.json')
streamer = JSONStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize'],
expand_facets=_expand_facets if expand_facets else None,
callback=request.params.get('callback'))
return streamer.response()
b = Browser(**params)
try:
b.execute()
except SolrException, e:
return {'errors': [unicode(e)]}
stats, facets, entries = b.get_stats(), b.get_facets(), b.get_entries()
_entries = []
for dataset, entry in entries:
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
_entries.append(entry)
if format == 'csv':
return write_csv(_entries, response, filename='entries.csv')
if expand_facets and len(datasets) == 1:
_expand_facets(facets, datasets[0])
return to_jsonp({
'stats': stats,
'facets': facets,
'results': _entries
})
class Api2Controller(BaseController):
def aggregate(self):
parser = AggregateParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return {'errors': errors}
params['cuts'] = params.pop('cut')
params['drilldowns'] = params.pop('drilldown')
dataset = params.pop('dataset')
format = params.pop('format')
require.dataset.read(dataset)
try:
cache = AggregationCache(dataset)
result = cache.aggregate(**params)
if 'drilldown' in result:
result['drilldown'] = drilldowns_apply_links(
dataset.name, result['drilldown'])
response.last_modified = dataset.updated_at
if cache.cache_enabled and 'cache_key' in result['summary']:
etag_cache(result['summary']['cache_key'])
except (KeyError, ValueError) as ve:
log.exception(ve)
response.status = 400
return {'errors': ['Invalid aggregation query: %r' % ve]}
if format == 'csv':
return write_csv(result['drilldown'],
response,
filename=dataset.name + '.csv')
return to_jsonp(result)
@jsonpify
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return {'errors': errors}
expand_facets = params.pop('expand_facet_dimensions')
datasets = params.pop('dataset', None)
if datasets is None or not len(datasets):
q = model.Dataset.all_by_account(c.account)
if params.get('category'):
q = q.filter_by(category=params.pop('category'))
datasets = q.all()
expand_facets = False
if not len(datasets):
return {'errors': [_("No dataset available.")]}
params['filter']['dataset'] = []
for dataset in datasets:
require.dataset.read(dataset)
params['filter']['dataset'].append(dataset.name)
response.last_modified = max([d.updated_at for d in datasets])
etag_cache_keygen(parser.key(), response.last_modified)
b = Browser(**params)
try:
stats, facets, entries = b.execute()
except SolrException, e:
return {'errors': [unicode(e)]}
_entries = []
for dataset, entry in entries:
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
_entries.append(entry)
if expand_facets and len(datasets) == 1:
_expand_facets(facets, datasets[0])
return {'stats': stats, 'facets': facets, 'results': _entries}
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return to_jsonp({'errors': errors})
expand_facets = params.pop('expand_facet_dimensions')
format = params.pop('format')
if format == 'csv':
params['stats'] = False
params['facet_field'] = None
datasets = params.pop('dataset', None)
if datasets is None or not datasets:
q = Dataset.all_by_account(c.account)
if params.get('category'):
q = q.filter_by(category=params.pop('category'))
datasets = q.all()
expand_facets = False
if not datasets:
return {'errors': ["No dataset available."]}
params['filter']['dataset'] = []
for dataset in datasets:
require.dataset.read(dataset)
params['filter']['dataset'].append(dataset.name)
response.last_modified = max([d.updated_at for d in datasets])
etag_cache_keygen(parser.key(), response.last_modified)
if params['pagesize'] > parser.defaults['pagesize']:
# http://wiki.nginx.org/X-accel#X-Accel-Buffering
response.headers['X-Accel-Buffering'] = 'no'
if format == 'csv':
csv_headers(response, 'entries.csv')
streamer = CSVStreamingResponse(
datasets, params, pagesize=parser.defaults['pagesize'])
return streamer.response()
else:
json_headers(filename='entries.json')
streamer = JSONStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize'],
expand_facets=util.expand_facets
if expand_facets else None,
callback=request.params.get('callback'))
return streamer.response()
solr_browser = Browser(**params)
try:
solr_browser.execute()
except SolrException as e:
return {'errors': [unicode(e)]}
entries = []
for dataset, entry in solr_browser.get_entries():
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
entries.append(entry)
if format == 'csv':
return write_csv(entries, response, filename='entries.csv')
if expand_facets and len(datasets) == 1:
facets = solr_browser.get_expanded_facets(datasets[0])
else:
facets = solr_browser.get_facets()
return to_jsonp({
'stats': solr_browser.get_stats(),
'facets': facets,
'results': entries
})
def view(self, dataset, id, format='html'):
"""
Get a specific entry in the dataset, identified by the id. Entry
can be return as html (default), json or csv.
"""
# Generate the dataset
self._get_dataset(dataset)
# Get the entry that matches the given id. c.dataset.entries is
# a generator so we create a list from it's responses based on the
# given constraint
entries = list(c.dataset.entries(c.dataset.alias.c.id == id))
# Since we're trying to get a single entry the list should only
# contain one entry, if not then we return an error
if not len(entries) == 1:
abort(404, _('Sorry, there is no entry %r') % id)
# Add urls to the dataset and assign assign it as a context variable
c.entry = entry_apply_links(dataset, entries.pop())
# Get and set some context variables from the entry
# This shouldn't really be necessary but it's here so nothing gets
# broken
c.id = c.entry.get('id')
c.from_ = c.entry.get('from')
c.to = c.entry.get('to')
c.currency = c.entry.get('currency', c.dataset.currency).upper()
c.time = c.entry.get('time')
# Get the amount for the entry
amount = c.entry.get('amount')
# We adjust for inflation if the user as asked for this to be inflated
if request.params.has_key('inflate'):
try:
# Inflate the amount. Target date is provided in request.params
# as value for inflate and reference date is the date of the
# entry. We also provide a list of the territories to extract
# a single country for which to do the inflation
c.inflation = h.inflate(amount, request.params['inflate'],
c.time, c.dataset.territories)
# The amount to show should be the inflated amount
# and overwrite the entry's amount as well
c.amount = c.inflation['inflated']
c.entry['amount'] = c.inflation['inflated']
# We include the inflation response in the entry's dict
# HTML description assumes every dict value for the entry
# includes a label so we include a default "Inflation
# adjustment" for it to work.
c.inflation['label'] = 'Inflation adjustment'
c.entry['inflation_adjustment'] = c.inflation
except:
# If anything goes wrong in the try clause (and there's a lot
# that can go wrong). We just say that we can't adjust for
# inflation and set the context amount as the original amount
h.flash_notice(_('Unable to adjust for inflation'))
c.amount = amount
else:
# If we haven't been asked to inflate then we just use the
# original amount
c.amount = amount
# Add custom html for the dataset entry if the dataset has some
# custom html
# 2013-11-17 disabled this as part of removal of genshi as depended on
# a genshi specific helper.
# TODO: reinstate if important
# c.custom_html = h.render_entry_custom_html(c.dataset, c.entry)
# Add the rest of the dimensions relating to this entry into a
# extras dictionary. We first need to exclude all dimensions that
# are already shown and then we can loop through the dimensions
excluded_keys = ('time', 'amount', 'currency', 'from',
'to', 'dataset', 'id', 'name', 'description')
c.extras = {}
if c.dataset:
# Create a dictionary of the dataset dimensions
c.desc = dict([(d.name, d) for d in c.dataset.dimensions])
# Loop through dimensions of the entry
for key in c.entry:
# Entry dimension must be a dataset dimension and not in
# the predefined excluded keys
if key in c.desc and \
not key in excluded_keys:
c.extras[key] = c.entry[key]
# Return entry based on
if format == 'json':
return to_jsonp(c.entry)
elif format == 'csv':
return write_csv([c.entry], response)
else:
return templating.render('entry/view.html')
def search(self):
parser = SearchParamParser(request.params)
params, errors = parser.parse()
if errors:
response.status = 400
return to_jsonp({'errors': errors})
expand_facets = params.pop('expand_facet_dimensions')
format = params.pop('format')
if format == 'csv':
params['stats'] = False
params['facet_field'] = None
datasets = params.pop('dataset', None)
if datasets is None or not datasets:
q = Dataset.all_by_account(c.account)
if params.get('category'):
q = q.filter_by(category=params.pop('category'))
datasets = q.all()
expand_facets = False
if not datasets:
return {'errors': ["No dataset available."]}
params['filter']['dataset'] = []
for dataset in datasets:
require.dataset.read(dataset)
params['filter']['dataset'].append(dataset.name)
response.last_modified = max([d.updated_at for d in datasets])
etag_cache_keygen(parser.key(), response.last_modified)
if params['pagesize'] > parser.defaults['pagesize']:
# http://wiki.nginx.org/X-accel#X-Accel-Buffering
response.headers['X-Accel-Buffering'] = 'no'
if format == 'csv':
csv_headers(response, 'entries.csv')
streamer = CSVStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize']
)
return streamer.response()
else:
json_headers(filename='entries.json')
streamer = JSONStreamingResponse(
datasets,
params,
pagesize=parser.defaults['pagesize'],
expand_facets=util.expand_facets
if expand_facets else None,
callback=request.params.get('callback')
)
return streamer.response()
solr_browser = Browser(**params)
try:
solr_browser.execute()
except SolrException as e:
return {'errors': [unicode(e)]}
entries = []
for dataset, entry in solr_browser.get_entries():
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
entries.append(entry)
if format == 'csv':
return write_csv(entries, response,
filename='entries.csv')
if expand_facets and len(datasets) == 1:
facets = solr_browser.get_expanded_facets(datasets[0])
else:
facets = solr_browser.get_facets()
return to_jsonp({
'stats': solr_browser.get_stats(),
'facets': facets,
'results': entries
})
def make_entries(self, entries):
for dataset, entry in entries:
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
yield entry
def view(self, dataset, id, format='html'):
"""
Get a specific entry in the dataset, identified by the id. Entry
can be return as html (default), json or csv.
"""
# Generate the dataset
self._get_dataset(dataset)
# Get the entry that matches the given id. c.dataset.entries is
# a generator so we create a list from it's responses based on the
# given constraint
entries = list(c.dataset.entries(c.dataset.alias.c.id == id))
# Since we're trying to get a single entry the list should only
# contain one entry, if not then we return an error
if not len(entries) == 1:
abort(404, _('Sorry, there is no entry %r') % id)
# Add urls to the dataset and assign assign it as a context variable
c.entry = entry_apply_links(dataset, entries.pop())
# Get and set some context variables from the entry
# This shouldn't really be necessary but it's here so nothing gets
# broken
c.id = c.entry.get('id')
c.from_ = c.entry.get('from')
c.to = c.entry.get('to')
c.currency = c.entry.get('currency', c.dataset.currency).upper()
c.time = c.entry.get('time')
# Get the amount for the entry
amount = c.entry.get('amount')
# We adjust for inflation if the user as asked for this to be inflated
if 'inflate' in request.params:
try:
# Inflate the amount. Target date is provided in request.params
# as value for inflate and reference date is the date of the
# entry. We also provide a list of the territories to extract
# a single country for which to do the inflation
c.inflation = h.inflate(amount, request.params['inflate'],
c.time, c.dataset.territories)
# The amount to show should be the inflated amount
# and overwrite the entry's amount as well
c.amount = c.inflation['inflated']
c.entry['amount'] = c.inflation['inflated']
# We include the inflation response in the entry's dict
# HTML description assumes every dict value for the entry
# includes a label so we include a default "Inflation
# adjustment" for it to work.
c.inflation['label'] = 'Inflation adjustment'
c.entry['inflation_adjustment'] = c.inflation
except:
# If anything goes wrong in the try clause (and there's a lot
# that can go wrong). We just say that we can't adjust for
# inflation and set the context amount as the original amount
h.flash_notice(_('Unable to adjust for inflation'))
c.amount = amount
else:
# If we haven't been asked to inflate then we just use the
# original amount
c.amount = amount
# Add custom html for the dataset entry if the dataset has some
# custom html
# 2013-11-17 disabled this as part of removal of genshi as depended on
# a genshi specific helper.
# TODO: reinstate if important
# c.custom_html = h.render_entry_custom_html(c.dataset, c.entry)
# Add the rest of the dimensions relating to this entry into a
# extras dictionary. We first need to exclude all dimensions that
# are already shown and then we can loop through the dimensions
excluded_keys = ('time', 'amount', 'currency', 'from', 'to', 'dataset',
'id', 'name', 'description')
c.extras = {}
if c.dataset:
# Create a dictionary of the dataset dimensions
c.desc = dict([(d.name, d) for d in c.dataset.dimensions])
# Loop through dimensions of the entry
for key in c.entry:
# Entry dimension must be a dataset dimension and not in
# the predefined excluded keys
if key in c.desc and \
key not in excluded_keys:
c.extras[key] = c.entry[key]
# Return entry based on
if format == 'json':
return to_jsonp(c.entry)
elif format == 'csv':
return write_csv([c.entry], response)
else:
return templating.render('entry/view.html')
def make_entries(self, entries):
for dataset, entry in entries:
entry = entry_apply_links(dataset.name, entry)
entry['dataset'] = dataset_apply_links(dataset.as_dict())
yield entry
