def before_view(self, pkg_dict):
"""
Extend the group controller to show resource information
The resource information will come from elastic search
"""
# use r as query string
r = c.r = request.params.get(
'r', default=None) # unicode format (decoded from utf8)
# format r to send to elastic search
if r is None:
query = {
"sort": {
"data": {
"order": "desc"
},
},
"query": {
"match_all": {}
}
}
else:
query = {
"sort": {
"data": {
"order": "desc"
},
},
"query": {r}
}
# Now send query to elastic search
self._load_elastic_config()
client = DataStoreClient(urlparse.urljoin(self.url, pkg_dict['name']))
headers = dict()
headers['Authorization'] = self.user.get('apikey')
#req = urllib2.Request(webstore_request_url, post_data, headers)
client._headers = headers
response = client.query(query)
# Now we have to parse the result back to package dict
hits = response.get('hits')
resources = list()
for res in hits['hits']:
# Store it in extras
resources.append(res['_source'])
# Add a new field on pkg_dict
pkg_dict['elastic_resources'] = resources
return pkg_dict
def before_view(self,pkg_dict):
"""
Extend the group controller to show resource information
The resource information will come from elastic search
"""
# use r as query string
r = c.r = request.params.get('r', default=None) # unicode format (decoded from utf8)
# format r to send to elastic search
if r is None:
query = {
"sort": {
"data" : {"order" : "desc"},
},
"query": {
"match_all":{}
}
}
else:
query = {
"sort": {
"data" : {"order" : "desc"},
},
"query":{r}
}
# Now send query to elastic search
self._load_elastic_config()
client = DataStoreClient(urlparse.urljoin(self.url, pkg_dict['name']))
headers = dict()
headers['Authorization'] = self.user.get('apikey')
#req = urllib2.Request(webstore_request_url, post_data, headers)
client._headers = headers
response = client.query(query)
# Now we have to parse the result back to package dict
hits = response.get('hits')
resources = list()
for res in hits['hits']:
# Store it in extras
resources.append(res['_source'])
# Add a new field on pkg_dict
pkg_dict['elastic_resources'] = resources
return pkg_dict
def datatable(self, resource_name, resource_id):
from ckanext.recombinant.tables import get_chromo
t = get_chromo(resource_name)
echo = int(request.params['sEcho'])
search_text = unicode(request.params['sSearch'])
offset = int(request.params['iDisplayStart'])
limit = int(request.params['iDisplayLength'])
sort_cols = int(request.params['iSortingCols'])
if sort_cols:
sort_by_num = int(request.params['iSortCol_0'])
sort_order = 'desc' if request.params['sSortDir_0'] == 'desc' else 'asc'
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['datastore_id'] for f in t['fields']]
sort_str = ''
if sort_cols:
sort_str = cols[sort_by_num] + ' ' + sort_order
response = lc.action.datastore_search(
q=search_text,
resource_id=resource_id,
fields=cols,
offset=offset,
limit=limit,
sort=sort_str)
return json.dumps({
'sEcho': echo,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': [
[row[colname] for colname in cols]
for row in response['records']],
})
def before_view_resource(self,pkg_name,resource_id):
"""
Extend the group controller to show resource information
The resource information will come from elastic search
"""
query = {
"query": {
"query_string": {
"default_field": "id",
"query": resource_id
}
},
"size":"1"
}
# Now send query to elastic search
self._load_elastic_config()
self.url = urlparse.urljoin(self.url, pkg_name)
client = DataStoreClient(self.url)
headers = dict()
headers['Authorization'] = self.user.get('apikey')
#req = urllib2.Request(webstore_request_url, post_data, headers)
client._headers = headers
response = client.query(query)
# Now we have to parse the result back to package dict
hits = response.get('hits')
resources = dict()
for res in hits['hits']:
# Store it in extras
resources = res['_source']
# Return only first item
break
# Check not found
if not resources:
abort(404, _('Resource not found'))
else:
return resources
def before_view_resource(self, pkg_name, resource_id):
"""
Extend the group controller to show resource information
The resource information will come from elastic search
"""
query = {
"query": {
"query_string": {
"default_field": "id",
"query": resource_id
}
},
"size": "1"
}
# Now send query to elastic search
self._load_elastic_config()
self.url = urlparse.urljoin(self.url, pkg_name)
client = DataStoreClient(self.url)
headers = dict()
headers['Authorization'] = self.user.get('apikey')
#req = urllib2.Request(webstore_request_url, post_data, headers)
client._headers = headers
response = client.query(query)
# Now we have to parse the result back to package dict
hits = response.get('hits')
resources = dict()
for res in hits['hits']:
# Store it in extras
resources = res['_source']
# Return only first item
break
# Check not found
if not resources:
abort(404, _('Resource not found'))
else:
return resources
def datatable(self, resource_id):
draw = int(request.params['draw'])
search_text = unicode(request.params['search[value]'])
offset = int(request.params['start'])
limit = int(request.params['length'])
sort_by_num = int(request.params['order[0][column]'])
sort_order = ('desc' if request.params['order[0][dir]'] == 'desc'
else 'asc'
)
lc = LocalCKAN(username=c.user)
unfiltered_response = lc.action.datastore_search(
resource_id=resource_id,
limit=1,
)
cols = [f['id'] for f in unfiltered_response['fields']][1:]
sort_str = cols[sort_by_num] + ' ' + sort_order
response = lc.action.datastore_search(
q=search_text,
resource_id=resource_id,
offset=offset,
limit=limit,
sort=sort_str
)
return json.dumps({
'draw': draw,
'iTotalRecords': unfiltered_response.get('total', 0),
'iTotalDisplayRecords': response.get('total', 0),
'aaData': [
[row[colname] for colname in cols]
for row in response['records']
],
})
def before_view(self,pkg_dict):
"""
Extend the group controller to show resource information
The resource information will come from elastic search
"""
# use r as query string
q = c.q = request.params.get('q', default=None) # unicode format (decoded from utf8)
page = c.page = request.params.get('page', default=None)
# TODO: put this as a parameter
rows = 20
if page is None:
start = 0
else:
# Start with the first element in this page
start = ((int(page) * rows) - rows)
# format q to send to elastic search
if q is None or q is '*:*':
query = {
"sort": {
"data" : {"order" : "desc"},
},
"query": {
"match_all":{}
},
"size" : rows,
"from" : start
}
else:
query = {
"sort": {
"data" : {"order" : "desc"},
},
"query": {
"query_string": {
"query": q,
"default_operator": "AND"
}
},
"size" : rows,
"from" : start
}
# Now send query to elastic search
self._load_elastic_config()
client = DataStoreClient(urlparse.urljoin(self.url, pkg_dict['name']))
headers = dict()
headers['Authorization'] = self.user.get('apikey')
#req = urllib2.Request(webstore_request_url, post_data, headers)
client._headers = headers
# do not fail on search errors
try:
response = client.query(query)
except:
# there's an error in search params
import traceback
response = dict()
errmsg = 'Error searching query string \n %s \n Message\n%s' % (query,traceback.format_exc())
log.error(errmsg)
raise SearchError(errmsg)
# Now we have to parse the result back to package dict
hits = response.get('hits')
resources = list()
if hits is not None:
for res in hits['hits']:
# Store it in extras
resources.append(res['_source'])
# Add a new field on pkg_dict
pkg_dict['elastic_resources'] = resources
pkg_dict['elastic_hits'] = hits.get('total')
else:
# Add a new field on pkg_dict
pkg_dict['elastic_resources'] = dict()
pkg_dict['elastic_hits'] = 0
return pkg_dict
def before_view(self, pkg_dict):
"""
Extend the group controller to show resource information
The resource information will come from elastic search
"""
# use r as query string
q = c.q = request.params.get(
'q', default=None) # unicode format (decoded from utf8)
page = c.page = request.params.get('page', default=None)
# TODO: put this as a parameter
rows = 20
if page is None:
start = 0
else:
# Start with the first element in this page
start = ((int(page) * rows) - rows)
# format q to send to elastic search
if q is None or q is '*:*':
query = {
"sort": {
"data": {
"order": "desc"
},
},
"query": {
"match_all": {}
},
"size": rows,
"from": start
}
else:
query = {
"sort": {
"data": {
"order": "desc"
},
},
"query": {
"query_string": {
"query": q,
"default_operator": "AND"
}
},
"size": rows,
"from": start
}
# Now send query to elastic search
self._load_elastic_config()
client = DataStoreClient(urlparse.urljoin(self.url, pkg_dict['name']))
headers = dict()
headers['Authorization'] = self.user.get('apikey')
#req = urllib2.Request(webstore_request_url, post_data, headers)
client._headers = headers
# do not fail on search errors
try:
response = client.query(query)
except:
# there's an error in search params
import traceback
response = dict()
errmsg = 'Error searching query string \n %s \n Message\n%s' % (
query, traceback.format_exc())
log.error(errmsg)
raise SearchError(errmsg)
# Now we have to parse the result back to package dict
hits = response.get('hits')
resources = list()
if hits is not None:
for res in hits['hits']:
# Store it in extras
resources.append(res['_source'])
# Add a new field on pkg_dict
pkg_dict['elastic_resources'] = resources
pkg_dict['elastic_hits'] = hits.get('total')
else:
# Add a new field on pkg_dict
pkg_dict['elastic_resources'] = dict()
pkg_dict['elastic_hits'] = 0
return pkg_dict
class lightbaseReportsController(PackageController):
"""
Add controller to Reports
"""
def _load_elastic_config(self):
"""
Load elastic search configuration
"""
self.user = get_action('get_site_user')({
'model': model,
'ignore_auth': True
}, {})
ckan_url = config.get('ckan.site_url').rstrip('/')
self.url = '%s/api/data/' % (ckan_url)
# FIXME: Parametrize this
self.url = urlparse.urljoin(self.url, 'lbdf/')
def report_action(self, id, data=None, errors=None, error_summary=None):
"""
Reports action page
"""
# Setup dataset information
package_type = self._get_package_type(id.split('@')[0])
#context = {'model': model, 'session': model.Session,
# 'user': c.user or c.author, 'extras_as_string': True,
# 'schema': self._form_to_db_schema(package_type=package_type)}
context = {
'model': model,
'session': model.Session,
'save': 'save' in request.params,
'user': c.user or c.author,
'extras_as_string': True
}
data_dict = {'id': id}
#check if package exists
try:
c.pkg_dict = get_action('package_update')(context, data_dict)
c.pkg = context['package']
c.pkg_json = json.dumps(c.pkg_dict)
except NotFound:
abort(404, _('Dataset not found'))
except NotAuthorized:
abort(401, _('Unauthorized to edit package %s') % id)
data = data or clean_dict(
unflatten(
tuplize_dict(
parse_params(request.params, ignore_keys=[CACHE_PARAMETER
]))))
errors = errors or {}
error_summary = error_summary or {}
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
# Save data if requested
if context.get('save') and data:
result = self._save_report_action_new(data)
# Return error message
if not result:
errors['name'] = 'Insertion error'
error_summary[_(u'Ação')] = _(
u'Erro inserindo elemento na base de dados')
# Add actual reports actions information
data['actions'] = model.Session.query(lightbaseDatasetActions).all()
self._setup_template_variables(context, {'id': id})
c.form = render('package/reports/actions_new_form.html',
extra_vars=vars)
return render('package/reports/actions_new.html')
def _save_report_action_new(self, data):
"""
Save report action data
"""
# Save everything or fail everything
try:
# First save clean element
action_elm = data.pop('name', None)
if action_elm:
data_elm = lightbaseDatasetActions(action_name=action_elm)
model.Session.add(data_elm)
log.warning('Adding element %s' % data_elm)
# Get elements to be updated
for key in data.keys():
# find items to update
field, separator, identifier = key.partition(':')
if field == 'name' and separator == ':':
data_elm = model.Session.query(lightbaseDatasetActions).\
filter_by(dataset_action_id=identifier).all()[0]
# Assing new value to this object
if data_elm:
data_elm.action_name = data.get(key)
# Now find elements to be deleted
if isinstance(data.get('deleted'), (list, tuple)):
# Use it to iterate throught the objects
for elm in data.get('deleted'):
data_elm = model.Session.query(lightbaseDatasetActions).\
filter_by(dataset_action_id=elm).all()
report_elm = model.Session.query(lightbaseDatasetReports).\
filter_by(dataset_action_id=elm).all()
# Remove it from reports first
for report in report_elm:
log.warning('Removing report %s' % report)
model.Session.delete(report)
model.Session.commit()
if len(data_elm) > 0:
data_elm = data_elm[0]
log.warning('Removing element %s' % elm)
# As it's not in the list anymore, remove it
model.Session.delete(data_elm)
elif data.get('deleted'):
elm = data.get('deleted')
data_elm = model.Session.query(lightbaseDatasetActions).\
filter_by(dataset_action_id=elm).all()
report_elm = model.Session.query(lightbaseDatasetReports).\
filter_by(dataset_action_id=elm).all()
# Remove it from reports first
for report in report_elm:
log.warning('Removing report %s' % report)
model.Session.delete(report)
model.Session.commit()
if len(data_elm) > 0:
data_elm = data_elm[0]
log.warning('Removing element %s' % elm)
# As it's not in the list anymore, remove it
model.Session.delete(data_elm)
# Commit changes
model.Session.commit()
return True
except:
import traceback
# For now just fail in any error
log.error('Error adding value.\n%s' % traceback.format_exc())
return False
def report_new(self, id, data=None, errors=None, error_summary=None):
"""
Controller to add a new report
"""
# Setup dataset information
package_type = self._get_package_type(id.split('@')[0])
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,
'schema': self._form_to_db_schema(package_type=package_type),
'save': 'save' in request.params
}
data_dict = {'id': id}
#check if package exists
try:
c.pkg_dict = get_action('package_show')(context, data_dict)
c.pkg = context['package']
c.pkg_json = json.dumps(c.pkg_dict)
except NotFound:
abort(404, _('Dataset not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % id)
# Only package admins can see this page
try:
check_access('package_update', context)
except NotAuthorized, e:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
# Load Elastic Search parameters and configuration
self._load_elastic_config()
self.url = self.url + c.pkg_dict.get('name')
client = DataStoreClient(self.url)
headers = dict()
headers['Authorization'] = self.user.get('apikey')
client._headers = headers
# Now return mapping info
try:
response = client.mapping()
except:
# Error retrieving information from elastic search
abort(404, _('Index not found on Elastic Search'))
# In this list every resource has an specific set of items
pkg_resources = response.get(c.pkg_dict.get('name'))
if pkg_resources is None:
# Dataset not found on elastic search. Abort
abort(404, _('Dataset not found'))
key_dict = dict()
# We loop throught every key on every resource
for key in pkg_resources.get('properties').keys():
if key not in key_dict:
# Add i18n translation of field to the dict
key_aux = {'i18n': _(key)}
key_dict[key] = key_aux
# Get mapping from reports
session = model.Session()
results = session.query(lightbaseDatasetReports,lightbaseDatasetActions.action_name).\
filter(lightbaseDatasetReports.dataset_action_id == lightbaseDatasetActions.dataset_action_id).\
filter(lightbaseDatasetReports.dataset_id == c.pkg_dict.get('id')).all()
# This dict will contain all reports and mapped fields
report_dict = dict()
for report in results:
# Store the action and fields necessary for this dataset
if report[0].source:
source = report[0].source
else:
source = None
report_dict[report[0].dataset_id] = {
report[1]: {
'fields_list': report[0].fields_list,
'source': source
}
}
# Map actions options
report_actions = session.query(lightbaseDatasetActions).all()
# This is required to save data
data = data or clean_dict(
unflatten(
tuplize_dict(
parse_params(request.params, ignore_keys=[CACHE_PARAMETER
]))))
vars = {
'data': data,
'errors': errors,
'error_summary': error_summary,
'report': report_dict,
'keys': key_dict,
'report_actions': report_actions
}
# Save data if requested
if context.get('save') and data:
result = self._save_report_new(data, key_dict)
# Return error message
if result:
# Redirect
h.redirect_to(
controller=
'ckanext.lightbase.controllers.reports:lightbaseReportsController',
action='report_read',
id=id,
report_id=self.report_id)
else:
# Show error message
errors['name'] = 'Insertion error'
error_summary[_(u'Ação')] = _(
u'Erro inserindo elemento na base de dados')
errors = errors or {}
error_summary = error_summary or {}
c.form = render('package/reports/new_form.html', extra_vars=vars)
return render('package/reports/new.html')