#all_periods_tuples = [period.split('-') for period in all_periods
# if '-' in period]
#all_periods_tuples.sort(key=lambda x: x[0])
#all_periods_by_year = [
# (year, [p for y, p in year_periods])
# for year, year_periods in groupby(all_periods_tuples, lambda x: x[0])]
return {
'table': rows,
'all periods': all_periods,
#'all periods by year': all_periods_by_year
}
def publisher_report_option_combinations():
return ({
'metric': metric
} for metric in ('views', 'visits', 'downloads', 'viewsdownloads'))
publisher_report_info = {
'name': 'site-usage-publisher',
'title': 'Site usage by publisher',
'description':
'Usage statistics, by publisher for each month. Data is from Google Analytics.',
'option_defaults': OrderedDict([('metric', 'views')]),
'option_combinations': publisher_report_option_combinations,
'generate': publisher_report,
'template': 'report/publisher.html',
}
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {
'model': model,
'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True,
'extras_as_string': True
}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if not param in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
include_private = False
user_member_of_orgs = [
org['id'] for org in h.organizations_available('read')
]
if (c.group and c.group.id in user_member_of_orgs):
include_private = True
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
if 'capacity' in facets and (group_type != 'organization'
or not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': include_private,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict(
(k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.group_dict['package_count'] = query['count']
c.facets = query['facets']
maintain.deprecate_context_item('facets',
'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(
request.params.get('_%s_limit' % facet,
g.facets_default_number))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
class GroupController(base.BaseController):
group_type = 'group'
## hooks for subclasses
def _group_form(self, group_type=None):
return lookup_group_plugin(group_type).group_form()
def _form_to_db_schema(self, group_type=None):
return lookup_group_plugin(group_type).form_to_db_schema()
def _db_to_form_schema(self, group_type=None):
'''This is an interface to manipulate data from the database
into a format suitable for the form (optional)'''
return lookup_group_plugin(group_type).db_to_form_schema()
def _setup_template_variables(self, context, data_dict, group_type=None):
return lookup_group_plugin(group_type).\
setup_template_variables(context, data_dict)
def _new_template(self, group_type):
return lookup_group_plugin(group_type).new_template()
def _index_template(self, group_type):
return lookup_group_plugin(group_type).index_template()
def _about_template(self, group_type):
return lookup_group_plugin(group_type).about_template()
def _read_template(self, group_type):
return lookup_group_plugin(group_type).read_template()
def _history_template(self, group_type):
return lookup_group_plugin(group_type).history_template()
def _edit_template(self, group_type):
return lookup_group_plugin(group_type).edit_template()
def _activity_template(self, group_type):
return lookup_group_plugin(group_type).activity_template()
def _admins_template(self, group_type):
return lookup_group_plugin(group_type).admins_template()
def _bulk_process_template(self, group_type):
return lookup_group_plugin(group_type).bulk_process_template()
## end hooks
def _replace_group_org(self, string):
''' substitute organization for group if this is an org'''
if self.group_type == 'organization':
string = re.sub('^group', 'organization', string)
return string
def _action(self, action_name):
''' select the correct group/org action '''
return get_action(self._replace_group_org(action_name))
def _check_access(self, action_name, *args, **kw):
''' select the correct group/org check_access '''
return check_access(self._replace_group_org(action_name), *args, **kw)
def _render_template(self, template_name):
''' render the correct group/org template '''
return render(self._replace_group_org(template_name))
def _redirect_to(self, *args, **kw):
''' wrapper to ensue the correct controller is used '''
if self.group_type == 'organization' and 'controller' in kw:
kw['controller'] = 'organization'
return h.redirect_to(*args, **kw)
def _url_for(self, *args, **kw):
''' wrapper to ensue the correct controller is used '''
if self.group_type == 'organization' and 'controller' in kw:
kw['controller'] = 'organization'
return h.url_for(*args, **kw)
def _guess_group_type(self, expecting_name=False):
"""
Guess the type of group from the URL handling the case
where there is a prefix on the URL (such as /data/organization)
"""
parts = [x for x in request.path.split('/') if x]
idx = -1
if expecting_name:
idx = -2
gt = parts[idx]
if gt == 'group':
gt = None
return gt
def index(self):
group_type = self._guess_group_type()
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'with_private': False
}
q = c.q = request.params.get('q', '')
data_dict = {'all_fields': True, 'q': q}
sort_by = c.sort_by_selected = request.params.get('sort')
if sort_by:
data_dict['sort'] = sort_by
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = self._action('group_list')(context, data_dict)
c.amount_group_diplayed = amount_group_displayed
c.groups = results[:amount_group_displayed]
c.hasmoregroups = len(results) > amount_group_displayed
#c.page = h.Page(
# collection=results,
# page=request.params.get('page', 1),
# url=h.pager_url,
# items_per_page=21
#)
return render(self._index_template(group_type))
def read(self, id, limit=20):
if request.GET.get('ext_boolean') in ['all', 'any', 'exact']:
base.session['ext_boolean'] = request.GET['ext_boolean']
base.session.save()
group_type = self._get_group_type(id.split('@')[0])
if (group_type != self.group_type) and (group_type !=
"eurovoc_domain"):
abort(404, _('Incorrect group type'))
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True
}
# Do not query for the group datasets when dictizing, as they will
# be ignored and get requested on the controller anyway
data_dict = {'id': id, 'include_datasets': False}
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', '')
try:
c.group_dict = self._action('group_show')(context, data_dict)
c.group = context['group']
except NotFound:
abort(404, _('Group not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read group %s') % id)
self._read(id, limit)
return render(self._read_template(c.group_dict['type']))
def _read(self, id, limit):
''' This is common code used by both read and bulk_process'''
group_type = self._get_group_type(id.split('@')[0])
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True,
'extras_as_string': True
}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = h.render_markdown(
c.group_dict.get('description'))
context['return_query'] = True
# c.group_admins is used by CKAN's legacy (Genshi) templates only,
# if we drop support for those then we can delete this line.
c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id)
try:
page = int(request.params.get('page', 1))
except ValueError, e:
abort(400, ('"page" parameter must be an integer'))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
new_params_nopage = []
for key, value in params_nopage:
if key == 'eurovoc_domains':
new_params_nopage.append(('groups', value))
else:
new_params_nopage.append((key, value))
params_nopage = new_params_nopage
#sort_by = request.params.get('sort', 'name asc')
sort_by = request.params.get('sort', None)
def search_url(params):
if group_type == 'organization':
if c.action == 'bulk_process':
url = self._url_for(controller='organization',
action='bulk_process',
id=id)
else:
url = self._url_for(controller='organization',
action='read',
id=id)
else:
url = self._url_for(controller='group', action='read', id=id)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
if c.group_dict.get('is_organization'):
return h.remove_url_param(
key,
value=value,
replace=replace,
controller='organization',
action='read',
extras=dict(id=c.group_dict.get('id')))
else:
return h.remove_url_param(
key,
value=value,
replace=replace,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if not param in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
param = 'groups' if (param
== 'eurovoc_domains') else param
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
fq = 'capacity:"public"'
user_member_of_orgs = [
org['id'] for org in h.organizations_available('read')
]
if (c.group and c.group.id in user_member_of_orgs):
fq = ''
context['ignore_capacity_check'] = True
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in plugins.PluginImplementations(plugins.IFacets):
if self.group_type == 'organization':
facets = plugin.organization_facets(
facets, self.group_type, None)
else:
facets = plugin.group_facets(facets, self.group_type, None)
if 'capacity' in facets and (self.group_type != 'organization'
or not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict(
(k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.search_url_params = urlencode(_encode_params(params_nopage))
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.group_dict['package_count'] = query['count']
c.facets = query['facets']
maintain.deprecate_context_item('facets',
'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(
request.params.get('_%s_limit' % facet,
g.facets_default_number))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
c.search_url_params = ''
class DateType(object):
'''Utils for handling dates in forms.
* Full or partial dates
* User inputs in form DD/MM/YYYY and it is stored in db as YYYY-MM-DD.
'''
format_types = ('form', 'db')
datetime_fields = OrderedDict([
('year', (1000, 2100, 4, 'YYYY')),
('month', (1, 12, 2, 'MM')),
('day', (1, 31, 2, 'DD')),
('hour', (0, 23, 2, 'HH')),
('minute', (0, 59, 2, 'MM')),
])
datetime_fields_indexes = {
'min': 0,
'max': 1,
'digits': 2,
'format_code': 3
}
date_fields_order = {
'db': ('year', 'month', 'day'),
'form': ('day', 'month', 'year')
}
parsing_separators = {'date': '-/', 'time': ':\.'}
default_separators = {
'db': {
'date': '-',
'time': ':'
},
'form': {
'date': '/',
'time': ':'
},
}
field_code_map = {
'year': 'YYYY',
'month': 'MM',
'day': 'DD',
'hour': 'HH',
'minute': 'MM'
}
word_match = re.compile('[A-Za-z]+')
timezone_match = re.compile('(\s[A-Z]{3})|(\s[+-]\d\d:?\d\d)')
months_abbreviated = [month[:3] for month in months]
@classmethod
def parse_timedate(cls, timedate_str, format_type):
'''Takes a timedate and returns a dictionary of the fields.
* Little validation is done.
* If it can\'t understand the layout it raises DateConvertError
'''
assert format_type in cls.format_types
if not hasattr(cls, 'matchers'):
# build up a list of re matches for the different
# acceptable ways of expressing the time and date
cls.matchers = {}
cls.readable_formats = {}
for format_type_ in cls.format_types:
finished_regexps = []
readable_formats = [] # analogous to the regexps,
# but human readable
year_re = '(?P<%s>\d{2,4})'
month_re = '(?P<%s>\w+)'
two_digit_decimal_re = '(?P<%s>\d{1,2})'
time_re = '%s[%s]%s' % (two_digit_decimal_re % 'hour',
cls.parsing_separators['time'],
two_digit_decimal_re % 'minute')
time_readable = '%s%s%s' % (
cls.datetime_fields['hour'][
cls.datetime_fields_indexes['format_code']],
cls.default_separators[format_type_]['time'],
cls.datetime_fields['minute'][
cls.datetime_fields_indexes['format_code']])
date_field_re = {
'year': year_re % 'year',
'month': month_re % 'month',
'day': two_digit_decimal_re % 'day'
}
date_fields = list(cls.date_fields_order[format_type_])
for how_specific in ('day', 'month', 'year'):
date_sep_re = '[%s]' % cls.parsing_separators['date']
date_sep_readable = cls.default_separators[format_type_][
'date']
date_field_regexps = [
date_field_re[field] for field in date_fields
]
date_field_readable = [
cls.datetime_fields[field][
cls.datetime_fields_indexes['format_code']]
for field in date_fields
]
date_re = date_sep_re.join(date_field_regexps)
date_readable = date_sep_readable.join(date_field_readable)
finished_regexps.append(date_re)
readable_formats.append(date_readable)
date_fields.remove(how_specific)
full_date_re = finished_regexps[0]
full_date_readable = readable_formats[0]
# Allow time to be before or after the date
for format_ in ('%(time_re)s%(sep)s%(full_date_re)s',
'%(full_date_re)s%(sep)s%(time_re)s'):
finished_regexps.insert(
0, format_ % {
'time_re': time_re,
'sep': '\s',
'full_date_re': full_date_re
})
readable_formats.insert(
0, format_ % {
'time_re': time_readable,
'sep': ' ',
'full_date_re': full_date_readable
})
cls.matchers[format_type_] = [
re.compile('^%s$' % regexp) for regexp in finished_regexps
]
cls.readable_formats[format_type_] = readable_formats
#print format_type_, finished_regexps, readable_formats
for index, matcher in enumerate(cls.matchers[format_type]):
match = matcher.match(timedate_str)
if match:
timedate_dict = match.groupdict()
timedate_dict = cls.int_timedate(timedate_dict)
timedate_dict['readable_format'] = cls.readable_formats[
format_type][index]
return timedate_dict
else:
acceptable_formats = ', '.join([
"'%s'" % format_
for format_ in cls.readable_formats[format_type]
])
raise DateConvertError(
"Cannot parse %s date '%s'. Acceptable formats: %s" %
(format_type, timedate_str, acceptable_formats))
@classmethod
def int_timedate(cls, timedate_dict):
# Convert timedate string values to integers
int_timedate_dict = timedate_dict.copy()
for field in cls.datetime_fields.keys():
if timedate_dict.has_key(field):
val = timedate_dict[field]
if field == 'year':
if len(val) == 2:
# Deal with 2 digit dates
try:
int_val = int(val)
except ValueError:
raise DateConvertError(
'Expecting integer for %s value: %s' %
(field, val))
val = cls.add_centurys_to_two_digit_year(int_val)
elif len(val) == 3:
raise DateConvertError(
'Expecting 2 or 4 digit year: "%s"' % (val))
if field == 'month':
# Deal with months expressed as words
if val in months:
val = months.index(val) + 1
if val in cls.months_abbreviated:
val = cls.months_abbreviated.index(val) + 1
try:
int_timedate_dict[field] = int(val)
except ValueError:
raise DateConvertError(
'Expecting integer for %s value: %s' % (field, val))
return int_timedate_dict
@classmethod
def iso_to_db(cls, iso_date, format):
# e.g. 'Wed, 06 Jan 2010 09:30:00'
# '%a, %d %b %Y %H:%M:%S'
assert isinstance(iso_date, (unicode, str))
try:
date_tuple = time.strptime(iso_date, format)
except ValueError, e:
raise DateConvertError(
'Could not read date as ISO format "%s". Date provided: "%s"' %
(format, iso_date))
date_obj = datetime.datetime(*date_tuple[:4])
date_str = cls.date_to_db(date_obj)
return date_str
def dataset_facets(self, facets_dict, package_type):
'''Only show tags for Showcase search list.'''
if package_type != DATASET_TYPE_NAME:
return facets_dict
return OrderedDict({'tags': _('Tags')})
def _add_dataset_search(self, showcase_id, showcase_name):
'''
Search logic for discovering datasets to add to a showcase.
'''
from ckan.lib.search import SearchError
package_type = 'dataset'
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
try:
page = self._get_page_number(request.params)
except AttributeError:
# in CKAN >= 2.5 _get_page_number has been moved
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return self._search_url(params, showcase_name)
c.sort_by = _sort_by
if sort_by is None:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return self._search_url(params, showcase_name)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not tk.asbool(
config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
# Only search for packages that aren't already associated with the
# Showcase
associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(
showcase_id)
# flatten resulting list to space separated string
if associated_package_ids:
associated_package_ids_str = \
' OR '.join([id[0] for id in associated_package_ids])
fq += ' !id:({0})'.format(associated_package_ids_str)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
# for CKAN-Versions that do not provide the facets-method from
# helper-context, import facets from ckan.common
if hasattr(h, 'facets'):
current_facets = h.facets()
else:
from ckan.common import g
current_facets = g.facets
for facet in current_facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
from ckan.common import OrderedDict
from pylons import config
from ckan.plugins import toolkit
import ckanapi
groups = OrderedDict([
('health', 'Health'),
('education', 'Education'),
('finance', 'Finance'),
('transport', 'Transport'),
('environment', 'Environment & agriculture'),
('property', 'Property & land'),
('economy', 'Economy, industry & employment'),
('tourism', 'Tourism, leisure, culture & arts'),
('population', 'Population & society'),
])
class CreateFeaturedGroups(toolkit.CkanCommand):
'''Create the nine featured groups
Usage:
create_featured_groups - create featured groups
'''
summary = __doc__.split('\n')[0]
usage = __doc__
max_args = 0
min_args = 0
def view(self):
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
# Obtenemos parametros de configuracion
site_url = config.get('ckan.site_url') + config.get(
'ckan.root_path').replace('{{LANG}}', '')
# Inicializamos variables
now = d.datetime.now()
year_from = 1989
year_to = now.year
# Comprobamos si es un usuario identificado o no
logged_in = False
if 'user' in context and context['user']:
logged_in = True
# Si se pide el catalogo publico ignoramos el usuario identificado
if logged_in and 'public' in request.params and request.params.get(
'public') == 'true':
logged_in = False
if not logged_in:
# Obtenemos el catalogo para usuarios no identificados
packages = t.get_action('package_search')(context, {
'include_private': False,
'rows': 1000,
'sort': 'name asc'
})
packages = packages['results']
elif c.userobj.sysadmin:
# Obtenemos el catalogo para sysadmin
packages = t.get_action('package_search')(context, {
'include_private': True,
'rows': 1000,
'sort': 'name asc'
})
packages = packages['results']
else:
# Obtenemos el catalogo para usuarios identificados: solo los de sus organizaciones
user_org = t.get_action('organization_list_for_user')(
context, {
'permission': 'create_dataset'
})
org_facets = ''
for org in user_org:
if org_facets != '':
org_facets = org_facets + ' OR '
org_facets = org_facets + org['name']
packages = t.get_action('package_search')(
context, {
'fq': 'organization:(' + org_facets + ')',
'include_private': True,
'rows': 1000,
'sort': 'name asc'
})
packages = packages['results']
# obtenemos los formatos
formats = t.get_action('format_autocomplete')(context, {
'q': '',
'limit': 50
})
# puede devolver formatos duplicados, lo convertimos a un set que eliminara los elementos
# duplicados y de nuevo a una lista
formats = list(set(formats))
for format in formats:
format_strip = format.strip()
if not format_strip:
formats.remove(format)
# Realizamos conexion a la BBDD de Drupal para obtener el numero de comentarios de cada dataset y almacenamos los valores en un array
dbc = parse_db_config('ckan.drupal.url')
ckan_conn_string = "host='%s' port='%s' dbname='%s' user='******' password='******'" % (
dbc['db_host'], dbc['db_port'], dbc['db_name'], dbc['db_user'],
dbc['db_pass'])
ckan_conn = psycopg2.connect(ckan_conn_string)
ckan_cursor = ckan_conn.cursor()
ckan_cursor.execute(
"""SELECT OP.pkg_name, COUNT(*) FROM opendata_package OP INNER JOIN node N ON N.tnid = OP.pkg_node_id INNER JOIN comment C ON C.nid = N.nid WHERE N.tnid != 0 GROUP BY OP.pkg_name;"""
)
comments = {}
for row in ckan_cursor:
comments.update({row[0]: row[1]})
ckan_cursor.close()
ckan_conn.close()
sql_downloads = '''select sum(count) AS downloads, sum(count_absolute) AS downloads_absolute, t.tracking_type, p.name from tracking_summary t
inner join resource r ON r.id = t.resource_id
inner join package p ON p.id = r.package_id
GROUP BY p.name, t.tracking_type;'''
results_downloads = model.Session.execute(sql_downloads)
downloads = {}
downloads_absolute = {}
api_access_number = {}
api_access_number_absolute = {}
for row in results_downloads:
if row.tracking_type == 'resource':
downloads.update({row.name: row.downloads})
downloads_absolute.update({row.name: row.downloads_absolute})
else:
api_access_number.update({row.name: row.downloads})
api_access_number_absolute.update(
{row.name: row.downloads_absolute})
sql_views = '''SELECT t.tracking_date, t.running_total, t.recent_views, t.package_id
FROM tracking_summary t
INNER JOIN
(SELECT package_id, MAX(tracking_date) AS tracking_date
FROM tracking_summary
GROUP BY package_id) t2
ON t.package_id = t2.package_id
INNER JOIN package p ON p.id = t.package_id
AND t.tracking_date = t2.tracking_date;'''
results_views = model.Session.execute(sql_views)
tracking_total = {}
tracking_recent = {}
for row in results_views:
tracking_total.update({row.package_id: row.running_total})
tracking_recent.update({row.package_id: row.recent_views})
# Incluimos la informacion que necesitamos mostrar para cada dataset
for package in packages:
for key in package['notes_translated']:
if package['notes_translated'][key]:
package['notes_translated'][key] = package[
'notes_translated'][key].replace('\n', ' ').replace(
'\r', ' ')
# Obtenemos un string con las etiquetas
tags = ''
for tag in package['tags']:
tags = tags + ' ' + tag['display_name']
package['flattened_tags'] = tags
# Obtenemos un string con los formatos de sus recursos, el total de descargas y el valor de openness_score del dataset
# y si el dataset esta automatizado
flattened_formats = ','
qa = 0
automatic = 'N'
if 'update_string' in package and package['update_string']:
automatic = 'S'
for resource in package['resources']:
if resource['format'].lower() not in flattened_formats:
# Lo rodeamos con otros caracteres para que los strings contenidos en otros no den resultado "true" (ej: XLS y XLSX)
flattened_formats = flattened_formats + resource[
'format'].lower() + ','
if automatic == 'N':
if (not resource['url_type'] == 'upload'
and not '/resources/opendata/' in resource['url']
and not '/resource/' + resource['id'] +
'/download/' in resource['url']):
automatic = 'S'
if 'qa' in resource:
resource_qa = ast.literal_eval(resource['qa'])
if (resource_qa['openness_score'] > qa):
qa = int(resource_qa['openness_score'])
package['flattened_formats'] = flattened_formats
package['automatic'] = automatic
package['qa'] = qa
# Establecemos la tabla de formatos para cada dataset
package['formats'] = OrderedDict()
for format in formats:
format_value = 'N'
if ',' + format + ',' in flattened_formats:
format_value = 'S'
package['formats'][format] = format_value
# Establecemos la tabla de anyos para cada dataset
package['years'] = OrderedDict()
for year in range(year_from, year_to + 1):
year_value = 'N'
if 'Any ' + str(year) in package['flattened_tags']:
year_value = 'S'
package['years'][year] = year_value
# Escapamos los campos de texto
self.escape_text(package)
self.escape_translated_text(package)
# Obtenemos numero comentarios
if (package['name'] in comments):
package['comments'] = comments[package['name']]
else:
package['comments'] = 0
if (package['name'] in downloads):
package['downloads'] = downloads[package['name']]
else:
package['downloads'] = 0
if (package['name'] in downloads_absolute):
package['downloads_absolute'] = downloads_absolute[
package['name']]
else:
package['downloads_absolute'] = 0
if (package['name'] in api_access_number):
package['api_access_number'] = api_access_number[
package['name']]
else:
package['api_access_number'] = 0
if (package['name'] in api_access_number_absolute):
package[
'api_access_number_absolute'] = api_access_number_absolute[
package['name']]
else:
package['api_access_number_absolute'] = 0
if (package['id'] in tracking_total):
package['tracking_total'] = tracking_total[package['id']]
else:
package['tracking_total'] = 0
if (package['id'] in tracking_recent):
package['tracking_recent'] = tracking_recent[package['id']]
else:
package['tracking_recent'] = 0
curdate = d.datetime.now().strftime('%Y-%m-%d_%H-%M')
t.response.headers['Content-Type'] = 'application/csv; charset=utf-8'
t.response.headers[
'Content-Disposition'] = 'attachment; filename=catalegBCN_' + curdate + '.csv'
return t.render('cataleg.csv',
extra_vars={
'site_url': site_url,
'packages': packages,
'logged_in': logged_in,
'formats': formats,
'year_from': year_from,
'year_to': year_to,
'user': c.user,
'auth_user_obj': c.userobj,
'request': request
})
top_packages = PackageStats.get_top(limit=last)
return {
'table' : top_packages.get("packages")
}
def google_analytics_dataset_option_combinations():
options = [20,25,30,35,40,45,50]
for option in options:
yield { 'last': option }
googleanalytics_dataset_report_info = {
'name': 'google-analytics-dataset',
'title': 'Most popular datasets',
'description': 'Google analytics showing top datasets with most views',
'option_defaults': OrderedDict((('last',20),)),
'option_combinations': google_analytics_dataset_option_combinations,
'generate': google_analytics_dataset_report,
'template': 'report/dataset_analytics.html',
}
def google_analytics_resource_report(last):
'''
Generates report based on google analytics data. number of views per package
'''
# get resource objects corresponding to popular GA content
top_resources = ResourceStats.get_top(limit=last)
return {
'table' : top_resources.get("resources")
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
org = request.params.get('organization', None)
c.query_error = False
page = h.get_page_number(request.params)
try:
limit = int(config.get('ckan.datasets_per_page', 20))
except:
limit = 20
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search',
alternative_url=package_type)
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get('ckan.search.show_all_types',
'dataset')
search_all = False
map_results = None
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = 'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not package_type:
package_type = 'dataset'
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
limit,
'start': (page - 1) * limit,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
# loop the search query and get all the results
# this workaround the 1000 rows solr hard limit
HARD_LIMIT = 1000
conn = get_connection_redis()
pager = 0
# crank up the pager limit high as using points cluster for better performance
PAGER_LIMIT = 10000
data_dict_full_result = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
HARD_LIMIT,
'start':
0,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
if not org:
# if no q, it is an init load or direct visit on / dataset
log.info('### Not org ###')
if not q:
log.info('### Not q ###')
if not conn.exists('redis_full_results'):
log.info('### generating full results ###')
# get full results and add to redis when there is not full results in redis
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT,
q, fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
# adding to redis
log.info('adding full results to redis')
conn.set('redis_full_results', json.dumps(map_results))
# log.info(c.full_results)
else:
log.info('### using cached full results ###')
map_results = json.loads(
conn.get('redis_full_results'))
# log.info(c.full_results)
else:
log.info('### With q ###')
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT, q,
fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
else:
log.info('### With org ###')
if not q:
log.info('### Not q ###')
if not conn.exists('redis_full_results_%s' % org):
log.info('### generating %s results ###' % org)
# get full results and add to redis when there is not full results in redis
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT,
q, fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
# adding to redis
log.info('adding %s results to redis' % org)
conn.set('redis_full_results_%s' % org,
json.dumps(map_results))
# log.info(c.full_results)
else:
log.info('### using cached %s results ###' % org)
map_results = json.loads(
conn.get('redis_full_results_%s' % org))
# log.info(c.full_results)
else:
log.info('### With q ###')
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT, q,
fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
# log.info(c.full_results)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(
400,
_('Invalid search query: {error_message}').format(
error_message=str(se)))
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
c.search_facets_limits = {}
for facet in c.search_facets.keys():
try:
limit = int(
request.params.get(
'_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
except ValueError:
abort(
400,
_('Parameter "{parameter_name}" is not '
'an integer').format(parameter_name='_%s_limit' % facet))
c.search_facets_limits[facet] = limit
self._setup_template_variables(context, {}, package_type=package_type)
return render(self._search_template(package_type),
extra_vars={
'dataset_type': package_type,
'map_results': map_results
})
'average_tags_per_package': average_tags_per_package,
}
def tagless_report_option_combinations():
for organization in lib.all_organizations(include_none=True):
for include_sub_organizations in (False, True):
yield {
'organization': organization,
'include_sub_organizations': include_sub_organizations
}
tagless_report_info = {
'name':
'tagless-datasets',
'description':
'Datasets which have no tags.',
'option_defaults':
OrderedDict((
('organization', None),
('include_sub_organizations', False),
)),
'option_combinations':
tagless_report_option_combinations,
'generate':
tagless_report,
'template':
'report/tagless-datasets.html',
}
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {'model': model, 'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key, value=value, replace=replace,
controller=controller, action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
def collection_information(collection_id=None):
from ckanext.opensearch import config
collections = config.load_settings("collections_list")
collection_items = collections.items()
for collection in collection_items:
if collection[0] == collection_id:
return dict(collection[1])
group_extras = c.group_dict.get('extras')
group_collections = []
for extra in group_extras:
if extra['key'] == 'collections':
col_value = extra['value'].split(", ")
for a in col_value:
group_collections.append(a)
c.collections = []
for collection_id in group_collections:
c.collections.append(collection_information(collection_id))
collection_params = ''
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
facets = OrderedDict()
search_parameters = OrderedDict()
default_facet_titles = {
'collection_name': _('Data Collections'),
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
sentinel_facet_titles = {
'FamilyName': _('Family Name'),
'ProductType': _('Product Type'),
'OrbitDirection': _('Orbit Direction'),
'Swath': _('Acquisition Mode'),
'TransmitterReceiverPolarisation': _('Polarisation'),
}
default_search_titles = {
'timerange_start': _('Timerange Start'),
'timerange_end': _('Timerange End'),
}
collection_names = ['Sentinel-1 Level-1 (SLC)', 'Sentinel-1 Level-1 (GRD)', 'Sentinel-1 Level-2 (OCN)',
'Sentinel-2 Level-1C', 'Sentinel-2 Level-2A', 'Sentinel-3 SRAL Level-1 SRA']
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for search_param in helpers.search_params():
if search_param in default_search_titles:
search_parameters[search_param] = default_search_titles[search_param]
# Facet titles
self._update_facet_titles(facets, group_type)
if collection_params not in collection_names:
for f in facets:
if f in sentinel_facet_titles:
del facets[f]
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': True,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items()
if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.group_dict['collection_count'] = len(c.collections)
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
config.get('search.facets.default', 10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.page = h.Page(collection=[])
class package(PackageController):
def search(self):
'''
See ckan.controllers.package:PackageController for source, this is HACKed to fix broken scandic parameters.
'''
from ckan.lib.search import SearchError
package_type = self._guess_package_type()
try:
context = {'model': model, 'user': c.user or c.author}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
try:
page = int(request.params.get('page', 1))
except ValueError, e:
abort(400, ('"page" parameter must be an integer'))
limit = g.datasets_per_page
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
# HACK When removing scandic error prone values, check for their broken couterpart's presence in the URL parameters.
# If found, replace given fixed value with broken one, otherwise remove_url_param won't find it and crashes.
if value == u'Kaavat ja kiinteistöt' and ('categories', u'Kaavat ja kiinteist' + u"\ufffd" + 't') in request.params.items():
value = u'Kaavat ja kiinteist' + u"\ufffd" + 't'
if value == u'Työ ja elinkeinot' and ('categories', u'Ty' + u"\ufffd" + u' ja elinkeinot') in request.params.items():
value = u'Ty' + u"\ufffd" + u' ja elinkeinot'
if value == u'Väestö' and ('categories', u'V' + u"\ufffd" + u'est' + u"\ufffd") in request.params.items():
value = u'V' + u"\ufffd" + u'est' + u"\ufffd"
if value == u'Ympäristö ja luonto' and ('categories', u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto') in request.params.items():
value = u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto'
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if sort_by is None:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
# HACK Hardcoded replace for parameters with scandics that sometimes get replaced with unknown char (u"\ufffd").
if param == 'categories':
if value == u'Kaavat ja kiinteist' + u"\ufffd" + 't':
value = u'Kaavat ja kiinteistöt'
if value == u'Ty' + u"\ufffd" + u' ja elinkeinot':
value = u'Työ ja elinkeinot'
if value == u'V' + u"\ufffd" + u'est' + u"\ufffd":
value = u'Väestö'
if value == u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto':
value = u'Ympäristö ja luonto'
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('License'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def publisher_report(metric):
orgs = dict(model.Session.query(model.Group.name, model.Group)\
.filter_by(state='active').all())
org_counts = collections.defaultdict(dict)
if metric in ('views', 'viewsdownloads', 'visits'):
if metric == 'views' or metric == 'viewsdownloads':
sql_function = 'sum(pageviews::int)'
elif metric == 'visits':
sql_function = 'sum(visits::int)'
q = '''
select department_id, period_name, %s metric
from ga_url
where department_id <> ''
and package_id <> ''
group by department_id, period_name
order by department_id
''' % sql_function
org_period_count = model.Session.connection().execute(q)
for org_name, period_name, count in org_period_count:
org_counts[org_name][period_name] = count
if metric in ('downloads', 'viewsdownloads'):
q = '''
select g.name as org_name, s.period_name, sum(s.value::int) as downloads
from GA_Stat as s
join Package as p on s.key=p.name
join "group" as g on p.owner_org=g.id
where stat_name='Downloads'
and g.state='active'
group by org_name, s.period_name
order by downloads desc;
'''
org_period_count = model.Session.connection().execute(q)
if metric == 'viewsdownloads':
# add it onto the existing counts
for org_name, period_name, count in org_period_count:
org_counts[org_name][period_name] = count + \
org_counts[org_name].get(period_name, 0)
org_counts[org_name]['All'] = count + \
org_counts[org_name].get('All', 0)
else:
for org_name, period_name, count in org_period_count:
org_counts[org_name][period_name] = count
org_counts[org_name]['All'] = count + \
org_counts[org_name].get('All', 0)
org_counts = sorted(org_counts.items(), key=lambda x: -x[1].get('All', 0))
all_periods = [
res[0] for res in model.Session.query(GA_Url.period_name).group_by(
GA_Url.period_name).order_by(GA_Url.period_name).all()
]
rows = []
for org_name, counts in org_counts:
org = orgs.get(org_name)
if not org:
continue
top_org = list(go_up_tree(org))[-1]
row = OrderedDict((
('organization title', org.title),
('organization name', org.name),
('top-level organization title', top_org.title),
('top-level organization name', top_org.name),
))
for period_name in all_periods:
row[period_name] = counts.get(period_name, 0)
rows.append(row)
# Group the periods by year, to help the template draw the table nicely
#all_periods_tuples = [period.split('-') for period in all_periods
# if '-' in period]
#all_periods_tuples.sort(key=lambda x: x[0])
#all_periods_by_year = [
# (year, [p for y, p in year_periods])
# for year, year_periods in groupby(all_periods_tuples, lambda x: x[0])]
return {
'table': rows,
'all periods': all_periods,
#'all periods by year': all_periods_by_year
}
def dashboard_facets(self, facets_dict, package_type):
facets_dict = OrderedDict()
facets_dict['capacity'] = u'ecodp.common.privacy_state'
facets_dict = self._common_facets(facets_dict)
return facets_dict
def report_resources_by_organization(context, data_dict):
"""
Returns a list of OrderedDicts (one for each dataset in an organization)
sorted by the last modified date, then creation date
(if no modifications have been made yet).
Each OrderedDict contains the following keys:
dataset_name, dataset_url, resource_name, resource_url,
dataset_organisation, dataset_organisation_url, resource_created,
resource_last_modified, resource_view_count, resource_download_count
:return: a sorted list of OrderedDicts
:rtype: list
"""
user = toolkit.c.user or context.get('name')
org = data_dict.get('org_name') or context.get('org')
report = []
if not helpers.verify_datasets_exist(org):
return report
if 'org_name' in data_dict:
del data_dict['org_name']
if not helpers.is_admin(user, org):
toolkit.abort(
403,
_('You are not authorized to access this \
report or the organization does not exist.'))
data_dict['include_private'] = True
data_dict['q'] = 'organization:{0}'.format(org)
results = toolkit.get_action('package_search')({}, data_dict)
for item in results['results']:
resources = item['resources']
organization = item['organization']
for resource in resources:
# resource_view_count depends on tracking_summary, which
# doesn't seem to be enabled. Once it's enabled,
# resource_view_count will come from
# resource.get('tracking_summary').get('total')
# For now, there's a shortened version to avoid errors.
# resource_download_count will also need to be looked into
# when tracking_summary is enabled.
report.append(
OrderedDict([
('dataset_name', item.get('title')),
('dataset_url', (config.get('ckan.site_url') +
'/dataset/{0}'.format(item.get('name')))),
('resource_name', resource.get('name')),
('resource_url', resource.get('url')),
('dataset_organization', organization.get('name')),
('dataset_organization_url',
(config.get('ckan.site_url') +
'/organization/{0}'.format(organization.get('name')))),
('resource_created', resource.get('created')),
('resource_last_modified', resource.get('last_modified')),
('resource_view_count',
resource.get('tracking_summary', 0)),
('resource_download_count', resource.get('downloads', 0))
]))
return sorted(report,
key=lambda x:
(x['resource_last_modified'], x['resource_created']),
reverse=True)
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = g.datasets_per_page
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
# This is the only difference between this and core implementation
sort_by = request.params.get('sort', 'metadata_modified desc') # <-
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(config.get('ckan.search.show_all_types',
'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
limit,
'start': (page - 1) * limit,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError, se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(
400,
_('Invalid search query: {error_message}').format(
error_message=str(se)))
class ECODPGroupController(GroupController):
def index(self):
start = time.time()
group_type = self._guess_group_type()
language = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en')
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
q = c.q = request.params.get('q', '')
data_dict = {'all_fields': True, 'q': q}
sort_by = c.sort_by_selected = request.params.get('sort')
if sort_by:
data_dict['sort'] = sort_by
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = self._action('group_list')(context, data_dict)
c.amount_group_displayed = amount_group_displayed
c.groups = results[:amount_group_displayed]
c.hasmoregroups = len(results) > amount_group_displayed
c.themes = self._action('theme_list')(context, {})
c.catalogs = CatalogDcatApOp.get_ui_list_catalogs(config.get('ckan.locale_default', 'en'))
c.amount_catalog_displayed = amount_catalog_displayed
#c.page = h.Page(
# collection=results,
# page=request.params.get('page', 1),
# url=h.pager_url,
# items_per_page=21
#)
# @cache.cache('cached_render', expire=3600)
def cached_render(user, languge, group_type):
_render = base.render(group_type)
return _render
start_render = time.time()
_render = cached_render(context.get('user'), language, self._index_template(group_type))
duration_render= time.time() - start_render
log.info("Duration index render. {0}".format(duration_render))
duration = time.time() - start
log.info("Duration index. {0}".format(duration))
return _render
def read(self, id, limit=20):
start = time.time()
if request.GET.get('ext_boolean') in ['all', 'any', 'exact']:
base.session['ext_boolean'] = request.GET['ext_boolean']
base.session.save()
language = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en')
group_type = self._get_group_type(id.split('@')[0])
if (group_type != self.group_type) and (group_type != "eurovoc_domain"):
abort(404, _('Incorrect group type'))
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True}
# Do not query for the group datasets when dictizing, as they will
# be ignored and get requested on the controller anyway
data_dict = {'id': id, 'include_datasets': False}
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', '')
try:
stratqs = time.time()
if 'eurovoc_domain' in data_dict.get('id','') :
raise NotFound('EurovocDomains are not available any more')
result = self._action('group_show_read')(context, data_dict)
c.group_dict = result.get('group_dict', None)
context["group"] = result.get('group')
durationgs = time.time() - stratqs
log.info("Duration group show read. {0}".format(durationgs))
c.group = context['group']
except NotFound:
abort(404, _('Group not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read group %s') % id)
self._read(id, limit)
start_render = time.time()
# @cache.cache('render_cached_read', expire=3600)
def render_cached(user, id, language, group_type):
_render = base.render(self._read_template(c.group_dict['type']))
return _render
_render = render_cached(context.get('user'), id, language, c.group_dict['type'])
duration_render = time.time() - start_render
# _render = base.render(self._read_template(c.group_dict['type']))
duration = time.time() - start
log.info("Duration read_group. {0}".format(duration))
return _render
def _read(self, id, limit=20):
import time
language = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en')
start = time.time()
''' This is common code used by both read and bulk_process'''
group_type = self._get_group_type(id.split('@')[0])
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if q != u'':
qTab = q.split(' ')
checkbox = request.params.get('ext_boolean')
if checkbox == 'all':
q = '(' + ' AND '.join(qTab) + ')'
elif checkbox == 'any':
q = '(' + ' OR '.join(qTab) + ')'
else: #checkbox == 'exact'
q = '"' + q + '"'
c.description_formatted = h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
# c.group_admins is used by CKAN's legacy (Genshi) templates only,
# if we drop support for those then we can delete this line.
c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id)
try:
page = int(request.params.get('page', 1))
except ValueError, e:
abort(400, ('"page" parameter must be an integer'))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
new_params_nopage = []
for key, value in params_nopage:
if key == 'eurovoc_domains':
new_params_nopage.append(('groups', value))
else:
new_params_nopage.append((key,value))
params_nopage = new_params_nopage
#sort_by = request.params.get('sort', 'name asc')
sort_by = request.params.get('sort', None)
@cache.cache('search_url', expire=3600)
def search_url(params):
if group_type == 'organization':
if c.action == 'bulk_process':
url = self._url_for(controller='organization',
action='bulk_process',
id=id)
else:
url = self._url_for(controller='organization',
action='read',
id=id)
else:
url = self._url_for(controller='group', action='read', id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
@cache.cache('drill_down_url', expire=3600)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
if c.group_dict.get('is_organization'):
return h.remove_url_param(key, value=value, replace=replace,
controller='organization', action='read',
extras=dict(id=c.group_dict.get('id')))
else:
return h.remove_url_param(key, value=value, replace=replace,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if not param in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
param = 'eurovoc_domains' if (param == 'eurovoc_domains') else param;
q += ' %s:"%s"' % (param, value)
else:
search_extras[param] = value
fq = ''
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
elif c.group_dict.get('name') not in q:
q += ' groups:"%s"' % c.group_dict.get('name')
fq = fq + ' capacity:"public"'
user_member_of_orgs = [org['id'] for org
in h.organizations_available('read')]
if (c.group and c.group.id in user_member_of_orgs):
fq = ''
context['ignore_capacity_check'] = True
facets = OrderedDict()
default_facet_titles = {'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in plugins.PluginImplementations(plugins.IFacets):
if self.group_type == 'organization':
facets = plugin.organization_facets(
facets, self.group_type, None)
else:
facets = plugin.group_facets(
facets, self.group_type, None)
if 'capacity' in facets and (self.group_type != 'organization' or
not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras,
'defType': 'edismax'
}
active_cache = config.get('ckan.cache.active', 'false')
context_ = dict((k, v) for (k, v) in context.items() if k != 'schema')
has_result = False
dict_as_pickle = None
if active_cache == 'true':
dict_as_pickle = pickle.dumps(data_dict)
query_json = redis.get_from_cache(dict_as_pickle, pool=redis.MISC_POOL)
if query_json:
query = pickle.loads(query_json)
has_result = True
if has_result == False:
query = get_action('package_search')(context_, data_dict)
if active_cache == 'true':
redis.set_value_in_cache(dict_as_pickle, pickle.dumps(query), pool=redis.MISC_POOL)
c.search_url_params = urlencode(_encode_params(params_nopage))
c.page = page_util.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.facets = query['facets']
#maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
g.facets_default_number))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
duration = time.time() - start
log.info("Duration _read. {0}".format(duration))
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = page_util.Page(collection=[])
c.search_url_params = ''
class EDCOrganizationController(OrganizationController):
def index(self):
# FIXME: index copied from GroupController and modified to
# show only parent groups
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'with_private': False
}
data_dict = {'all_fields': False}
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
search_result = self._action('organization_list')(context, data_dict)
org_model = context['model']
#Get the list of all groups of type "organization" that have no parents.
top_level_orgs = org_model.Group.get_top_level_groups(
type="organization")
top_results = [
org for org in top_level_orgs if org.name in search_result
]
facets = OrderedDict()
facets['organization'] = _('Organizations')
data_dict = {
'facet.field': facets.keys(),
}
query = get_action('package_search')(context, data_dict)
c.org_pkg_count = query['facets'].get('organization')
c.top_orgs_items = top_results
return render('organization/index.html')
def _read(self, id, limit, group_type='organization'):
# FIXME: copied and modified from GroupController to collect
# sub organizations, create c.fields_grouped and hard-code
# search facets
''' This is common code used by both read and bulk_process'''
group_type = self._get_group_type(id.split('@')[0])
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True,
'extras_as_string': True
}
# Get the subgorgs of this org
org_id = c.group_dict.get('id')
q = c.q = request.params.get('q', '')
# XXX: unfortunate hack, copy sort default behaviour from
# before_search because we're using the q parameter below
# even when no text query was submitted
if not q and request.params.get('sort') in (None, 'rank'):
sort_by = 'record_publish_date desc, metadata_modified desc'
else:
sort_by = request.params.get('sort', None)
suborgs = ['"' + org + '"' for org in get_suborgs(org_id)]
if suborgs != []:
q += ' owner_org:("' + org_id + '" OR ' + ' OR '.join(
suborgs) + ')'
else:
q += ' owner_org:"%s"' % org_id
c.description_formatted = h.render_markdown(
c.group_dict.get('description'))
context['return_query'] = True
try:
page = int(request.params.get('page', 1))
except ValueError, e:
abort(400, ('"page" parameter must be an integer'))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def search_url(params):
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller='organization', action=action, id=id)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='organization',
action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='organization',
action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
c.fields_grouped = {}
for (param, value) in request.params.items():
if not param in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
fq = 'capacity:"public"'
user_member_of_orgs = [
org['id'] for org in h.organizations_available('read')
]
if (c.group and c.group.id in user_member_of_orgs):
fq = ''
context['ignore_capacity_check'] = True
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'edc_state': _('States'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
'type': _('Dataset types')
}
for facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.group_dict['package_count'] = query['count']
c.facets = query['facets']
maintain.deprecate_context_item('facets',
'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(
request.params.get('_%s_limit' % facet,
g.facets_default_number))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
class ECPortalUserController(core_user.UserController):
def logged_in(self):
#override the logged_in method after the access was granted or denied
#if login was successfull check the group membership
if c.user:
util.add_missing_user_to_all_groups()
return super(ECPortalUserController, self).logged_in()
def login_user(self, tup):
if tup:
user = tup[0]
user_object = model.User.get(user)
if not user_object:
user_object = model.User.get('api')
c.userobj = user_object
c.user = '******'
try:
# Create user
user_attributes = tup[1]
user_name = user
user_email = user_attributes["email"].text
user_password = "******"
user_fullname = user_attributes[
"firstName"].text + " " + user_attributes[
"lastName"].text
h.get_action(
'user_create', {
"name": user_name,
"email": user_email,
"password": user_password,
"fullname": user_fullname
})
rememberer = request.environ['repoze.who.plugins'][
'friendlyform']
identity = {'repoze.who.userid': user}
response.headerlist += rememberer.remember(
request.environ, identity)
except logic.ActionError:
base.abort(401, _('Error while creating new user'))
else:
c.userobj = user_object
c.user = user
rememberer = request.environ['repoze.who.plugins'][
'friendlyform']
identity = {'repoze.who.userid': user}
response.headerlist += rememberer.remember(
request.environ, identity)
self.logged_in()
def read(self, id=None):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'for_view': True,
'return_minimal': True,
'save': 'save' in request.params
}
data_dict = {'id': id, 'user_obj': c.userobj}
context['with_related'] = True
self._setup_user_template_variables(context, data_dict)
if context['save']:
request.POST.pop('save')
data = ecportal_logic.transform_to_data_dict(request.POST)
credential_validator = Validator({
'contact_mailbox': [email]
#'contact_phone_number': [IntPhoneNumberRule]
})
errors = {}
succeed = credential_validator.validate(data, results=errors)
if succeed is True:
contact_info = Package_contact_info.get_by_user(c.userobj.id)
if not contact_info:
contact_info = Package_contact_info(c.userobj.id)
contact_info.from_dict(data)
contact_info.save()
h.flash_success(_('ecodp.user.save.success'))
elif errors:
h.flash_error(_('ecodp.user.save.error'))
c.user_dict.update(data)
c.errors = errors
# The legacy templates have the user's activity stream on the user
# profile page, new templates do not.
if h.asbool(config.get('ckan.legacy_templates', False)):
c.user_activity_stream = get_action('user_activity_list_html')(
context, {
'id': c.user_dict['id']
})
#vars = {'data': {}, 'errors': {}, 'error_summary': {}}
#c.form = base.render('user/edit_contact_info_form.html', extra_vars=vars)
return base.render('user/read.html')
def read_contact_info(self, id=None):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'for_view': True,
'return_minimal': True,
'save': 'save' in request.params
}
data_dict = {'id': id, 'user_obj': c.userobj}
context['with_related'] = True
self._setup_user_template_variables(context, data_dict)
if context['save']:
request.POST.pop('save')
data = ecportal_logic.transform_to_data_dict(request.POST)
credential_validator = Validator({
'contact_mailbox': [email]
#'contact_phone_number': [IntPhoneNumberRule]
})
errors = {}
succeed = credential_validator.validate(data, results=errors)
if succeed is True:
contact_info = Package_contact_info.get_by_user(c.userobj.id)
if not contact_info:
contact_info = Package_contact_info(c.userobj.id)
contact_info.from_dict(data)
contact_info.save()
h.flash_success(_('ecodp.user.save.success'))
elif errors:
h.flash_error(_('ecodp.user.save.error'))
c.user_dict.update(data)
c.errors = errors
# The legacy templates have the user's activity stream on the user
# profile page, new templates do not.
if h.asbool(config.get('ckan.legacy_templates', False)):
c.user_activity_stream = get_action('user_activity_list_html')(
context, {
'id': c.user_dict['id']
})
#vars = {'data': {}, 'errors': {}, 'error_summary': {}}
#c.form = base.render('user/edit_contact_info_form.html', extra_vars=vars)
return base.render('user/read_contact_info.html')
def dashboard(self):
def query_for_datasets():
query = ''
if not new_authz.is_sysadmin(c.user):
organisation_ids = self.__get_users_organizations_ids(c.user)
if not organisation_ids:
# return an empty query result with the needed (i.e.: accessed later) elements
return {'results': [], 'count': 0}
query = build_owner_filter(organisation_ids)
if c.q != u'':
search = c.q.strip().split(' ')
result = ''
if request.params.get('ext_boolean') == 'any':
result = build_search_for_any_words(search)
elif request.params.get('ext_boolean') == 'all':
result = build_search_for_all_words(search)
elif request.params.get('ext_boolean') == 'exact':
result = '("%s")' % (c.q)
log.info("%s" % (request.params.get('ext_boolean')))
query = query + " + title:" + result
return query
log.debug("Entering MDT dashboard")
c.is_sysadmin = new_authz.is_sysadmin(c.user)
context = {
'for_view': True,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'return_minimal': True,
'ignore_capacity_check': True # Display also private datasets
}
user_data = {"id": c.userobj.id, 'user_obj': c.userobj}
try:
user_dict = logic.get_action('user_show')(context, user_data)
except logic.NotFound:
base.abort(404, _('User not found'))
except logic.NotAuthorized:
base.abort(401, _('Not authorized to see this page'))
c.user_dict = user_dict
c.is_myself = user_dict['name'] == c.user
c.about_formatted = h.render_markdown(user_dict['about'])
c.search_choice = request.params.get('ext_boolean')
# Save the used search paramters in a proper format to the exchange variable
user = user_dict['name']
log.debug("Start getting the datasets")
# ---------------------------------------------------------------------------------------------------------------
# Get the datasets
from ckan.lib.search import SearchError
package_type = self._guess_package_type()
if request.GET.get('ext_boolean') in ['all', 'any', 'exact']:
session['ext_boolean'] = request.GET['ext_boolean']
session.save()
# unicode format (decoded from utf8)
c.q = request.params.get('q', u'')
c.query_error = False
q = query_for_datasets()
try:
page = int(request.params.get('page', 1))
except ValueError, e:
abort(400, ('"page" parameter must be an integer'))
limit = g.datasets_per_page
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
new_params_nopage = []
for key, value in params_nopage:
if key == 'eurovoc_domains':
new_params_nopage.append(('groups', value))
else:
new_params_nopage.append((key, value))
params_nopage = new_params_nopage
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(
key,
value=value,
replace=replace,
controller=
'ckanext.ecportal.controllers.user:ECPortalUserController',
action='dashboard')
c.remove_field = remove_field
sort_by = request.params.get('sort', 'views_total desc')
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(
core_package._encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
# if request.params.get('private'):
# fq = 'private: %s' % request.params.get('private')
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort', 'id'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
# paramFQ = 'groups' if (param == 'eurovoc_domains') else param;
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
# if package_type and package_type != 'dataset':
# Only show datasets of this particular type
# fq += ' +dataset_type:{type}'.format(type=package_type)
# else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
# if not asbool(config.get('ckan.search.show_all_types', 'False')):
# fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dashboard_facets(facets, package_type)
c.facet_titles = facets
facet_fields = facets.keys()
facet_fields.append('private')
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facet_fields,
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
if sort_by == 'modified_date desc':
# This is the customized part for ODP-570
# add the group parameter to the solr query
data_dict['group'] = 'true'
data_dict['group.query'] = [
'-organization:estat AND -organization:comp AND -organization:grow',
'organization:estat', 'organization:comp',
'organization:grow'
]
data_dict['group.format'] = 'simple'
data_dict['rows'] = 2147483646
query = get_action('custom_package_search')(context, data_dict)
cached_result = []
for name, group in query['groups'].iteritems():
cached_result += group['doclist']['docs']
start = (page - 1) * limit
result_list = customsearch.check_solr_result(
context, cached_result[start:], limit)
else:
query = get_action('package_search')(context, data_dict)
result_list = query['results']
c.sort_by_selected = query['sort']
c.page = page_util.Page(collection=result_list,
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = result_list
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = page_util.Page(collection=[])
# coding=utf8
# '''Settings and constants for Kata CKAN extension'''
from ckan.common import OrderedDict
from pylons.i18n.translation import gettext_noop as N_
PID_TYPES = ['primary', 'relation']
# Overridden CKAN role permissions
ROLE_PERMISSIONS = OrderedDict([
('admin', ['admin']),
('editor', ['admin']),
('member', ['read', 'create_dataset']),
])
ORGANIZATION_MEMBER_PERMISSIONS = {
# ORGANIZATION_MEMBER_PERMISSIONS format:
# (user role, target original role, target final role, user == target): has permission to modify role
# Permission to delete a role is given if one has permission to change role to 'member'.
('admin', 'admin', 'admin', True):
False,
('admin', 'admin', 'admin', False):
False,
('admin', 'admin', 'editor', True):
False,
('admin', 'admin', 'editor', False):
False,
('admin', 'admin', 'member', True):
False, # Admin should not be able to delete oneself
('admin', 'admin', 'member', False):
# -*- coding: utf-8 -*-
from ckan.common import OrderedDict
from pylons import config
from ckan.plugins import toolkit
import ckanapi
groups = OrderedDict([
('local-governments', ('Local Governments', 'Municipios')),
('health', ('Health', 'Salud')),
('business', ('Business', 'Comercio e Industrias')),
('transportation', ('Transportation', 'Transporte y Logística')),
('education', ('Education', 'Educación')),
('justice', ('Justice', 'Justicia')),
('social-development', ('Social Development', 'Desarrollo Social')),
('finance', ('Finance', 'Finanzas Publicas')),
('labour', ('Labour', 'Empleo')),
('environment', ('Environment', 'Ambiente')),
])
class CreateFeaturedGroups(toolkit.CkanCommand):
'''Create featured groups
Usage:
create_featured_groups - create featured groups
'''
summary = __doc__.split('\n')[0]
usage = __doc__
max_args = 0
default roles with custom roles and decorates
has_user_permission_for_group_or_org to allow an approver to editor groups,
Approveer can act as editor plus accession to all the process states, but
have no other sysadmin powers
'''
from ckan import authz, model
from ckan.common import OrderedDict
from ckan.plugins import toolkit
# these are the permissions that roles have
authz.ROLE_PERMISSIONS = OrderedDict([
('admin', ['admin']),
('approver', [
'read', 'delete_dataset', 'create_dataset', 'update_dataset',
'manage_group', 'workflow'
]),
('editor', [
'read', 'delete_dataset', 'create_dataset', 'update_dataset',
'manage_group'
]),
('member', ['read', 'manage_group']),
])
def _trans_role_approver():
return toolkit._('Workflow Approver')
authz._trans_role_approver = _trans_role_approver
def is_approver_decorator(method):
def dataset_facets(self, facets_dict, package_type):
facets_dict = OrderedDict()
facets_dict = self._common_facets(facets_dict)
return facets_dict
item_count=query['count'],
items_per_page=int(limit))
c.page.items = query['results']
c.results = c.page.items
c.form = self.render_package_form()
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.query_count = 0
c.facets = {}
c.page = h.Page(collection=[])
# Facets for our sidebar.
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('License'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
def group_facets(self, facets_dict, group_type, package_type):
facets_dict = OrderedDict()
facets_dict = self._common_facets(facets_dict)
return facets_dict
def _get_fields_types(context, data_dict):
all_fields = _get_fields(context, data_dict)
all_fields.insert(0, {'id': '_id', 'type': 'int'})
field_types = OrderedDict([(f['id'], f['type']) for f in all_fields])
return field_types
def organization_facets(self, facets_dict, organization_type,
package_type):
facets_dict = OrderedDict()
facets_dict = self._common_facets(facets_dict)
return facets_dict
def _read(id, limit, group_type):
u''' This is common code used by both read and bulk_process'''
extra_vars = {}
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'schema': _db_to_form_schema(group_type=group_type),
u'for_view': True,
u'extras_as_string': True
}
q = request.params.get(u'q', u'')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
# Search within group
if g.group_dict.get(u'is_organization'):
fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
else:
fq = u' groups:"%s"' % g.group_dict.get(u'name')
extra_vars["q"] = q
g.description_formatted = \
h.render_markdown(g.group_dict.get(u'description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
sort_by = request.params.get(u'sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = u'bulk_process' if getattr(
g, u'action', u'') == u'bulk_process' else u'read'
url = h.url_for(u'.'.join([controller, action]), id=id)
params = [(k, v.encode(u'utf-8')
if isinstance(v, string_types) else str(v))
for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(
alternative_url=None,
controller=u'group',
action=u'read',
extras=dict(id=g.group_dict.get(u'name')),
new_params=by)
extra_vars["drill_down_url"] = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(
key,
value=value,
replace=replace,
controller=controller,
action=u'read',
extras=dict(id=g.group_dict.get(u'name')))
extra_vars["remove_field"] = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append((u'page', page))
return search_url(params)
try:
extra_vars["fields"] = fields = []
extra_vars["fields_grouped"] = fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
q += u' %s: "%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.fields = fields
g.fields_grouped = fields_grouped
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
_update_facet_titles(facets, group_type)
extra_vars["facet_titles"] = facets
data_dict = {
u'q': q,
u'fq': fq,
u'include_private': True,
u'facet.field': facets.keys(),
u'rows': limit,
u'sort': sort_by,
u'start': (page - 1) * limit,
u'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
query = get_action(u'package_search')(context_, data_dict)
extra_vars["page"] = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict['package_count'] = query['count']
extra_vars["search_facets"] = g.search_facets = query['search_facets']
extra_vars["search_facets_limits"] = g.search_facets_limits = {}
for facet in g.search_facets.keys():
limit = int(
request.params.get(u'_%s_limit' % facet,
config.get(u'search.facets.default', 10)))
g.search_facets_limits[facet] = limit
extra_vars["page"].items = query['results']
extra_vars["sort_by_selected"] = sort_by
except search.SearchError as se:
log.error(u'Group search error: %r', se.args)
extra_vars["query_error"] = True
extra_vars["page"] = h.Page(collection=[])
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.facet_titles = facets
g.page = extra_vars["page"]
extra_vars["group_type"] = group_type
_setup_template_variables(context, {u'id': id}, group_type=group_type)
return extra_vars
class EDCUserController(UserController):
def dashboard_unpublished(self):
if not c.userobj:
abort(401, _('You must be logged-in to access the dashboard.'))
user_id = c.userobj.id
fq = ' +edc_state:("DRAFT" OR "PENDING PUBLISH" OR "REJECTED")'
# Get the list of organizations that this user is the admin
if not c.userobj.sysadmin:
user_orgs = get_orgs_user_can_edit(c.userobj)
if len(user_orgs) > 0:
fq += ' +owner_org:(' + ' OR '.join(user_orgs) + ')'
self._user_datasets('dashboard_unpublished', c.userobj.id, fq)
return render('user/dashboard_unpublished.html')
def dashboard_datasets(self):
if not c.userobj:
abort(401, _('You must be logged-in to access the dashboard.'))
fq = ' +author:("%s")' % (c.userobj.id)
self._user_datasets('dashboard_datasets', c.userobj.id, fq)
return render('user/dashboard_datasets.html')
def read(self, id=None):
if c.userobj and c.userobj.sysadmin == True:
fq = ''
else:
fq = ' +(edc_state:("PUBLISHED" OR "PENDING ARCHIVE")'
if c.userobj:
user_id = c.userobj.id
user_orgs = get_orgs_user_can_edit(c.userobj)
if len(user_orgs) > 0:
fq += ' OR owner_org:(' + ' OR '.join(user_orgs) + ')'
fq += ')'
self._user_datasets('read', id, fq)
return render('user/read.html')
def _user_datasets(self, action, id=None, filter_query=None):
from ckan.lib.search import SearchError
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'for_view': True
}
user_dict = {
'id': id,
'user_obj': c.userobj,
'include_datasets': False
}
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
context['return_query'] = True
try:
page = int(request.params.get('page', 1))
except ValueError, e:
abort(400, ('"page" parameter must be an integer'))
limit = int(toolkit.config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
base_url = config.get('ckan.site_url')
if action == 'dashboard_datasets':
url = base_url + '/dashboard/datasets'
elif action == 'dashboard_unpublished':
url = base_url + '/dashboard/unpublished'
elif action == 'read':
url = h.url_for(controller='user', action=action, id=id)
else:
url = h.url_for(controller='user', action=action)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='user',
action=action,
extras=dict(id=c.userobj.id),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='user',
action=action,
extras=dict(id=c.userobj.id))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s:"%s"' % (param, value)
else:
search_extras[param] = value
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'edc_state': _('States'),
'tags': _('Tags'),
'res_format': _('Formats'),
}
for facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
c.facet_titles = facets
fq = filter_query or ''
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
user_dict['package_count'] = query['count']
c.facets = query['facets']
#maintain.deprecate_context_item('facets',
# 'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(
request.params.get(
'_%s_limit' % facet,
int(toolkit.config.get('search.facets.default', 10))))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except SearchError, se:
log.error('User search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
