def dashboard_organizations(self):
context = {'model': model, 'session': model.Session,
'for_view': True, 'user': c.user or c.author,
'auth_user_obj': c.userobj}
data_dict = {'user_obj': c.userobj}
self._setup_template_variables(context, data_dict)
(user_orgs, usr_suborgs) = get_user_toporgs(c.userobj.id)
facets = OrderedDict()
#Add the organization facet to get the number of records for each organization
facets['organization'] = _('Organizations')
data_dict = {
'facet.field': facets.keys(),
}
query = get_action('package_search')(context, data_dict)
c.org_pkg_count = query['facets'].get('organization')
c.top_orgs_items = user_orgs
c.suborgs_items = usr_suborgs
return render('user/dashboard_organizations.html')
def cioos_get_facets(package_type='dataset'):
''' get all dataset for the given package type, including private ones.
This function works similarly to code found in ckan/ckan/controllers/package.py
in that it does a search of all datasets and populates the following
globals for later use:
c.facet_titles
c.search_facets
'''
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in toolkit.h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'facet.field':
facets.keys(),
'rows':
0,
'include_private':
asbool(config.get('ckan.search.default_include_private', True)),
}
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
query = get_action('package_search')(context, data_dict)
c.search_facets = query['search_facets']
def datasets_without_resources(organization, include_sub_organizations=False):
pkg_dicts = []
query = model.Session.query(model.Package)\
.filter_by(state='active')\
.order_by(model.Package.title)
if organization:
query = lib.filter_by_organizations(query, organization,
include_sub_organizations)
for pkg in query.all():
if len(pkg.resources) != 0 or \
pkg.extras.get('unpublished', '').lower() == 'true':
continue
deleted, url = last_resource_deleted(pkg)
pkg_dict = OrderedDict((
('name', pkg.name),
('title', pkg.title),
('metadata created', pkg.metadata_created.isoformat()),
('metadata modified', pkg.metadata_modified.isoformat()),
('last resource deleted',
deleted.isoformat() if deleted else None),
('last resource url', url),
('dataset_notes', lib.dataset_notes(pkg)),
))
pkg_dicts.append(pkg_dict)
return {'table': pkg_dicts}
def update_report():
with open(update_log_location) as f:
content = f.readlines()
full_history = []
grouped_history = []
failed_history = []
group_dict = {}
names = {}
for line in content:
json_line = json.loads(line)
id = json_line["id"]
name = json_line["name"]
time = json_line["time"]
success = json_line["success"]
details = json_line["details"]
table_line = OrderedDict((("id", id), ("name", name), ("time", time),
("success", success), ("details", details)))
group_line = OrderedDict(
(("time", time), ("success", success), ("details", details)))
if id in group_dict.keys():
group_dict[id].append(group_line)
else:
group_dict[id] = [group_line]
names[id] = name
full_history.append(table_line)
if success == "Fail":
failed_history.append(table_line)
for id in group_dict.keys():
if len(group_dict[id]) > 10:
group_dict[id] = group_dict[id][len(group_dict[id]) -
11:len(group_dict[id]) - 1]
grouped_history.append(
OrderedDict((("id", id), ("name", names[id]), ("history",
group_dict[id]))))
return {
'table': list(reversed(full_history)),
'fail_table': failed_history,
'grouped_table': grouped_history
}
def tagless_report(organization, include_sub_organizations=False):
'''
Produces a report on packages without tags.
Returns something like this:
{
'table': [
{'name': 'river-levels', 'title': 'River levels', 'notes': 'Harvested', 'user': '******',
'created': '2008-06-13T10:24:59.435631'},
{'name': 'co2-monthly', 'title' 'CO2 monthly', 'notes': '', 'user': '******',
'created': '2009-12-14T08:42:45.473827'},
],
'num_packages': 56,
'packages_without_tags_percent': 4,
'average_tags_per_package': 3.5,
}
'''
# Find the packages without tags
q = model.Session.query(model.Package) \
.outerjoin(model.PackageTag) \
.filter(
model.PackageTag.id == None # noqa: E711
)
if organization:
q = lib.filter_by_organizations(q, organization,
include_sub_organizations)
tagless_pkgs = [
OrderedDict((
('name', pkg.name),
('title', pkg.title),
('notes', lib.dataset_notes(pkg)),
('user', pkg.creator_user_id),
('created', pkg.metadata_created.isoformat()),
)) for pkg in q.slice(0, 100)
] # First 100 only for this demo
# Average number of tags per package
q = model.Session.query(model.Package)
q = lib.filter_by_organizations(q, organization, include_sub_organizations)
num_packages = q.count()
q = q.join(model.PackageTag)
num_taggings = q.count()
if num_packages:
average_tags_per_package = round(float(num_taggings) / num_packages, 1)
else:
average_tags_per_package = None
packages_without_tags_percent = lib.percent(len(tagless_pkgs),
num_packages)
return {
'table': tagless_pkgs,
'num_packages': num_packages,
'packages_without_tags_percent': packages_without_tags_percent,
'average_tags_per_package': average_tags_per_package,
}
def html_datasets_report(organization, include_sub_organizations=False):
'''
Returns datasets that only have an HTML link, by organization.
'''
# Get packages
query = model.Session.query(model.Package)\
.filter_by(state='active')
if organization:
query = lib.filter_by_organizations(query, organization,
include_sub_organizations)
pkgs = query.all()
# See if HTML
num_datasets_published = 0
num_datasets_only_html = 0
datasets_only_html = []
# use yield_per, otherwise memory use just goes up til the script is killed
# by the os.
for pkg in pkgs:
if p.toolkit.asbool(pkg.extras.get('unpublished')):
continue
num_datasets_published += 1
formats = set([
res.format.lower() for res in pkg.resources
if res.resource_type != 'documentation'
])
if 'html' not in formats:
continue
#org = pkg.get_organization().name
data_formats = formats - set(('asp', '', None))
if data_formats == set(('html', )):
num_datasets_only_html += 1
datasets_only_html.append(pkg)
rows = []
for pkg in datasets_only_html:
row = OrderedDict((
('name', pkg.name),
('title', pkg.title),
('metadata created', pkg.metadata_created.isoformat()),
('metadata modified', pkg.metadata_modified.isoformat()),
('dataset_notes', lib.dataset_notes(pkg)),
))
rows.append(row)
return {
'table': rows,
'num_datasets_published': num_datasets_published,
'num_datasets_only_html': num_datasets_only_html,
}
def index(self):
# FIXME: index copied from GroupController and modified to
# show only parent groups
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
data_dict = {'all_fields': False}
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
search_result = self._action('organization_list')(context, data_dict)
org_model = context['model']
#Get the list of all groups of type "organization" that have no parents.
top_level_orgs = org_model.Group.get_top_level_groups(type="organization")
top_results = [org for org in top_level_orgs if org.name in search_result ]
facets = OrderedDict()
facets['organization'] = _('Organizations')
data_dict = {
'facet.field': facets.keys(),
}
query = get_action('package_search')(context, data_dict)
c.org_pkg_count = query['facets'].get('organization')
c.top_orgs_items = top_results
return render('organization/index.html')
def _add_facets(facets_dict, group=False):
new_facets_dict = OrderedDict([
('keywords_facet', ''),
facets_dict.items()[2],
('data_type_facet', p.toolkit._('Data Type')),
('organization', p.toolkit._('Projects')),
('groups', p.toolkit._('Topics')),
('language_facet', p.toolkit._('Language'))
] + facets_dict.items()[3:])
if not group:
new_facets_dict['date_facet'] = ''
return new_facets_dict
def dump(self, dump_file_obj):
query = model.Session.query(model.User)
query = query.order_by(model.User.created.asc())
columns = (('id', 'name', 'openid', 'fullname', 'email', 'created',
'about'))
row_dicts = []
for user in query:
row = OrderedDict()
for col in columns:
value = getattr(user, col)
if not value:
value = ''
if col == 'created':
value = str(value) # or maybe dd/mm/yyyy?
row[col] = value
row_dicts.append(row)
writer = CsvWriter(row_dicts)
writer.save(dump_file_obj)
dump_file_obj.close()
search_extras = {}
fq = ''
if c.facet_choose_item:
# Build query: in dataset browsing the query is limited to a single facet only
fq = ' %s:"%s"' % (c.facet_choose_title, c.facet_choose_item)
c.fields.append((c.facet_choose_title, c.facet_choose_item))
c.fields_grouped[c.facet_choose_title] = [c.facet_choose_item]
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True}
if not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('License'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
# Get package type name
package_type = self._guess_package_type()[:-1]
c.package_type = package_type
# Get page content from Wordpress
# =========================================
import ckanext.dadosgovbr.helpers.wordpress as wp
wp_page_slug = 'scheming_'+package_type+'s'
c.wp_page = type('Nothing', (object,), {})
c.wp_page.content = type('Nothing', (object,), {})
c.wp_page.content.rendered = "Conteudo da pagina nao encontrado..."
try:
c.wp_page = wp.page(wp_page_slug)
except:
pass
# DEBUG
# from pprint import pprint
# pprint(c.concursos)
# Package type facets (filters)
# =========================================
package_type_facets = u'organization groups tags res_format license_id'
if(package_type == 'inventario'):
package_type_facets = u'organization situacao_base informacoes_sigilosas_base informacoes_publicas_base atualizacoes_base dados_abertos_base'
if(package_type == 'concurso'):
package_type_facets = u'organization datasets_used'
if(package_type == 'aplicativo'):
package_type_facets = u'organization groups tags res_format license_id'
try:
context = {'model': model, 'user': c.user,
'auth_user_obj': c.userobj}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(
config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
# Default package
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
# Inventário package
'situacao_base': _(u'Situação da base'),
'informacoes_sigilosas_base': _(u'Base possui informações sigilosas?'),
'vocab_sim': _(u'Sim'),
'vocab_nao': _(u'Não'),
'informacoes_publicas_base': _(u'Base possui informações públicas?'),
'informacoes_publicas_base_publico': _(u'Público'),
'atualizacoes_base': _(u'Período de atualização dos dados'),
'dados_abertos_base': _(u'Exporta para dados abertos?'),
# Concurso package
'datasets_used': _(u'Dados utilizados'),
'tags': _(u'Tags'),
'date': _(u'Data de início'),
'end_date': _(u'Data final'),
'publico': _(u'Público'),
'sim': _(u'Sim'),
'nao': _(u'Não'),
}
for facet in config.get(u'search.facets', package_type_facets.split()):
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras,
'include_private': asbool(config.get(
'ckan.search.default_include_private', True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError, se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(400, _('Invalid search query: {error_message}')
.format(error_message=str(se)))
def search(self):
from ckan.lib.search import SearchError
package_type = self._guess_package_type()
if package_type == 'search':
package_type = 'dataset'
try:
context = {'model': model, 'user': c.user or c.author,
'auth_user_obj': c.userobj}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = self._page_number()
params_nopage = self._params_nopage()
# Commenting below parts as it doesn't seem to be used
# def drill_down_url(alternative_url=None, **by):
# return h.add_url_param(alternative_url=alternative_url,
# controller='package', action='search',
# new_params=by)
#
# c.drill_down_url = drill_down_url
self._set_remove_field_function()
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return self._search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return self._search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
self._set_other_links()
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
#limit = g.datasets_per_page
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
self._set_filters_are_selected_flag()
limit = self._allowed_num_of_items(search_extras)
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
self._which_tab_is_selected(search_extras)
self._performing_search(q, fq, facets.keys(), limit, page, sort_by,
search_extras, pager_url, context)
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {'model': model, 'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key, value=value, replace=replace,
controller=controller, action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
facets = OrderedDict()
default_facet_titles = {'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': True,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items()
if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
config.get('search.facets.default', 10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.page = h.Page(collection=[])
for (param, value) in request.params.items():
if not param in ["q", "page", "sort"] and len(value) and not param.startswith("_"):
if not param.startswith("ext_"):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
fq = 'capacity:"public"'
user_member_of_orgs = [org["id"] for org in h.organizations_available("read")]
if c.group and c.group.id in user_member_of_orgs:
fq = ""
context["ignore_capacity_check"] = True
facets = OrderedDict()
default_facet_titles = {
"groups": _("Groups"),
"tags": _("Tags"),
"res_format": _("Formats"),
"license": _("Licence"),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
try:
context = {'model': model, 'user': c.user,
'auth_user_obj': c.userobj}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(
config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in ckan.plugins.PluginImplementations(ckan.plugins.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
# Set the custom default sort parameter here
# Might need a rewrite when ckan is updated
if not sort_by:
sort_by = 'metadata_created desc'
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras,
'include_private': asbool(config.get(
'ckan.search.default_include_private', True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError, se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(400, _('Invalid search query: {error_message}')
.format(error_message=str(se)))
def _add_dataset_search(self, showcase_id, showcase_name):
'''
Search logic for discovering datasets to add to a showcase.
'''
from ckan.lib.search import SearchError
package_type = 'dataset'
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
try:
page = self._get_page_number(request.params)
except AttributeError:
# in CKAN >= 2.5 _get_page_number has been moved
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return self._search_url(params, showcase_name)
c.sort_by = _sort_by
if sort_by is None:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return self._search_url(params, showcase_name)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not tk.asbool(config.get('ckan.search.show_all_types',
'False')):
fq += ' +dataset_type:dataset'
# Only search for packages that aren't already associated with the
# Showcase
associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(showcase_id)
# flatten resulting list to space separated string
if associated_package_ids:
associated_package_ids_str = \
' OR '.join([id[0] for id in associated_package_ids])
fq += ' !id:({0})'.format(associated_package_ids_str)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
# for CKAN-Versions that do not provide the facets-method from
# helper-context, import facets from ckan.common
if hasattr(h, 'facets'):
current_facets = h.facets()
else:
from ckan.common import g
current_facets = g.facets
for facet in current_facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def search(self):
package_type = self._guess_package_type()
try:
context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
q = c.q = request.params.get('q', u'')
c.query_error = False
page = self._get_page_number(request.params)
limit = g.datasets_per_page
params_nopage = [(k, v) for k, v in request.params.items() if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(
alternative_url=alternative_url,
controller='package',
action='search',
new_params=by
)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
if param != 'organization':
fq += ' %s:"%s"' % (param, value)
else:
fq += custom_organization_filter(value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
fq += ' +dataset_type:{type}'.format(type=package_type)
elif not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def search(self):
from ckan.lib.search import SearchError
package_type = self._guess_package_type()
try:
context = {'model': model, 'user': c.user or c.author,
'auth_user_obj': c.userobj}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = self._get_page_number(request.params)
limit = g.datasets_per_page
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if sort_by is None:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
api_search_url_params = None
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
##### EXTENDING ORIGINAL <<<<<<<<<<
# API Params
api_url_params = {
'q': q,
'fq': fq.strip(),
'rows': limit,
'start': (page - 1) * limit,
}
if sort_by:
api_url_params['sort'] = sort_by
if facets.keys():
api_url_params['facet.field'] = json.dumps(facets.keys())
# if search_extras:
# api_url_params['extras'] = ','.join(search_extras)
api_search_url_params = urlencode(_encode_params(api_url_params.items()))
##### EXTENDING ORIGINAL >>>>>>>>>>>>>>
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
# c.group_admins is used by CKAN's legacy (Genshi) templates only,
# if we drop support for those then we can delete this line.
c.group_admins = authz.get_group_or_org_admin_ids(c.group.id)
page = self._get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if not param in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
fq = 'capacity:"public"'
user_member_of_orgs = [org['id'] for org
in h.organizations_available('read')]
if (c.group and c.group.id in user_member_of_orgs):
fq = ''
context['ignore_capacity_check'] = True
facets = OrderedDict()
default_facet_titles = {'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
if 'capacity' in facets and (group_type != 'organization' or
not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
lang = get_lang()[0]
q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.group_id == c.group_dict.get('id'), GroupMultilang.lang == lang).all()
if q_results:
for result in q_results:
c.group_dict[result.field] = result.text
if result.field == 'title':
c.group_dict['display_name'] = result.text
c.facets = query['facets']
maintain.deprecate_context_item('facets',
'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
g.facets_default_number))
c.search_facets_limits[facet] = limit
# MULTILANG - Localizing Organizations display names in Facet list
organizations = c.search_facets.get('organization')
for org in organizations.get('items'):
q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.name == org.get('name'), GroupMultilang.lang == lang).all()
if q_results:
for result in q_results:
if result.field == 'title':
org['display_name'] = result.text
# MULTILANG - Localizing Groups display names in Facet list
groups = c.search_facets.get('groups')
for group in groups.get('items'):
q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.name == group.get('name'), GroupMultilang.lang == lang).all()
if q_results:
for result in q_results:
if result.field == 'title':
group['display_name'] = result.text
# MULTILANG - Localizing Tags display names in Facet list
tags = c.search_facets.get('tags')
for tag in tags.get('items'):
localized_tag = TagMultilang.by_name(tag.get('name'), lang)
if localized_tag:
tag['display_name'] = localized_tag.text
c.page.items = query['results']
# MULTILANG - Localizing Datasets names and descriptions in search list
log.info(':::::::::::: Retrieving the corresponding localized title and abstract :::::::::::::::')
for item in c.page.items:
lang = get_lang()[0]
q_results = model.Session.query(PackageMultilang).filter(PackageMultilang.package_id == item.get('id'), PackageMultilang.lang == lang).all()
if q_results:
for result in q_results:
item[result.field] = result.text
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
def _read(self, id, limit, group_type):
""" This is common code used by both read and bulk_process"""
context = {
"model": model,
"session": model.Session,
"user": c.user,
"schema": self._db_to_form_schema(group_type=group_type),
"for_view": True,
"extras_as_string": True,
}
q = c.q = request.params.get("q", "")
# Search within group
if c.group_dict.get("is_organization"):
q += ' owner_org:"%s"' % c.group_dict.get("id")
else:
q += ' groups:"%s"' % c.group_dict.get("name")
c.description_formatted = h.render_markdown(c.group_dict.get("description"))
context["return_query"] = True
page = self._get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items() if k != "page"]
sort_by = request.params.get("sort", None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = "bulk_process" if c.action == "bulk_process" else "read"
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode("utf-8") if isinstance(v, basestring) else str(v)) for k, v in params]
return url + u"?" + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(
alternative_url=None,
controller="group",
action="read",
extras=dict(id=c.group_dict.get("name")),
new_params=by,
)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(
key,
value=value,
replace=replace,
controller="group",
action="read",
extras=dict(id=c.group_dict.get("name")),
)
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(("page", page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if not param in ["q", "page", "sort"] and len(value) and not param.startswith("_"):
if not param.startswith("ext_"):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
fq = 'capacity:"public"'
user_member_of_orgs = [org["id"] for org in h.organizations_available("read")]
if c.group and c.group.id in user_member_of_orgs:
fq = ""
context["ignore_capacity_check"] = True
facets = OrderedDict()
default_facet_titles = {
"organization": _("Organizations"),
"groups": _("Groups"),
"tags": _("Tags"),
"res_format": _("Formats"),
"license_id": _("Licenses"),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
if "capacity" in facets and (group_type != "organization" or not user_member_of_orgs):
del facets["capacity"]
c.facet_titles = facets
data_dict = {
"q": q,
"fq": fq,
"facet.field": facets.keys(),
"rows": limit,
"sort": sort_by,
"start": (page - 1) * limit,
"extras": search_extras,
}
context_ = dict((k, v) for (k, v) in context.items() if k != "schema")
query = get_action("package_search")(context_, data_dict)
c.page = h.Page(
collection=query["results"], page=page, url=pager_url, item_count=query["count"], items_per_page=limit
)
c.group_dict["package_count"] = query["count"]
c.facets = query["facets"]
maintain.deprecate_context_item("facets", "Use `c.search_facets` instead.")
c.search_facets = query["search_facets"]
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(request.params.get("_%s_limit" % facet, g.facets_default_number))
c.search_facets_limits[facet] = limit
c.page.items = query["results"]
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error("Group search error: %r", se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
def _read(self, id, limit):
''' This is common code used by both read and bulk_process'''
group_type = self._get_group_type(id.split('@')[0])
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
description = c.group_dict.get('description')
if description:
if group_type == 'group':
c.description_formatted = h.render_markdown(description.get(h.lang()))
else:
c.description_formatted = h.render_markdown(description)
context['return_query'] = True
# c.group_admins is used by CKAN's legacy (Genshi) templates only,
# if we drop support for those then we can delete this line.
c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id)
page = self._get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
if group_type == 'organization':
if c.action == 'bulk_process':
url = self._url_for(controller='organization',
action='bulk_process',
id=id)
else:
url = self._url_for(controller='organization',
action='read',
id=id)
else:
url = self._url_for(controller='group', action='read', id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if not param in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
fq = 'capacity:"public"'
user_member_of_orgs = [org['id'] for org
in h.organizations_available('read')]
if (c.group and c.group.id in user_member_of_orgs):
fq = ''
context['ignore_capacity_check'] = True
facets = OrderedDict()
default_facet_titles = {'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in plugins.PluginImplementations(plugins.IFacets):
if self.group_type == 'organization':
facets = plugin.organization_facets(
facets, self.group_type, None)
else:
facets = plugin.group_facets(
facets, self.group_type, None)
if 'capacity' in facets and (self.group_type != 'organization' or
not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.facets = query['facets']
maintain.deprecate_context_item('facets',
'Use `c.search_facets` instead.')
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
g.facets_default_number))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
def _read(id, limit, group_type):
u''' This is common code used by both read and bulk_process'''
extra_vars = {}
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'schema': _db_to_form_schema(group_type=group_type),
u'for_view': True,
u'extras_as_string': True
}
q = request.params.get(u'q', u'')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
# Search within group
if g.group_dict.get(u'is_organization'):
fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
else:
fq = u' groups:"%s"' % g.group_dict.get(u'name')
extra_vars["q"] = q
g.description_formatted = \
h.render_markdown(g.group_dict.get(u'description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
sort_by = request.params.get(u'sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = u'bulk_process' if getattr(
g, u'action', u'') == u'bulk_process' else u'read'
url = h.url_for(u'.'.join([controller, action]), id=id)
params = [(k, v.encode(u'utf-8')
if isinstance(v, string_types) else str(v))
for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(
alternative_url=None,
controller=u'group',
action=u'read',
extras=dict(id=g.group_dict.get(u'name')),
new_params=by)
extra_vars["drill_down_url"] = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(
key,
value=value,
replace=replace,
controller=controller,
action=u'read',
extras=dict(id=g.group_dict.get(u'name')))
extra_vars["remove_field"] = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append((u'page', page))
return search_url(params)
try:
extra_vars["fields"] = fields = []
extra_vars["fields_grouped"] = fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
q += u' %s: "%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.fields = fields
g.fields_grouped = fields_grouped
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
_update_facet_titles(facets, group_type)
extra_vars["facet_titles"] = facets
data_dict = {
u'q': q,
u'fq': fq,
u'include_private': True,
u'facet.field': facets.keys(),
u'rows': limit,
u'sort': sort_by,
u'start': (page - 1) * limit,
u'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
query = get_action(u'package_search')(context_, data_dict)
extra_vars["page"] = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict['package_count'] = query['count']
extra_vars["search_facets"] = g.search_facets = query['search_facets']
extra_vars["search_facets_limits"] = g.search_facets_limits = {}
for facet in g.search_facets.keys():
limit = int(
request.params.get(u'_%s_limit' % facet,
config.get(u'search.facets.default', 10)))
g.search_facets_limits[facet] = limit
extra_vars["page"].items = query['results']
extra_vars["sort_by_selected"] = sort_by
except search.SearchError as se:
log.error(u'Group search error: %r', se.args)
extra_vars["query_error"] = True
extra_vars["page"] = h.Page(collection=[])
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.facet_titles = facets
g.page = extra_vars["page"]
extra_vars["group_type"] = group_type
_setup_template_variables(context, {u'id': id}, group_type=group_type)
return extra_vars
def show(self):
#This is based on the CKAN package search controller
#There are probably still some pieces we don't need
#And using the facets to get the info we need is not ideal
#because normal facet limits are then not applied on the page
from ckan.lib.search import SearchError
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
params_nopage = [(k, v) for k, v in request.params.items()]
def pager_url(q=None, page=None):
params = list(params_nopage)
return self.search_url(params, 'dataset')
c.search_url_params = urlencode(self._encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in g.facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, 'dataset')
c.facet_titles = facets
#We use the organization facet to get the list of organizations detected for this query
#The actual data is irrelevant
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'facet.limit': -1,
'extras': search_extras,
'rows': 1
}
query = get_action('package_search')(context, data_dict)
orglist = query['facets']['organization'].keys()
#Now get the data concerning the orgs
q = Session.query(Group).filter(Group.name.in_(orglist))
values = []
qres = q.all()
#Combine in extras, add counts
#Make the data look like what we get with a org search
for d in qres:
dobj = d.as_dict()
dobj['extras'] = d.extras
dobj['package_count'] = query['facets']['organization'][dobj['name']]
values.append(dobj)
c.page = h.Page(
collection=query['results'],
url=pager_url,
item_count=query['count']
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search (map) error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
def get_pkg_dict(cls, resource, ldr):
from ckan import model
pkg_dict = OrderedDict()
extras = OrderedDict()
uri = str(resource.identifier)
pkg_dict['title'] = unicode(resource[RDFS.label].next())
extras['registry_uri'] = uri
# Create or update?
pkg = model.Session.query(model.Package) \
.filter_by(state='active') \
.join(model.PackageExtra) \
.filter_by(state='active') \
.filter_by(key='registry_uri') \
.filter_by(value=uri).first()
if pkg:
pkg_dict['id'] = pkg.id
pkg_dict['name'] = pkg.name
action = 'update'
else:
pkg_dict['id'] = unicode(uuid.uuid4())
pkg_dict['name'] = cls._gen_new_name(pkg_dict['title'])
action = 'new'
dgu_type = cls.get_dgu_type(resource)
extras['data_standard_type'] = dgu_type
resources = []
for format_display, format_extension, format_dgu in (
('RDF ttl', 'ttl', 'RDF'),
('RDF/XML', 'rdf', 'RDF'),
('JSON-LD', 'jsonld', 'JSON')):
url = uri + '?_format=%s' % format_extension
resources.append({'description': '%s as %s' % (dgu_type, format_display),
'url': url,
'format': format_dgu,
'resource_type': 'file'})
resources.append({'description': '%s and metadata as %s' % (dgu_type, format_display),
'url': url + METADATA_PARAM,
'format': format_dgu,
'resource_type': 'file'})
pkg_dict['notes'] = unicode(resource[DCT.description].next())
licence_url = str(resource[DCT.license].next())
if 'open-government-licence' in licence_url:
pkg_dict['licence_id'] = 'uk-ogl'
else:
extras['licence_url'] = licence_url
# not sure how this will display as just as URL
pkg_dict['owner_org'] = cls.get_publisher(resource).id
resources.append({'description': 'Web page for this %s on a Linked Data Registry' % dgu_type,
'url': uri,
'format': 'HTML',
'resource_type': 'documentation'})
metadata = cls.get_resource_metadata(uri)
status = metadata[REG.status].next()
extras['status'] = str(status).split('#')[-1]
extras['harvested_version'] = str(metadata[OWL.versionInfo].next())
extras['data_standard_type'] = dgu_type
pkg_dict['type'] = 'data-standard'
pkg_dict['extras'] = [{'key': k, 'value': v} for k, v in extras.items()]
pkg_dict['resources'] = resources
return pkg_dict, action