#all_periods_tuples = [period.split('-') for period in all_periods # if '-' in period] #all_periods_tuples.sort(key=lambda x: x[0]) #all_periods_by_year = [ # (year, [p for y, p in year_periods]) # for year, year_periods in groupby(all_periods_tuples, lambda x: x[0])] return { 'table': rows, 'all periods': all_periods, #'all periods by year': all_periods_by_year } def publisher_report_option_combinations(): return ({ 'metric': metric } for metric in ('views', 'visits', 'downloads', 'viewsdownloads')) publisher_report_info = { 'name': 'site-usage-publisher', 'title': 'Site usage by publisher', 'description': 'Usage statistics, by publisher for each month. Data is from Google Analytics.', 'option_defaults': OrderedDict([('metric', 'views')]), 'option_combinations': publisher_report_option_combinations, 'generate': publisher_report, 'template': 'report/publisher.html', }
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = { 'model': model, 'session': model.Session, 'user': c.user, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True } q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value include_private = False user_member_of_orgs = [ org['id'] for org in h.organizations_available('read') ] if (c.group and c.group.id in user_member_of_orgs): include_private = True facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses') } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) if 'capacity' in facets and (group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': '', 'include_private': include_private, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict( (k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int( request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
class GroupController(base.BaseController): group_type = 'group' ## hooks for subclasses def _group_form(self, group_type=None): return lookup_group_plugin(group_type).group_form() def _form_to_db_schema(self, group_type=None): return lookup_group_plugin(group_type).form_to_db_schema() def _db_to_form_schema(self, group_type=None): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' return lookup_group_plugin(group_type).db_to_form_schema() def _setup_template_variables(self, context, data_dict, group_type=None): return lookup_group_plugin(group_type).\ setup_template_variables(context, data_dict) def _new_template(self, group_type): return lookup_group_plugin(group_type).new_template() def _index_template(self, group_type): return lookup_group_plugin(group_type).index_template() def _about_template(self, group_type): return lookup_group_plugin(group_type).about_template() def _read_template(self, group_type): return lookup_group_plugin(group_type).read_template() def _history_template(self, group_type): return lookup_group_plugin(group_type).history_template() def _edit_template(self, group_type): return lookup_group_plugin(group_type).edit_template() def _activity_template(self, group_type): return lookup_group_plugin(group_type).activity_template() def _admins_template(self, group_type): return lookup_group_plugin(group_type).admins_template() def _bulk_process_template(self, group_type): return lookup_group_plugin(group_type).bulk_process_template() ## end hooks def _replace_group_org(self, string): ''' substitute organization for group if this is an org''' if self.group_type == 'organization': string = re.sub('^group', 'organization', string) return string def _action(self, action_name): ''' select the correct group/org action ''' return get_action(self._replace_group_org(action_name)) def _check_access(self, action_name, *args, **kw): ''' select the correct group/org check_access ''' return check_access(self._replace_group_org(action_name), *args, **kw) def _render_template(self, template_name): ''' render the correct group/org template ''' return render(self._replace_group_org(template_name)) def _redirect_to(self, *args, **kw): ''' wrapper to ensue the correct controller is used ''' if self.group_type == 'organization' and 'controller' in kw: kw['controller'] = 'organization' return h.redirect_to(*args, **kw) def _url_for(self, *args, **kw): ''' wrapper to ensue the correct controller is used ''' if self.group_type == 'organization' and 'controller' in kw: kw['controller'] = 'organization' return h.url_for(*args, **kw) def _guess_group_type(self, expecting_name=False): """ Guess the type of group from the URL handling the case where there is a prefix on the URL (such as /data/organization) """ parts = [x for x in request.path.split('/') if x] idx = -1 if expecting_name: idx = -2 gt = parts[idx] if gt == 'group': gt = None return gt def index(self): group_type = self._guess_group_type() context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False } q = c.q = request.params.get('q', '') data_dict = {'all_fields': True, 'q': q} sort_by = c.sort_by_selected = request.params.get('sort') if sort_by: data_dict['sort'] = sort_by try: self._check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin results = self._action('group_list')(context, data_dict) c.amount_group_diplayed = amount_group_displayed c.groups = results[:amount_group_displayed] c.hasmoregroups = len(results) > amount_group_displayed #c.page = h.Page( # collection=results, # page=request.params.get('page', 1), # url=h.pager_url, # items_per_page=21 #) return render(self._index_template(group_type)) def read(self, id, limit=20): if request.GET.get('ext_boolean') in ['all', 'any', 'exact']: base.session['ext_boolean'] = request.GET['ext_boolean'] base.session.save() group_type = self._get_group_type(id.split('@')[0]) if (group_type != self.group_type) and (group_type != "eurovoc_domain"): abort(404, _('Incorrect group type')) context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True } # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict = {'id': id, 'include_datasets': False} # unicode format (decoded from utf8) q = c.q = request.params.get('q', '') try: c.group_dict = self._action('group_show')(context, data_dict) c.group = context['group'] except NotFound: abort(404, _('Group not found')) except NotAuthorized: abort(401, _('Unauthorized to read group %s') % id) self._read(id, limit) return render(self._read_template(c.group_dict['type'])) def _read(self, id, limit): ''' This is common code used by both read and bulk_process''' group_type = self._get_group_type(id.split('@')[0]) context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True } q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = h.render_markdown( c.group_dict.get('description')) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id) try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] new_params_nopage = [] for key, value in params_nopage: if key == 'eurovoc_domains': new_params_nopage.append(('groups', value)) else: new_params_nopage.append((key, value)) params_nopage = new_params_nopage #sort_by = request.params.get('sort', 'name asc') sort_by = request.params.get('sort', None) def search_url(params): if group_type == 'organization': if c.action == 'bulk_process': url = self._url_for(controller='organization', action='bulk_process', id=id) else: url = self._url_for(controller='organization', action='read', id=id) else: url = self._url_for(controller='group', action='read', id=id) params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): if c.group_dict.get('is_organization'): return h.remove_url_param( key, value=value, replace=replace, controller='organization', action='read', extras=dict(id=c.group_dict.get('id'))) else: return h.remove_url_param( key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) param = 'groups' if (param == 'eurovoc_domains') else param q += ' %s: "%s"' % (param, value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [ org['id'] for org in h.organizations_available('read') ] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses') } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in plugins.PluginImplementations(plugins.IFacets): if self.group_type == 'organization': facets = plugin.organization_facets( facets, self.group_type, None) else: facets = plugin.group_facets(facets, self.group_type, None) if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict( (k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.search_url_params = urlencode(_encode_params(params_nopage)) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int( request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[]) c.search_url_params = ''
class DateType(object): '''Utils for handling dates in forms. * Full or partial dates * User inputs in form DD/MM/YYYY and it is stored in db as YYYY-MM-DD. ''' format_types = ('form', 'db') datetime_fields = OrderedDict([ ('year', (1000, 2100, 4, 'YYYY')), ('month', (1, 12, 2, 'MM')), ('day', (1, 31, 2, 'DD')), ('hour', (0, 23, 2, 'HH')), ('minute', (0, 59, 2, 'MM')), ]) datetime_fields_indexes = { 'min': 0, 'max': 1, 'digits': 2, 'format_code': 3 } date_fields_order = { 'db': ('year', 'month', 'day'), 'form': ('day', 'month', 'year') } parsing_separators = {'date': '-/', 'time': ':\.'} default_separators = { 'db': { 'date': '-', 'time': ':' }, 'form': { 'date': '/', 'time': ':' }, } field_code_map = { 'year': 'YYYY', 'month': 'MM', 'day': 'DD', 'hour': 'HH', 'minute': 'MM' } word_match = re.compile('[A-Za-z]+') timezone_match = re.compile('(\s[A-Z]{3})|(\s[+-]\d\d:?\d\d)') months_abbreviated = [month[:3] for month in months] @classmethod def parse_timedate(cls, timedate_str, format_type): '''Takes a timedate and returns a dictionary of the fields. * Little validation is done. * If it can\'t understand the layout it raises DateConvertError ''' assert format_type in cls.format_types if not hasattr(cls, 'matchers'): # build up a list of re matches for the different # acceptable ways of expressing the time and date cls.matchers = {} cls.readable_formats = {} for format_type_ in cls.format_types: finished_regexps = [] readable_formats = [] # analogous to the regexps, # but human readable year_re = '(?P<%s>\d{2,4})' month_re = '(?P<%s>\w+)' two_digit_decimal_re = '(?P<%s>\d{1,2})' time_re = '%s[%s]%s' % (two_digit_decimal_re % 'hour', cls.parsing_separators['time'], two_digit_decimal_re % 'minute') time_readable = '%s%s%s' % ( cls.datetime_fields['hour'][ cls.datetime_fields_indexes['format_code']], cls.default_separators[format_type_]['time'], cls.datetime_fields['minute'][ cls.datetime_fields_indexes['format_code']]) date_field_re = { 'year': year_re % 'year', 'month': month_re % 'month', 'day': two_digit_decimal_re % 'day' } date_fields = list(cls.date_fields_order[format_type_]) for how_specific in ('day', 'month', 'year'): date_sep_re = '[%s]' % cls.parsing_separators['date'] date_sep_readable = cls.default_separators[format_type_][ 'date'] date_field_regexps = [ date_field_re[field] for field in date_fields ] date_field_readable = [ cls.datetime_fields[field][ cls.datetime_fields_indexes['format_code']] for field in date_fields ] date_re = date_sep_re.join(date_field_regexps) date_readable = date_sep_readable.join(date_field_readable) finished_regexps.append(date_re) readable_formats.append(date_readable) date_fields.remove(how_specific) full_date_re = finished_regexps[0] full_date_readable = readable_formats[0] # Allow time to be before or after the date for format_ in ('%(time_re)s%(sep)s%(full_date_re)s', '%(full_date_re)s%(sep)s%(time_re)s'): finished_regexps.insert( 0, format_ % { 'time_re': time_re, 'sep': '\s', 'full_date_re': full_date_re }) readable_formats.insert( 0, format_ % { 'time_re': time_readable, 'sep': ' ', 'full_date_re': full_date_readable }) cls.matchers[format_type_] = [ re.compile('^%s$' % regexp) for regexp in finished_regexps ] cls.readable_formats[format_type_] = readable_formats #print format_type_, finished_regexps, readable_formats for index, matcher in enumerate(cls.matchers[format_type]): match = matcher.match(timedate_str) if match: timedate_dict = match.groupdict() timedate_dict = cls.int_timedate(timedate_dict) timedate_dict['readable_format'] = cls.readable_formats[ format_type][index] return timedate_dict else: acceptable_formats = ', '.join([ "'%s'" % format_ for format_ in cls.readable_formats[format_type] ]) raise DateConvertError( "Cannot parse %s date '%s'. Acceptable formats: %s" % (format_type, timedate_str, acceptable_formats)) @classmethod def int_timedate(cls, timedate_dict): # Convert timedate string values to integers int_timedate_dict = timedate_dict.copy() for field in cls.datetime_fields.keys(): if timedate_dict.has_key(field): val = timedate_dict[field] if field == 'year': if len(val) == 2: # Deal with 2 digit dates try: int_val = int(val) except ValueError: raise DateConvertError( 'Expecting integer for %s value: %s' % (field, val)) val = cls.add_centurys_to_two_digit_year(int_val) elif len(val) == 3: raise DateConvertError( 'Expecting 2 or 4 digit year: "%s"' % (val)) if field == 'month': # Deal with months expressed as words if val in months: val = months.index(val) + 1 if val in cls.months_abbreviated: val = cls.months_abbreviated.index(val) + 1 try: int_timedate_dict[field] = int(val) except ValueError: raise DateConvertError( 'Expecting integer for %s value: %s' % (field, val)) return int_timedate_dict @classmethod def iso_to_db(cls, iso_date, format): # e.g. 'Wed, 06 Jan 2010 09:30:00' # '%a, %d %b %Y %H:%M:%S' assert isinstance(iso_date, (unicode, str)) try: date_tuple = time.strptime(iso_date, format) except ValueError, e: raise DateConvertError( 'Could not read date as ISO format "%s". Date provided: "%s"' % (format, iso_date)) date_obj = datetime.datetime(*date_tuple[:4]) date_str = cls.date_to_db(date_obj) return date_str
def dataset_facets(self, facets_dict, package_type): '''Only show tags for Showcase search list.''' if package_type != DATASET_TYPE_NAME: return facets_dict return OrderedDict({'tags': _('Tags')})
def _add_dataset_search(self, showcase_id, showcase_name): ''' Search logic for discovering datasets to add to a showcase. ''' from ckan.lib.search import SearchError package_type = 'dataset' # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False try: page = self._get_page_number(request.params) except AttributeError: # in CKAN >= 2.5 _get_page_number has been moved page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return self._search_url(params, showcase_name) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return self._search_url(params, showcase_name) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj } if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not tk.asbool( config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' # Only search for packages that aren't already associated with the # Showcase associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase( showcase_id) # flatten resulting list to space separated string if associated_package_ids: associated_package_ids_str = \ ' OR '.join([id[0] for id in associated_package_ids]) fq += ' !id:({0})'.format(associated_package_ids_str) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } # for CKAN-Versions that do not provide the facets-method from # helper-context, import facets from ckan.common if hasattr(h, 'facets'): current_facets = h.facets() else: from ckan.common import g current_facets = g.facets for facet in current_facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
from ckan.common import OrderedDict from pylons import config from ckan.plugins import toolkit import ckanapi groups = OrderedDict([ ('health', 'Health'), ('education', 'Education'), ('finance', 'Finance'), ('transport', 'Transport'), ('environment', 'Environment & agriculture'), ('property', 'Property & land'), ('economy', 'Economy, industry & employment'), ('tourism', 'Tourism, leisure, culture & arts'), ('population', 'Population & society'), ]) class CreateFeaturedGroups(toolkit.CkanCommand): '''Create the nine featured groups Usage: create_featured_groups - create featured groups ''' summary = __doc__.split('\n')[0] usage = __doc__ max_args = 0 min_args = 0
def view(self): context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } # Obtenemos parametros de configuracion site_url = config.get('ckan.site_url') + config.get( 'ckan.root_path').replace('{{LANG}}', '') # Inicializamos variables now = d.datetime.now() year_from = 1989 year_to = now.year # Comprobamos si es un usuario identificado o no logged_in = False if 'user' in context and context['user']: logged_in = True # Si se pide el catalogo publico ignoramos el usuario identificado if logged_in and 'public' in request.params and request.params.get( 'public') == 'true': logged_in = False if not logged_in: # Obtenemos el catalogo para usuarios no identificados packages = t.get_action('package_search')(context, { 'include_private': False, 'rows': 1000, 'sort': 'name asc' }) packages = packages['results'] elif c.userobj.sysadmin: # Obtenemos el catalogo para sysadmin packages = t.get_action('package_search')(context, { 'include_private': True, 'rows': 1000, 'sort': 'name asc' }) packages = packages['results'] else: # Obtenemos el catalogo para usuarios identificados: solo los de sus organizaciones user_org = t.get_action('organization_list_for_user')( context, { 'permission': 'create_dataset' }) org_facets = '' for org in user_org: if org_facets != '': org_facets = org_facets + ' OR ' org_facets = org_facets + org['name'] packages = t.get_action('package_search')( context, { 'fq': 'organization:(' + org_facets + ')', 'include_private': True, 'rows': 1000, 'sort': 'name asc' }) packages = packages['results'] # obtenemos los formatos formats = t.get_action('format_autocomplete')(context, { 'q': '', 'limit': 50 }) # puede devolver formatos duplicados, lo convertimos a un set que eliminara los elementos # duplicados y de nuevo a una lista formats = list(set(formats)) for format in formats: format_strip = format.strip() if not format_strip: formats.remove(format) # Realizamos conexion a la BBDD de Drupal para obtener el numero de comentarios de cada dataset y almacenamos los valores en un array dbc = parse_db_config('ckan.drupal.url') ckan_conn_string = "host='%s' port='%s' dbname='%s' user='******' password='******'" % ( dbc['db_host'], dbc['db_port'], dbc['db_name'], dbc['db_user'], dbc['db_pass']) ckan_conn = psycopg2.connect(ckan_conn_string) ckan_cursor = ckan_conn.cursor() ckan_cursor.execute( """SELECT OP.pkg_name, COUNT(*) FROM opendata_package OP INNER JOIN node N ON N.tnid = OP.pkg_node_id INNER JOIN comment C ON C.nid = N.nid WHERE N.tnid != 0 GROUP BY OP.pkg_name;""" ) comments = {} for row in ckan_cursor: comments.update({row[0]: row[1]}) ckan_cursor.close() ckan_conn.close() sql_downloads = '''select sum(count) AS downloads, sum(count_absolute) AS downloads_absolute, t.tracking_type, p.name from tracking_summary t inner join resource r ON r.id = t.resource_id inner join package p ON p.id = r.package_id GROUP BY p.name, t.tracking_type;''' results_downloads = model.Session.execute(sql_downloads) downloads = {} downloads_absolute = {} api_access_number = {} api_access_number_absolute = {} for row in results_downloads: if row.tracking_type == 'resource': downloads.update({row.name: row.downloads}) downloads_absolute.update({row.name: row.downloads_absolute}) else: api_access_number.update({row.name: row.downloads}) api_access_number_absolute.update( {row.name: row.downloads_absolute}) sql_views = '''SELECT t.tracking_date, t.running_total, t.recent_views, t.package_id FROM tracking_summary t INNER JOIN (SELECT package_id, MAX(tracking_date) AS tracking_date FROM tracking_summary GROUP BY package_id) t2 ON t.package_id = t2.package_id INNER JOIN package p ON p.id = t.package_id AND t.tracking_date = t2.tracking_date;''' results_views = model.Session.execute(sql_views) tracking_total = {} tracking_recent = {} for row in results_views: tracking_total.update({row.package_id: row.running_total}) tracking_recent.update({row.package_id: row.recent_views}) # Incluimos la informacion que necesitamos mostrar para cada dataset for package in packages: for key in package['notes_translated']: if package['notes_translated'][key]: package['notes_translated'][key] = package[ 'notes_translated'][key].replace('\n', ' ').replace( '\r', ' ') # Obtenemos un string con las etiquetas tags = '' for tag in package['tags']: tags = tags + ' ' + tag['display_name'] package['flattened_tags'] = tags # Obtenemos un string con los formatos de sus recursos, el total de descargas y el valor de openness_score del dataset # y si el dataset esta automatizado flattened_formats = ',' qa = 0 automatic = 'N' if 'update_string' in package and package['update_string']: automatic = 'S' for resource in package['resources']: if resource['format'].lower() not in flattened_formats: # Lo rodeamos con otros caracteres para que los strings contenidos en otros no den resultado "true" (ej: XLS y XLSX) flattened_formats = flattened_formats + resource[ 'format'].lower() + ',' if automatic == 'N': if (not resource['url_type'] == 'upload' and not '/resources/opendata/' in resource['url'] and not '/resource/' + resource['id'] + '/download/' in resource['url']): automatic = 'S' if 'qa' in resource: resource_qa = ast.literal_eval(resource['qa']) if (resource_qa['openness_score'] > qa): qa = int(resource_qa['openness_score']) package['flattened_formats'] = flattened_formats package['automatic'] = automatic package['qa'] = qa # Establecemos la tabla de formatos para cada dataset package['formats'] = OrderedDict() for format in formats: format_value = 'N' if ',' + format + ',' in flattened_formats: format_value = 'S' package['formats'][format] = format_value # Establecemos la tabla de anyos para cada dataset package['years'] = OrderedDict() for year in range(year_from, year_to + 1): year_value = 'N' if 'Any ' + str(year) in package['flattened_tags']: year_value = 'S' package['years'][year] = year_value # Escapamos los campos de texto self.escape_text(package) self.escape_translated_text(package) # Obtenemos numero comentarios if (package['name'] in comments): package['comments'] = comments[package['name']] else: package['comments'] = 0 if (package['name'] in downloads): package['downloads'] = downloads[package['name']] else: package['downloads'] = 0 if (package['name'] in downloads_absolute): package['downloads_absolute'] = downloads_absolute[ package['name']] else: package['downloads_absolute'] = 0 if (package['name'] in api_access_number): package['api_access_number'] = api_access_number[ package['name']] else: package['api_access_number'] = 0 if (package['name'] in api_access_number_absolute): package[ 'api_access_number_absolute'] = api_access_number_absolute[ package['name']] else: package['api_access_number_absolute'] = 0 if (package['id'] in tracking_total): package['tracking_total'] = tracking_total[package['id']] else: package['tracking_total'] = 0 if (package['id'] in tracking_recent): package['tracking_recent'] = tracking_recent[package['id']] else: package['tracking_recent'] = 0 curdate = d.datetime.now().strftime('%Y-%m-%d_%H-%M') t.response.headers['Content-Type'] = 'application/csv; charset=utf-8' t.response.headers[ 'Content-Disposition'] = 'attachment; filename=catalegBCN_' + curdate + '.csv' return t.render('cataleg.csv', extra_vars={ 'site_url': site_url, 'packages': packages, 'logged_in': logged_in, 'formats': formats, 'year_from': year_from, 'year_to': year_to, 'user': c.user, 'auth_user_obj': c.userobj, 'request': request })
top_packages = PackageStats.get_top(limit=last) return { 'table' : top_packages.get("packages") } def google_analytics_dataset_option_combinations(): options = [20,25,30,35,40,45,50] for option in options: yield { 'last': option } googleanalytics_dataset_report_info = { 'name': 'google-analytics-dataset', 'title': 'Most popular datasets', 'description': 'Google analytics showing top datasets with most views', 'option_defaults': OrderedDict((('last',20),)), 'option_combinations': google_analytics_dataset_option_combinations, 'generate': google_analytics_dataset_report, 'template': 'report/dataset_analytics.html', } def google_analytics_resource_report(last): ''' Generates report based on google analytics data. number of views per package ''' # get resource objects corresponding to popular GA content top_resources = ResourceStats.get_top(limit=last) return { 'table' : top_resources.get("resources")
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() try: context = { 'model': model, 'user': c.user, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') org = request.params.get('organization', None) c.query_error = False page = h.get_page_number(request.params) try: limit = int(config.get('ckan.datasets_per_page', 20)) except: limit = 20 # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search', alternative_url=package_type) c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } # Unless changed via config options, don't show other dataset # types any search page. Potential alternatives are do show them # on the default search page (dataset) or on one other search page search_all_type = config.get('ckan.search.show_all_types', 'dataset') search_all = False map_results = None try: # If the "type" is set to True or False, convert to bool # and we know that no type was specified, so use traditional # behaviour of applying this only to dataset type search_all = asbool(search_all_type) search_all_type = 'dataset' # Otherwise we treat as a string representing a type except ValueError: search_all = True if not package_type: package_type = 'dataset' if not search_all or package_type != search_all_type: # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) # loop the search query and get all the results # this workaround the 1000 rows solr hard limit HARD_LIMIT = 1000 conn = get_connection_redis() pager = 0 # crank up the pager limit high as using points cluster for better performance PAGER_LIMIT = 10000 data_dict_full_result = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': HARD_LIMIT, 'start': 0, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } if not org: # if no q, it is an init load or direct visit on / dataset log.info('### Not org ###') if not q: log.info('### Not q ###') if not conn.exists('redis_full_results'): log.info('### generating full results ###') # get full results and add to redis when there is not full results in redis full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) # adding to redis log.info('adding full results to redis') conn.set('redis_full_results', json.dumps(map_results)) # log.info(c.full_results) else: log.info('### using cached full results ###') map_results = json.loads( conn.get('redis_full_results')) # log.info(c.full_results) else: log.info('### With q ###') full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) else: log.info('### With org ###') if not q: log.info('### Not q ###') if not conn.exists('redis_full_results_%s' % org): log.info('### generating %s results ###' % org) # get full results and add to redis when there is not full results in redis full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) # adding to redis log.info('adding %s results to redis' % org) conn.set('redis_full_results_%s' % org, json.dumps(map_results)) # log.info(c.full_results) else: log.info('### using cached %s results ###' % org) map_results = json.loads( conn.get('redis_full_results_%s' % org)) # log.info(c.full_results) else: log.info('### With q ###') full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) # log.info(c.full_results) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError as se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort( 400, _('Invalid search query: {error_message}').format( error_message=str(se))) except SearchError as se: # May be bad input from the user, but may also be more serious like # bad code causing a SOLR syntax error, or a problem connecting to # SOLR log.error('Dataset search error: %r', se.args) c.query_error = True c.search_facets = {} c.page = h.Page(collection=[]) except NotAuthorized: abort(403, _('Not authorized to see this page')) c.search_facets_limits = {} for facet in c.search_facets.keys(): try: limit = int( request.params.get( '_%s_limit' % facet, int(config.get('search.facets.default', 10)))) except ValueError: abort( 400, _('Parameter "{parameter_name}" is not ' 'an integer').format(parameter_name='_%s_limit' % facet)) c.search_facets_limits[facet] = limit self._setup_template_variables(context, {}, package_type=package_type) return render(self._search_template(package_type), extra_vars={ 'dataset_type': package_type, 'map_results': map_results })
'average_tags_per_package': average_tags_per_package, } def tagless_report_option_combinations(): for organization in lib.all_organizations(include_none=True): for include_sub_organizations in (False, True): yield { 'organization': organization, 'include_sub_organizations': include_sub_organizations } tagless_report_info = { 'name': 'tagless-datasets', 'description': 'Datasets which have no tags.', 'option_defaults': OrderedDict(( ('organization', None), ('include_sub_organizations', False), )), 'option_combinations': tagless_report_option_combinations, 'generate': tagless_report, 'template': 'report/tagless-datasets.html', }
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = {'model': model, 'session': model.Session, 'user': c.user, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) def collection_information(collection_id=None): from ckanext.opensearch import config collections = config.load_settings("collections_list") collection_items = collections.items() for collection in collection_items: if collection[0] == collection_id: return dict(collection[1]) group_extras = c.group_dict.get('extras') group_collections = [] for extra in group_extras: if extra['key'] == 'collections': col_value = extra['value'].split(", ") for a in col_value: group_collections.append(a) c.collections = [] for collection_id in group_collections: c.collections.append(collection_information(collection_id)) collection_params = '' try: c.fields = [] c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value facets = OrderedDict() search_parameters = OrderedDict() default_facet_titles = { 'collection_name': _('Data Collections'), 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } sentinel_facet_titles = { 'FamilyName': _('Family Name'), 'ProductType': _('Product Type'), 'OrbitDirection': _('Orbit Direction'), 'Swath': _('Acquisition Mode'), 'TransmitterReceiverPolarisation': _('Polarisation'), } default_search_titles = { 'timerange_start': _('Timerange Start'), 'timerange_end': _('Timerange End'), } collection_names = ['Sentinel-1 Level-1 (SLC)', 'Sentinel-1 Level-1 (GRD)', 'Sentinel-1 Level-2 (OCN)', 'Sentinel-2 Level-1C', 'Sentinel-2 Level-2A', 'Sentinel-3 SRAL Level-1 SRA'] for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet for search_param in helpers.search_params(): if search_param in default_search_titles: search_parameters[search_param] = default_search_titles[search_param] # Facet titles self._update_facet_titles(facets, group_type) if collection_params not in collection_names: for f in facets: if f in sentinel_facet_titles: del facets[f] c.facet_titles = facets data_dict = { 'q': q, 'fq': '', 'include_private': True, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.group_dict['collection_count'] = len(c.collections) c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int(request.params.get('_%s_limit' % facet, config.get('search.facets.default', 10))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.page = h.Page(collection=[])
class package(PackageController): def search(self): ''' See ckan.controllers.package:PackageController for source, this is HACKed to fix broken scandic parameters. ''' from ckan.lib.search import SearchError package_type = self._guess_package_type() try: context = {'model': model, 'user': c.user or c.author} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): # HACK When removing scandic error prone values, check for their broken couterpart's presence in the URL parameters. # If found, replace given fixed value with broken one, otherwise remove_url_param won't find it and crashes. if value == u'Kaavat ja kiinteistöt' and ('categories', u'Kaavat ja kiinteist' + u"\ufffd" + 't') in request.params.items(): value = u'Kaavat ja kiinteist' + u"\ufffd" + 't' if value == u'Työ ja elinkeinot' and ('categories', u'Ty' + u"\ufffd" + u' ja elinkeinot') in request.params.items(): value = u'Ty' + u"\ufffd" + u' ja elinkeinot' if value == u'Väestö' and ('categories', u'V' + u"\ufffd" + u'est' + u"\ufffd") in request.params.items(): value = u'V' + u"\ufffd" + u'est' + u"\ufffd" if value == u'Ympäristö ja luonto' and ('categories', u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto') in request.params.items(): value = u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto' return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): # HACK Hardcoded replace for parameters with scandics that sometimes get replaced with unknown char (u"\ufffd"). if param == 'categories': if value == u'Kaavat ja kiinteist' + u"\ufffd" + 't': value = u'Kaavat ja kiinteistöt' if value == u'Ty' + u"\ufffd" + u' ja elinkeinot': value = u'Työ ja elinkeinot' if value == u'V' + u"\ufffd" + u'est' + u"\ufffd": value = u'Väestö' if value == u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto': value = u'Ympäristö ja luonto' c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True} if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('License'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def publisher_report(metric): orgs = dict(model.Session.query(model.Group.name, model.Group)\ .filter_by(state='active').all()) org_counts = collections.defaultdict(dict) if metric in ('views', 'viewsdownloads', 'visits'): if metric == 'views' or metric == 'viewsdownloads': sql_function = 'sum(pageviews::int)' elif metric == 'visits': sql_function = 'sum(visits::int)' q = ''' select department_id, period_name, %s metric from ga_url where department_id <> '' and package_id <> '' group by department_id, period_name order by department_id ''' % sql_function org_period_count = model.Session.connection().execute(q) for org_name, period_name, count in org_period_count: org_counts[org_name][period_name] = count if metric in ('downloads', 'viewsdownloads'): q = ''' select g.name as org_name, s.period_name, sum(s.value::int) as downloads from GA_Stat as s join Package as p on s.key=p.name join "group" as g on p.owner_org=g.id where stat_name='Downloads' and g.state='active' group by org_name, s.period_name order by downloads desc; ''' org_period_count = model.Session.connection().execute(q) if metric == 'viewsdownloads': # add it onto the existing counts for org_name, period_name, count in org_period_count: org_counts[org_name][period_name] = count + \ org_counts[org_name].get(period_name, 0) org_counts[org_name]['All'] = count + \ org_counts[org_name].get('All', 0) else: for org_name, period_name, count in org_period_count: org_counts[org_name][period_name] = count org_counts[org_name]['All'] = count + \ org_counts[org_name].get('All', 0) org_counts = sorted(org_counts.items(), key=lambda x: -x[1].get('All', 0)) all_periods = [ res[0] for res in model.Session.query(GA_Url.period_name).group_by( GA_Url.period_name).order_by(GA_Url.period_name).all() ] rows = [] for org_name, counts in org_counts: org = orgs.get(org_name) if not org: continue top_org = list(go_up_tree(org))[-1] row = OrderedDict(( ('organization title', org.title), ('organization name', org.name), ('top-level organization title', top_org.title), ('top-level organization name', top_org.name), )) for period_name in all_periods: row[period_name] = counts.get(period_name, 0) rows.append(row) # Group the periods by year, to help the template draw the table nicely #all_periods_tuples = [period.split('-') for period in all_periods # if '-' in period] #all_periods_tuples.sort(key=lambda x: x[0]) #all_periods_by_year = [ # (year, [p for y, p in year_periods]) # for year, year_periods in groupby(all_periods_tuples, lambda x: x[0])] return { 'table': rows, 'all periods': all_periods, #'all periods by year': all_periods_by_year }
def dashboard_facets(self, facets_dict, package_type): facets_dict = OrderedDict() facets_dict['capacity'] = u'ecodp.common.privacy_state' facets_dict = self._common_facets(facets_dict) return facets_dict
def report_resources_by_organization(context, data_dict): """ Returns a list of OrderedDicts (one for each dataset in an organization) sorted by the last modified date, then creation date (if no modifications have been made yet). Each OrderedDict contains the following keys: dataset_name, dataset_url, resource_name, resource_url, dataset_organisation, dataset_organisation_url, resource_created, resource_last_modified, resource_view_count, resource_download_count :return: a sorted list of OrderedDicts :rtype: list """ user = toolkit.c.user or context.get('name') org = data_dict.get('org_name') or context.get('org') report = [] if not helpers.verify_datasets_exist(org): return report if 'org_name' in data_dict: del data_dict['org_name'] if not helpers.is_admin(user, org): toolkit.abort( 403, _('You are not authorized to access this \ report or the organization does not exist.')) data_dict['include_private'] = True data_dict['q'] = 'organization:{0}'.format(org) results = toolkit.get_action('package_search')({}, data_dict) for item in results['results']: resources = item['resources'] organization = item['organization'] for resource in resources: # resource_view_count depends on tracking_summary, which # doesn't seem to be enabled. Once it's enabled, # resource_view_count will come from # resource.get('tracking_summary').get('total') # For now, there's a shortened version to avoid errors. # resource_download_count will also need to be looked into # when tracking_summary is enabled. report.append( OrderedDict([ ('dataset_name', item.get('title')), ('dataset_url', (config.get('ckan.site_url') + '/dataset/{0}'.format(item.get('name')))), ('resource_name', resource.get('name')), ('resource_url', resource.get('url')), ('dataset_organization', organization.get('name')), ('dataset_organization_url', (config.get('ckan.site_url') + '/organization/{0}'.format(organization.get('name')))), ('resource_created', resource.get('created')), ('resource_last_modified', resource.get('last_modified')), ('resource_view_count', resource.get('tracking_summary', 0)), ('resource_download_count', resource.get('downloads', 0)) ])) return sorted(report, key=lambda x: (x['resource_last_modified'], x['resource_created']), reverse=True)
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() try: context = { 'model': model, 'user': c.user, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = h.get_page_number(request.params) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field # This is the only difference between this and core implementation sort_by = request.params.get('sort', 'metadata_modified desc') # <- params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError, se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort( 400, _('Invalid search query: {error_message}').format( error_message=str(se)))
class ECODPGroupController(GroupController): def index(self): start = time.time() group_type = self._guess_group_type() language = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en') context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False} q = c.q = request.params.get('q', '') data_dict = {'all_fields': True, 'q': q} sort_by = c.sort_by_selected = request.params.get('sort') if sort_by: data_dict['sort'] = sort_by try: self._check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin results = self._action('group_list')(context, data_dict) c.amount_group_displayed = amount_group_displayed c.groups = results[:amount_group_displayed] c.hasmoregroups = len(results) > amount_group_displayed c.themes = self._action('theme_list')(context, {}) c.catalogs = CatalogDcatApOp.get_ui_list_catalogs(config.get('ckan.locale_default', 'en')) c.amount_catalog_displayed = amount_catalog_displayed #c.page = h.Page( # collection=results, # page=request.params.get('page', 1), # url=h.pager_url, # items_per_page=21 #) # @cache.cache('cached_render', expire=3600) def cached_render(user, languge, group_type): _render = base.render(group_type) return _render start_render = time.time() _render = cached_render(context.get('user'), language, self._index_template(group_type)) duration_render= time.time() - start_render log.info("Duration index render. {0}".format(duration_render)) duration = time.time() - start log.info("Duration index. {0}".format(duration)) return _render def read(self, id, limit=20): start = time.time() if request.GET.get('ext_boolean') in ['all', 'any', 'exact']: base.session['ext_boolean'] = request.GET['ext_boolean'] base.session.save() language = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en') group_type = self._get_group_type(id.split('@')[0]) if (group_type != self.group_type) and (group_type != "eurovoc_domain"): abort(404, _('Incorrect group type')) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True} # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict = {'id': id, 'include_datasets': False} # unicode format (decoded from utf8) q = c.q = request.params.get('q', '') try: stratqs = time.time() if 'eurovoc_domain' in data_dict.get('id','') : raise NotFound('EurovocDomains are not available any more') result = self._action('group_show_read')(context, data_dict) c.group_dict = result.get('group_dict', None) context["group"] = result.get('group') durationgs = time.time() - stratqs log.info("Duration group show read. {0}".format(durationgs)) c.group = context['group'] except NotFound: abort(404, _('Group not found')) except NotAuthorized: abort(401, _('Unauthorized to read group %s') % id) self._read(id, limit) start_render = time.time() # @cache.cache('render_cached_read', expire=3600) def render_cached(user, id, language, group_type): _render = base.render(self._read_template(c.group_dict['type'])) return _render _render = render_cached(context.get('user'), id, language, c.group_dict['type']) duration_render = time.time() - start_render # _render = base.render(self._read_template(c.group_dict['type'])) duration = time.time() - start log.info("Duration read_group. {0}".format(duration)) return _render def _read(self, id, limit=20): import time language = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en') start = time.time() ''' This is common code used by both read and bulk_process''' group_type = self._get_group_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if q != u'': qTab = q.split(' ') checkbox = request.params.get('ext_boolean') if checkbox == 'all': q = '(' + ' AND '.join(qTab) + ')' elif checkbox == 'any': q = '(' + ' OR '.join(qTab) + ')' else: #checkbox == 'exact' q = '"' + q + '"' c.description_formatted = h.render_markdown(c.group_dict.get('description')) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id) try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] new_params_nopage = [] for key, value in params_nopage: if key == 'eurovoc_domains': new_params_nopage.append(('groups', value)) else: new_params_nopage.append((key,value)) params_nopage = new_params_nopage #sort_by = request.params.get('sort', 'name asc') sort_by = request.params.get('sort', None) @cache.cache('search_url', expire=3600) def search_url(params): if group_type == 'organization': if c.action == 'bulk_process': url = self._url_for(controller='organization', action='bulk_process', id=id) else: url = self._url_for(controller='organization', action='read', id=id) else: url = self._url_for(controller='group', action='read', id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) @cache.cache('drill_down_url', expire=3600) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): if c.group_dict.get('is_organization'): return h.remove_url_param(key, value=value, replace=replace, controller='organization', action='read', extras=dict(id=c.group_dict.get('id'))) else: return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) param = 'eurovoc_domains' if (param == 'eurovoc_domains') else param; q += ' %s:"%s"' % (param, value) else: search_extras[param] = value fq = '' if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') elif c.group_dict.get('name') not in q: q += ' groups:"%s"' % c.group_dict.get('name') fq = fq + ' capacity:"public"' user_member_of_orgs = [org['id'] for org in h.organizations_available('read')] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = {'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses')} for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in plugins.PluginImplementations(plugins.IFacets): if self.group_type == 'organization': facets = plugin.organization_facets( facets, self.group_type, None) else: facets = plugin.group_facets( facets, self.group_type, None) if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras, 'defType': 'edismax' } active_cache = config.get('ckan.cache.active', 'false') context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') has_result = False dict_as_pickle = None if active_cache == 'true': dict_as_pickle = pickle.dumps(data_dict) query_json = redis.get_from_cache(dict_as_pickle, pool=redis.MISC_POOL) if query_json: query = pickle.loads(query_json) has_result = True if has_result == False: query = get_action('package_search')(context_, data_dict) if active_cache == 'true': redis.set_value_in_cache(dict_as_pickle, pickle.dumps(query), pool=redis.MISC_POOL) c.search_url_params = urlencode(_encode_params(params_nopage)) c.page = page_util.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] #maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int(request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by duration = time.time() - start log.info("Duration _read. {0}".format(duration)) except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = page_util.Page(collection=[]) c.search_url_params = ''
class EDCOrganizationController(OrganizationController): def index(self): # FIXME: index copied from GroupController and modified to # show only parent groups context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False } data_dict = {'all_fields': False} try: self._check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin search_result = self._action('organization_list')(context, data_dict) org_model = context['model'] #Get the list of all groups of type "organization" that have no parents. top_level_orgs = org_model.Group.get_top_level_groups( type="organization") top_results = [ org for org in top_level_orgs if org.name in search_result ] facets = OrderedDict() facets['organization'] = _('Organizations') data_dict = { 'facet.field': facets.keys(), } query = get_action('package_search')(context, data_dict) c.org_pkg_count = query['facets'].get('organization') c.top_orgs_items = top_results return render('organization/index.html') def _read(self, id, limit, group_type='organization'): # FIXME: copied and modified from GroupController to collect # sub organizations, create c.fields_grouped and hard-code # search facets ''' This is common code used by both read and bulk_process''' group_type = self._get_group_type(id.split('@')[0]) context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True } # Get the subgorgs of this org org_id = c.group_dict.get('id') q = c.q = request.params.get('q', '') # XXX: unfortunate hack, copy sort default behaviour from # before_search because we're using the q parameter below # even when no text query was submitted if not q and request.params.get('sort') in (None, 'rank'): sort_by = 'record_publish_date desc, metadata_modified desc' else: sort_by = request.params.get('sort', None) suborgs = ['"' + org + '"' for org in get_suborgs(org_id)] if suborgs != []: q += ' owner_org:("' + org_id + '" OR ' + ' OR '.join( suborgs) + ')' else: q += ' owner_org:"%s"' % org_id c.description_formatted = h.render_markdown( c.group_dict.get('description')) context['return_query'] = True try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def search_url(params): action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller='organization', action=action, id=id) params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='organization', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='organization', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} c.fields_grouped = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [ org['id'] for org in h.organizations_available('read') ] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'edc_state': _('States'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), 'type': _('Dataset types') } for facet in default_facet_titles: facets[facet] = default_facet_titles[facet] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int( request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
class ECPortalUserController(core_user.UserController): def logged_in(self): #override the logged_in method after the access was granted or denied #if login was successfull check the group membership if c.user: util.add_missing_user_to_all_groups() return super(ECPortalUserController, self).logged_in() def login_user(self, tup): if tup: user = tup[0] user_object = model.User.get(user) if not user_object: user_object = model.User.get('api') c.userobj = user_object c.user = '******' try: # Create user user_attributes = tup[1] user_name = user user_email = user_attributes["email"].text user_password = "******" user_fullname = user_attributes[ "firstName"].text + " " + user_attributes[ "lastName"].text h.get_action( 'user_create', { "name": user_name, "email": user_email, "password": user_password, "fullname": user_fullname }) rememberer = request.environ['repoze.who.plugins'][ 'friendlyform'] identity = {'repoze.who.userid': user} response.headerlist += rememberer.remember( request.environ, identity) except logic.ActionError: base.abort(401, _('Error while creating new user')) else: c.userobj = user_object c.user = user rememberer = request.environ['repoze.who.plugins'][ 'friendlyform'] identity = {'repoze.who.userid': user} response.headerlist += rememberer.remember( request.environ, identity) self.logged_in() def read(self, id=None): context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'for_view': True, 'return_minimal': True, 'save': 'save' in request.params } data_dict = {'id': id, 'user_obj': c.userobj} context['with_related'] = True self._setup_user_template_variables(context, data_dict) if context['save']: request.POST.pop('save') data = ecportal_logic.transform_to_data_dict(request.POST) credential_validator = Validator({ 'contact_mailbox': [email] #'contact_phone_number': [IntPhoneNumberRule] }) errors = {} succeed = credential_validator.validate(data, results=errors) if succeed is True: contact_info = Package_contact_info.get_by_user(c.userobj.id) if not contact_info: contact_info = Package_contact_info(c.userobj.id) contact_info.from_dict(data) contact_info.save() h.flash_success(_('ecodp.user.save.success')) elif errors: h.flash_error(_('ecodp.user.save.error')) c.user_dict.update(data) c.errors = errors # The legacy templates have the user's activity stream on the user # profile page, new templates do not. if h.asbool(config.get('ckan.legacy_templates', False)): c.user_activity_stream = get_action('user_activity_list_html')( context, { 'id': c.user_dict['id'] }) #vars = {'data': {}, 'errors': {}, 'error_summary': {}} #c.form = base.render('user/edit_contact_info_form.html', extra_vars=vars) return base.render('user/read.html') def read_contact_info(self, id=None): context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'for_view': True, 'return_minimal': True, 'save': 'save' in request.params } data_dict = {'id': id, 'user_obj': c.userobj} context['with_related'] = True self._setup_user_template_variables(context, data_dict) if context['save']: request.POST.pop('save') data = ecportal_logic.transform_to_data_dict(request.POST) credential_validator = Validator({ 'contact_mailbox': [email] #'contact_phone_number': [IntPhoneNumberRule] }) errors = {} succeed = credential_validator.validate(data, results=errors) if succeed is True: contact_info = Package_contact_info.get_by_user(c.userobj.id) if not contact_info: contact_info = Package_contact_info(c.userobj.id) contact_info.from_dict(data) contact_info.save() h.flash_success(_('ecodp.user.save.success')) elif errors: h.flash_error(_('ecodp.user.save.error')) c.user_dict.update(data) c.errors = errors # The legacy templates have the user's activity stream on the user # profile page, new templates do not. if h.asbool(config.get('ckan.legacy_templates', False)): c.user_activity_stream = get_action('user_activity_list_html')( context, { 'id': c.user_dict['id'] }) #vars = {'data': {}, 'errors': {}, 'error_summary': {}} #c.form = base.render('user/edit_contact_info_form.html', extra_vars=vars) return base.render('user/read_contact_info.html') def dashboard(self): def query_for_datasets(): query = '' if not new_authz.is_sysadmin(c.user): organisation_ids = self.__get_users_organizations_ids(c.user) if not organisation_ids: # return an empty query result with the needed (i.e.: accessed later) elements return {'results': [], 'count': 0} query = build_owner_filter(organisation_ids) if c.q != u'': search = c.q.strip().split(' ') result = '' if request.params.get('ext_boolean') == 'any': result = build_search_for_any_words(search) elif request.params.get('ext_boolean') == 'all': result = build_search_for_all_words(search) elif request.params.get('ext_boolean') == 'exact': result = '("%s")' % (c.q) log.info("%s" % (request.params.get('ext_boolean'))) query = query + " + title:" + result return query log.debug("Entering MDT dashboard") c.is_sysadmin = new_authz.is_sysadmin(c.user) context = { 'for_view': True, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'return_minimal': True, 'ignore_capacity_check': True # Display also private datasets } user_data = {"id": c.userobj.id, 'user_obj': c.userobj} try: user_dict = logic.get_action('user_show')(context, user_data) except logic.NotFound: base.abort(404, _('User not found')) except logic.NotAuthorized: base.abort(401, _('Not authorized to see this page')) c.user_dict = user_dict c.is_myself = user_dict['name'] == c.user c.about_formatted = h.render_markdown(user_dict['about']) c.search_choice = request.params.get('ext_boolean') # Save the used search paramters in a proper format to the exchange variable user = user_dict['name'] log.debug("Start getting the datasets") # --------------------------------------------------------------------------------------------------------------- # Get the datasets from ckan.lib.search import SearchError package_type = self._guess_package_type() if request.GET.get('ext_boolean') in ['all', 'any', 'exact']: session['ext_boolean'] = request.GET['ext_boolean'] session.save() # unicode format (decoded from utf8) c.q = request.params.get('q', u'') c.query_error = False q = query_for_datasets() try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] new_params_nopage = [] for key, value in params_nopage: if key == 'eurovoc_domains': new_params_nopage.append(('groups', value)) else: new_params_nopage.append((key, value)) params_nopage = new_params_nopage def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param( key, value=value, replace=replace, controller= 'ckanext.ecportal.controllers.user:ECPortalUserController', action='dashboard') c.remove_field = remove_field sort_by = request.params.get('sort', 'views_total desc') params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode( core_package._encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' # if request.params.get('private'): # fq = 'private: %s' % request.params.get('private') for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort', 'id'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) # paramFQ = 'groups' if (param == 'eurovoc_domains') else param; fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value # if package_type and package_type != 'dataset': # Only show datasets of this particular type # fq += ' +dataset_type:{type}'.format(type=package_type) # else: # Unless changed via config options, don't show non standard # dataset types on the default search page # if not asbool(config.get('ckan.search.show_all_types', 'False')): # fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dashboard_facets(facets, package_type) c.facet_titles = facets facet_fields = facets.keys() facet_fields.append('private') data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facet_fields, 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } if sort_by == 'modified_date desc': # This is the customized part for ODP-570 # add the group parameter to the solr query data_dict['group'] = 'true' data_dict['group.query'] = [ '-organization:estat AND -organization:comp AND -organization:grow', 'organization:estat', 'organization:comp', 'organization:grow' ] data_dict['group.format'] = 'simple' data_dict['rows'] = 2147483646 query = get_action('custom_package_search')(context, data_dict) cached_result = [] for name, group in query['groups'].iteritems(): cached_result += group['doclist']['docs'] start = (page - 1) * limit result_list = customsearch.check_solr_result( context, cached_result[start:], limit) else: query = get_action('package_search')(context, data_dict) result_list = query['results'] c.sort_by_selected = query['sort'] c.page = page_util.Page(collection=result_list, page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = result_list except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = page_util.Page(collection=[])
# coding=utf8 # '''Settings and constants for Kata CKAN extension''' from ckan.common import OrderedDict from pylons.i18n.translation import gettext_noop as N_ PID_TYPES = ['primary', 'relation'] # Overridden CKAN role permissions ROLE_PERMISSIONS = OrderedDict([ ('admin', ['admin']), ('editor', ['admin']), ('member', ['read', 'create_dataset']), ]) ORGANIZATION_MEMBER_PERMISSIONS = { # ORGANIZATION_MEMBER_PERMISSIONS format: # (user role, target original role, target final role, user == target): has permission to modify role # Permission to delete a role is given if one has permission to change role to 'member'. ('admin', 'admin', 'admin', True): False, ('admin', 'admin', 'admin', False): False, ('admin', 'admin', 'editor', True): False, ('admin', 'admin', 'editor', False): False, ('admin', 'admin', 'member', True): False, # Admin should not be able to delete oneself ('admin', 'admin', 'member', False):
# -*- coding: utf-8 -*- from ckan.common import OrderedDict from pylons import config from ckan.plugins import toolkit import ckanapi groups = OrderedDict([ ('local-governments', ('Local Governments', 'Municipios')), ('health', ('Health', 'Salud')), ('business', ('Business', 'Comercio e Industrias')), ('transportation', ('Transportation', 'Transporte y Logística')), ('education', ('Education', 'Educación')), ('justice', ('Justice', 'Justicia')), ('social-development', ('Social Development', 'Desarrollo Social')), ('finance', ('Finance', 'Finanzas Publicas')), ('labour', ('Labour', 'Empleo')), ('environment', ('Environment', 'Ambiente')), ]) class CreateFeaturedGroups(toolkit.CkanCommand): '''Create featured groups Usage: create_featured_groups - create featured groups ''' summary = __doc__.split('\n')[0] usage = __doc__ max_args = 0
default roles with custom roles and decorates has_user_permission_for_group_or_org to allow an approver to editor groups, Approveer can act as editor plus accession to all the process states, but have no other sysadmin powers ''' from ckan import authz, model from ckan.common import OrderedDict from ckan.plugins import toolkit # these are the permissions that roles have authz.ROLE_PERMISSIONS = OrderedDict([ ('admin', ['admin']), ('approver', [ 'read', 'delete_dataset', 'create_dataset', 'update_dataset', 'manage_group', 'workflow' ]), ('editor', [ 'read', 'delete_dataset', 'create_dataset', 'update_dataset', 'manage_group' ]), ('member', ['read', 'manage_group']), ]) def _trans_role_approver(): return toolkit._('Workflow Approver') authz._trans_role_approver = _trans_role_approver def is_approver_decorator(method):
def dataset_facets(self, facets_dict, package_type): facets_dict = OrderedDict() facets_dict = self._common_facets(facets_dict) return facets_dict
item_count=query['count'], items_per_page=int(limit)) c.page.items = query['results'] c.results = c.page.items c.form = self.render_package_form() except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.query_count = 0 c.facets = {} c.page = h.Page(collection=[]) # Facets for our sidebar. facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('License'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet
def group_facets(self, facets_dict, group_type, package_type): facets_dict = OrderedDict() facets_dict = self._common_facets(facets_dict) return facets_dict
def _get_fields_types(context, data_dict): all_fields = _get_fields(context, data_dict) all_fields.insert(0, {'id': '_id', 'type': 'int'}) field_types = OrderedDict([(f['id'], f['type']) for f in all_fields]) return field_types
def organization_facets(self, facets_dict, organization_type, package_type): facets_dict = OrderedDict() facets_dict = self._common_facets(facets_dict) return facets_dict
def _read(id, limit, group_type): u''' This is common code used by both read and bulk_process''' extra_vars = {} context = { u'model': model, u'session': model.Session, u'user': g.user, u'schema': _db_to_form_schema(group_type=group_type), u'for_view': True, u'extras_as_string': True } q = request.params.get(u'q', u'') # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.q = q # Search within group if g.group_dict.get(u'is_organization'): fq = u' owner_org:"%s"' % g.group_dict.get(u'id') else: fq = u' groups:"%s"' % g.group_dict.get(u'name') extra_vars["q"] = q g.description_formatted = \ h.render_markdown(g.group_dict.get(u'description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != u'page'] sort_by = request.params.get(u'sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = u'bulk_process' if getattr( g, u'action', u'') == u'bulk_process' else u'read' url = h.url_for(u'.'.join([controller, action]), id=id) params = [(k, v.encode(u'utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param( alternative_url=None, controller=u'group', action=u'read', extras=dict(id=g.group_dict.get(u'name')), new_params=by) extra_vars["drill_down_url"] = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param( key, value=value, replace=replace, controller=controller, action=u'read', extras=dict(id=g.group_dict.get(u'name'))) extra_vars["remove_field"] = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append((u'page', page)) return search_url(params) try: extra_vars["fields"] = fields = [] extra_vars["fields_grouped"] = fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in [u'q', u'page', u'sort'] \ and len(value) and not param.startswith(u'_'): if not param.startswith(u'ext_'): fields.append((param, value)) q += u' %s: "%s"' % (param, value) if param not in fields_grouped: fields_grouped[param] = [value] else: fields_grouped[param].append(value) else: search_extras[param] = value # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.fields = fields g.fields_grouped = fields_grouped facets = OrderedDict() default_facet_titles = { u'organization': _(u'Organizations'), u'groups': _(u'Groups'), u'tags': _(u'Tags'), u'res_format': _(u'Formats'), u'license_id': _(u'Licenses') } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles _update_facet_titles(facets, group_type) extra_vars["facet_titles"] = facets data_dict = { u'q': q, u'fq': fq, u'include_private': True, u'facet.field': facets.keys(), u'rows': limit, u'sort': sort_by, u'start': (page - 1) * limit, u'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != u'schema') query = get_action(u'package_search')(context_, data_dict) extra_vars["page"] = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.group_dict['package_count'] = query['count'] extra_vars["search_facets"] = g.search_facets = query['search_facets'] extra_vars["search_facets_limits"] = g.search_facets_limits = {} for facet in g.search_facets.keys(): limit = int( request.params.get(u'_%s_limit' % facet, config.get(u'search.facets.default', 10))) g.search_facets_limits[facet] = limit extra_vars["page"].items = query['results'] extra_vars["sort_by_selected"] = sort_by except search.SearchError as se: log.error(u'Group search error: %r', se.args) extra_vars["query_error"] = True extra_vars["page"] = h.Page(collection=[]) # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.facet_titles = facets g.page = extra_vars["page"] extra_vars["group_type"] = group_type _setup_template_variables(context, {u'id': id}, group_type=group_type) return extra_vars
class EDCUserController(UserController): def dashboard_unpublished(self): if not c.userobj: abort(401, _('You must be logged-in to access the dashboard.')) user_id = c.userobj.id fq = ' +edc_state:("DRAFT" OR "PENDING PUBLISH" OR "REJECTED")' # Get the list of organizations that this user is the admin if not c.userobj.sysadmin: user_orgs = get_orgs_user_can_edit(c.userobj) if len(user_orgs) > 0: fq += ' +owner_org:(' + ' OR '.join(user_orgs) + ')' self._user_datasets('dashboard_unpublished', c.userobj.id, fq) return render('user/dashboard_unpublished.html') def dashboard_datasets(self): if not c.userobj: abort(401, _('You must be logged-in to access the dashboard.')) fq = ' +author:("%s")' % (c.userobj.id) self._user_datasets('dashboard_datasets', c.userobj.id, fq) return render('user/dashboard_datasets.html') def read(self, id=None): if c.userobj and c.userobj.sysadmin == True: fq = '' else: fq = ' +(edc_state:("PUBLISHED" OR "PENDING ARCHIVE")' if c.userobj: user_id = c.userobj.id user_orgs = get_orgs_user_can_edit(c.userobj) if len(user_orgs) > 0: fq += ' OR owner_org:(' + ' OR '.join(user_orgs) + ')' fq += ')' self._user_datasets('read', id, fq) return render('user/read.html') def _user_datasets(self, action, id=None, filter_query=None): from ckan.lib.search import SearchError context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'for_view': True } user_dict = { 'id': id, 'user_obj': c.userobj, 'include_datasets': False } # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') context['return_query'] = True try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = int(toolkit.config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): base_url = config.get('ckan.site_url') if action == 'dashboard_datasets': url = base_url + '/dashboard/datasets' elif action == 'dashboard_unpublished': url = base_url + '/dashboard/unpublished' elif action == 'read': url = h.url_for(controller='user', action=action, id=id) else: url = h.url_for(controller='user', action=action) params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urlencode(params) def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='user', action=action, extras=dict(id=c.userobj.id), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='user', action=action, extras=dict(id=c.userobj.id)) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s:"%s"' % (param, value) else: search_extras[param] = value facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'edc_state': _('States'), 'tags': _('Tags'), 'res_format': _('Formats'), } for facet in default_facet_titles: facets[facet] = default_facet_titles[facet] c.facet_titles = facets fq = filter_query or '' data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) user_dict['package_count'] = query['count'] c.facets = query['facets'] #maintain.deprecate_context_item('facets', # 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int( request.params.get( '_%s_limit' % facet, int(toolkit.config.get('search.facets.default', 10)))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except SearchError, se: log.error('User search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])