def index(self): c.q = request.params.get('q', '') context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True } # This is the one difference from ckan core. # No need for all vocab / translation stuff, so save the massive # effort of dictizing every tag. This improves page load from # 60s to 10s. data_dict = {'all_fields': False} if c.q: try: page = int(request.params.get('page', 1)) except ValueError: abort(404, _('Not found')) data_dict['q'] = c.q data_dict['limit'] = LIMIT data_dict['offset'] = (page - 1) * LIMIT data_dict['return_objects'] = True result_dict = get_action('tag_search')(context, data_dict) def pager_url(q=None, page=None): return h.url_for( controller='ckanext.dgu.controllers.tag:TagController', action='index', q=request.params['q'], page=page) c.page = h.Page( collection=result_dict['results'], page=page, item_count=result_dict['count'], items_per_page=LIMIT, url=pager_url, ) c.page.items = [ tag_dict['name'] for tag_dict in result_dict['results'] ] else: results = get_action('tag_list')(context, data_dict) c.page = AlphaPageLarge( collection=results, page=request.params.get('page', 'A'), alpha_attribute='name', other_text=_('Other'), controller_name='ckanext.dgu.controllers.tag:TagController', ) return render('tag/index.html')
def report_users(self): context = {'model': model, 'session': model.Session, 'user': c.user or c.author} try: check_access('group_create', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) q = model.Session.query(model.User).order_by(model.User.created.desc()) c.count = q.count() c.page = h.Page( collection=q, page=int(request.params.get('page', 1)), url=h.pager_url, items_per_page=report_limit, ) return render('publisher/report_users.html')
def report_organisations_and_users(self): context = { 'model': model, 'session': model.Session, 'user': c.user or c.author } try: check_access('group_create', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) q = model.Group.all('organization') c.count = q.count() c.page = h.Page( collection=q, page=int(request.params.get('page', 1)), url=h.pager_url, items_per_page=report_limit, ) return render('organisation/report_organisations_and_users.html')
class OrganizationController(GroupController): def source_list(self, id, limit=20): self.group_type = 'organization' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=self.group_type), 'for_view': True} data_dict = {'id': id} # unicode format (decoded from utf8) q = c.q = request.params.get('q', '') try: c.group_dict = self._action('group_show')(context, data_dict) c.group = context['group'] except p.toolkit.ObjectNotFound: abort(404, p.toolkit._('Group not found')) except p.toolkit.NotAuthorized: abort(401, p.toolkit._('Unauthorized to read group %s') % id) self._read(id, limit, dataset_type=DATASET_TYPE_NAME) return render('source/org_source_list.html') def _read(self, id, limit, dataset_type=None): ''' This is common code used by both read and bulk_process''' self.group_type = 'organization' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=self.group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org: "%s"' % c.group_dict.get('id') else: q += ' groups: "%s"' % c.group_dict.get('name') context['return_query'] = True try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] #sort_by = request.params.get('sort', 'name asc') sort_by = request.params.get('sort', None) def search_url(params): if self.group_type == 'organization': if c.action == 'bulk_process': url = self._url_for(controller='organization', action='bulk_process', id=id) elif c.action == 'source_list': url = self._url_for('harvest_org_list', id=id) else: url = self._url_for(controller='organization', action='read', id=id) else: url = self._url_for(controller='group', action='read', id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [org['id'] for org in h.organizations_available('read')] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = {'groups': p.toolkit._('Groups'), 'tags': p.toolkit._('Tags'), 'res_format': p.toolkit._('Formats'), 'license': p.toolkit._('Licence'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet if dataset_type: fq = fq + 'dataset_type:"{dataset_type}"'.format(dataset_type=dataset_type) # Facet titles for plugin in p.PluginImplementations(p.IFacets): if self.group_type == 'organization': facets = plugin.organization_facets( facets, self.group_type, dataset_type) else: facets = plugin.group_facets( facets, self.group_type, dataset_type) if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } query = p.toolkit.get_action('package_search')(context, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] maintain.deprecate_context_item( 'facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int(request.params.get('_%s_limit' % facet, 10)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
class PackageController(BaseController): ## hooks for subclasses package_form = 'package/new_package_form.html' def _form_to_db_schema(self): return package_form_schema() def _db_to_form_schema(self): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' def _check_data_dict(self, data_dict): '''Check if the return data is correct, mostly for checking out if spammers are submitting only part of the form''' # Resources might not exist yet (eg. Add Dataset) surplus_keys_schema = [ '__extras', '__junk', 'state', 'groups', 'extras_validation', 'save', 'return_to', 'resources' ] schema_keys = package_form_schema().keys() keys_in_schema = set(schema_keys) - set(surplus_keys_schema) missing_keys = keys_in_schema - set(data_dict.keys()) if missing_keys: #print data_dict #print missing_keys log.info('incorrect form fields posted') raise DataError(data_dict) def _setup_template_variables(self, context, data_dict): c.groups_authz = get_action('group_list_authz')(context, data_dict) data_dict.update({'available_only': True}) c.groups_available = get_action('group_list_authz')(context, data_dict) c.licences = [('', '')] + model.Package.get_license_options() c.is_sysadmin = Authorizer().is_sysadmin(c.user) ## This is messy as auths take domain object not data_dict context_pkg = context.get('package', None) pkg = context_pkg or c.pkg if pkg: try: if not context_pkg: context['package'] = pkg check_access('package_change_state', context) c.auth_for_change_state = True except NotAuthorized: c.auth_for_change_state = False ## end hooks authorizer = ckan.authz.Authorizer() def search(self): try: context = {'model': model, 'user': c.user or c.author} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) q = c.q = request.params.get('q', u'') # unicode format (decoded from utf8) c.query_error = False try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = 20 # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(**by): params = list(params_nopage) params.extend(by.items()) return search_url(set(params)) c.drill_down_url = drill_down_url def remove_field(key, value): params = list(params_nopage) params.remove((key, value)) return search_url(params) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author } data_dict = { 'q': q, 'facet.field': g.facets, 'rows': limit, 'start': (page - 1) * limit, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.page.items = query['results'] except SearchError, se: c.query_error = True c.facets = {} c.page = h.Page(collection=[])
data_dict = { 'q': q, 'fq': fq, 'facet.field': g.facets, 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] maintain.deprecate_context_item( 'facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.facet_titles = {'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license': _('Licence'), } c.search_facets_limits = {}
class PackageController(BaseController): def _package_form(self, package_type=None): return lookup_package_plugin(package_type).package_form() def _form_to_db_schema(self, package_type=None): return lookup_package_plugin(package_type).form_to_db_schema() def _db_to_form_schema(self, package_type=None): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' return lookup_package_plugin(package_type).db_to_form_schema() def _check_data_dict(self, data_dict, package_type=None): '''Check if the return data is correct, mostly for checking out if spammers are submitting only part of the form''' return lookup_package_plugin(package_type).check_data_dict(data_dict) def _setup_template_variables(self, context, data_dict, package_type=None): return lookup_package_plugin(package_type).\ setup_template_variables(context, data_dict) def _new_template(self, package_type): return lookup_package_plugin(package_type).new_template() def _comments_template(self, package_type): return lookup_package_plugin(package_type).comments_template() def _search_template(self, package_type): return lookup_package_plugin(package_type).search_template() def _read_template(self, package_type): return lookup_package_plugin(package_type).read_template() def _history_template(self, package_type): return lookup_package_plugin(package_type).history_template() def _guess_package_type(self, expecting_name=False): """ Guess the type of package from the URL handling the case where there is a prefix on the URL (such as /data/package) """ # Special case: if the rot URL '/' has been redirected to the package # controller (e.g. by an IRoutes extension) then there's nothing to do # here. if request.path == '/': return 'dataset' parts = [x for x in request.path.split('/') if x] idx = -1 if expecting_name: idx = -2 pt = parts[idx] if pt == 'package': pt = 'dataset' return pt authorizer = ckan.authz.Authorizer() def search(self): from ckan.lib.search import SearchError package_type = self._guess_package_type() try: context = {'model': model, 'user': c.user or c.author} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): params = set(params_nopage) params |= set(by.items()) if alternative_url: return url_with_params(alternative_url, params) return search_url(params) c.drill_down_url = drill_down_url def remove_field(key, value): params = list(params_nopage) params.remove((key, value)) return search_url(params) c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True } data_dict = { 'q': q, 'fq': fq, 'facet.field': g.facets, 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
class PackageController(BaseController): def _package_form(self, package_type=None): return lookup_package_plugin(package_type).package_form() def _form_to_db_schema(self, package_type=None): return lookup_package_plugin(package_type).form_to_db_schema() def _db_to_form_schema(self, package_type=None): '''This is an interface to manipulate data from the database into a format suitable for the form (optional)''' return lookup_package_plugin(package_type).db_to_form_schema() def _check_data_dict(self, data_dict, package_type=None): '''Check if the return data is correct, mostly for checking out if spammers are submitting only part of the form''' return lookup_package_plugin(package_type).check_data_dict(data_dict) def _setup_template_variables(self, context, data_dict, package_type=None): return lookup_package_plugin(package_type).setup_template_variables( context, data_dict) authorizer = ckan.authz.Authorizer() def search(self): from ckan.lib.search import SearchError try: context = {'model': model, 'user': c.user or c.author} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) q = c.q = request.params.get('q', u'') # unicode format (decoded from utf8) c.query_error = False try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = 20 # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(**by): params = list(params_nopage) params.extend(by.items()) return search_url(set(params)) c.drill_down_url = drill_down_url def remove_field(key, value): params = list(params_nopage) params.remove((key, value)) return search_url(params) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} fq = '' for (param, value) in request.params.items(): if not param in ['q', 'page'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True } data_dict = { 'q': q, 'fq': fq, 'facet.field': g.facets, 'rows': limit, 'start': (page - 1) * limit, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) c.query_error = False try: # Change returned package dict c.pkg_dict = self.before_view(c.pkg_dict) c.page = h.Page(collection=c.pkg_dict.get('elastic_resources'), page=page, url=pager_url, item_count=c.pkg_dict.get('elastic_hits'), items_per_page=limit) except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[]) # TODO: Find another way to get this action action_name = 'read' # Get fields list for this action report = model.Session.query(lightbaseDatasetReports).\ filter(lightbaseDatasetReports.dataset_action_id == lightbaseDatasetActions.dataset_action_id).\
def index(self): group_type = self._guess_group_type() page = h.get_page_number(toolkit.request.params) or 1 items_per_page = 50 context = {'model': model, 'session': model.Session, 'user': toolkit.c.user, 'for_view': True, 'with_private': False} q = toolkit.c.q = toolkit.request.params.get('q', '') sort_by = toolkit.c.sort_by_selected = toolkit.request.params.get('sort') startswith = toolkit.request.params.get('startswith') try: self._check_access('site_read', context) self._check_access('group_list', context) except toolkit.NotAuthorized: toolkit.abort(403, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if toolkit.c.userobj: context['user_id'] = toolkit.c.userobj.id context['user_is_admin'] = toolkit.c.userobj.sysadmin try: data_dict_global_results = { 'all_fields': True, 'q': q, 'sort': sort_by, 'type': group_type or 'group', 'include_extras': True, } global_results = self._action('group_list')( context, data_dict_global_results) except toolkit.ValidationError as e: if e.error_dict and e.error_dict.get('message'): msg = e.error_dict['message'] else: msg = str(e) h.flash_error(msg) toolkit.c.page = h.Page([], 0) return toolkit.render( self._index_template(group_type), extra_vars={'group_type': group_type} ) start = items_per_page * (page - 1) end = start + items_per_page page_results = global_results if startswith is not None: page_results = [x for x in global_results if x['name'][0].lower() == startswith] toolkit.c.page = h.Page( collection=page_results, page=page, url=h.pager_url, items_per_page=items_per_page, ) toolkit.c.page.items = page_results[start:end] return toolkit.render( self._index_template(group_type), extra_vars={ 'total_orgs': len(global_results), 'group_type': group_type, 'org_initials': set([x['name'][0].lower() for x in global_results]), } )
class lightbaseAdminController(AdminController): """ Add controller to log page """ def log(self): import sqlalchemy from ckan import plugins, model from ckanext.datadaemon.model import setup as setup_model from ckanext.datadaemon.model import ErrorRepository # Adding pagination q = c.q = request.params.get('q', u'') # unicode format (decoded from utf8) try: page = int(request.params.get('page', 1)) except ValueError, e: abort(400, ('"page" parameter must be an integer')) limit = 20 # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def search_url(params): url = '/ckan-admin/log' params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) \ for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): params = list(params_nopage) params.extend(by.items()) return search_url(set(params)) c.drill_down_url = drill_down_url def remove_field(key, value): params = list(params_nopage) params.remove((key, value)) return search_url(params) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) params = request.params setup_model() session = ckan.model.Session # Variáveis de entrada: # data: Postagem. Valores: 24, 168, 720 # data_inicio # data_final # tipo: Tipo de Error. Valores: None, ParsingError, FileRetrievalError, FileCollectionError # Construir uma consulta com base nas variáveis de entrada # 1 - Criar consulta que traz todos os valores query = "SELECT e.* FROM dt_errors e WHERE 1 = 1" # 2 - Filtrar pela variável data. Ou eu filtro pelo período ou por data de início # e data de fim value_inicio = "" value_final = "" if params.get('data'): query = query + "AND creation_date >= (now() - interval '%s hours')" % params.get( 'data') else: if params.get('data_inicio'): query = query + "AND creation_date >= '%s'" % params.get( 'data_inicio') value_inicio = params.get('data_inicio') if params.get('data_final'): query = query + "AND creation_date <= '%s'" % params.get( 'data_final') value_final = params.get('data_final') # 3 - Filtrar por tipo de erro if params.get('tipo') == 'None': query = query + "AND error_type is NULL " elif params.get('tipo'): query = query + "AND error_type = '%s'" % params.get('tipo') # 4 - Adicionar paginação à consulta query_pagination = query + "ORDER BY creation_date DESC LIMIT %s OFFSET %s" % ( limit * page, (limit * page) - limit) error_list = session.query(ErrorRepository).from_statement( query_pagination).all() error_list2 = session.query(ErrorRepository).from_statement( query).all() error_list3 = session.query( ErrorRepository.original_file).from_statement( query_pagination).all() from os.path import basename retorno = list() for lista in error_list: lista.original_file = basename(lista.original_file) retorno.append(lista) tipos_de_erros = session.query( ErrorRepository.error_type).distinct().all() data = params.get('data') tipo = params.get('tipo') x = { 'valor': retorno, 'valor2': tipos_de_erros, 'valor3': value_inicio, 'valor4': value_final, 'valor5': data, 'valor6': tipo } c.page = h.Page(collection=error_list2, page=page, url=pager_url, item_count=len(error_list2), items_per_page=limit) #--------------Download do arquivo------------- from paste.fileapp import FileApp import mimetypes if params.get('arquivo'): filepath = session.query(ErrorRepository.original_file).filter( ErrorRepository.hash == params.get('arquivo')).all() content_type = mimetypes.guess_type(str(filepath[0][0])) if content_type: headers = [ ('Content-Disposition', 'attachment; filename=\"' + str(filepath[0][0]) + '\"'), ('Content-Type', '\'' + str(content_type[0]) + '\'') ] else: headers = [ ('Content-Disposition', 'attachment; filename=\"' + str(filepath[0][0]) + '\"'), ('Content-Type', 'aplication/octet-stream') ] fapp = FileApp(str(filepath[0][0]), headers) return fapp(request.environ, self.start_response) else: return render('admin/log.html', extra_vars=x)