def update_config(self, config): u''' Load config and set up the resource library, public directory and template directory for the view ''' # https://datatables.net/reference/option/lengthMenu self.page_length_choices = toolkit.aslist( config.get(u'ckan.datatables.page_length_choices', DEFAULT_PAGE_LENGTH_CHOICES)) self.page_length_choices = [int(i) for i in self.page_length_choices] self.top_pagination_controls = toolkit.asbool( config.get(u'ckan.datatables.top_pagination_controls', False)) self.search_delay = toolkit.asint( config.get(u'ckan.datatables.search_delay', DEFAULT_SEARCH_DELAY)) self.state_saving = toolkit.asbool( config.get(u'ckan.datatables.state_saving', True)) # https://datatables.net/reference/option/stateDuration self.state_duration = toolkit.asint( config.get(u'ckan.datatables.state_duration', DEFAULT_STATE_DURATION)) self.data_dictionary_labels = toolkit.asbool( config.get(u'ckan.datatables.data_dictionary_labels', True)) toolkit.add_template_directory(config, u'templates') toolkit.add_public_directory(config, u'public') toolkit.add_resource(u'public', u'ckanext-datatablesview')
def parse_dataset(self, dataset_dict, dataset_ref): dataset_dict['frequency'] = 'notPlanned' dataset_dict['topic_category'] = 'location' dataset_dict['lineage'] = '-' dataset_dict['contact_name'] = 'Nial Mc Sorley' dataset_dict[ 'contact_email'] = '*****@*****.**' dataset_dict['license_id'] = 'uk-ogl' _remove_extra('contact_name', dataset_dict) _remove_extra('contact_email', dataset_dict) # Ping the ArcGIS server so the processing of the files # starts identifier = None avoid = [] if toolkit.asbool( config.get('ckanext.opendatani.harvest.ping_arcgis_urls')): for extra in dataset_dict.get('extras', []): if extra['key'] == 'identifier' and extra['value']: identifier = extra['value'] if identifier: query = toolkit.get_action('package_search')( {}, { 'q': 'guid:"{0}"'.format(identifier) }) if query['count']: current_dataset = query['results'][0] for current_resource in current_dataset.get( 'resources', []): if ('requested' in current_resource and toolkit.asbool( current_resource['requested'])): avoid.append(current_resource['url']) for resource in dataset_dict.get('resources', []): if resource['format'] == 'OGC WMS': resource['format'] = 'WMS' if resource['format'] == 'ZIP': resource['format'] = 'SHP' resource['requested'] = False file_formats = ('geojson', 'kml', 'zip', 'csv') if resource['url'] in avoid: resource['requested'] = True elif resource['format'].lower() in file_formats: try: requests.head(resource['url']) resource['requested'] = True log.debug( 'Requested resource to start the processing: {0}'. format(resource['url'])) except Exception, e: log.debug('Error requesting resource: {0}\n{1}'.format( resource['url'], e)) pass
def _can_edit(state: str, by_author: bool = False) -> bool: if state == Comment.State.draft: if by_author: return tk.asbool( tk.config.get( const.CONFIG_DRAFT_EDITS_BY_AUTHOR, const.DEFAULT_DRAFT_EDITS_BY_AUTHOR, )) return tk.asbool( tk.config.get( const.CONFIG_DRAFT_EDITS, const.DEFAULT_DRAFT_EDITS, )) elif state == Comment.State.approved: if by_author: return tk.asbool( tk.config.get( const.CONFIG_APPROVED_EDITS_BY_AUTHOR, const.DEFAULT_APPROVED_EDITS_BY_AUTHOR, )) return tk.asbool( tk.config.get( const.CONFIG_APPROVED_EDITS, const.DEFAULT_APPROVED_EDITS, )) log.warning("Unexpected comment state: %s", state) return False
def issues_enabled(dataset): '''Returns whether issues are enabled for the given dataset (dict)''' # config options allow you to only enable issues for particular datasets or # organizations datasets_with_issues_enabled = set(toolkit.aslist( config.get('ckanext.issues.enabled_for_datasets') )) organizations_with_issues_enabled = set(toolkit.aslist( config.get('ckanext.issues.enabled_for_organizations') )) if datasets_with_issues_enabled or organizations_with_issues_enabled: if datasets_with_issues_enabled and \ dataset['name'] in datasets_with_issues_enabled: return True elif organizations_with_issues_enabled and \ (dataset.get('organization') or {}).get('name') in \ organizations_with_issues_enabled: return True return False else: extras = dataset.get('extras', []) for extra in extras: if extra.get('key') == 'issues_enabled': return toolkit.asbool(extra.get('value')) else: return toolkit.asbool( config.get('ckanext.issues.enabled_without_extra', True) )
def update_config(self, config): u''' Set up the resource library, public directory and template directory for the view ''' # https://datatables.net/reference/option/lengthMenu self.page_length_choices = toolkit.aslist( config.get(u'ckan.datatables.page_length_choices', DEFAULT_PAGE_LENGTH_CHOICES)) self.page_length_choices = [int(i) for i in self.page_length_choices] self.state_saving = toolkit.asbool( config.get(u'ckan.datatables.state_saving', True)) # https://datatables.net/reference/option/stateDuration self.state_duration = toolkit.asint( config.get(u'ckan.datatables.state_duration', DEFAULT_STATE_DURATION)) self.data_dictionary_labels = toolkit.asbool( config.get(u'ckan.datatables.data_dictionary_labels', True)) self.ellipsis_length = toolkit.asint( config.get(u'ckan.datatables.ellipsis_length', DEFAULT_ELLIPSIS_LENGTH)) self.date_format = config.get(u'ckan.datatables.date_format', DEFAULT_DATE_FORMAT) self.default_view = config.get(u'ckan.datatables.default_view', 'table') toolkit.add_template_directory(config, u'templates') toolkit.add_public_directory(config, u'public') toolkit.add_resource(u'public', u'ckanext-datatablesview')
def issues_enabled(dataset): '''Returns whether issues are enabled for the given dataset (dict)''' # config options allow you to only enable issues for particular datasets or # organizations datasets_with_issues_enabled = set( toolkit.aslist(config.get('ckanext.issues.enabled_for_datasets'))) organizations_with_issues_enabled = set( toolkit.aslist(config.get('ckanext.issues.enabled_for_organizations'))) if datasets_with_issues_enabled or organizations_with_issues_enabled: if datasets_with_issues_enabled and \ dataset['name'] in datasets_with_issues_enabled: return True elif organizations_with_issues_enabled and \ (dataset.get('organization') or {}).get('name') in \ organizations_with_issues_enabled: return True return False else: extras = dataset.get('extras', []) for extra in extras: if extra.get('key') == 'issues_enabled': return toolkit.asbool(extra.get('value')) else: return toolkit.asbool( config.get('ckanext.issues.enabled_without_extra', True))
def notify(self, entity, operation=None): if isinstance(entity, model.Resource): resource_dict = model.Resource.get(entity.id).as_dict() if helpers.is_spatially_ingestible_resource(resource_dict): d_type = model.domain_object.DomainObjectOperation auto_ingest = toolkit.asbool( config.get('ckan.spatialingestor.auto_ingest', 'False')) is_spatial_parent = toolkit.asbool( resource_dict.get('spatial_parent', 'False')) if operation == d_type.deleted or entity.state == 'deleted': helpers.log.error(">>>>>>> Registered Purge Trigger") toolkit.get_action( 'spatialingestor_purge_resource_datastores')( {}, resource_dict) package_dict = model.Package.get( resource_dict['package_id']).as_dict() if package_dict['state'] != 'deleted': helpers.log.error( ">>>>>>> Registered Orphan Delete Trigger") toolkit.get_action( 'spatialingestor_delete_orphaned_resources')( {}, package_dict) elif (is_spatial_parent and (operation == d_type.changed or not operation)) or ( operation == d_type.new and auto_ingest): helpers.log.error(">>>>>>> Registered Ingest Trigger") toolkit.get_action('spatialingestor_ingest_resource')( {}, resource_dict)
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes, ssl_cert, ssl_key): u"""Runs the Werkzeug development server""" if tk.asbool(config.get("debug")): warnings.filterwarnings("default", category=CkanDeprecationWarning) # Reloading use_reloader = not disable_reloader config_extra_files = tk.aslist( config.get(u"ckan.devserver.watch_patterns") ) extra_files = list(extra_files) + [ config[u"__file__"] ] + config_extra_files # Threads and processes threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded")) processes = processes or tk.asint( config.get(u"ckan.devserver.multiprocess", 1) ) if threaded and processes > 1: tk.error_shout(u"Cannot have a multithreaded and multi process server") raise click.Abort() # SSL cert_file = ssl_cert or config.get('ckan.devserver.ssl_cert') key_file = ssl_key or config.get('ckan.devserver.ssl_key') if cert_file and key_file: if cert_file == key_file == 'adhoc': ssl_context = 'adhoc' else: ssl_context = (ssl_cert, ssl_key) else: ssl_context = None host = host or config.get('ckan.devserver.host', DEFAULT_HOST) port = port or config.get('ckan.devserver.port', DEFAULT_PORT) try: port = int(port) except ValueError: tk.error_shout(u"Server port must be an integer, not {}".format(port)) raise click.Abort() log.info(u"Running CKAN on {scheme}://{host}:{port}".format( scheme='https' if ssl_context else 'http', host=host, port=port)) run_simple( host, port, ctx.obj.app, use_reloader=use_reloader, use_evalex=True, threaded=threaded, processes=processes, extra_files=extra_files, ssl_context=ssl_context, )
def parse_dataset(self, dataset_dict, dataset_ref): # TODO: if there is more than one source with different defaults, # modify accordingly dataset_dict['frequency'] = 'notPlanned' dataset_dict['topic_category'] = 'location' dataset_dict['lineage'] = '-' dataset_dict['contact_name'] = 'OSNI Mapping Helpdesk' dataset_dict['contact_email'] = '*****@*****.**' dataset_dict['license_id'] = 'uk-ogl' _remove_extra('contact_name', dataset_dict) _remove_extra('contact_email', dataset_dict) # Ping the ArcGIS server so the processing of the files # starts identifier = None avoid = [] if toolkit.asbool( config.get('ckanext.opendatani.harvest.ping_arcgis_urls')): for extra in dataset_dict.get('extras', []): if extra['key'] == 'identifier' and extra['value']: identifier = extra['value'] if identifier: query = toolkit.get_action('package_search')( {}, {'q': 'guid:"{0}"'.format(identifier)}) if query['count']: current_dataset = query['results'][0] for current_resource in current_dataset.get('resources', []): if ('requested' in current_resource and toolkit.asbool(current_resource['requested'])): avoid.append(current_resource['url']) for resource in dataset_dict.get('resources', []): if resource['format'] == 'OGC WMS': resource['format'] = 'WMS' resource['requested'] = False file_formats = ('geojson', 'kml', 'zip', 'csv') if resource['url'] in avoid: resource['requested'] = True elif resource['format'].lower() in file_formats: try: requests.head(resource['url']) resource['requested'] = True log.debug( 'Requested resource to start the processing: {0}' .format(resource['url'])) except Exception, e: log.debug( 'Error requesting resource: {0}\n{1}' .format(resource['url'], e)) pass
def notify(context, issue, email_template): notify_admin = toolkit.asbool( config.get("ckanext.issues.notify_admin", False)) notify_owner = toolkit.asbool( config.get("ckanext.issues.notify_owner", False)) if not notify_admin and not notify_owner: return user_obj = model.User.get(issue.user_id) dataset = model.Package.get(issue.dataset_id) extra_vars = { 'issue': issue, 'title': dataset.title, 'username': user_obj.name, 'email': user_obj.email, 'site_url': h.url_for(controller='ckanext.issues.controller:IssueController', action='show', id=issue.id, package_id=issue.dataset_id, qualified=True) } if notify_owner: contact_name = dataset.author or dataset.maintainer contact_email = dataset.author_email or dataset.maintainer_email email_msg = "TODO" #render(email_template,extra_vars=extra_vars,loader_class=NewTextTemplate) send_email(contact_name, contact_email, email_msg) if notify_admin: # retrieve organization's admins (or the role specified on ckanext.issues.minimun_role_required) to notify organization = action.get.organization_show( context, data_dict={'id': dataset.owner_org}) minimun_role_required = config.get( "ckanext.issues.minimun_role_required", "Anonymous") for user in organization['users']: if user['capacity'] == role_mapper[minimun_role_required] and user[ 'activity_streams_email_notifications']: admin_user = model.User.get(user['id']) admin_name = admin_user.name admin_email = admin_user.email if admin_email != contact_email: email_msg = "TODO" #render(email_template,extra_vars=extra_vars,loader_class=NewTextTemplate) send_email(admin_name, admin_email, email_msg)
def build_pages_nav_main(*args): about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', False)) group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', False)) org_menu = toolkit.asbool( config.get('ckanext.pages.organization_menu', True)) new_args = [] for arg in args: if arg[0] == 'home.about' and not about_menu: continue if arg[0] == 'organizations_index' and not org_menu: continue if arg[0] == 'group_index' and not group_menu: continue new_args.append(arg) output = h.build_nav_main(*new_args) # add forum link forum_url = config.get('ckan.pages.forum.link', False) if forum_url: link = h.literal(u'<a href="/{}/{}">{}</a>'.format( h.lang(), 'forum', "Forum")) li = h.literal('<li>') + link + h.literal('</li>') output = output + li # do not display any private datasets in menu even for sysadmins pages_list = toolkit.get_action('ckanext_pages_list')(None, { 'order': True, 'private': False }) page_name = '' if (toolkit.c.action in ('pages_show', 'blog_show') and toolkit.c.controller == 'ckanext.pages.controller:PagesController'): page_name = toolkit.c.environ['routes.url'].current().split('/')[-1] for page in pages_list: type_ = 'blog' if page['page_type'] == 'blog' else 'pages' name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8') title = cgi.escape(pages_page_title(h.lang(), page)) if h.lang(): link = h.literal(u'<a href="/{}/{}/{}">{}</a>'.format( h.lang(), type_, name, title)) else: link = h.literal(u'<a href="/{}/{}">{}</a>'.format( type_, name, title)) if page['name'] == page_name: li = h.literal('<li class="active">') + link + h.literal('</li>') else: li = h.literal('<li>') + link + h.literal('</li>') output = output + li return output
def update_config(self, config): toolkit.add_template_directory(config, 'templates') facets_inputs = config.get(self.ADDITIONAL_FACETS_CONFIG, '').split() # get additional facets from the input self.additional_facets = loader.get_additional_facets(facets_inputs[0]) self.display_facets_on_group_page = toolkit.asbool(config.get(self.DISPLAY_FACETS_ON_GROUPS_PAGE, False)) self.display_facets_on_org_page = toolkit.asbool(config.get(self.DISPLAY_FACETS_ON_ORG_PAGE, False)) self.clear_default_facets = toolkit.asbool(config.get(self.CLEAR_DEFAULT_FACETS, False))
def configure(self, config): """Load config settings for this extension from config file. See IConfigurable. """ if "googleanalytics.id" not in config: msg = "Missing googleanalytics.id in config" raise GoogleAnalyticsException(msg) self.googleanalytics_id = config["googleanalytics.id"] self.googleanalytics_domain = config.get( "googleanalytics.domain", "auto" ) self.googleanalytics_fields = ast.literal_eval( config.get("googleanalytics.fields", "{}") ) googleanalytics_linked_domains = config.get( "googleanalytics.linked_domains", "" ) self.googleanalytics_linked_domains = [ x.strip() for x in googleanalytics_linked_domains.split(",") if x ] if self.googleanalytics_linked_domains: self.googleanalytics_fields["allowLinker"] = "true" # If resource_prefix is not in config file then write the default value # to the config dict, otherwise templates seem to get 'true' when they # try to read resource_prefix from config. if "googleanalytics_resource_prefix" not in config: config[ "googleanalytics_resource_prefix" ] = DEFAULT_RESOURCE_URL_TAG self.googleanalytics_resource_prefix = config[ "googleanalytics_resource_prefix" ] self.show_downloads = tk.asbool( config.get("googleanalytics.show_downloads", True) ) self.track_events = tk.asbool( config.get("googleanalytics.track_events", False) ) self.enable_user_id = tk.asbool( config.get("googleanalytics.enable_user_id", False) ) p.toolkit.add_resource("../assets", "ckanext-googleanalytics") # spawn a pool of 5 threads, and pass them queue instance for i in range(5): t = AnalyticsPostThread(self.analytics_queue) t.setDaemon(True) t.start()
def build_pages_nav_main(*args): about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True)) group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True)) org_menu = toolkit.asbool( config.get('ckanext.pages.organization_menu', True)) new_args = [] for arg in args: if arg[0] == 'about' and not about_menu: continue if arg[0] == 'organizations_index' and not org_menu: continue if arg[0] == 'group_index' and not group_menu: continue new_args.append(arg) output = h.build_nav_main(*new_args) # do not display any private datasets in menu even for sysadmins pages_list = toolkit.get_action('ckanext_pages_list')(None, { 'order': True, 'private': False }) page_name = '' if (toolkit.c.action in ('pages_show', 'blog_show') and toolkit.c.controller == 'ckanext.pages.controller:PagesController'): page_name = toolkit.c.environ['routes.url'].current().split('/')[-1] desired_lang_code = pylons.request.environ['CKAN_LANG'] acceptable_lang_codes = [ desired_lang_code, desired_lang_code.split('_', 1)[0] ] for page in pages_list: if page.get('lang') and page.get('lang') not in acceptable_lang_codes: continue type_ = 'blog' if page['page_type'] == 'blog' else 'pages' name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8') title = cgi.escape(page['title']) link = h.literal(u'<a href="/{}/{}">{}</a>'.format(type_, name, title)) if page['name'] == page_name: li = h.literal('<li class="active">') + link + h.literal('</li>') else: li = h.literal('<li>') + link + h.literal('</li>') output = output + li return output
def build_pages_nav_main(*args): about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True)) group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True)) org_menu = toolkit.asbool( config.get('ckanext.pages.organization_menu', True)) new_args = [] for arg in args: if arg[0] == 'about' and not about_menu: continue if arg[0] == 'organizations_index' and not org_menu: continue if arg[0] == 'group_index' and not group_menu: continue new_args.append(arg) output = h.build_nav_main(*new_args) # do not display any private datasets in menu even for sysadmins pages_list = toolkit.get_action('ckanext_pages_list')(None, { 'order': True, 'private': False }) page_name = '' try: if (toolkit.c.action in ('pages_show', 'blog_show') and toolkit.c.controller == 'ckanext.pages.controller:PagesController'): page_name = toolkit.c.environ['routes.url'].current().split( '/')[-1] except AttributeError: # NOTE(e0ne): we don't have 'action' attribute in Flask context. # We can igrore if it's Flask Bluprint-bases plugin pass for page in pages_list: if page['page_type'] == 'blog': link = h.literal('<a href="/blog/%s">%s</a>' % (str(page['name']), str(page['title']))) else: link = h.literal('<a href="/pages/%s">%s</a>' % (str(page['name']), str(page['title']))) if page['name'] == page_name: li = h.literal('<li class="active">') + link + h.literal('</li>') else: li = h.literal('<li>') + link + h.literal('</li>') output = output + li return output
def insert_or_update_pkg(self, pkg_dict, upload=None): registry = ckanapi.LocalCKAN(username=self.username) allow_duplicates = tk.asbool( tk.config.get('ckanext.ddi.allow_duplicates', False) ) override_datasets = tk.asbool( tk.config.get('ckanext.ddi.override_datasets', False) ) visibility = pkg_dict.pop('visibility', 'restricted') try: existing_pkg = registry.call_action('package_show', pkg_dict) if not allow_duplicates and not override_datasets: raise ContentDuplicateError( 'Dataset already exists and duplicates are not allowed.' ) if override_datasets: pkg_dict.pop('id', None) pkg_dict.pop('name', None) existing_pkg.update(pkg_dict) pkg_dict = existing_pkg registry.call_action('package_update', pkg_dict) else: raise ckanapi.NotFound() except ckanapi.NotFound: pkg_dict.pop('id', None) pkg_dict['name'] = self._gen_new_name(pkg_dict['name']) registry.call_action('package_create', pkg_dict) if upload is not None: try: registry.call_action( 'resource_create', { 'package_id': pkg_dict['name'], 'upload': upload, 'name': 'DDI XML', 'format': 'xml', 'url': '', 'type': 'attachment', 'file_type': 'other', 'visibility': visibility } ) except Exception as e: raise UploadError( 'Could not upload file: %s' % str(e) ) log.debug(pkg_dict['name']) return pkg_dict['name']
def configure(self, config_): '''Load config settings for this extension from config file. See IConfigurable. ''' if 'googleanalytics.id' not in config_: msg = "Missing googleanalytics.id in config" log.error(msg) raise CscGAException(msg) self.googleanalytics_id = config_['googleanalytics.id'] self.googleanalytics_domain = config_.get('googleanalytics.domain', 'auto') self.googleanalytics_fields = ast.literal_eval( config_.get('googleanalytics.fields', '{}')) googleanalytics_linked_domains = config_.get( 'googleanalytics.linked_domains', '') self.googleanalytics_linked_domains = [ x.strip() for x in googleanalytics_linked_domains.split(',') if x ] if self.googleanalytics_linked_domains: self.googleanalytics_fields['allowLinker'] = 'true' # If resource_prefix is not in config file then write the default value # to the config dict, otherwise templates seem to get 'true' when they # try to read resource_prefix from config. if 'googleanalytics_resource_prefix' not in config_: config_['googleanalytics_resource_prefix'] = ( DEFAULT_RESOURCE_URL_TAG) self.googleanalytics_resource_prefix = config_[ 'googleanalytics_resource_prefix'] self.show_downloads = toolkit.asbool( config_.get('googleanalytics.show_downloads', True)) self.track_events = toolkit.asbool( config_.get('googleanalytics.track_events', False)) self.ga_send_stats = toolkit.asbool( config_.get('ckanext.csc_ga.send_stats', True)) log.debug('ga_send_stats=%s', self.ga_send_stats) toolkit.add_resource('fanstatic', 'csc_ga') # spawn a pool of 5 threads, and pass them queue instance for i in range(5): t = AnalyticsPostThread(self.analytics_queue) t.setDaemon(True) t.start()
def build_pages_nav_main(*args): about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True)) group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True)) org_menu = toolkit.asbool( config.get('ckanext.pages.organization_menu', True)) # Different CKAN versions use different route names - gotta catch em all! about_menu_routes = ['about', 'home.about'] group_menu_routes = ['group_index', 'home.group_index'] org_menu_routes = ['organizations_index', 'home.organizations_index'] new_args = [] for arg in args: if arg[0] in about_menu_routes and not about_menu: continue if arg[0] in org_menu_routes and not org_menu: continue if arg[0] in group_menu_routes and not group_menu: continue new_args.append(arg) output = h.build_nav_main(*new_args) # do not display any private datasets in menu even for sysadmins pages_list = toolkit.get_action('ckanext_pages_list')(None, { 'order': True, 'private': False }) page_name = '' if (hasattr(toolkit.c, 'action') and toolkit.c.action in ('pages_show', 'blog_show') and toolkit.c.controller == 'ckanext.pages.controller:PagesController'): page_name = toolkit.c.environ['routes.url'].current().split('/')[-1] for page in pages_list: type_ = 'blog' if page['page_type'] == 'blog' else 'pages' name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8') title = cgi.escape(page['title']) link = h.literal(u'<a href="/{}/{}">{}</a>'.format(type_, name, title)) if page['name'] == page_name: li = h.literal('<li class="active">') + link + h.literal('</li>') else: li = h.literal('<li>') + link + h.literal('</li>') output = output + li return output
def notify(context,issue,email_template): notify_admin = toolkit.asbool(config.get("ckanext.issues.notify_admin", False)) notify_owner = toolkit.asbool(config.get("ckanext.issues.notify_owner", False)) if not notify_admin and not notify_owner: return user_obj = model.User.get(issue.user_id) dataset = model.Package.get(issue.dataset_id) extra_vars = { 'issue': issue, 'title': dataset.title, 'username': user_obj.name, 'email': user_obj.email, 'site_url': h.url_for( controller='ckanext.issues.controller:IssueController', action='show', id=issue.id, package_id=issue.dataset_id, qualified=True ) } if notify_owner: contact_name = dataset.author or dataset.maintainer contact_email = dataset.author_email or dataset.maintainer_email email_msg = render(email_template,extra_vars=extra_vars) send_email(contact_name,contact_email,email_msg) if notify_admin: # retrieve organization's admins (or the role specified on ckanext.issues.minimun_role_required) to notify organization = action.get.organization_show(context,data_dict={'id':dataset.owner_org}) minimun_role_required = config.get("ckanext.issues.minimun_role_required", "Anonymous") for user in organization['users']: if user['capacity'] == role_mapper[minimun_role_required] and user['activity_streams_email_notifications']: admin_user = model.User.get(user['id']) admin_name = admin_user.name admin_email = admin_user.email if admin_email != contact_email: email_msg = render(email_template,extra_vars=extra_vars) send_email(admin_name,admin_email,email_msg)
def resource_create(context, data_dict): if not tk.asbool(config.get('ckan.cloud_storage_enable')) or data_dict.get('url'): return origin.resource_create(context, data_dict) model = context['model'] user = context['user'] package_id = _get_or_bust(data_dict, 'package_id') data_dict.pop('package_id') pkg_dict = _get_action('package_show')(context, {'id': package_id}) _check_access('resource_create', context, data_dict) if not 'resources' in pkg_dict: pkg_dict['resources'] = [] upload = uploader.S3Upload(data_dict) pkg_dict['resources'].append(data_dict) try: context['defer_commit'] = True context['use_cache'] = False _get_action('package_update')(context, pkg_dict) context.pop('defer_commit') except ValidationError, e: errors = e.error_dict['resources'][-1] raise ValidationError(errors)
def update_config(self, config): self.organization_pages = toolkit.asbool(config.get('ckanext.pages.organization', False)) self.group_pages = toolkit.asbool(config.get('ckanext.pages.group', False)) toolkit.add_template_directory(config, 'theme/templates_main') if self.group_pages: toolkit.add_template_directory(config, 'theme/templates_group') if self.organization_pages: toolkit.add_template_directory(config, 'theme/templates_organization') toolkit.add_resource('fanstatic', 'pages') toolkit.add_public_directory(config, 'public') toolkit.add_resource('theme/public', 'ckanext-pages') toolkit.add_resource('theme/resources', 'pages-theme') toolkit.add_public_directory(config, 'theme/public')
def fetch_stage(self, harvest_object): api_url = config.get('ckanext.glasgow.metadata_api', '').rstrip('/') # NB: this end point does not seem to support the $skip parameter api_endpoint = api_url + '/Metadata/Organisation/{0}/Dataset/{1}/File' try: content = json.loads(harvest_object.content) org = content['OrganisationId'] dataset = content['Id'] verify_ssl = toolkit.asbool( config.get('ckanext.glasgow.verify_ssl_certs', True) ) request = requests.get(api_endpoint.format(org, dataset), verify=verify_ssl) if request.status_code == 404: result = False log.debug('No files for dataset {0}'.format(dataset)) else: result = _fetch_from_ec(request) except requests.exceptions.RequestException, e: self._save_object_error( 'Error fetching file metadata for package {0}: {1}'.format( harvest_object.guid, str(e)), harvest_object, 'Fetch') return False
def cadasta_api_action(context, data_dict): # we actually always want to call check access # development option that should be removed later if toolkit.asbool(config.get('ckanext.cadasta.enforce_permissions', True)): toolkit.check_access(action, context, data_dict) string_arguments = [a[1] for a in string.Formatter().parse(cadasta_endpoint.url) if a[1]] cadasta_dict = {} for k, v in data_dict.items(): cadasta_dict[k] = cadasta_endpoint.convert_argument(k, v) error_dict = {} endpoint = cadasta_endpoint.url for arg in string_arguments: if arg not in data_dict.keys() or not data_dict.get(arg): error_dict[arg] = ['Missing value'] else: arg_value = '' if cadasta_endpoint.keep_param_key is True: arg_value = cadasta_dict.get(arg, '') else: arg_value = cadasta_dict.pop(arg, '') arg_value = re.sub('[^\-\.0-9a-zA-Z]+', '', str(arg_value)) endpoint_arg = ''.join(['{', arg, '}']) endpoint = endpoint.replace(endpoint_arg, arg_value) if error_dict: raise toolkit.ValidationError(error_dict) return cadasta_api_func(endpoint, cadasta_dict, cadasta_endpoint.upload_fields)
def create_orgs(organization_id, site_user): api_url = config.get('ckanext.glasgow.metadata_api', '').rstrip('/') api_endpoint = '{}/Metadata/Organisation/{}'.format(api_url, organization_id) verify_ssl = toolkit.asbool(config.get('ckanext.glasgow.verify_ssl_certs', True)) request = requests.get(api_endpoint, verify=verify_ssl) try: result = _fetch_from_ec(request) org = result['MetadataResultSet'][0] except (KeyError, IndexError): print 'failed to fetch org {} from EC. Response {}'.format(organization_id, str(result)) return context = { 'model': model, 'session': model.Session, 'user': site_user, 'local_action': True, } org_name = get_org_name(org, 'Title') data_dict = { 'id': org['Id'], 'title': org['Title'], 'name': org_name, } try: toolkit.get_action('organization_create')(context, data_dict) context.pop('local_action', None) return toolkit.get_action('organization_show')(context, {id: 'organization_id'}) except toolkit.ValidationError: print 'failed to create org {}'.format(organization_id)
def cadasta_api_action(context, data_dict): # we actually always want to call check access # development option that should be removed later if toolkit.asbool(config.get('ckanext.cadasta.enforce_permissions', True)): toolkit.check_access(action, context, data_dict) string_arguments = [a[1] for a in string.Formatter().parse(cadasta_endpoint.url) if a[1]] cadasta_dict = {} for k, v in data_dict.items(): cadasta_dict[k] = cadasta_endpoint.convert_argument(k, v) error_dict = {} for arg in string_arguments: if arg not in data_dict.keys(): error_dict[arg] = ['Missing value'] cadasta_dict.pop(arg, None) if error_dict: raise toolkit.ValidationError(error_dict) endpoint = cadasta_endpoint.url.format(**data_dict) return cadasta_api_func(endpoint, cadasta_dict, cadasta_endpoint.upload_fields)
def comment_create(context, data_dict): tk.check_access("comments_comment_create", context, data_dict) thread_data = { "subject_id": data_dict["subject_id"], "subject_type": data_dict["subject_type"], } try: thread_dict = tk.get_action("comments_thread_show")( context.copy(), thread_data ) except tk.ObjectNotFound: if not data_dict["create_thread"]: raise thread_dict = tk.get_action("comments_thread_create")( context.copy(), thread_data ) author_id = data_dict.get("author_id") can_set_author_id = ( context.get("ignore_auth") or context["auth_user_obj"].sysadmin ) if not author_id or not can_set_author_id: author_id = context["user"] reply_to_id = data_dict.get("reply_to_id") if reply_to_id: parent = tk.get_action("comments_comment_show")( context.copy(), {"id": reply_to_id} ) if parent["thread_id"] != thread_dict["id"]: raise tk.ValidationError( {"reply_to_id": ["Coment is owned by different thread"]} ) comment = Comment( thread_id=thread_dict["id"], content=data_dict["content"], author_type=data_dict["author_type"], author_id=author_id, reply_to_id=reply_to_id, ) author = comment.get_author() if author is None: raise tk.ObjectNotFound("Cannot find author for comment") # make sure we are not messing up with name_or_id comment.author_id = author.id if not tk.asbool( tk.config.get( const.CONFIG_REQUIRE_APPROVAL, const.DEFAULT_REQUIRE_APPROVAL ) ): comment.approve() context['session'].add(comment) context['session'].commit() comment_dict = get_dictizer(type(comment))(comment, context) return comment_dict
def render_content(content): allow_html = toolkit.asbool(config.get('ckanext.pages.allow_html', False)) try: return h.render_markdown(content, allow_html=allow_html) except TypeError: # allow_html is only available in CKAN >= 2.3 return h.render_markdown(content)
def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) # Auto-set some values based on configuration from pylons import config if toolkit.asbool(config.get('ckan.comments.moderation', 'true')): self.approval_status = COMMENT_PENDING else: # If user wants first comment moderated and the user who wrote this hasn't # got another comment, put it into moderation, otherwise approve if toolkit.asbool(config.get('ckan.comments.moderation.first_only', 'true')) and \ Comment.count_for_user(self.user, COMMENT_APPROVED) == 0: self.approval_status = COMMENT_PENDING else: self.approval_status = COMMENT_APPROVED
def before_search(self, search_params): deployment_mode = toolkit.asbool( config.get('ckan.ab_scheming.deployment', False)) if deployment_mode: return search_params user_member_of_orgs = [ org['id'] for org in h.organizations_available('read') ] if (c.group and c.group.id in user_member_of_orgs): # added for more control on result datasets # based on our requirement search_params.update({ 'include_private': True, 'include_drafts': True }) if c.process_state: # for the filters in user dashboard and organization. if search_params.get("q"): search_params['q'] += ' %s: "%s"' % ('process_state', c.process_state) else: search_params['q'] = '%s: "%s"' % ('process_state', c.process_state) if search_params.get('fq'): if re.search(r'dataset_type', search_params['fq']): # this is for general search search_params.update({ 'include_private': True, 'include_drafts': True }) return search_params
def _get_filepath_for_resource(res): """Returns a filepath for a resource that will be indexed""" res_id = res['id'] res_url = res['url'] if res["url_type"] == "upload": uploader = get_resource_uploader(res) # TODO temporary workaround for ckanext-cloudstorage support if p.plugin_loaded('cloudstorage'): url = uploader.get_url_from_filename(res_id, res_url) filepath = _download_remote_file(res_id, url) return filepath path = uploader.get_path(res_id) if not os.path.exists(path): log.warn('Resource "{res_id}" refers to unexisting path "{path}"') return return path if not tk.asbool(tk.config.get("ckanext.resource_indexer.allow_remote")): return filepath = _download_remote_file(res_id, res_url) return filepath
def harvest_source_index_clear(context, data_dict): ''' Clears all datasets, jobs and objects related to a harvest source, but keeps the source itself. This is useful to clean history of long running harvest sources to start again fresh. :param id: the id of the harvest source to clear :type id: string ''' check_access('harvest_source_clear', context, data_dict) harvest_source_id = data_dict.get('id') source = HarvestSource.get(harvest_source_id) if not source: log.error('Harvest source %s does not exist', harvest_source_id) raise NotFound('Harvest source %s does not exist' % harvest_source_id) harvest_source_id = source.id conn = make_connection() query = ''' +%s:"%s" +site_id:"%s" ''' % ( 'harvest_source_id', harvest_source_id, config.get('ckan.site_id')) solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true')) if toolkit.check_ckan_version(max_version='2.5.99'): # conn is solrpy try: conn.delete_query(query) if solr_commit: conn.commit() except Exception, e: log.exception(e) raise SearchIndexError(e) finally:
def add_defaults_to_options(options, defaults): '''Returns the options, using option values passed in and falling back to the default values for that report. When a option needs a boolean, an option passed in as 0 or 1 are converted to True/False, which suits when the options passed in are URL parameters. ''' defaulted_options = copy.deepcopy(defaults) for key in defaulted_options: if key not in options: if defaulted_options[key] is True: # Checkboxes don't submit a value when False, so cannot # default to True. i.e. to get a True value, you always # need be explicit in the params. defaulted_options[key] = False continue value = options[key] if isinstance(defaulted_options[key], bool): try: defaulted_options[key] = asbool(value) except ValueError: pass # leave it as default else: defaulted_options[key] = value for key in set(options) - set(defaulted_options): defaulted_options[key] = options[key] return defaulted_options
def issue_deleted_from_dataset(data_dict): review_system = toolkit.asbool( config.get("ckanext.issues.review_system", False)) log.debug("review_system issue_deleted: %s %s", data_dict, review_system) if review_system: issue_count = toolkit.get_action('issue_count')( data_dict={ 'dataset_id': data_dict['dataset_id'], 'status': issuemodel.ISSUE_STATUS.open }) if issue_count == 0: try: logic.get_action('package_patch')(data_dict={ 'id': data_dict['dataset_id'], 'private': False }) except logic.NotAuthorized: abort(401, _('Not authorized to modify the dataset')) h.flash_success(_('The dataset has now been made public.')) log.debug("Dataset %s made public", data_dict['dataset_id'])
def ingest_resource(context, resource_dict): if toolkit.asbool(resource_dict.get('spatial_parent', 'False')): try: task = toolkit.get_action('task_status_show')( { 'ignore_auth': True }, { 'entity_id': resource_dict['id'], 'task_type': 'spatial_ingest', 'key': 'spatialingestor' }) if task.get('state') in ['pending']: # There already is a pending Spatialingestor submission, # skip this one ... log.debug( 'Skipping Spatial Ingestor submission for resource {0}'. format(resource_dict['id'])) return except toolkit.ObjectNotFound: pass try: log.debug('Submitting resource {0} to Spatial Ingestor'.format( resource_dict['id'])) toolkit.get_action('spatialingestor_job_submit')( context, { 'resource_id': resource_dict['id'], 'job_type': 'spatial_ingest' }) except toolkit.ValidationError, e: log.error(e)
def update(context, data_dict): '''Update a PowerView. By default, only sysadmins can update PowerViews. But the setting `ckanext.powerview.allow_user_create` allows any logged in user to update powerviews they own. ''' if not toolkit.asbool(config.get('ckanext.powerview.allow_user_create', False)): return {'success': False} else: user = context.get('user') user_obj = model.User.get(user) powerview = PowerView.get(id=data_dict['id']) # Check resources for res_id in data_dict.get('resources', []): try: toolkit.check_access('resource_show', context, data_dict={'id': res_id}) except NotAuthorized: return { 'success': False, 'msg': _('User {0} not authorized to read resource {1}' .format(user, res_id)) } if powerview and user and powerview.created_by == user_obj.id: return {'success': True} return {'success': False}
def check_access_user(context, data_dict): if c.user: return {'success': True} else: allow_rating = toolkit.asbool( config.get('rating.enabled_for_unauthenticated_users', True)) return {'success': allow_rating}
def create(context, data_dict): '''Create a PowerView. By default, only sysadmins can create a PowerView. But the setting `ckanext.powerview.allow_user_create` allows any logged in user to create powerviews. ''' allow_user_create = toolkit.asbool( config.get('ckanext.powerview.allow_user_create', False)) if not allow_user_create: return {'success': False} user = context.get('user') # Check resources for res_id in data_dict.get('resources', []): try: toolkit.check_access('resource_show', context, data_dict={'id': res_id}) except NotAuthorized: return { 'success': False, 'msg': _('User {0} not authorized to read resource {1}' .format(user, res_id)) } return {'success': True}
def download(id, resource_id, filename=None): """Download resource blueprint This calls all registered download handlers in order, until a response is returned to the user """ context = get_context() resource = None try: resource = toolkit.get_action('resource_show')(context, { 'id': resource_id }) if id != resource['package_id']: return toolkit.abort( 404, toolkit._('Resource not found belonging to package')) package = toolkit.get_action('package_show')(context, {'id': id}) except toolkit.ObjectNotFound: return toolkit.abort(404, toolkit._('Resource not found')) except toolkit.NotAuthorized: return toolkit.abort( 401, toolkit._('Not authorized to read resource {0}'.format(id))) activity_id = request.args.get('activity_id') inline = toolkit.asbool(request.args.get('preview')) if activity_id and toolkit.check_ckan_version(min_version='2.9'): try: activity = toolkit.get_action(u'activity_show')( context, { u'id': activity_id, u'include_data': True }) activity_dataset = activity['data']['package'] assert activity_dataset['id'] == id activity_resources = activity_dataset['resources'] for r in activity_resources: if r['id'] == resource_id: resource = r package = activity_dataset break except toolkit.NotFound: toolkit.abort(404, toolkit._(u'Activity not found')) try: resource = call_pre_download_handlers(resource, package, activity_id=activity_id) return call_download_handlers(resource, package, filename, inline, activity_id=activity_id) except toolkit.ObjectNotFound: return toolkit.abort(404, toolkit._('Resource not found')) except toolkit.NotAuthorized: return toolkit.abort( 401, toolkit._( 'Not authorized to read resource {0}'.format(resource_id)))
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes): u"""Runs the Werkzeug development server""" use_reloader = not disable_reloader threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded")) processes = processes or tk.asint( config.get(u"ckan.devserver.multiprocess", 1) ) if threaded and processes > 1: tk.error_shout(u"Cannot have a multithreaded and multi process server") raise click.Abort() log.info(u"Running server {0} on port {1}".format(host, port)) config_extra_files = tk.aslist( config.get(u"ckan.devserver.watch_patterns") ) extra_files = list(extra_files) + [ config[u"__file__"] ] + config_extra_files run_simple( host, port, ctx.obj.app, use_reloader=use_reloader, use_evalex=True, threaded=threaded, processes=processes, extra_files=extra_files, )
def comment_update(context, data_dict): model = context['model'] logic.check_access("comment_update", context, data_dict) cid = logic.get_or_bust(data_dict, 'id') comment = comment_model.Comment.get(cid) if not comment: abort(404) # Validate that we have the required fields. if not all([data_dict.get('comment')]): raise logic.ValidationError("Comment text is required") # Cleanup the comment cleaned_comment = util.clean_input(data_dict.get('comment')) # Run profanity check if toolkit.asbool(config.get('ckan.comments.check_for_profanity', False)) \ and (helpers.profanity_check(cleaned_comment) or helpers.profanity_check(data_dict.get('subject', ''))): raise logic.ValidationError("Comment blocked due to profanity.") comment.subject = data_dict.get('subject') comment.comment = cleaned_comment comment.modified_date = datetime.datetime.utcnow() comment.flagged = data_dict.get('flagged') model.Session.add(comment) model.Session.commit() return comment.as_dict()
def create_zip(ckan_ini_filepath, package_id, queue='bulk'): load_config(ckan_ini_filepath) register_translator() log = create_zip.get_logger() context = {'model': model, 'ignore_auth': True, 'session': model.Session} pkg = get_action('package_show')(context, {'id': package_id}) extras = dict([(d['key'], d['value'],) for d in pkg['extras']]) unpublished = t.asbool(extras.get('unpublished', False)) if unpublished: log.info("Skipping unpublished dataset: %s", pkg['name']) return directory = config.get('ckanext.packagezip.destination_dir') if not os.path.exists(directory): log.info('Creating packagezip directory: %s' % directory) os.mkdir(directory) filename = "{0}.zip".format(pkg['name']) filepath = os.path.join(directory, filename) with zipfile.ZipFile(filepath, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as zipf: try: datapackage = get_action('datapackage_show')(context, {'id': package_id}) except KeyError, e: log.error('Cannot find action - check this plugin is enabled: %s', e) raise any_have_data = False for res in datapackage['resources']: if res['has_data']: any_have_data = True if res['cache_filepath'] and os.path.exists(res['cache_filepath']): zipf.write(res['cache_filepath'], res['path']) res['included_in_zip'] = True else: res['included_in_zip'] = False env = Environment(loader=PackageLoader('ckanext.packagezip', 'templates')) env.filters['datetimeformat'] = datetimeformat template = env.get_template('index.html') zipf.writestr('index.html', template.render(datapackage=datapackage, date=datetime.datetime.now()).encode('utf8')) # Strip out unnecessary data from datapackage for res in datapackage['resources']: del res['has_data'] if 'cache_filepath' in res: del res['cache_filepath'] if 'reason' in res: del res['reason'] if 'detected_format' in res: del res['detected_format'] zipf.writestr('datapackage.json', json.dumps(datapackage, indent=4))
def counter_on_off(context, data_dict=None): # Get the value of the ckan.open_alberta.counter_on # setting from the CKAN config file as a string, or False if the setting # isn't in the config file. counter_on = config.get('ckan.open_alberta.counter_on', False) # Convert the value from a string to a boolean. counter_on = toolkit.asbool(counter_on) return {"counter_on": counter_on}
def package_create(context, data_dict): deployment_mode = toolkit.asbool(config.get('ckan.ab_scheming.deployment', False)) # need to change data_dict if import from ckanapi if deployment_mode: data_dict = change_pkg_dict_for_import_deployment(data_dict, 'create') if data_dict['type'] in ['publications', 'opendata']: context['defer_commit'] = True return create.package_create(context, data_dict)
def issues_enabled(dataset): datasets_with_issues_enabled = set(toolkit.aslist( config.get('ckanext.issues.enabled_for_datasets') )) # if the config option 'ckanext.issues.enabled_for_dataset' is # set with a list of datasets, only enabled for the listed datasets if datasets_with_issues_enabled: if dataset['name'] in datasets_with_issues_enabled: return True else: extras = dataset.get('extras') if extras is not None: for extra in extras: if extra.get('key') == 'issues_enabled': return toolkit.asbool(extra.get('value')) else: return toolkit.asbool(config.get('ckanext.issues.enabled_per_dataset_default', True))
def _create_user_on_login(): '''Return whether to automatically create users on login defaults to true''' try: return toolkit.asbool( pylons.config['ckanext.oauth2waad.create_users_on_login'] ) except KeyError: return True
def command(self): self._load_config() import ckan.model as model model.Session.remove() model.Session.configure(bind=model.meta.engine) model.repo.new_revision() self.log.info("Database access initialised") root_url = config.get('ckan.site_url', 'http://data.gov.uk') self.log.debug("Root url is " + root_url) if toolkit.asbool(config.get('debug')): root_url = "http://data.gov.uk" self.log.debug("Overriding root_url in DEBUG ") data = self._make_request() for d in data: # Eventually we might handle other types. if d['type'] != 'App': continue related_url = urljoin(root_url, d['path']) package = model.Package.get(d['ckan_id']) if not package: continue found = False current_related = model.Related.get_for_dataset(package) for current in current_related: if current.related.url == related_url: self.log.info("Skipping existing related") found = True if not found: self.log.info("Adding related item [%s] to dataset [%s]" % (d['title'], package.name)) related = model.Related() related.type = 'App' related.title = d['title'] related.description = "" related.url = related_url thumb = d.get('thumbnail', '').replace("://", "/") if thumb: thumb_url = urljoin(root_url, "/sites/default/files/styles/medium/") thumb_url = urljoin(thumb_url, thumb) else: thumb_url = '' related.image_url = thumb_url model.Session.add(related) model.Session.commit() related_item = model.RelatedDataset(dataset_id=package.id, related_id=related.id) model.Session.add(related_item) model.Session.commit()
def _setup_webassets(app): app.use_x_sendfile = toolkit.asbool( config.get('ckan.webassets.use_x_sendfile') ) webassets_folder = get_webassets_path() @app.route('/webassets/<path:path>') def webassets(path): return send_from_directory(webassets_folder, path)
def update_config(self, config): if self.instance: # reloading plugins, probably in WebTest _SchemingMixin._helpers_loaded = False self._store_instance(self) self._add_template_directory(config) self._is_fallback = asbool(config.get(self.FALLBACK_OPTION, False)) self._schema_urls = config.get(self.SCHEMA_OPTION, "").split() self._schemas = _load_schemas(self._schema_urls, self.SCHEMA_TYPE_FIELD)
def resources(self, id=None): try: c.include_sub_publishers = t.asbool(t.request.params.get('include_sub_publishers') or False) except ValueError: abort(400, 'include_sub_publishers parameter value must be boolean') c.publisher_name = id or 'department-for-culture-media-and-sport' c.query = reports.organisation_resources c.data = c.query(organisation_name=c.publisher_name, include_sub_organisations=c.include_sub_publishers) return t.render('reports/resources.html')
def geo_resource_form(context, data): """ Returns the form for the OFI Manager create/edit of OFI resource. """ if toolkit.asbool(ofi_config.get('convert_to_single_res', False)): file_formats = toolkit.get_action(u'file_formats')({}, {}) data.update(file_formats=file_formats) return toolkit.render('ofi/snippets/geo_resource_form.html', extra_vars=data)
def configure(self, config): '''Store the YouCKAN configuration''' if not toolkit.check_ckan_version('2.1'): log.warn('This extension has only been tested on CKAN 2.1!') self.use_auth = toolkit.asbool(config.get('youckan.use_auth', False)) if self.use_auth: self.logout_url = config['youckan.logout_url'] model_setup()
def _setup_ofi(id, context=None, pkg_dict=None, open_modal=True): # Setup for OFI if not context: context = {} try: if not pkg_dict: pkg_dict = get_action('package_show')(context, {'id': id, 'include_tracking':True}) except NotFound: abort(404, _('Dataset not found')) except NotAuthorized: abort(401, _('Unauthorized to read dataset %s') % id) ofi_data = {} if 'type' in pkg_dict and pkg_dict[u'type'] == 'Geographic': ofi_config = edc_h.get_ofi_config() # dev_secure_call is for working around missing siteminer session, for vagrant use only secure_call = toolkit.asbool(ofi_config.get('dev_secure_call', True)) ofi_resources = [] for resource in pkg_dict[u'resources']: if 'ofi' in resource and resource[u'ofi']: ofi_resources.append(resource) if len(ofi_resources) > 0: projections = get_action(u'crs_types')(context, {}) ofi_data.update({ u'object_name': pkg_dict.get(u'object_name', ""), u'ofi_resources': ofi_resources, u'ofi_exists': True, u'secure': secure_call, u'projections': projections, u'open_modal': open_modal }) elif pkg_dict.get('object_name', False): obj_data = { u'object_name': pkg_dict[u'object_name'], u'secure': secure_call, } ofi_data.update(get_action('check_object_name')(context, obj_data)) ofi_data.update(open_modal=open_modal) else: ofi_data.update({ u'object_name': '', u'secure': False, u'open_modal': False }) return ofi_data
def command(self): #if not correct args, show help if not self.args or self.args[0] in ['--help', '-h', 'help']: print self.usage return cmd = self.args[0] self._load_config() #update command if cmd == 'update': url_id = config.get('ckan.piwik.url').split('/piwik.php') piwik_url = url_id[0] use_https = toolkit.asbool(config.get('ckan.piwik.https', False)) if use_https: piwik_url = 'https:' + piwik_url else: piwik_url = 'http:' + piwik_url recent_date = date.today() - timedelta(days=int(config.get('ckan.piwik.recent_days', '14'))) piwik_date_opts = {'total': '2011-01-01,today', 'recent': recent_date.strftime('%Y-%m-%d') + ',today'} #setup piwik api GET request params piwik_params = { 'idSite': config.get('ckan.piwik.site_id'), 'token_auth': config.get('ckan.piwik.auth_token'), 'module': 'API', 'method': 'Actions.getPageUrl', 'format': 'json', 'period': 'range', 'pageUrl': None, 'date': None } if len(self.args) > 1: #update a single package self.update_package_stats(self.args[1], piwik_params, piwik_date_opts, piwik_url) else: #get all the packages pkgs_list = toolkit.get_action('package_list')(context=None, data_dict={}) for pkg in pkgs_list: self.update_package_stats(pkg, piwik_params, piwik_date_opts, piwik_url) else: print 'command: {cmd} not recognised'.format(cmd=cmd)
def build_pages_nav_main(*args): about_menu = toolkit.asbool(config.get('ckanext.pages.about_menu', True)) group_menu = toolkit.asbool(config.get('ckanext.pages.group_menu', True)) org_menu = toolkit.asbool(config.get('ckanext.pages.organization_menu', True)) new_args = [] for arg in args: if arg[0] == 'about' and not about_menu: continue if arg[0] == 'organizations_index' and not org_menu: continue if arg[0] == 'group_index' and not group_menu: continue new_args.append(arg) output = h.build_nav_main(*new_args) # do not display any private datasets in menu even for sysadmins pages_list = toolkit.get_action('ckanext_pages_list')(None, {'order': True, 'private': False}) page_name = '' if (toolkit.c.action in ('pages_show', 'blog_show') and toolkit.c.controller == 'ckanext.pages.controller:PagesController'): page_name = toolkit.c.environ['routes.url'].current().split('/')[-1] for page in pages_list: type_ = 'blog' if page['page_type'] == 'blog' else 'pages' name = urllib.quote(page['name'].encode('utf-8')).decode('utf-8') title = cgi.escape(page['title']) link = h.literal(u'<a href="/{}/{}">{}</a>'.format(type_, name, title)) if page['name'] == page_name: li = h.literal('<li class="active">') + link + h.literal('</li>') else: li = h.literal('<li>') + link + h.literal('</li>') output = output + li return output
def related_pkgs(cls): # Parameter extractNum = int(config.get('ckan.data_recommended.extract_num', '5')) byTag = asbool(config.get('ckan.data_recommended.by_tag', 'true')) byTitle = asbool(config.get('ckan.data_recommended.by_title', 'true')) renderMod = 'front_end' # fetch pkg info pkg_name = request.environ['PATH_INFO'].split('/')[-1] pkg_title = toolkit.get_action('package_show')({}, {'id':pkg_name})['title'] pkg_tags = [pkg_tag['name'] for pkg_tag in toolkit.get_action('package_show')({}, {'id':pkg_name})['tags']] # related_tag_titles related_tag_titles = set() if byTag: related_tag_titles.update(set(pkg_tags)) if byTitle: related_tag_titles.update( set( jieba.analyse.extract_tags(pkg_title, topK=extractNum) ) ) related_pkgs = {} if renderMod == 'back_end': related_pkgs['mod'] = renderMod related_pkgs['results'] = dict() for related_tag_title in related_tag_titles: related_pkg_results = toolkit.get_action('package_search')({}, {'q': related_tag_title, 'rows': 3})['results'] related_pkgs['results'][related_tag_title] = related_pkg_results return related_pkgs elif renderMod == 'front_end': related_pkgs['results'] = related_tag_titles related_pkgs['mod'] = renderMod return related_pkgs
def after_create(self, context, pkg_dict): dataset_type = context['package'].type if 'package' in context else pkg_dict['type'] if dataset_type == 'dataset': log.info('after_create: %s', pkg_dict['name']) odm_dataset_helper.session['last_dataset'] = pkg_dict odm_dataset_helper.session.save() # Create default Issue review_system = toolkit.asbool(config.get("ckanext.issues.review_system", False)) if review_system: if pkg_dict['type'] == 'dataset': odm_dataset_helper.create_default_issue_dataset(pkg_dict)