def config_tool(config_filepath: str, options: list[str], section: str, edit: bool, merge_filepath: str) -> None: u'''Tool for editing options in a CKAN config file ckan config-tool <default.ini> <key>=<value> [<key>=<value> ...] ckan config-tool <default.ini> -f <custom_options.ini> Examples: ckan config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC' ckan config-tool default.ini -s server:main -e port=8080 ckan config-tool default.ini -f custom_options.ini ''' if merge_filepath: ct.config_edit_using_merge_file(config_filepath, merge_filepath) if not (options or merge_filepath): error_shout(u'No options provided') raise click.Abort() try: ct.config_edit_using_option_strings(config_filepath, options, section, edit=edit) except ct.ConfigToolError as e: error_shout(e) raise click.Abort()
def make_config(output_path): u"""Generate a new CKAN configuration ini file.""" # Output to current directory if no path is specified if u'/' not in output_path: output_path = os.path.join(os.getcwd(), output_path) cur_loc = os.path.dirname(os.path.abspath(__file__)) template_loc = os.path.join(cur_loc, u'..', u'config', u'deployment.ini_tmpl') template_variables = { u'app_instance_uuid': uuid.uuid4(), u'app_instance_secret': secrets.token_urlsafe(20)[:25] } with open(template_loc, u'r') as file_in: template = string.Template(file_in.read()) try: with open(output_path, u'w') as file_out: file_out.writelines(template.substitute(template_variables)) except IOError as e: error_shout(e) raise click.Abort()
def submit(package: str, yes: bool): u'''Submits resources from package. If no package ID/name specified, submits all resources from all packages. ''' confirm(yes) if not package: ids = tk.get_action(u'package_list')(cast( Context, { u'model': model, u'ignore_auth': True }), {}) else: ids = [package] for id in ids: package_show = tk.get_action(u'package_show') try: pkg = package_show( cast(Context, { u'model': model, u'ignore_auth': True }), {u'id': id}) except Exception as e: error_shout(e) error_shout(u"Package '{}' was not found".format(package)) raise click.Abort() if not pkg[u'resources']: continue resource_ids = [r[u'id'] for r in pkg[u'resources']] _submit(resource_ids)
def config_tool(config_filepath, options, section, edit, merge_filepath): u'''Tool for editing options in a CKAN config file paster config-tool <default.ini> <key>=<value> [<key>=<value> ...] paster config-tool <default.ini> -f <custom_options.ini> Examples: paster config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC' paster config-tool default.ini -s server:main -e port=8080 paster config-tool default.ini -f custom_options.ini ''' if merge_filepath: ct.config_edit_using_merge_file( config_filepath, merge_filepath ) if not (options or merge_filepath): return error_shout(u'No options provided') try: ct.config_edit_using_option_strings( config_filepath, options, section, edit=edit ) except ct.ConfigToolError as e: error_shout(e)
def list_tokens(username): u"""List all API Tokens for the given user""" try: tokens = plugin.toolkit.get_action(u"api_token_list")( {u"ignore_auth": True}, {u"user": username} ) except plugin.toolkit.ObjectNotFound as e: error_shout(e) raise click.Abort() if not tokens: click.secho(u"No tokens have been created for user yet", fg=u"red") return click.echo(u"Tokens([id] name - lastAccess):") for token in tokens: last_access = token[u"last_access"] if last_access: accessed = plugin.toolkit.h.date_str_to_datetime( last_access ).isoformat(u" ", u"seconds") else: accessed = u"Never" click.echo( u"\t[{id}] {name} - {accessed}".format( name=token[u"name"], id=token[u"id"], accessed=accessed ) )
def _update_search_params(search_data_dict: dict[str, Any], search: str): """ Update the `package_search` data dict with the user provided parameters Supported fields are `q`, `fq` and `fq_list`. If the provided JSON object can not be parsed the process stops with an error. Returns the updated data dict """ if not search: return search_data_dict try: user_search_params = json.loads(search) except ValueError as e: error_shout(u"Unable to parse JSON search parameters: {0}".format(e)) return None if user_search_params.get(u"q"): search_data_dict[u"q"] = user_search_params[u"q"] if user_search_params.get(u"fq"): if search_data_dict[u"fq"]: search_data_dict[u"fq"] += u" " + user_search_params[u"fq"] else: search_data_dict[u"fq"] = user_search_params[u"fq"] if user_search_params.get(u"fq_list") and isinstance( user_search_params[u"fq_list"], list): search_data_dict[u"fq_list"].extend(user_search_params[u"fq_list"]) return search_data_dict
def less(): command = (u'npm', u'bin') process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) output = process.communicate() directory = output[0].strip() if not directory: error_shout(u'Command "{}" returned nothing. Check that npm is ' u'installed.'.format(' '.join(command))) less_bin = os.path.join(directory, u'lessc') public = config.get(u'ckan.base_public_folder') root = os.path.join(os.path.dirname(__file__), u'..', public, u'base') root = os.path.abspath(root) custom_less = os.path.join(root, u'less', u'custom.less') for color in _custom_css: f = open(custom_less, u'w') f.write(_custom_css[color]) f.close() _compile_less(root, less_bin, color) f = open(custom_less, u'w') f.write(u'// This file is needed in order for ./bin/less to ' u'compile in less 1.3.1+\n') f.close() _compile_less(root, less_bin, u'main')
def show_user(username): import ckan.model as model if not username: error_shout(u'Please specify the username for the user') return user = model.User.get(text_type(username)) click.secho(u'User: %s' % user)
def make_config(output_path, include_plugin): u"""Generate a new CKAN configuration ini file.""" # Output to current directory if no path is specified if u'/' not in output_path: output_path = os.path.join(os.getcwd(), output_path) cur_loc = os.path.dirname(os.path.abspath(__file__)) template_loc = os.path.join(cur_loc, u'..', u'config', u'deployment.ini_tmpl') config_declaration._reset() config_declaration.load_core_declaration() for plugin in include_plugin: config_declaration.load_plugin(plugin) variables = {"declaration": config_declaration.into_ini(False, False)} with open(template_loc, u'r') as file_in: template = string.Template(file_in.read()) try: with open(output_path, u'w') as file_out: file_out.writelines(template.substitute(variables)) except IOError as e: error_shout(e) raise click.Abort()
def update_tracking_solr(engine, start_date): sql = u'''SELECT package_id FROM tracking_summary where package_id!='~~not~found~~' and tracking_date >= %s;''' results = engine.execute(sql, start_date) package_ids = set() for row in results: package_ids.add(row[u'package_id']) total = len(package_ids) not_found = 0 click.echo('{} package index{} to be rebuilt starting from {}'.format( total, '' if total < 2 else 'es', start_date)) from ckan.lib.search import rebuild for package_id in package_ids: try: rebuild(package_id) except logic.NotFound: click.echo(u'Error: package {} not found.'.format(package_id)) not_found += 1 except KeyboardInterrupt: click.echo(u'Stopped.') return except Exception as e: error_shout(e) click.echo(u'search index rebuilding done.' + (u' {} not found.'.format(not_found) if not_found else u''))
def add_token(username: str, token_name: str, extras: list[str], json_str: str): """Create a new API Token for the given user. Arbitrary fields can be passed in the form `key=value` or using the --json option, containing a JSON encoded object. When both provided, `key=value` fields will take precedence and will replace the corresponding keys from the --json object. Example: ckan user token add john_doe new_token x=y --json '{"prop": "value"}' """ data_dict = json.loads(json_str) for chunk in extras: try: key, value = chunk.split(u"=") except ValueError: error_shout( u"Extras must be passed as `key=value`. Got: {}".format(chunk)) raise click.Abort() data_dict[key] = value data_dict.update({u"user": username, u"name": token_name}) try: token = logic.get_action(u"api_token_create")({ u"ignore_auth": True }, data_dict) except logic.NotFound as e: error_shout(e) raise click.Abort() click.secho(u"API Token created:", fg=u"green") click.echo(u"\t", nl=False) click.echo(token[u"token"])
def profile_url(url): # noqa try: app.get(url, status=[200], extra_environ={u"REMOTE_USER": str(user)}) except KeyboardInterrupt: raise except Exception: error_shout(traceback.format_exc())
def downgradedb(version): u'''Downgrading the database''' try: import ckan.model as model model.repo.downgrade_db(version) except Exception as e: error_shout(e) else: click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True)
def downgradedb(version): u'''Downgrading the database''' try: import ckan.model as model model.repo.downgrade_db(version) except Exception as e: error_shout(e) else: click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True)
def cleandb(): u'''Cleaning the database''' try: import ckan.model as model model.repo.clean_db() except Exception as e: error_shout(e) else: click.secho(u'Cleaning DB: SUCCESS', fg=u'green', bold=True)
def cleandb(): u'''Cleaning the database''' try: import ckan.model as model model.repo.clean_db() except Exception as e: error_shout(e) else: click.secho(u'Cleaning DB: SUCCESS', fg=u'green', bold=True)
def initdb(): u'''Initialising the database''' log.info(u"Initialize the Database") try: import ckan.model as model model.repo.init_db() except Exception as e: error_shout(e) else: click.secho(u'Initialising DB: SUCCESS', fg=u'green', bold=True)
def remove_user(ctx: click.Context, username: str): if not username: error_shout(u'Please specify the username to be removed') return site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {}) context: Context = {u'user': site_user[u'name']} with ctx.meta['flask_app'].test_request_context(): logic.get_action(u'user_delete')(context, {u'id': username}) click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True)
def initdb(): u'''Initialising the database''' log.info(u"Initialize the Database") try: import ckan.model as model model.repo.init_db() except Exception as e: error_shout(e) else: click.secho(u'Initialising DB: SUCCESS', fg=u'green', bold=True)
def version(): u'''Return current version''' log.info(u"Returning current DB version") try: from ckan.model import Session ver = Session.execute(u'select version from ' u'migrate_version;').fetchall() click.secho(u"Latest data schema version: {0}".format(ver[0][0]), bold=True) except Exception as e: error_shout(e)
def add_user(ctx, username, args): u'''Add new user if we use ckan sysadmin add or ckan user add ''' # parse args into data_dict data_dict = {u'name': username} for arg in args: try: field, value = arg.split(u'=', 1) data_dict[field] = value except ValueError: raise ValueError( u'Could not parse arg: %r (expected "<option>=<value>)"' % arg) # Required if u'email' not in data_dict: data_dict['email'] = click.prompt(u'Email address ').strip() if u'password' not in data_dict: data_dict['password'] = click.prompt(u'Password ', hide_input=True, confirmation_prompt=True) # Optional if u'fullname' in data_dict: data_dict['fullname'] = data_dict['fullname'].decode( sys.getfilesystemencoding()) # pprint(u'Creating user: %r' % username) try: import ckan.logic as logic import ckan.model as model site_user = logic.get_action(u'get_site_user')({ u'model': model, u'ignore_auth': True }, {}) context = { u'model': model, u'session': model.Session, u'ignore_auth': True, u'user': site_user['name'], } flask_app = ctx.meta['flask_app'] # Current user is tested agains sysadmin role during model # dictization, thus we need request context with flask_app.test_request_context(): user_dict = logic.get_action(u'user_create')(context, data_dict) click.secho(u"Successfully created user: %s" % user_dict['name'], fg=u'green', bold=True) except logic.ValidationError as e: error_shout(e) raise click.Abort()
def remove_user(ctx, username): import ckan.model as model if not username: error_shout(u'Please specify the username to be removed') return flask_app = ctx.obj.app.apps['flask_app']._wsgi_app site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {}) context = {u'user': site_user[u'name']} with flask_app.test_request_context(): plugin.toolkit.get_action(u'user_delete')(context, {u'id': username}) click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True)
def _version_hash_to_ordinal(version): if u'base' == version: return 0 versions_dir = os.path.join(os.path.dirname(migration_repo.__file__), u'versions') versions = sorted(os.listdir(versions_dir)) # latest version looks like `123abc (head)` if version.endswith(u'(head)'): return int(versions[-1].split(u'_')[0]) for name in versions: if version in name: return int(name.split(u'_')[0]) error_shout(u'Version `{}` was not found in {}'.format( version, versions_dir))
def cancel(id): """Cancel a specific job. Jobs can only be canceled while they are enqueued. Once a worker has started executing a job it cannot be aborted anymore. """ try: p.toolkit.get_action(u"job_cancel")( {u"ignore_auth": True}, {u"id": id} ) except logic.NotFound: error_shout(u'There is no job with ID "{}"'.format(id)) raise click.Abort() click.secho(u"Cancelled job {}".format(id), fg=u"green")
def rebuild(ctx, verbose, force, refresh, only_missing, quiet, commit_each): u''' Rebuild search index ''' flask_app = ctx.obj.app.apps['flask_app']._wsgi_app from ckan.lib.search import rebuild, commit try: with flask_app.test_request_context(): rebuild(only_missing=only_missing, force=force, refresh=refresh, defer_commit=(not commit_each), quiet=quiet) except Exception as e: error_shout(e) if not commit_each: commit()
def rebuild(ctx, verbose, force, refresh, only_missing, quiet, commit_each): u''' Rebuild search index ''' flask_app = ctx.obj.app.apps['flask_app']._wsgi_app from ckan.lib.search import rebuild, commit try: with flask_app.test_request_context(): rebuild(only_missing=only_missing, force=force, refresh=refresh, defer_commit=(not commit_each), quiet=quiet) except Exception as e: error_shout(e) if not commit_each: commit()
def remove(username): user = model.User.by_name(text_type(username)) if not user: return error_shout(u'Error: user "%s" not found!' % username) user.sysadmin = False model.repo.commit_and_remove() click.secho(u"Removed %s from sysadmins" % username, fg=u"green")
def set_password(username): import ckan.model as model if not username: error_shout(u'Need name of the user.') return user = model.User.get(username) if not user: error_shout(u"User not found!") return click.secho(u'Editing user: %r' % user.name, fg=u'yellow') password = click.prompt(u'Password', hide_input=True, confirmation_prompt=True) user.password = password model.repo.commit_and_remove() click.secho(u'Password updated!', fg=u'green', bold=True)
def clean(): u'''Will clear out the cache, which after a while can grow quite large.''' try: script.main(['clean'], webassets_tools.env) except BundleError as e: return error_shout(e) click.secho(u'Clear cache: SUCCESS', fg=u'green', bold=True)
def show(id): try: job = p.toolkit.get_action(u"job_show")( {u"ignore_auth": True}, {u"id": id} ) except logic.NotFound: error_shout(u'There is no job with ID "{}"'.format(id)) raise click.Abort() click.secho(u"ID: {}".format(job[u"id"])) if job[u"title"] is None: title = u"None" else: title = u'"{}"'.format(job[u"title"]) click.secho(u"Title: {}".format(title)) click.secho(u"Created: {}".format(job[u"created"])) click.secho(u"Queue: {}".format(job[u"queue"]))
def _version_hash_to_ordinal(version): if u'base' == version: return 0 versions_dir = os.path.join( os.path.dirname(migration_repo.__file__), u'versions' ) versions = sorted(os.listdir(versions_dir)) # latest version looks like `123abc (head)` if version.endswith(u'(head)'): return int(versions[-1].split(u'_')[0]) for name in versions: if version in name: return int(name.split(u'_')[0]) error_shout(u'Version `{}` was not found in {}'.format( version, versions_dir ))
def _get_view_plugins(view_plugin_types: list[str], get_datastore_views: bool = False): """Returns the view plugins that were succesfully loaded Views are provided as a list of ``view_plugin_types``. If no types are provided, the default views defined in the ``ckan.views.default_views`` will be created. Only in this case (when the default view plugins are used) the `get_datastore_views` parameter can be used to get also view plugins that require data to be in the DataStore. If any of the provided plugins could not be loaded (eg it was not added to `ckan.plugins`) the command will stop. Returns a list of loaded plugin names. """ view_plugins = [] if not view_plugin_types: click.secho(u"No view types provided, using default types") view_plugins = get_default_view_plugins() if get_datastore_views: view_plugins.extend( get_default_view_plugins(get_datastore_views=True) ) else: view_plugins = get_view_plugins(view_plugin_types) loaded_view_plugins = [ view_plugin.info()[u"name"] for view_plugin in view_plugins ] plugins_not_found = list(set(view_plugin_types) - set(loaded_view_plugins)) if plugins_not_found: error_shout( u"View plugin(s) not found : {0}. ".format(plugins_not_found) + u"Have they been added to the `ckan.plugins` configuration" + u" option?" ) return None return loaded_view_plugins
def migration(plugin: str, message: str): """Create new alembic revision for DB migration. """ if not config: error_shout(u'Config is not loaded') raise click.Abort() alembic_config = CKANAlembicConfig(_resolve_alembic_config(plugin)) assert alembic_config.config_file_name migration_dir = os.path.dirname(alembic_config.config_file_name) alembic_config.set_main_option("sqlalchemy.url", "") alembic_config.set_main_option(u'script_location', migration_dir) if not os.path.exists(os.path.join(migration_dir, u'script.py.mako')): alembic.command.init(alembic_config, migration_dir) rev = alembic.command.revision(alembic_config, message) rev_path = rev.path # type: ignore click.secho( f"Revision file created. Now, you need to update it: \n\t{rev_path}", fg=u"green")
def migration(plugin, message): """Create new alembic revision for DB migration. """ import ckan.model if not config: error_shout(u'Config is not loaded') raise click.Abort() alembic_config = CKANAlembicConfig(_resolve_alembic_config(plugin)) migration_dir = os.path.dirname(alembic_config.config_file_name) alembic_config.set_main_option(u"sqlalchemy.url", str(ckan.model.repo.metadata.bind.url)) alembic_config.set_main_option(u'script_location', migration_dir) if not os.path.exists(os.path.join(migration_dir, u'script.py.mako')): alembic.command.init(alembic_config, migration_dir) rev = alembic.command.revision(alembic_config, message) click.secho( u"Revision file created. Now, you need to update it: \n\t{}".format( rev.path), fg=u"green")
def send_emails(): """ Sends an email to users notifying about new activities. As currently implemented, it will only send notifications from dashboard activity list if users have `activity_streams_email_notifications` set in their profile. It will send emails with updates depending on the `ckan.email_notifications_since` config. (default: 2 days.) """ import ckan.logic as logic import ckan.lib.mailer as mailer from ckan.types import Context from typing import cast site_user = logic.get_action("get_site_user")({"ignore_auth": True}, {}) context = cast(Context, {"user": site_user["name"]}) try: logic.get_action("send_email_notifications")(context, {}) except (NotAuthorized, ValidationError, mailer.MailerException) as e: error_shout(e) except KeyError: error_shout("`activity` plugin is not enabled")