def config_tool(config_filepath: str, options: list[str], section: str,
edit: bool, merge_filepath: str) -> None:
u'''Tool for editing options in a CKAN config file
ckan config-tool <default.ini> <key>=<value> [<key>=<value> ...]
ckan config-tool <default.ini> -f <custom_options.ini>
Examples:
ckan config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC'
ckan config-tool default.ini -s server:main -e port=8080
ckan config-tool default.ini -f custom_options.ini
'''
if merge_filepath:
ct.config_edit_using_merge_file(config_filepath, merge_filepath)
if not (options or merge_filepath):
error_shout(u'No options provided')
raise click.Abort()
try:
ct.config_edit_using_option_strings(config_filepath,
options,
section,
edit=edit)
except ct.ConfigToolError as e:
error_shout(e)
raise click.Abort()
def make_config(output_path):
u"""Generate a new CKAN configuration ini file."""
# Output to current directory if no path is specified
if u'/' not in output_path:
output_path = os.path.join(os.getcwd(), output_path)
cur_loc = os.path.dirname(os.path.abspath(__file__))
template_loc = os.path.join(cur_loc, u'..', u'config',
u'deployment.ini_tmpl')
template_variables = {
u'app_instance_uuid': uuid.uuid4(),
u'app_instance_secret': secrets.token_urlsafe(20)[:25]
}
with open(template_loc, u'r') as file_in:
template = string.Template(file_in.read())
try:
with open(output_path, u'w') as file_out:
file_out.writelines(template.substitute(template_variables))
except IOError as e:
error_shout(e)
raise click.Abort()
def submit(package: str, yes: bool):
u'''Submits resources from package.
If no package ID/name specified, submits all resources from all
packages.
'''
confirm(yes)
if not package:
ids = tk.get_action(u'package_list')(cast(
Context, {
u'model': model,
u'ignore_auth': True
}), {})
else:
ids = [package]
for id in ids:
package_show = tk.get_action(u'package_show')
try:
pkg = package_show(
cast(Context, {
u'model': model,
u'ignore_auth': True
}), {u'id': id})
except Exception as e:
error_shout(e)
error_shout(u"Package '{}' was not found".format(package))
raise click.Abort()
if not pkg[u'resources']:
continue
resource_ids = [r[u'id'] for r in pkg[u'resources']]
_submit(resource_ids)
def config_tool(config_filepath, options, section, edit, merge_filepath):
u'''Tool for editing options in a CKAN config file
paster config-tool <default.ini> <key>=<value> [<key>=<value> ...]
paster config-tool <default.ini> -f <custom_options.ini>
Examples:
paster config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC'
paster config-tool default.ini -s server:main -e port=8080
paster config-tool default.ini -f custom_options.ini
'''
if merge_filepath:
ct.config_edit_using_merge_file(
config_filepath, merge_filepath
)
if not (options or merge_filepath):
return error_shout(u'No options provided')
try:
ct.config_edit_using_option_strings(
config_filepath, options, section, edit=edit
)
except ct.ConfigToolError as e:
error_shout(e)
def list_tokens(username):
u"""List all API Tokens for the given user"""
try:
tokens = plugin.toolkit.get_action(u"api_token_list")(
{u"ignore_auth": True}, {u"user": username}
)
except plugin.toolkit.ObjectNotFound as e:
error_shout(e)
raise click.Abort()
if not tokens:
click.secho(u"No tokens have been created for user yet", fg=u"red")
return
click.echo(u"Tokens([id] name - lastAccess):")
for token in tokens:
last_access = token[u"last_access"]
if last_access:
accessed = plugin.toolkit.h.date_str_to_datetime(
last_access
).isoformat(u" ", u"seconds")
else:
accessed = u"Never"
click.echo(
u"\t[{id}] {name} - {accessed}".format(
name=token[u"name"], id=token[u"id"], accessed=accessed
)
)
def _update_search_params(search_data_dict: dict[str, Any], search: str):
"""
Update the `package_search` data dict with the user provided parameters
Supported fields are `q`, `fq` and `fq_list`.
If the provided JSON object can not be parsed the process stops with
an error.
Returns the updated data dict
"""
if not search:
return search_data_dict
try:
user_search_params = json.loads(search)
except ValueError as e:
error_shout(u"Unable to parse JSON search parameters: {0}".format(e))
return None
if user_search_params.get(u"q"):
search_data_dict[u"q"] = user_search_params[u"q"]
if user_search_params.get(u"fq"):
if search_data_dict[u"fq"]:
search_data_dict[u"fq"] += u" " + user_search_params[u"fq"]
else:
search_data_dict[u"fq"] = user_search_params[u"fq"]
if user_search_params.get(u"fq_list") and isinstance(
user_search_params[u"fq_list"], list):
search_data_dict[u"fq_list"].extend(user_search_params[u"fq_list"])
return search_data_dict
def less():
command = (u'npm', u'bin')
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
output = process.communicate()
directory = output[0].strip()
if not directory:
error_shout(u'Command "{}" returned nothing. Check that npm is '
u'installed.'.format(' '.join(command)))
less_bin = os.path.join(directory, u'lessc')
public = config.get(u'ckan.base_public_folder')
root = os.path.join(os.path.dirname(__file__), u'..', public, u'base')
root = os.path.abspath(root)
custom_less = os.path.join(root, u'less', u'custom.less')
for color in _custom_css:
f = open(custom_less, u'w')
f.write(_custom_css[color])
f.close()
_compile_less(root, less_bin, color)
f = open(custom_less, u'w')
f.write(u'// This file is needed in order for ./bin/less to '
u'compile in less 1.3.1+\n')
f.close()
_compile_less(root, less_bin, u'main')
def show_user(username):
import ckan.model as model
if not username:
error_shout(u'Please specify the username for the user')
return
user = model.User.get(text_type(username))
click.secho(u'User: %s' % user)
def make_config(output_path, include_plugin):
u"""Generate a new CKAN configuration ini file."""
# Output to current directory if no path is specified
if u'/' not in output_path:
output_path = os.path.join(os.getcwd(), output_path)
cur_loc = os.path.dirname(os.path.abspath(__file__))
template_loc = os.path.join(cur_loc, u'..', u'config',
u'deployment.ini_tmpl')
config_declaration._reset()
config_declaration.load_core_declaration()
for plugin in include_plugin:
config_declaration.load_plugin(plugin)
variables = {"declaration": config_declaration.into_ini(False, False)}
with open(template_loc, u'r') as file_in:
template = string.Template(file_in.read())
try:
with open(output_path, u'w') as file_out:
file_out.writelines(template.substitute(variables))
except IOError as e:
error_shout(e)
raise click.Abort()
def update_tracking_solr(engine, start_date):
sql = u'''SELECT package_id FROM tracking_summary
where package_id!='~~not~found~~'
and tracking_date >= %s;'''
results = engine.execute(sql, start_date)
package_ids = set()
for row in results:
package_ids.add(row[u'package_id'])
total = len(package_ids)
not_found = 0
click.echo('{} package index{} to be rebuilt starting from {}'.format(
total, '' if total < 2 else 'es', start_date))
from ckan.lib.search import rebuild
for package_id in package_ids:
try:
rebuild(package_id)
except logic.NotFound:
click.echo(u'Error: package {} not found.'.format(package_id))
not_found += 1
except KeyboardInterrupt:
click.echo(u'Stopped.')
return
except Exception as e:
error_shout(e)
click.echo(u'search index rebuilding done.' +
(u' {} not found.'.format(not_found) if not_found else u''))
def add_token(username: str, token_name: str, extras: list[str],
json_str: str):
"""Create a new API Token for the given user.
Arbitrary fields can be passed in the form `key=value` or using
the --json option, containing a JSON encoded object. When both provided,
`key=value` fields will take precedence and will replace the
corresponding keys from the --json object.
Example:
ckan user token add john_doe new_token x=y --json '{"prop": "value"}'
"""
data_dict = json.loads(json_str)
for chunk in extras:
try:
key, value = chunk.split(u"=")
except ValueError:
error_shout(
u"Extras must be passed as `key=value`. Got: {}".format(chunk))
raise click.Abort()
data_dict[key] = value
data_dict.update({u"user": username, u"name": token_name})
try:
token = logic.get_action(u"api_token_create")({
u"ignore_auth": True
}, data_dict)
except logic.NotFound as e:
error_shout(e)
raise click.Abort()
click.secho(u"API Token created:", fg=u"green")
click.echo(u"\t", nl=False)
click.echo(token[u"token"])
def profile_url(url): # noqa
try:
app.get(url,
status=[200],
extra_environ={u"REMOTE_USER": str(user)})
except KeyboardInterrupt:
raise
except Exception:
error_shout(traceback.format_exc())
def downgradedb(version):
u'''Downgrading the database'''
try:
import ckan.model as model
model.repo.downgrade_db(version)
except Exception as e:
error_shout(e)
else:
click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True)
def downgradedb(version):
u'''Downgrading the database'''
try:
import ckan.model as model
model.repo.downgrade_db(version)
except Exception as e:
error_shout(e)
else:
click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True)
def cleandb():
u'''Cleaning the database'''
try:
import ckan.model as model
model.repo.clean_db()
except Exception as e:
error_shout(e)
else:
click.secho(u'Cleaning DB: SUCCESS', fg=u'green', bold=True)
def cleandb():
u'''Cleaning the database'''
try:
import ckan.model as model
model.repo.clean_db()
except Exception as e:
error_shout(e)
else:
click.secho(u'Cleaning DB: SUCCESS', fg=u'green', bold=True)
def initdb():
u'''Initialising the database'''
log.info(u"Initialize the Database")
try:
import ckan.model as model
model.repo.init_db()
except Exception as e:
error_shout(e)
else:
click.secho(u'Initialising DB: SUCCESS', fg=u'green', bold=True)
def remove_user(ctx: click.Context, username: str):
if not username:
error_shout(u'Please specify the username to be removed')
return
site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {})
context: Context = {u'user': site_user[u'name']}
with ctx.meta['flask_app'].test_request_context():
logic.get_action(u'user_delete')(context, {u'id': username})
click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True)
def initdb():
u'''Initialising the database'''
log.info(u"Initialize the Database")
try:
import ckan.model as model
model.repo.init_db()
except Exception as e:
error_shout(e)
else:
click.secho(u'Initialising DB: SUCCESS', fg=u'green', bold=True)
def version():
u'''Return current version'''
log.info(u"Returning current DB version")
try:
from ckan.model import Session
ver = Session.execute(u'select version from '
u'migrate_version;').fetchall()
click.secho(u"Latest data schema version: {0}".format(ver[0][0]),
bold=True)
except Exception as e:
error_shout(e)
def add_user(ctx, username, args):
u'''Add new user if we use ckan sysadmin add
or ckan user add
'''
# parse args into data_dict
data_dict = {u'name': username}
for arg in args:
try:
field, value = arg.split(u'=', 1)
data_dict[field] = value
except ValueError:
raise ValueError(
u'Could not parse arg: %r (expected "<option>=<value>)"' % arg)
# Required
if u'email' not in data_dict:
data_dict['email'] = click.prompt(u'Email address ').strip()
if u'password' not in data_dict:
data_dict['password'] = click.prompt(u'Password ',
hide_input=True,
confirmation_prompt=True)
# Optional
if u'fullname' in data_dict:
data_dict['fullname'] = data_dict['fullname'].decode(
sys.getfilesystemencoding())
# pprint(u'Creating user: %r' % username)
try:
import ckan.logic as logic
import ckan.model as model
site_user = logic.get_action(u'get_site_user')({
u'model': model,
u'ignore_auth': True
}, {})
context = {
u'model': model,
u'session': model.Session,
u'ignore_auth': True,
u'user': site_user['name'],
}
flask_app = ctx.meta['flask_app']
# Current user is tested agains sysadmin role during model
# dictization, thus we need request context
with flask_app.test_request_context():
user_dict = logic.get_action(u'user_create')(context, data_dict)
click.secho(u"Successfully created user: %s" % user_dict['name'],
fg=u'green',
bold=True)
except logic.ValidationError as e:
error_shout(e)
raise click.Abort()
def remove_user(ctx, username):
import ckan.model as model
if not username:
error_shout(u'Please specify the username to be removed')
return
flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {})
context = {u'user': site_user[u'name']}
with flask_app.test_request_context():
plugin.toolkit.get_action(u'user_delete')(context, {u'id': username})
click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True)
def _version_hash_to_ordinal(version):
if u'base' == version:
return 0
versions_dir = os.path.join(os.path.dirname(migration_repo.__file__),
u'versions')
versions = sorted(os.listdir(versions_dir))
# latest version looks like `123abc (head)`
if version.endswith(u'(head)'):
return int(versions[-1].split(u'_')[0])
for name in versions:
if version in name:
return int(name.split(u'_')[0])
error_shout(u'Version `{}` was not found in {}'.format(
version, versions_dir))
def cancel(id):
"""Cancel a specific job. Jobs can only be canceled while they are
enqueued. Once a worker has started executing a job it cannot be
aborted anymore.
"""
try:
p.toolkit.get_action(u"job_cancel")(
{u"ignore_auth": True}, {u"id": id}
)
except logic.NotFound:
error_shout(u'There is no job with ID "{}"'.format(id))
raise click.Abort()
click.secho(u"Cancelled job {}".format(id), fg=u"green")
def rebuild(ctx, verbose, force, refresh, only_missing, quiet, commit_each):
u''' Rebuild search index '''
flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
from ckan.lib.search import rebuild, commit
try:
with flask_app.test_request_context():
rebuild(only_missing=only_missing,
force=force,
refresh=refresh,
defer_commit=(not commit_each),
quiet=quiet)
except Exception as e:
error_shout(e)
if not commit_each:
commit()
def rebuild(ctx, verbose, force, refresh, only_missing, quiet, commit_each):
u''' Rebuild search index '''
flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
from ckan.lib.search import rebuild, commit
try:
with flask_app.test_request_context():
rebuild(only_missing=only_missing,
force=force,
refresh=refresh,
defer_commit=(not commit_each),
quiet=quiet)
except Exception as e:
error_shout(e)
if not commit_each:
commit()
def remove(username):
user = model.User.by_name(text_type(username))
if not user:
return error_shout(u'Error: user "%s" not found!' % username)
user.sysadmin = False
model.repo.commit_and_remove()
click.secho(u"Removed %s from sysadmins" % username, fg=u"green")
def set_password(username):
import ckan.model as model
if not username:
error_shout(u'Need name of the user.')
return
user = model.User.get(username)
if not user:
error_shout(u"User not found!")
return
click.secho(u'Editing user: %r' % user.name, fg=u'yellow')
password = click.prompt(u'Password', hide_input=True,
confirmation_prompt=True)
user.password = password
model.repo.commit_and_remove()
click.secho(u'Password updated!', fg=u'green', bold=True)
def clean():
u'''Will clear out the cache, which after a while can grow quite large.'''
try:
script.main(['clean'], webassets_tools.env)
except BundleError as e:
return error_shout(e)
click.secho(u'Clear cache: SUCCESS', fg=u'green', bold=True)
def show(id):
try:
job = p.toolkit.get_action(u"job_show")(
{u"ignore_auth": True}, {u"id": id}
)
except logic.NotFound:
error_shout(u'There is no job with ID "{}"'.format(id))
raise click.Abort()
click.secho(u"ID: {}".format(job[u"id"]))
if job[u"title"] is None:
title = u"None"
else:
title = u'"{}"'.format(job[u"title"])
click.secho(u"Title: {}".format(title))
click.secho(u"Created: {}".format(job[u"created"]))
click.secho(u"Queue: {}".format(job[u"queue"]))
def _version_hash_to_ordinal(version):
if u'base' == version:
return 0
versions_dir = os.path.join(
os.path.dirname(migration_repo.__file__), u'versions'
)
versions = sorted(os.listdir(versions_dir))
# latest version looks like `123abc (head)`
if version.endswith(u'(head)'):
return int(versions[-1].split(u'_')[0])
for name in versions:
if version in name:
return int(name.split(u'_')[0])
error_shout(u'Version `{}` was not found in {}'.format(
version, versions_dir
))
def _get_view_plugins(view_plugin_types: list[str],
get_datastore_views: bool = False):
"""Returns the view plugins that were succesfully loaded
Views are provided as a list of ``view_plugin_types``. If no types
are provided, the default views defined in the
``ckan.views.default_views`` will be created. Only in this case
(when the default view plugins are used) the `get_datastore_views`
parameter can be used to get also view plugins that require data
to be in the DataStore.
If any of the provided plugins could not be loaded (eg it was not
added to `ckan.plugins`) the command will stop.
Returns a list of loaded plugin names.
"""
view_plugins = []
if not view_plugin_types:
click.secho(u"No view types provided, using default types")
view_plugins = get_default_view_plugins()
if get_datastore_views:
view_plugins.extend(
get_default_view_plugins(get_datastore_views=True)
)
else:
view_plugins = get_view_plugins(view_plugin_types)
loaded_view_plugins = [
view_plugin.info()[u"name"] for view_plugin in view_plugins
]
plugins_not_found = list(set(view_plugin_types) - set(loaded_view_plugins))
if plugins_not_found:
error_shout(
u"View plugin(s) not found : {0}. ".format(plugins_not_found)
+ u"Have they been added to the `ckan.plugins` configuration"
+ u" option?"
)
return None
return loaded_view_plugins
def migration(plugin: str, message: str):
"""Create new alembic revision for DB migration.
"""
if not config:
error_shout(u'Config is not loaded')
raise click.Abort()
alembic_config = CKANAlembicConfig(_resolve_alembic_config(plugin))
assert alembic_config.config_file_name
migration_dir = os.path.dirname(alembic_config.config_file_name)
alembic_config.set_main_option("sqlalchemy.url", "")
alembic_config.set_main_option(u'script_location', migration_dir)
if not os.path.exists(os.path.join(migration_dir, u'script.py.mako')):
alembic.command.init(alembic_config, migration_dir)
rev = alembic.command.revision(alembic_config, message)
rev_path = rev.path # type: ignore
click.secho(
f"Revision file created. Now, you need to update it: \n\t{rev_path}",
fg=u"green")
def migration(plugin, message):
"""Create new alembic revision for DB migration.
"""
import ckan.model
if not config:
error_shout(u'Config is not loaded')
raise click.Abort()
alembic_config = CKANAlembicConfig(_resolve_alembic_config(plugin))
migration_dir = os.path.dirname(alembic_config.config_file_name)
alembic_config.set_main_option(u"sqlalchemy.url",
str(ckan.model.repo.metadata.bind.url))
alembic_config.set_main_option(u'script_location', migration_dir)
if not os.path.exists(os.path.join(migration_dir, u'script.py.mako')):
alembic.command.init(alembic_config, migration_dir)
rev = alembic.command.revision(alembic_config, message)
click.secho(
u"Revision file created. Now, you need to update it: \n\t{}".format(
rev.path),
fg=u"green")
def send_emails():
""" Sends an email to users notifying about new activities.
As currently implemented, it will only send notifications from dashboard
activity list if users have `activity_streams_email_notifications` set
in their profile. It will send emails with updates depending
on the `ckan.email_notifications_since` config. (default: 2 days.)
"""
import ckan.logic as logic
import ckan.lib.mailer as mailer
from ckan.types import Context
from typing import cast
site_user = logic.get_action("get_site_user")({"ignore_auth": True}, {})
context = cast(Context, {"user": site_user["name"]})
try:
logic.get_action("send_email_notifications")(context, {})
except (NotAuthorized, ValidationError, mailer.MailerException) as e:
error_shout(e)
except KeyError:
error_shout("`activity` plugin is not enabled")