def purge(opts):
"""Purge environment database and uploaded files
Usage:
datacats purge [--delete-environment] [ENVIRONMENT]
Options:
--delete-environment Delete environment directory as well as its data
ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
try:
environment = Environment.load(opts['ENVIRONMENT'])
except DatacatsError as e:
environment = Environment.load(opts['ENVIRONMENT'], data_only=True)
environment.stop_web()
environment.stop_postgres_and_solr()
if opts['--delete-environment']:
if not environment.target:
print 'Failed to load environment.',
print 'Not deleting environment directory.'
else:
environment.fix_project_permissions()
rmtree(environment.target)
environment.purge_data()
def purge(opts):
"""Purge environment database and uploaded files
Usage:
datacats purge [-s NAME | --delete-environment] [-y] [ENVIRONMENT]
Options:
--delete-environment Delete environment directory as well as its data, as
well as the data for **all** sites.
-s --site=NAME Specify a site to be purge [default: primary]
-y --yes Respond yes to all prompts (i.e. force)
ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
old = False
try:
environment = Environment.load(opts['ENVIRONMENT'], opts['--site'])
except DatacatsError:
environment = Environment.load(opts['ENVIRONMENT'],
opts['--site'],
data_only=True)
if get_format_version(environment.datadir) == 1:
old = True
environment = Environment.load(opts['ENVIRONMENT'],
opts['--site'],
allow_old=True)
# We need a valid site if they don't want to blow away everything.
if not opts['--delete-environment'] and not old:
environment.require_valid_site()
sites = [opts['--site']
] if not opts['--delete-environment'] else environment.sites
if not opts['--yes']:
y_or_n_prompt('datacats purge will delete all stored data')
environment.stop_ckan()
environment.stop_supporting_containers()
environment.purge_data(sites)
if opts['--delete-environment']:
if environment.target:
rmtree(environment.target)
else:
DatacatsError(
("Unable to find the environment source"
" directory so that it can be deleted.\n"
"Chances are it's because it already does not exist"))
def install(environment, opts):
"""Install or reinstall Python packages within this environment
Usage:
datacats install [-cq] [--address=IP] [ENVIRONMENT]
Options:
--address=IP The address to bind to when reloading after install [default: 127.0.0.1]
-c --clean Reinstall packages into a clean virtualenv
-q --quiet Do not show output from installing packages and requirements.
ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
environment.require_data()
install_all(environment, opts['--clean'], verbose=not opts['--quiet'])
for site in environment.sites:
environment = Environment.load(environment.name, site)
if 'web' in environment.containers_running():
# FIXME: reload without changing debug setting?
manage.reload_(environment, {
'--address': opts['--address'],
'--background': False,
'--no-watch': False,
'--production': False,
'PORT': None,
'--syslog': False,
'--site-url': None
})
def paster(opts):
"""Run a paster command from the current directory
Usage:
datacats paster [-d] [-s NAME] [COMMAND...]
Options:
-s --site=NAME Specify a site to run this paster command on [default: primary]
-d --detach Run the resulting container in the background
You must be inside a datacats environment to run this. The paster command will
run within your current directory inside the environment. You don't need to
specify the --plugin option. The --config option also need not be specified.
"""
environment = Environment.load('.')
environment.require_data()
environment.start_supporting_containers()
if not opts['COMMAND']:
opts['COMMAND'] = ['--', 'help']
assert opts['COMMAND'][0] == '--'
return environment.interactive_shell(
opts['COMMAND'][1:],
paster=True,
detach=opts['--detach']
)
def paster(opts):
"""Run a paster command from the current directory
Usage:
datacats paster [-d] [-s NAME] [COMMAND...]
Options:
-s --site=NAME Specify a site to run this paster command on [default: primary]
-d --detach Run the resulting container in the background
You must be inside a datacats environment to run this. The paster command will
run within your current directory inside the environment. You don't need to
specify the --plugin option. The --config option also need not be specified.
"""
environment = Environment.load('.')
environment.require_data()
environment.start_supporting_containers()
if not opts['COMMAND']:
opts['COMMAND'] = ['--', 'help']
assert opts['COMMAND'][0] == '--'
return environment.interactive_shell(opts['COMMAND'][1:],
paster=True,
detach=opts['--detach'])
def install(environment, opts):
"""Install or reinstall Python packages within this environment
Usage:
datacats install [-cq] [--address=IP] [ENVIRONMENT]
Options:
--address=IP The address to bind to when reloading after install
-c --clean Reinstall packages into a clean virtualenv
-q --quiet Do not show output from installing packages and requirements.
ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
environment.require_data()
install_all(environment, opts['--clean'], verbose=not opts['--quiet'])
for site in environment.sites:
environment = Environment.load(environment.name, site)
if 'web' in environment.containers_running():
# FIXME: reload without changing debug setting?
manage.reload_(environment, {
'--address': opts['--address'],
'--background': False,
'--no-watch': False,
'--production': False,
'PORT': None,
'--syslog': False,
'--site-url': None
})
def purge(opts):
"""Purge environment database and uploaded files
Usage:
datacats purge [-s NAME | --delete-environment] [-y] [ENVIRONMENT]
Options:
--delete-environment Delete environment directory as well as its data, as
well as the data for **all** sites.
-s --site=NAME Specify a site to be purge [default: primary]
-y --yes Respond yes to all prompts (i.e. force)
ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
old = False
try:
environment = Environment.load(opts['ENVIRONMENT'], opts['--site'])
except DatacatsError:
environment = Environment.load(opts['ENVIRONMENT'], opts['--site'], data_only=True)
if get_format_version(environment.datadir) == 1:
old = True
environment = Environment.load(opts['ENVIRONMENT'], opts['--site'], allow_old=True)
# We need a valid site if they don't want to blow away everything.
if not opts['--delete-environment'] and not old:
environment.require_valid_site()
sites = [opts['--site']] if not opts['--delete-environment'] else environment.sites
if not opts['--yes']:
y_or_n_prompt('datacats purge will delete all stored data')
environment.stop_ckan()
environment.stop_supporting_containers()
environment.purge_data(sites)
if opts['--delete-environment']:
if environment.target:
rmtree(environment.target)
else:
DatacatsError(("Unable to find the environment source"
" directory so that it can be deleted.\n"
"Chances are it's because it already does not exist"))
def create_environment(environment_dir, port, ckan_version, create_skin,
site_name, start_web, create_sysadmin, address, log_syslog=False,
datapusher=True, quiet=False, site_url=None, interactive=False,
init_db=True):
if not init_db:
print 'Since the database will not be initialized, we will not copy datapusher.'
datapusher = False
environment = Environment.new(environment_dir, ckan_version, site_name,
address=address, port=port)
try:
# There are a lot of steps we can/must skip if we're making a sub-site only
making_full_environment = not environment.data_exists()
if not quiet:
write('Creating environment "{0}/{1}"'.format(environment.name, environment.site_name))
steps = [
lambda: environment.create_directories(making_full_environment),
environment.create_bash_profile
] + \
([
environment.create_virtualenv,
environment.save,
lambda: environment.create_source(datapusher),
environment.create_ckan_ini] if making_full_environment else []
) + \
[
environment.save_site,
environment.start_supporting_containers,
environment.fix_storage_permissions,
lambda: environment.update_ckan_ini(skin=create_skin),
]
if create_skin and making_full_environment:
steps.append(environment.create_install_template_skin)
for fn in steps:
fn()
if not quiet:
write('.')
if not quiet:
write('\n')
return finish_init(environment, start_web, create_sysadmin,
log_syslog=log_syslog, site_url=site_url,
interactive=interactive, init_db=init_db)
except:
# Make sure that it doesn't get printed right after the dots
# by printing a newline
# i.e. Creating environment 'hello'.....ERROR MESSAGE
if not quiet:
print
raise
def main():
args = sys.argv[1:]
help_ = False
# Find subcommand without docopt so that subcommand options may appear
# anywhere
for i, a in enumerate(args):
if a.startswith('-'):
continue
if a == 'help':
help_ = True
continue
command_fn = COMMANDS.get(a)
break
else:
if help_:
args = ['--help']
return docopt(__doc__, args,
version=pkg_resources.require("datacats")[0].version)
if not command_fn:
return docopt(__doc__, ['--help'])
# shell, paster are special: options might belong to the command being
# executed
if command_fn == shell.shell:
# assume commands don't start with '-' and that those options
# are intended for datacats
for j, a in enumerate(args[i + 2:], i + 2):
if not a.startswith('-'):
# -- makes docopt parse the rest as positional args
args = args[:j] + ['--'] + args[j:]
break
if command_fn == shell.paster:
args = args[:i + 1] + ['--'] + args[i + 1:]
if help_:
args.insert(1, '--help')
try:
opts = docopt(command_fn.__doc__, args)
option_not_yet_implemented(opts, '--ckan')
option_not_yet_implemented(opts, '--remote')
# purge handles loading differently
if command_fn != purge.purge and 'ENVIRONMENT' in opts:
environment = Environment.load(opts['ENVIRONMENT'] or '.')
return command_fn(environment, opts)
return command_fn(opts)
except DatacatsError as e:
print e
return 1
def main():
"""
The main entry point for datacats cli tool
(as defined in setup.py's entry_points)
It parses the cli arguments for corresponding options
and runs the corresponding command
"""
# pylint: disable=bare-except
try:
command_fn, opts = _parse_arguments(sys.argv[1:])
# purge handles loading differently
# 1 - Bail and just call the command if it doesn't have ENVIRONMENT.
if command_fn == purge.purge or 'ENVIRONMENT' not in opts:
return command_fn(opts)
environment = Environment.load(
opts['ENVIRONMENT'] or '.',
opts['--site'] if '--site' in opts else 'primary')
if command_fn not in COMMANDS_THAT_USE_SSH:
return command_fn(environment, opts)
# for commands that communicate with a remote server
# we load UserProfile and test our communication
user_profile = UserProfile()
user_profile.test_ssh_key(environment)
return command_fn(environment, opts, user_profile)
except DatacatsError as e:
_error_exit(e)
except SystemExit:
raise
except:
exc_info = "\n".join([line.rstrip()
for line in traceback.format_exception(*sys.exc_info())])
user_message = ("Something that should not"
" have happened happened when attempting"
" to run this command:\n"
" datacats {args}\n\n"
"It is seems to be a bug.\n"
"Please report this issue to us by"
" creating an issue ticket at\n\n"
" https://github.com/datacats/datacats/issues\n\n"
"so that we would be able to look into that "
"and fix the issue."
).format(args=" ".join(sys.argv[1:]))
_error_exit(DatacatsError(user_message,
parent_exception=UndocumentedError(exc_info)))
def wrapper():
if 'name' not in request.values:
site_name = 'primary'
else:
site_name = request.values.get('name')
try:
environment = Environment.load(MAIN_ENV_NAME, site_name)
if require_valid_site:
environment.require_valid_site()
return func(environment)
except DatacatsError as e:
return jsonify({'error': str(e)}), CLIENT_ERROR_CODE
def paster(opts):
"""Run a paster command from the current directory
Usage:
datacats paster [COMMAND...]
You must be inside a datacats environment to run this. The paster command will
run within your current directory inside the environment. You don't need to
specify the --plugin option. The --config option also need not be specified.
"""
environment = Environment.load('.')
environment.require_data()
environment.start_postgres_and_solr()
assert opts['COMMAND'][0] == '--'
return environment.interactive_shell(opts['COMMAND'][1:], paster=True)
def __init__(self, name, display_name, finished_create=True, sort=True):
"""
Initializes a site object. Also places it into the global `sites` list.
:param name: The name of the site to be created.
:param display_name: The name of the site to be shown to the user.
:param sort: True if we should maintain the sorted order of the `sites` list.
For all uses except internal ones this should ALWAYS be True or
Bad Things Will Happen (TM).
"""
self.name = name
self.display_name = display_name
self.environment = Environment.load(MAIN_ENV_NAME, name)
self.port = self.environment.port
self.finished_create = finished_create
self.celery_task = None
def init(opts):
"""Initialize a purged environment or copied environment directory
Usage:
datacats init [-in] [ENVIRONMENT_DIR [PORT]]
Options:
-i --image-only Create the environment but don't start containers
-n --no-sysadmin Don't prompt for an initial sysadmin user account
ENVIRONMENT_DIR is an existing datacats environment directory. Defaults to '.'
"""
environment_dir = opts['ENVIRONMENT_DIR']
port = opts['PORT']
start_web = not opts['--image-only']
create_sysadmin = not opts['--no-sysadmin']
environment_dir = abspath(environment_dir or '.')
try:
environment = Environment.load(environment_dir)
if port:
environment.port = int(port)
except DatacatsError as e:
print e
return 1
write('Creating from existing environment directory "{0}"'.format(
environment.name))
steps = [
lambda: environment.create_directories(create_project_dir=False),
environment.save,
environment.create_virtualenv,
environment.start_postgres_and_solr,
environment.fix_storage_permissions,
environment.fix_project_permissions,
]
for fn in steps:
fn()
write('.')
write('\n')
return finish_init(environment, start_web, create_sysadmin)
def create_environment(environment_dir, port, ckan_version, create_skin,
start_web, create_sysadmin):
try:
# FIXME: only 2.3 preload supported at the moment
environment = Environment.new(environment_dir, '2.3', port)
except DatacatsError as e:
print e
return 1
if not valid_deploy_name(environment.name):
print "WARNING: When deploying you will need to choose a"
print "target name that is at least 5 characters long"
print
write('Creating environment "{0}"'.format(environment.name))
steps = [
environment.create_directories,
environment.create_bash_profile,
environment.save,
environment.create_virtualenv,
environment.create_source,
environment.start_postgres_and_solr,
environment.fix_storage_permissions,
environment.create_ckan_ini,
lambda: environment.update_ckan_ini(skin=create_skin),
environment.fix_project_permissions,
]
if create_skin:
steps.append(environment.create_install_template_skin)
steps.append(environment.ckan_db_init)
for fn in steps:
fn()
write('.')
write('\n')
return finish_init(environment, start_web, create_sysadmin)
def main():
opts = docopt(__doc__, version=__version__)
environment = Environment.load(opts['ENVIRONMENT'] or '.')
env_path = environment.target
less_paths = [
path_join(env_path, 'ckan', 'ckan', 'public', 'base', 'less'),
path_join(env_path, 'ckan', 'ckan', 'public', 'base', 'vendor')
]
if not env_path or not all(exists(less_path) for less_path in less_paths):
print 'No source code to watch found'
return
observer = Observer()
event_handler = LessCompileEventHandler(environment)
for less_path in less_paths:
observer.schedule(event_handler, less_path, recursive=True)
observer.start()
# HACK: We make it so that the OS doesn't consult us and just kills us.
signal.signal(signal.SIGINT, signal.SIG_DFL)
observer.join()
def init(opts, no_install=False, quiet=False):
"""Initialize a purged environment or copied environment directory
Usage:
datacats init [-in] [--syslog] [-s NAME] [--address=IP]
[--site-url SITE_URL] [ENVIRONMENT_DIR [PORT]]
Options:
--address=IP Address to listen on (Linux-only) [default: 127.0.0.1]
-i --image-only Create the environment but don't start containers
-n --no-sysadmin Don't prompt for an initial sysadmin user account
-s --site=NAME Pick a site to initialize [default: primary]
--site-url SITE_URL The site_url to use in API responses (e.g. http://example.org:{port}/)
--syslog Log to the syslog
ENVIRONMENT_DIR is an existing datacats environment directory. Defaults to '.'
"""
environment_dir = opts['ENVIRONMENT_DIR']
port = opts['PORT']
address = opts['--address']
start_web = not opts['--image-only']
create_sysadmin = not opts['--no-sysadmin']
site_name = opts['--site']
site_url = opts['--site-url']
environment_dir = abspath(environment_dir or '.')
log_syslog = opts['--syslog']
environment = Environment.load(environment_dir, site_name)
environment.address = address
if port:
environment.port = int(port)
if site_url:
environment.site_url = site_url
try:
if environment.sites and site_name in environment.sites:
raise DatacatsError('Site named {0} already exists.'
.format(site_name))
# There are a couple of steps we can/must skip if we're making a sub-site only
making_full_environment = not environment.data_exists()
if not quiet:
write('Creating environment {0}/{1} '
'from existing environment directory "{0}"'
.format(environment.name, environment.site_name))
steps = [
lambda: environment.create_directories(create_project_dir=False)] + ([
environment.save,
environment.create_virtualenv
] if making_full_environment else []) + [
environment.save_site,
environment.start_supporting_containers,
environment.fix_storage_permissions,
]
for fn in steps:
fn()
if not quiet:
write('.')
if not quiet:
write('\n')
except:
if not quiet:
print
raise
return finish_init(environment, start_web, create_sysadmin, address,
log_syslog=log_syslog, do_install=not no_install,
quiet=quiet, site_url=site_url)