def _machine_check_connectivity():
"""
This method calls to docker-machine on the command line and
makes sure that it is up and ready.
Potential improvements to be made:
- Support multiple machine names (run a `docker-machine ls` and then
see which machines are active. Use a priority list)
"""
with open(devnull, 'w') as devnull_f:
try:
status = subprocess.check_output(
['docker-machine', 'status', 'dev'], stderr=devnull_f).strip()
if status == 'Stopped':
raise DatacatsError('Please start your docker-machine '
'VM with "docker-machine start dev"')
# XXX HACK: This exists because of
# http://github.com/datacats/datacats/issues/63,
# as a temporary fix.
if 'tls' in _docker_kwargs:
# It will print out messages to the user otherwise.
_docker_kwargs['tls'].assert_hostname = False
except subprocess.CalledProcessError:
raise DatacatsError('Please create a docker-machine with '
'"docker-machine start dev"')
def _get_docker():
global _docker
if not _docker:
if sys.platform.startswith('darwin'):
_machine_check_connectivity()
# Create the Docker client
version_client = Client(version=MINIMUM_API_VERSION, **_docker_kwargs)
try:
api_version = version_client.version()['ApiVersion']
except ConnectionError:
try:
# workaround for connection issue when old version specified
# on some clients
version_client = Client(**_docker_kwargs)
api_version = version_client.version()['ApiVersion']
except:
raise DatacatsError(DOCKER_FAIL_STRING)
except:
raise DatacatsError(DOCKER_FAIL_STRING)
version = get_api_version(DEFAULT_DOCKER_API_VERSION, api_version)
_docker = Client(version=version, **_docker_kwargs)
return _docker
def new_environment_check(srcpath, site_name, ckan_version):
"""
Check if a new environment or site can be created at the given path.
Returns (name, datadir, sitedir, srcdir) or raises DatacatsError
"""
docker.require_images()
workdir, name = path.split(path.abspath(path.expanduser(srcpath)))
if not validate.valid_name(name):
raise DatacatsError(
'Please choose an environment name starting'
' with a letter and including only lowercase letters'
' and digits')
if not path.isdir(workdir):
raise DatacatsError('Parent directory for environment'
' does not exist')
datadir = path.expanduser('~/.datacats/' + name)
sitedir = datadir + '/sites/' + site_name
# We track through the datadir to the target if we are just making a
# site
if path.isdir(datadir):
with open(datadir + '/project-dir') as pd:
srcdir = pd.read()
else:
srcdir = workdir + '/' + name
if ckan_version not in SUPPORTED_PRELOADS:
raise DatacatsError(
'''Datacats does not currently support CKAN version {}.
Versions that are currently supported are: {}'''.format(
ckan_version, ', '.join(SUPPORTED_PRELOADS)))
preload_name = str(ckan_version)
# Get all the versions from the tags
downloaded_versions = [tag for tag in docker.get_tags('datacats/ckan')]
if ckan_version not in downloaded_versions:
retrying_pull_image('datacats/ckan:{}'.format(preload_name))
if path.isdir(sitedir):
raise DatacatsError(
'Site data directory {0} already exists'.format(sitedir))
# This is the case where the data dir has been removed,
if path.isdir(srcdir) and not path.isdir(datadir):
raise DatacatsError(
'Environment directory exists, but data directory does not.\n'
'If you simply want to recreate the data directory, run '
'"datacats init" in the environment directory.')
return name, datadir, srcdir
def find_environment_dirs(environment_name=None, data_only=False):
"""
:param environment_name: exising environment name, path or None to
look in current or parent directories for project
returns (srcdir, extension_dir, datadir)
extension_dir is the name of extension directory user was in/referenced,
default: 'ckan'. This value is used by the paster cli command.
datadir will be None if environment_name was a path or None (not a name)
"""
docker.require_images()
if environment_name is None:
environment_name = '.'
extension_dir = 'ckan'
if validate.valid_name(environment_name) and path.isdir(
path.expanduser('~/.datacats/' + environment_name)):
# loading from a name
datadir = path.expanduser('~/.datacats/' + environment_name)
with open(datadir + '/project-dir') as pd:
srcdir = pd.read()
if not data_only and not path.exists(srcdir +
'/.datacats-environment'):
raise DatacatsError(
'Environment data found but environment directory is'
' missing. Try again from the new environment directory'
' location or remove the environment data with'
' "datacats purge"')
return srcdir, extension_dir, datadir
# loading from a path
srcdir = path.abspath(environment_name)
if not path.isdir(srcdir):
raise DatacatsError('No environment found with that name')
wd = srcdir
oldwd = None
while not path.exists(wd + '/.datacats-environment'):
oldwd = wd
wd, _ = path.split(wd)
if wd == oldwd:
raise DatacatsError(
'Environment not found in {0} or above'.format(srcdir))
srcdir = wd
if oldwd:
_, extension_dir = path.split(oldwd)
return srcdir, extension_dir, None
def install_all(environment, clean, verbose=False, quiet=False, packages=None):
logs = check_connectivity()
if logs.strip():
raise DatacatsError(logs)
if clean:
clean_pyc(environment, quiet)
srcdirs = set()
reqdirs = set()
for d in listdir(environment.target):
fulld = environment.target + '/' + d
if not isdir(fulld):
continue
if not exists(fulld + '/setup.py'):
continue
if packages and d not in packages:
continue
srcdirs.add(d)
if (exists(fulld + '/requirements.txt')
or exists(fulld + '/pip-requirements.txt')):
reqdirs.add(d)
try:
if not packages or 'ckan' in packages:
srcdirs.remove('ckan')
reqdirs.remove('ckan')
srcdirs = ['ckan'] + sorted(srcdirs)
reqdirs = ['ckan'] + sorted(reqdirs)
except KeyError:
raise DatacatsError('ckan not found in environment directory')
if clean:
environment.clean_virtualenv()
environment.install_extra()
for s in srcdirs:
if verbose:
print colored.yellow('Installing ' + s + '\n')
elif not quiet:
print 'Installing ' + s
environment.install_package_develop(
s, sys.stdout if verbose and not quiet else None)
if verbose and not quiet:
print
for s in reqdirs:
if verbose:
print colored.yellow('Installing ' + s + ' requirements' + '\n')
elif not quiet:
print 'Installing ' + s + ' requirements'
environment.install_package_requirements(
s, sys.stdout if verbose and not quiet else None)
if verbose:
print
def _get_docker():
global _docker
# HACK: We determine from commands if we're boot2docker
# Needed cause this method is called from is_boot2docker...
boot2docker = False
if not _docker:
# First, check if boot2docker is powered on.
try:
with open(devnull, 'w') as devnull_f:
status = subprocess.check_output(
['boot2docker', 'status'],
# Don't show boot2docker message
# to the user... it's ugly!
stderr=devnull_f).strip()
if status == 'poweroff':
raise DatacatsError('boot2docker is not powered on.'
' Please run "boot2docker up".')
boot2docker = True
except OSError:
# We're on Linux, or boot2docker isn't installed.
pass
except subprocess.CalledProcessError:
raise DatacatsError('You have not created your boot2docker VM. '
'Please run "boot2docker init" to do so.')
# XXX HACK: This exists because of
# http://github.com/datacats/datacats/issues/63,
# as a temporary fix.
if 'tls' in _docker_kwargs and boot2docker:
import warnings
# It will print out messages to the user otherwise.
warnings.filterwarnings("ignore", category=InsecureRequestWarning)
warnings.filterwarnings("ignore", category=InsecurePlatformWarning)
_docker_kwargs['tls'].verify = False
# Create the Docker client
version_client = Client(version=MINIMUM_API_VERSION, **_docker_kwargs)
try:
api_version = version_client.version()['ApiVersion']
except ConnectionError:
# workaround for connection issue when old version specified
# on some clients
version_client = Client(**_docker_kwargs)
api_version = version_client.version()['ApiVersion']
version = get_api_version(DEFAULT_DOCKER_API_VERSION, api_version)
_docker = Client(version=version, **_docker_kwargs)
return _docker
def require_data(self):
"""
raise a DatacatsError if the datadir or volumes are missing or damaged
"""
files = task.source_missing(self.target)
if files:
raise DatacatsError('Missing files in source directory:\n' +
'\n'.join(files))
if not self.data_exists():
raise DatacatsError('Environment datadir missing. '
'Try "datacats init".')
if not self.data_complete():
raise DatacatsError('Environment datadir damaged or volumes '
'missing. '
'To reset and discard all data use '
'"datacats reset"')
def wait_for_web_available(self):
"""
Wait for the web server to become available or raise DatacatsError
if it fails to start.
"""
try:
if not wait_for_service_available(self._get_container_name('web'),
self.web_address(),
WEB_START_TIMEOUT_SECONDS):
raise DatacatsError(
'Error while starting web container:\n' + container_logs(
self._get_container_name('web'), "all", False, None))
except ServiceTimeout:
raise DatacatsError('Timeout while starting web container. Logs:' +
container_logs(self._get_container_name('web'),
"all", False, None))
def create_directories(datadir, sitedir, srcdir=None):
"""
Create expected directories in datadir, sitedir
and optionally srcdir
"""
# It's possible that the datadir already exists
# (we're making a secondary site)
if not path.isdir(datadir):
os.makedirs(datadir, mode=0o700)
try:
# This should take care if the 'site' subdir if needed
os.makedirs(sitedir, mode=0o700)
except OSError:
raise DatacatsError("Site already exists.")
# venv isn't site-specific, the rest are.
if not docker.is_boot2docker():
if not path.isdir(datadir + '/venv'):
os.makedirs(datadir + '/venv')
os.makedirs(sitedir + '/postgres')
os.makedirs(sitedir + '/solr')
os.makedirs(sitedir + '/files')
os.makedirs(sitedir + '/run')
if srcdir:
os.makedirs(srcdir)
def _subcommand_arguments(args):
"""
Return (subcommand, (possibly adjusted) arguments for that subcommand)
Returns (None, args) when no subcommand is found
Parsing our arguments is hard. Each subcommand has its own docopt
validation, and some subcommands (paster and shell) have positional
options (some options passed to datacats and others passed to
commands run inside the container)
"""
skip_site = False
# Find subcommand without docopt so that subcommand options may appear
# anywhere
for i, a in enumerate(args):
if skip_site:
skip_site = False
continue
if a.startswith('-'):
if a == '-s' or a == '--site':
skip_site = True
continue
if a == 'help':
return _subcommand_arguments(args[:i] + ['--help'] + args[i + 1:])
if a not in COMMANDS:
raise DatacatsError(
"\'{0}\' command is not recognized. \n"
"See \'datacats help\' for the list of available commands".
format(a))
command = a
break
else:
return None, args
if command != 'shell' and command != 'paster':
return command, args
# shell requires the environment name, paster does not
remaining_positional = 2 if command == 'shell' else 1
# i is where the subcommand starts.
# shell, paster are special: options might belong to the command being
# find where the the inner command starts and insert a '--' before
# so that we can separate inner options from ones we need to parse
while i < len(args):
a = args[i]
if a.startswith('-'):
if a == '-s' or a == '--site':
# site name is coming
i += 2
continue
i += 1
continue
if remaining_positional:
remaining_positional -= 1
i += 1
continue
return command, args[:i] + ['--'] + args[i:]
return command, args
def migrate(opts):
"""Migrate an environment to a given revision of the datadir format.
Usage:
datacats migrate [-y] [-r VERSION] [ENVIRONMENT_DIR]
Options:
-r --revision=VERSION The version of the datadir format you want
to convert to [default: 2]
-y --yes Answer yes to all questions.
Defaults to '.' if ENVIRONMENT_DIR isn't specified.
"""
try:
version = int(opts['--revision'])
except:
raise DatacatsError('--revision parameter must be an integer.')
always_yes = opts['--yes']
if 'ENVIRONMENT_DIR' not in opts or not opts['ENVIRONMENT_DIR']:
cwd = getcwd()
# Get the dirname
opts['ENVIRONMENT_DIR'] = split(cwd if cwd[-1] != '/' else cwd[:-1])[1]
datadir = expanduser('~/.datacats/' + opts['ENVIRONMENT_DIR'])
if needs_format_conversion(datadir, version):
convert_environment(datadir, version, always_yes)
print 'Successfully converted datadir {} to format version {}'.format(
datadir, version)
else:
print 'datadir {} is already at version {}.'.format(datadir, version)
def _next_port(self, port):
"""
Return another port from the 5000-5999 range
"""
port = 5000 + (port + 1) % 1000
if port == self.port:
raise DatacatsError('Too many instances running')
return port
def needs_datapusher(self):
cp = SafeConfigParser()
try:
cp.read(self.target + '/development.ini')
return ('datapusher' in cp.get('app:main', 'ckan.plugins')
and isdir(self.target + '/datapusher'))
except ConfigParserError as e:
raise DatacatsError('Failed to read and parse development.ini: ' + str(e))
def require_images():
"""
Raises a DatacatsError if the images required to use Datacats don't exist.
"""
if (not image_exists('datacats/web') or not image_exists('datacats/solr')
or not image_exists('datacats/postgres')):
raise DatacatsError(
'You do not have the needed Docker images. Please run "datacats pull"'
)
def deploy(self, environment, target_name, stream_output=None):
"""
Return True if deployment was successful
"""
try:
remote_server_command(
[
"rsync", "-lrv", "--safe-links", "--munge-links",
"--delete", "--inplace", "--chmod=ugo=rwX",
"--exclude=.datacats-environment", "--exclude=.git",
"/project/.", environment.deploy_target + ':' + target_name
],
environment,
self,
include_project_dir=True,
stream_output=stream_output,
clean_up=True,
)
except WebCommandError as e:
raise DatacatsError(
"Unable to deploy `{0}` to remote server for some reason:\n"
" datacats was not able to copy data to the remote server".
format((target_name, )),
parent_exception=e)
try:
remote_server_command(
[
"ssh",
environment.deploy_target,
"install",
target_name,
],
environment,
self,
clean_up=True,
)
return True
except WebCommandError as e:
raise DatacatsError(
"Unable to deploy `{0}` to remote server for some reason:\n"
"datacats copied data to the server but failed to register\n"
"(or `install`) the new catalog".format((target_name, )),
parent_exception=e)
def load_site(srcdir, datadir, site_name=None):
"""
Load configuration values for a site.
Returns (port, address, site_url, passwords)
"""
if site_name is None:
site_name = 'primary'
if not validate.valid_name(site_name):
raise DatacatsError('{} is not a valid site name.'.format(site_name))
cp = ConfigParser.SafeConfigParser()
try:
cp.read([srcdir + '/.datacats-environment'])
except ConfigParser.Error:
raise DatacatsError('Error reading environment information')
site_section = 'site_' + site_name
try:
port = cp.getint(site_section, 'port')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
port = None
try:
address = cp.get(site_section, 'address')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
address = None
try:
site_url = cp.get(site_section, 'site_url')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
site_url = None
passwords = {}
cp = ConfigParser.SafeConfigParser()
cp.read(datadir + '/sites/' + site_name + '/passwords.ini')
try:
pw_options = cp.options('passwords')
except ConfigParser.NoSectionError:
pw_options = []
for n in pw_options:
passwords[n.upper()] = cp.get('passwords', n)
return port, address, site_url, passwords
def main():
"""
The main entry point for datacats cli tool
(as defined in setup.py's entry_points)
It parses the cli arguments for corresponding options
and runs the corresponding command
"""
# pylint: disable=bare-except
try:
command_fn, opts = _parse_arguments(sys.argv[1:])
# purge handles loading differently
# 1 - Bail and just call the command if it doesn't have ENVIRONMENT.
if command_fn == purge.purge or 'ENVIRONMENT' not in opts:
return command_fn(opts)
environment = Environment.load(
opts['ENVIRONMENT'] or '.',
opts['--site'] if '--site' in opts else 'primary')
if command_fn not in COMMANDS_THAT_USE_SSH:
return command_fn(environment, opts)
# for commands that communicate with a remote server
# we load UserProfile and test our communication
user_profile = UserProfile()
user_profile.test_ssh_key(environment)
return command_fn(environment, opts, user_profile)
except DatacatsError as e:
_error_exit(e)
except SystemExit:
raise
except:
exc_info = "\n".join([
line.rstrip()
for line in traceback.format_exception(*sys.exc_info())
])
user_message = ("Something that should not"
" have happened happened when attempting"
" to run this command:\n"
" datacats {args}\n\n"
"It is seems to be a bug.\n"
"Please report this issue to us by"
" creating an issue ticket at\n\n"
" https://github.com/datacats/datacats/issues\n\n"
"so that we would be able to look into that "
"and fix the issue.").format(
args=" ".join(sys.argv[1:]))
_error_exit(
DatacatsError(user_message,
parent_exception=UndocumentedError(exc_info)))
def convert_environment(datadir, version, always_yes):
"""
Converts an environment TO the version specified by `version`.
:param datadir: The datadir to convert.
:param version: The version to convert TO.
:param always_yes: True if the user shouldn't be prompted about the migration.
"""
# Since we don't call either load() or new() we have to call require_images ourselves.
require_images()
inp = None
old_version = _get_current_format(datadir)
migration_func = migrations[(old_version, version)]
if version > CURRENT_FORMAT_VERSION:
raise DatacatsError('Cannot migrate to a version higher than the '
'current one.')
if version < 1:
raise DatacatsError('Datadir versioning starts at 1.')
if not always_yes:
while inp != 'y' and inp != 'n':
inp = raw_input(migration_func.__doc__.format(version))
if inp == 'n':
sys.exit(1)
lockfile = LockFile(path_join(datadir, '.migration_lock'))
lockfile.acquire()
try:
# FIXME: If we wanted to, we could find a set of conversions which
# would bring us up to the one we want if there's no direct path.
# This isn't necessary with just two formats, but it may be useful
# at 3.
# Call the appropriate conversion function
migration_func(datadir)
finally:
lockfile.release()
def finish_init(environment,
start_web,
create_sysadmin,
log_syslog=False,
do_install=True,
quiet=False,
site_url=None,
interactive=False,
init_db=True):
"""
Common parts of create and init: Install, init db, start site, sysadmin
"""
if not init_db:
start_web = False
create_sysadmin = False
if do_install:
install_all(environment, False, verbose=False, quiet=quiet)
if init_db:
if not quiet:
write('Initializing database')
environment.install_postgis_sql()
environment.ckan_db_init()
if not quiet:
write('\n')
if site_url:
try:
site_url = site_url.format(address=environment.address,
port=environment.port)
environment.site_url = site_url
environment.save_site(False)
except (KeyError, IndexError, ValueError) as e:
raise DatacatsError('Could not parse site_url: {}'.format(e))
if start_web:
environment.start_ckan(log_syslog=log_syslog)
if not quiet and not interactive:
write('Starting web server at {0} ...\n'.format(
environment.web_address()))
if create_sysadmin:
try:
adminpw = confirm_password()
environment.create_admin_set_password(adminpw)
except KeyboardInterrupt:
print
if not start_web:
environment.stop_supporting_containers()
def _create_run_ini(self,
port,
production,
output='development.ini',
source='development.ini',
override_site_url=True):
"""
Create run/development.ini in datadir with debug and site_url overridden
and with correct db passwords inserted
"""
cp = SafeConfigParser()
try:
cp.read([self.target + '/' + source])
except ConfigParserError:
raise DatacatsError('Error reading development.ini')
cp.set('DEFAULT', 'debug', 'false' if production else 'true')
if self.site_url:
site_url = self.site_url
else:
if is_boot2docker():
web_address = socket.gethostbyname(docker_host())
else:
web_address = self.address
site_url = 'http://{}:{}'.format(web_address, port)
if override_site_url:
cp.set('app:main', 'ckan.site_url', site_url)
cp.set(
'app:main', 'sqlalchemy.url',
'postgresql://*****:*****@db:5432/ckan'.format(
self.passwords['CKAN_PASSWORD']))
cp.set(
'app:main', 'ckan.datastore.read_url',
'postgresql://*****:*****@db:5432/ckan_datastore'.
format(self.passwords['DATASTORE_RO_PASSWORD']))
cp.set(
'app:main', 'ckan.datastore.write_url',
'postgresql://*****:*****@db:5432/ckan_datastore'.
format(self.passwords['DATASTORE_RW_PASSWORD']))
cp.set('app:main', 'solr_url', 'http://solr:8080/solr')
cp.set('app:main', 'beaker.session.secret',
self.passwords['BEAKER_SESSION_SECRET'])
if not isdir(self.sitedir + '/run'):
makedirs(self.sitedir + '/run') # upgrade old datadir
with open(self.sitedir + '/run/' + output, 'w') as runini:
cp.write(runini)
def pull_image(image_name):
sys.stdout.write('Pulling image ' + image_name)
sys.stdout.flush()
for s in pull_stream(image_name):
if 'status' not in s:
if 'error' in s:
# Line to make the error appear after the ...
print
raise DatacatsError(s['error'])
else:
print json.dumps(s)
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('\n')
def info(environment, opts):
"""Display information about environment and running containers
Usage:
datacats info [-qr] [ENVIRONMENT]
Options:
-q --quiet Echo only the web URL or nothing if not running
-r --remote Information about DataCats.com cloud instance
ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
damaged = False
sites = environment.sites
if not environment.sites:
sites = []
damaged = True
if opts['--quiet']:
if damaged:
raise DatacatsError('Damaged datadir: cannot get address.')
for site in sites:
environment.site_name = site
print '{}: {}'.format(site, environment.web_address())
return
datadir = environment.datadir
if not environment.data_exists():
datadir = ''
elif damaged:
datadir += ' (damaged)'
print 'Environment name: ' + environment.name
print ' Environment dir: ' + environment.target
print ' Data dir: ' + datadir
print ' Sites: ' + ' '.join(environment.sites)
for site in environment.sites:
print
environment.site_name = site
print ' Site: ' + site
print ' Containers: ' + ' '.join(environment.containers_running())
sitedir = environment.sitedir + (' (damaged)' if not environment.data_complete() else '')
print ' Site dir: ' + sitedir
addr = environment.web_address()
if addr:
print ' Available at: ' + addr
def create_profile(self):
self.ssh_private_key = self.profiledir + '/id_rsa'
self.ssh_public_key = self.profiledir + '/id_rsa.pub'
self.save()
self.generate_ssh_key()
user_error_message = (
"Your profile does not seem to have an ssh key\n"
" (which is an equivalent of your password so that datacats.io"
" could recognize you).\n"
"It is probably because this is your first time running"
" a remote command in which case welcome!\n"
"So we generated a new ssh key for you. \n "
"Please go to www.datacats.com/account/key"
" and add the following public key:"
" \n \n {public_key} \n \n"
" to your profile so that the server can recognize you as you."
).format(public_key=self.read_public_key())
raise DatacatsError(user_error_message)
def create(opts):
"""Create a new environment
Usage:
datacats create [-bin] [--interactive] [-s NAME] [--address=IP] [--syslog]
[--ckan=CKAN_VERSION] [--no-datapusher] [--site-url SITE_URL]
[--no-init-db] ENVIRONMENT_DIR [PORT]
Options:
--address=IP Address to listen on (Linux-only)
--ckan=CKAN_VERSION Use CKAN version CKAN_VERSION [default: 2.3]
-b --bare Bare CKAN site with no example extension
-i --image-only Create the environment but don't start containers
--interactive Doesn't detach from the web container
--no-datapusher Don't install/enable ckanext-datapusher
--no-init-db Don't initialize the database. Useful for importing CKANs.
-n --no-sysadmin Don't prompt for an initial sysadmin user account
-s --site=NAME Pick a site to create [default: primary]
--site-url SITE_URL The site_url to use in API responses (e.g. http://example.org:{port}/)
--syslog Log to the syslog
ENVIRONMENT_DIR is a path for the new environment directory. The last
part of this path will be used as the environment name.
"""
if opts['--address'] and is_boot2docker():
raise DatacatsError('Cannot specify address on boot2docker.')
return create_environment(
environment_dir=opts['ENVIRONMENT_DIR'],
port=opts['PORT'],
create_skin=not opts['--bare'],
start_web=not opts['--image-only'],
create_sysadmin=not opts['--no-sysadmin'],
site_name=opts['--site'],
ckan_version=opts['--ckan'],
address=opts['--address'],
log_syslog=opts['--syslog'],
datapusher=not opts['--no-datapusher'],
site_url=opts['--site-url'],
interactive=opts['--interactive'],
init_db=not opts['--no-init-db'],
)
def deploy(environment, opts, profile):
"""Deploy environment to production DataCats.com cloud service
Usage:
datacats deploy [--create] [ENVIRONMENT [TARGET_NAME]]
Options:
--create Create a new environment on DataCats.com instead
of updating an existing environment
ENVIRONMENT may be an environment name or a path to a environment directory.
Default: '.'
TARGET_NAME is the name of the environment on DataCats.com. Defaults to
the environment name.
"""
target_name = opts['TARGET_NAME']
if target_name is None:
target_name = environment.name
if not valid_deploy_name(target_name):
raise DatacatsError(
" `{target_name}` target name for deployment can't be accepted.\n"
"Can't have http://{target_name}.datacats.io for your datcat URL\n"
"Please choose a target name at least 5 characters long,\n"
"and containing only lowercase letters and numbers\n".format(
target_name=target_name))
if opts['--create']:
profile.create(environment, target_name)
profile.deploy(environment, target_name, stdout)
print "Deployed source to http://{0}.datacats.io".format(target_name)
if opts['--create']:
try:
pw = confirm_password()
profile.admin_password(environment, target_name, pw)
except KeyboardInterrupt:
pass
def add_extra_container(self, container, error_on_exists=False):
"""
Add a container as a 'extra'. These are running containers which are not necessary for
running default CKAN but are useful for certain extensions
:param container: The container name to add
:param error_on_exists: Raise a DatacatsError if the extra container already exists.
"""
if container in self.extra_containers:
if error_on_exists:
raise DatacatsError('{} is already added as an extra container.'.format(container))
else:
return
self.extra_containers.append(container)
cp = SafeConfigParser()
cp.read(self.target + '/.datacats-environment')
cp.set('datacats', 'extra_containers', ' '.join(self.extra_containers))
with open(self.target + '/.datacats-environment', 'w') as f:
cp.write(f)
def test_ssh_key(self, environment):
"""
Return True if this key is accepted by DataCats.com
"""
try:
remote_server_command(["ssh", environment.deploy_target, 'test'],
environment,
self,
clean_up=True)
except WebCommandError as e:
user_unrecognized_error_message = (
"Your ssh key "
"(which is an equivalent of your password so"
" that datacats.io could recognize you) "
"does not seem to be recognized by the datacats.io server. \n \n"
"Most likely it is because you need to go to"
" www.datacats.com/account/key"
" and add the following public key: \n \n {public_key} \n \n"
"to your profile so that datacat's server could recognize you."
" So maybe try that?\n"
"If the problem persists, please contact the developer team."
).format(public_key=self.read_public_key())
network_unreachable_error_message = (
"Unable to connect to the hosting server: {0} \n"
"Some of the reasons for that may be: \n"
" 1) the internet connection is down,\n"
" 2) the server is not up or functioning properly,\n"
" 3) there is a firewall block for the datacats application\n"
" or something of this sort.").format(
environment.deploy_target)
user_error_message = network_unreachable_error_message \
if "Network is unreachable" in e.logs \
else user_unrecognized_error_message
raise DatacatsError(user_error_message, parent_exception=e)
def _retry_func(func, param, num, retry_notif, error_msg):
"""
A function which retries a given function num times and calls retry_notif each
time the function is retried.
:param func: The function to retry num times.
:param num: The number of times to try before giving up.
:param retry_notif: Will be called with the same parameter as func if we have to retry the
function. Will also receive the number of retries so far as a second
parameter.
:param: error_msg: The message
Throws DatacatsError if we run out of retries. Returns otherwise.
"""
for retry_num in range(num):
if retry_num:
retry_notif(param, retry_num)
try:
func(param)
return
except DatacatsError:
pass
raise DatacatsError(error_msg)
def purge_data(self, which_sites=None, never_delete=False):
"""
Remove uploaded files, postgres db, solr index, venv
"""
# Default to the set of all sites
if not exists(self.datadir + '/.version'):
format_version = 1
else:
with open(self.datadir + '/.version') as f:
format_version = int(f.read().strip())
if format_version == 1:
print 'WARNING: Defaulting to old purge for version 1.'
datadirs = ['files', 'solr']
if is_boot2docker():
remove_container('datacats_pgdata_{}'.format(self.name))
remove_container('datacats_venv_{}'.format(self.name))
else:
datadirs += ['postgres', 'venv']
web_command(
command=['/scripts/purge.sh'] +
['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
shutil.rmtree(self.datadir)
elif format_version == 2:
if not which_sites:
which_sites = self.sites
datadirs = []
boot2docker = is_boot2docker()
if which_sites:
if self.target:
cp = SafeConfigParser()
cp.read([self.target + '/.datacats-environment'])
for site in which_sites:
if boot2docker:
remove_container(self._get_container_name('pgdata'))
else:
datadirs += [site + '/postgres']
# Always rm the site dir & solr & files
datadirs += [site, site + '/files', site + '/solr']
if self.target:
cp.remove_section('site_' + site)
self.sites.remove(site)
if self.target:
with open(self.target + '/.datacats-environment',
'w') as conf:
cp.write(conf)
datadirs = ['sites/' + datadir for datadir in datadirs]
if not self.sites and not never_delete:
datadirs.append('venv')
web_command(
command=['/scripts/purge.sh'] +
['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
if not self.sites and not never_delete:
shutil.rmtree(self.datadir)
else:
raise DatacatsError(
'Unknown format version {}'.format(format_version))
def _run_web_container(self,
port,
command,
address,
log_syslog=False,
datapusher=True,
interactive=False):
"""
Start web container on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = self._get_container_name('venv')
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
links = {
self._get_container_name('solr'): 'solr',
self._get_container_name('postgres'): 'db'
}
links.update({
self._get_container_name(container): container
for container in self.extra_containers
})
if datapusher:
if 'datapusher' not in self.containers_running():
raise DatacatsError(
container_logs(self._get_container_name('datapusher'),
"all", False, False))
links[self._get_container_name('datapusher')] = 'datapusher'
ro = dict(
{
self.target:
'/project/',
scripts.get_script_path('web.sh'):
'/scripts/web.sh',
scripts.get_script_path('adjust_devini.py'):
'/scripts/adjust_devini.py'
}, **ro)
rw = {
self.sitedir + '/files': '/var/www/storage',
self.sitedir + '/run/development.ini': '/project/development.ini'
}
try:
if not interactive:
run_container(name=self._get_container_name('web'),
image='datacats/web',
rw=rw,
ro=ro,
links=links,
volumes_from=volumes_from,
command=command,
port_bindings={
5000:
port if is_boot2docker() else (address, port)
},
log_syslog=log_syslog)
else:
# FIXME: share more code with interactive_shell
if is_boot2docker():
switches = [
'--volumes-from',
self._get_container_name('pgdata'), '--volumes-from',
self._get_container_name('venv')
]
else:
switches = []
switches += [
'--volume={}:{}:ro'.format(vol, ro[vol]) for vol in ro
]
switches += [
'--volume={}:{}'.format(vol, rw[vol]) for vol in rw
]
links = [
'--link={}:{}'.format(link, links[link]) for link in links
]
args = ['docker', 'run', '-it', '--name', self._get_container_name('web'),
'-p', '{}:5000'.format(port) if is_boot2docker()
else '{}:{}:5000'.format(address, port)] + \
switches + links + ['datacats/web', ] + command
subprocess.call(args)
except APIError as e:
if '409' in str(e):
raise DatacatsError('Web container already running. '
'Please stop_web before running.')
else:
raise
