def _machine_check_connectivity(): """ This method calls to docker-machine on the command line and makes sure that it is up and ready. Potential improvements to be made: - Support multiple machine names (run a `docker-machine ls` and then see which machines are active. Use a priority list) """ with open(devnull, 'w') as devnull_f: try: status = subprocess.check_output( ['docker-machine', 'status', 'dev'], stderr=devnull_f).strip() if status == 'Stopped': raise DatacatsError('Please start your docker-machine ' 'VM with "docker-machine start dev"') # XXX HACK: This exists because of # http://github.com/datacats/datacats/issues/63, # as a temporary fix. if 'tls' in _docker_kwargs: # It will print out messages to the user otherwise. _docker_kwargs['tls'].assert_hostname = False except subprocess.CalledProcessError: raise DatacatsError('Please create a docker-machine with ' '"docker-machine start dev"')
def _get_docker(): global _docker if not _docker: if sys.platform.startswith('darwin'): _machine_check_connectivity() # Create the Docker client version_client = Client(version=MINIMUM_API_VERSION, **_docker_kwargs) try: api_version = version_client.version()['ApiVersion'] except ConnectionError: try: # workaround for connection issue when old version specified # on some clients version_client = Client(**_docker_kwargs) api_version = version_client.version()['ApiVersion'] except: raise DatacatsError(DOCKER_FAIL_STRING) except: raise DatacatsError(DOCKER_FAIL_STRING) version = get_api_version(DEFAULT_DOCKER_API_VERSION, api_version) _docker = Client(version=version, **_docker_kwargs) return _docker
def new_environment_check(srcpath, site_name, ckan_version): """ Check if a new environment or site can be created at the given path. Returns (name, datadir, sitedir, srcdir) or raises DatacatsError """ docker.require_images() workdir, name = path.split(path.abspath(path.expanduser(srcpath))) if not validate.valid_name(name): raise DatacatsError( 'Please choose an environment name starting' ' with a letter and including only lowercase letters' ' and digits') if not path.isdir(workdir): raise DatacatsError('Parent directory for environment' ' does not exist') datadir = path.expanduser('~/.datacats/' + name) sitedir = datadir + '/sites/' + site_name # We track through the datadir to the target if we are just making a # site if path.isdir(datadir): with open(datadir + '/project-dir') as pd: srcdir = pd.read() else: srcdir = workdir + '/' + name if ckan_version not in SUPPORTED_PRELOADS: raise DatacatsError( '''Datacats does not currently support CKAN version {}. Versions that are currently supported are: {}'''.format( ckan_version, ', '.join(SUPPORTED_PRELOADS))) preload_name = str(ckan_version) # Get all the versions from the tags downloaded_versions = [tag for tag in docker.get_tags('datacats/ckan')] if ckan_version not in downloaded_versions: retrying_pull_image('datacats/ckan:{}'.format(preload_name)) if path.isdir(sitedir): raise DatacatsError( 'Site data directory {0} already exists'.format(sitedir)) # This is the case where the data dir has been removed, if path.isdir(srcdir) and not path.isdir(datadir): raise DatacatsError( 'Environment directory exists, but data directory does not.\n' 'If you simply want to recreate the data directory, run ' '"datacats init" in the environment directory.') return name, datadir, srcdir
def find_environment_dirs(environment_name=None, data_only=False): """ :param environment_name: exising environment name, path or None to look in current or parent directories for project returns (srcdir, extension_dir, datadir) extension_dir is the name of extension directory user was in/referenced, default: 'ckan'. This value is used by the paster cli command. datadir will be None if environment_name was a path or None (not a name) """ docker.require_images() if environment_name is None: environment_name = '.' extension_dir = 'ckan' if validate.valid_name(environment_name) and path.isdir( path.expanduser('~/.datacats/' + environment_name)): # loading from a name datadir = path.expanduser('~/.datacats/' + environment_name) with open(datadir + '/project-dir') as pd: srcdir = pd.read() if not data_only and not path.exists(srcdir + '/.datacats-environment'): raise DatacatsError( 'Environment data found but environment directory is' ' missing. Try again from the new environment directory' ' location or remove the environment data with' ' "datacats purge"') return srcdir, extension_dir, datadir # loading from a path srcdir = path.abspath(environment_name) if not path.isdir(srcdir): raise DatacatsError('No environment found with that name') wd = srcdir oldwd = None while not path.exists(wd + '/.datacats-environment'): oldwd = wd wd, _ = path.split(wd) if wd == oldwd: raise DatacatsError( 'Environment not found in {0} or above'.format(srcdir)) srcdir = wd if oldwd: _, extension_dir = path.split(oldwd) return srcdir, extension_dir, None
def install_all(environment, clean, verbose=False, quiet=False, packages=None): logs = check_connectivity() if logs.strip(): raise DatacatsError(logs) if clean: clean_pyc(environment, quiet) srcdirs = set() reqdirs = set() for d in listdir(environment.target): fulld = environment.target + '/' + d if not isdir(fulld): continue if not exists(fulld + '/setup.py'): continue if packages and d not in packages: continue srcdirs.add(d) if (exists(fulld + '/requirements.txt') or exists(fulld + '/pip-requirements.txt')): reqdirs.add(d) try: if not packages or 'ckan' in packages: srcdirs.remove('ckan') reqdirs.remove('ckan') srcdirs = ['ckan'] + sorted(srcdirs) reqdirs = ['ckan'] + sorted(reqdirs) except KeyError: raise DatacatsError('ckan not found in environment directory') if clean: environment.clean_virtualenv() environment.install_extra() for s in srcdirs: if verbose: print colored.yellow('Installing ' + s + '\n') elif not quiet: print 'Installing ' + s environment.install_package_develop( s, sys.stdout if verbose and not quiet else None) if verbose and not quiet: print for s in reqdirs: if verbose: print colored.yellow('Installing ' + s + ' requirements' + '\n') elif not quiet: print 'Installing ' + s + ' requirements' environment.install_package_requirements( s, sys.stdout if verbose and not quiet else None) if verbose: print
def _get_docker(): global _docker # HACK: We determine from commands if we're boot2docker # Needed cause this method is called from is_boot2docker... boot2docker = False if not _docker: # First, check if boot2docker is powered on. try: with open(devnull, 'w') as devnull_f: status = subprocess.check_output( ['boot2docker', 'status'], # Don't show boot2docker message # to the user... it's ugly! stderr=devnull_f).strip() if status == 'poweroff': raise DatacatsError('boot2docker is not powered on.' ' Please run "boot2docker up".') boot2docker = True except OSError: # We're on Linux, or boot2docker isn't installed. pass except subprocess.CalledProcessError: raise DatacatsError('You have not created your boot2docker VM. ' 'Please run "boot2docker init" to do so.') # XXX HACK: This exists because of # http://github.com/datacats/datacats/issues/63, # as a temporary fix. if 'tls' in _docker_kwargs and boot2docker: import warnings # It will print out messages to the user otherwise. warnings.filterwarnings("ignore", category=InsecureRequestWarning) warnings.filterwarnings("ignore", category=InsecurePlatformWarning) _docker_kwargs['tls'].verify = False # Create the Docker client version_client = Client(version=MINIMUM_API_VERSION, **_docker_kwargs) try: api_version = version_client.version()['ApiVersion'] except ConnectionError: # workaround for connection issue when old version specified # on some clients version_client = Client(**_docker_kwargs) api_version = version_client.version()['ApiVersion'] version = get_api_version(DEFAULT_DOCKER_API_VERSION, api_version) _docker = Client(version=version, **_docker_kwargs) return _docker
def require_data(self): """ raise a DatacatsError if the datadir or volumes are missing or damaged """ files = task.source_missing(self.target) if files: raise DatacatsError('Missing files in source directory:\n' + '\n'.join(files)) if not self.data_exists(): raise DatacatsError('Environment datadir missing. ' 'Try "datacats init".') if not self.data_complete(): raise DatacatsError('Environment datadir damaged or volumes ' 'missing. ' 'To reset and discard all data use ' '"datacats reset"')
def wait_for_web_available(self): """ Wait for the web server to become available or raise DatacatsError if it fails to start. """ try: if not wait_for_service_available(self._get_container_name('web'), self.web_address(), WEB_START_TIMEOUT_SECONDS): raise DatacatsError( 'Error while starting web container:\n' + container_logs( self._get_container_name('web'), "all", False, None)) except ServiceTimeout: raise DatacatsError('Timeout while starting web container. Logs:' + container_logs(self._get_container_name('web'), "all", False, None))
def create_directories(datadir, sitedir, srcdir=None): """ Create expected directories in datadir, sitedir and optionally srcdir """ # It's possible that the datadir already exists # (we're making a secondary site) if not path.isdir(datadir): os.makedirs(datadir, mode=0o700) try: # This should take care if the 'site' subdir if needed os.makedirs(sitedir, mode=0o700) except OSError: raise DatacatsError("Site already exists.") # venv isn't site-specific, the rest are. if not docker.is_boot2docker(): if not path.isdir(datadir + '/venv'): os.makedirs(datadir + '/venv') os.makedirs(sitedir + '/postgres') os.makedirs(sitedir + '/solr') os.makedirs(sitedir + '/files') os.makedirs(sitedir + '/run') if srcdir: os.makedirs(srcdir)
def _subcommand_arguments(args): """ Return (subcommand, (possibly adjusted) arguments for that subcommand) Returns (None, args) when no subcommand is found Parsing our arguments is hard. Each subcommand has its own docopt validation, and some subcommands (paster and shell) have positional options (some options passed to datacats and others passed to commands run inside the container) """ skip_site = False # Find subcommand without docopt so that subcommand options may appear # anywhere for i, a in enumerate(args): if skip_site: skip_site = False continue if a.startswith('-'): if a == '-s' or a == '--site': skip_site = True continue if a == 'help': return _subcommand_arguments(args[:i] + ['--help'] + args[i + 1:]) if a not in COMMANDS: raise DatacatsError( "\'{0}\' command is not recognized. \n" "See \'datacats help\' for the list of available commands". format(a)) command = a break else: return None, args if command != 'shell' and command != 'paster': return command, args # shell requires the environment name, paster does not remaining_positional = 2 if command == 'shell' else 1 # i is where the subcommand starts. # shell, paster are special: options might belong to the command being # find where the the inner command starts and insert a '--' before # so that we can separate inner options from ones we need to parse while i < len(args): a = args[i] if a.startswith('-'): if a == '-s' or a == '--site': # site name is coming i += 2 continue i += 1 continue if remaining_positional: remaining_positional -= 1 i += 1 continue return command, args[:i] + ['--'] + args[i:] return command, args
def migrate(opts): """Migrate an environment to a given revision of the datadir format. Usage: datacats migrate [-y] [-r VERSION] [ENVIRONMENT_DIR] Options: -r --revision=VERSION The version of the datadir format you want to convert to [default: 2] -y --yes Answer yes to all questions. Defaults to '.' if ENVIRONMENT_DIR isn't specified. """ try: version = int(opts['--revision']) except: raise DatacatsError('--revision parameter must be an integer.') always_yes = opts['--yes'] if 'ENVIRONMENT_DIR' not in opts or not opts['ENVIRONMENT_DIR']: cwd = getcwd() # Get the dirname opts['ENVIRONMENT_DIR'] = split(cwd if cwd[-1] != '/' else cwd[:-1])[1] datadir = expanduser('~/.datacats/' + opts['ENVIRONMENT_DIR']) if needs_format_conversion(datadir, version): convert_environment(datadir, version, always_yes) print 'Successfully converted datadir {} to format version {}'.format( datadir, version) else: print 'datadir {} is already at version {}.'.format(datadir, version)
def _next_port(self, port): """ Return another port from the 5000-5999 range """ port = 5000 + (port + 1) % 1000 if port == self.port: raise DatacatsError('Too many instances running') return port
def needs_datapusher(self): cp = SafeConfigParser() try: cp.read(self.target + '/development.ini') return ('datapusher' in cp.get('app:main', 'ckan.plugins') and isdir(self.target + '/datapusher')) except ConfigParserError as e: raise DatacatsError('Failed to read and parse development.ini: ' + str(e))
def require_images(): """ Raises a DatacatsError if the images required to use Datacats don't exist. """ if (not image_exists('datacats/web') or not image_exists('datacats/solr') or not image_exists('datacats/postgres')): raise DatacatsError( 'You do not have the needed Docker images. Please run "datacats pull"' )
def deploy(self, environment, target_name, stream_output=None): """ Return True if deployment was successful """ try: remote_server_command( [ "rsync", "-lrv", "--safe-links", "--munge-links", "--delete", "--inplace", "--chmod=ugo=rwX", "--exclude=.datacats-environment", "--exclude=.git", "/project/.", environment.deploy_target + ':' + target_name ], environment, self, include_project_dir=True, stream_output=stream_output, clean_up=True, ) except WebCommandError as e: raise DatacatsError( "Unable to deploy `{0}` to remote server for some reason:\n" " datacats was not able to copy data to the remote server". format((target_name, )), parent_exception=e) try: remote_server_command( [ "ssh", environment.deploy_target, "install", target_name, ], environment, self, clean_up=True, ) return True except WebCommandError as e: raise DatacatsError( "Unable to deploy `{0}` to remote server for some reason:\n" "datacats copied data to the server but failed to register\n" "(or `install`) the new catalog".format((target_name, )), parent_exception=e)
def load_site(srcdir, datadir, site_name=None): """ Load configuration values for a site. Returns (port, address, site_url, passwords) """ if site_name is None: site_name = 'primary' if not validate.valid_name(site_name): raise DatacatsError('{} is not a valid site name.'.format(site_name)) cp = ConfigParser.SafeConfigParser() try: cp.read([srcdir + '/.datacats-environment']) except ConfigParser.Error: raise DatacatsError('Error reading environment information') site_section = 'site_' + site_name try: port = cp.getint(site_section, 'port') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): port = None try: address = cp.get(site_section, 'address') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): address = None try: site_url = cp.get(site_section, 'site_url') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): site_url = None passwords = {} cp = ConfigParser.SafeConfigParser() cp.read(datadir + '/sites/' + site_name + '/passwords.ini') try: pw_options = cp.options('passwords') except ConfigParser.NoSectionError: pw_options = [] for n in pw_options: passwords[n.upper()] = cp.get('passwords', n) return port, address, site_url, passwords
def main(): """ The main entry point for datacats cli tool (as defined in setup.py's entry_points) It parses the cli arguments for corresponding options and runs the corresponding command """ # pylint: disable=bare-except try: command_fn, opts = _parse_arguments(sys.argv[1:]) # purge handles loading differently # 1 - Bail and just call the command if it doesn't have ENVIRONMENT. if command_fn == purge.purge or 'ENVIRONMENT' not in opts: return command_fn(opts) environment = Environment.load( opts['ENVIRONMENT'] or '.', opts['--site'] if '--site' in opts else 'primary') if command_fn not in COMMANDS_THAT_USE_SSH: return command_fn(environment, opts) # for commands that communicate with a remote server # we load UserProfile and test our communication user_profile = UserProfile() user_profile.test_ssh_key(environment) return command_fn(environment, opts, user_profile) except DatacatsError as e: _error_exit(e) except SystemExit: raise except: exc_info = "\n".join([ line.rstrip() for line in traceback.format_exception(*sys.exc_info()) ]) user_message = ("Something that should not" " have happened happened when attempting" " to run this command:\n" " datacats {args}\n\n" "It is seems to be a bug.\n" "Please report this issue to us by" " creating an issue ticket at\n\n" " https://github.com/datacats/datacats/issues\n\n" "so that we would be able to look into that " "and fix the issue.").format( args=" ".join(sys.argv[1:])) _error_exit( DatacatsError(user_message, parent_exception=UndocumentedError(exc_info)))
def convert_environment(datadir, version, always_yes): """ Converts an environment TO the version specified by `version`. :param datadir: The datadir to convert. :param version: The version to convert TO. :param always_yes: True if the user shouldn't be prompted about the migration. """ # Since we don't call either load() or new() we have to call require_images ourselves. require_images() inp = None old_version = _get_current_format(datadir) migration_func = migrations[(old_version, version)] if version > CURRENT_FORMAT_VERSION: raise DatacatsError('Cannot migrate to a version higher than the ' 'current one.') if version < 1: raise DatacatsError('Datadir versioning starts at 1.') if not always_yes: while inp != 'y' and inp != 'n': inp = raw_input(migration_func.__doc__.format(version)) if inp == 'n': sys.exit(1) lockfile = LockFile(path_join(datadir, '.migration_lock')) lockfile.acquire() try: # FIXME: If we wanted to, we could find a set of conversions which # would bring us up to the one we want if there's no direct path. # This isn't necessary with just two formats, but it may be useful # at 3. # Call the appropriate conversion function migration_func(datadir) finally: lockfile.release()
def finish_init(environment, start_web, create_sysadmin, log_syslog=False, do_install=True, quiet=False, site_url=None, interactive=False, init_db=True): """ Common parts of create and init: Install, init db, start site, sysadmin """ if not init_db: start_web = False create_sysadmin = False if do_install: install_all(environment, False, verbose=False, quiet=quiet) if init_db: if not quiet: write('Initializing database') environment.install_postgis_sql() environment.ckan_db_init() if not quiet: write('\n') if site_url: try: site_url = site_url.format(address=environment.address, port=environment.port) environment.site_url = site_url environment.save_site(False) except (KeyError, IndexError, ValueError) as e: raise DatacatsError('Could not parse site_url: {}'.format(e)) if start_web: environment.start_ckan(log_syslog=log_syslog) if not quiet and not interactive: write('Starting web server at {0} ...\n'.format( environment.web_address())) if create_sysadmin: try: adminpw = confirm_password() environment.create_admin_set_password(adminpw) except KeyboardInterrupt: print if not start_web: environment.stop_supporting_containers()
def _create_run_ini(self, port, production, output='development.ini', source='development.ini', override_site_url=True): """ Create run/development.ini in datadir with debug and site_url overridden and with correct db passwords inserted """ cp = SafeConfigParser() try: cp.read([self.target + '/' + source]) except ConfigParserError: raise DatacatsError('Error reading development.ini') cp.set('DEFAULT', 'debug', 'false' if production else 'true') if self.site_url: site_url = self.site_url else: if is_boot2docker(): web_address = socket.gethostbyname(docker_host()) else: web_address = self.address site_url = 'http://{}:{}'.format(web_address, port) if override_site_url: cp.set('app:main', 'ckan.site_url', site_url) cp.set( 'app:main', 'sqlalchemy.url', 'postgresql://*****:*****@db:5432/ckan'.format( self.passwords['CKAN_PASSWORD'])) cp.set( 'app:main', 'ckan.datastore.read_url', 'postgresql://*****:*****@db:5432/ckan_datastore'. format(self.passwords['DATASTORE_RO_PASSWORD'])) cp.set( 'app:main', 'ckan.datastore.write_url', 'postgresql://*****:*****@db:5432/ckan_datastore'. format(self.passwords['DATASTORE_RW_PASSWORD'])) cp.set('app:main', 'solr_url', 'http://solr:8080/solr') cp.set('app:main', 'beaker.session.secret', self.passwords['BEAKER_SESSION_SECRET']) if not isdir(self.sitedir + '/run'): makedirs(self.sitedir + '/run') # upgrade old datadir with open(self.sitedir + '/run/' + output, 'w') as runini: cp.write(runini)
def pull_image(image_name): sys.stdout.write('Pulling image ' + image_name) sys.stdout.flush() for s in pull_stream(image_name): if 'status' not in s: if 'error' in s: # Line to make the error appear after the ... print raise DatacatsError(s['error']) else: print json.dumps(s) sys.stdout.write('.') sys.stdout.flush() sys.stdout.write('\n')
def info(environment, opts): """Display information about environment and running containers Usage: datacats info [-qr] [ENVIRONMENT] Options: -q --quiet Echo only the web URL or nothing if not running -r --remote Information about DataCats.com cloud instance ENVIRONMENT may be an environment name or a path to an environment directory. Default: '.' """ damaged = False sites = environment.sites if not environment.sites: sites = [] damaged = True if opts['--quiet']: if damaged: raise DatacatsError('Damaged datadir: cannot get address.') for site in sites: environment.site_name = site print '{}: {}'.format(site, environment.web_address()) return datadir = environment.datadir if not environment.data_exists(): datadir = '' elif damaged: datadir += ' (damaged)' print 'Environment name: ' + environment.name print ' Environment dir: ' + environment.target print ' Data dir: ' + datadir print ' Sites: ' + ' '.join(environment.sites) for site in environment.sites: print environment.site_name = site print ' Site: ' + site print ' Containers: ' + ' '.join(environment.containers_running()) sitedir = environment.sitedir + (' (damaged)' if not environment.data_complete() else '') print ' Site dir: ' + sitedir addr = environment.web_address() if addr: print ' Available at: ' + addr
def create_profile(self): self.ssh_private_key = self.profiledir + '/id_rsa' self.ssh_public_key = self.profiledir + '/id_rsa.pub' self.save() self.generate_ssh_key() user_error_message = ( "Your profile does not seem to have an ssh key\n" " (which is an equivalent of your password so that datacats.io" " could recognize you).\n" "It is probably because this is your first time running" " a remote command in which case welcome!\n" "So we generated a new ssh key for you. \n " "Please go to www.datacats.com/account/key" " and add the following public key:" " \n \n {public_key} \n \n" " to your profile so that the server can recognize you as you." ).format(public_key=self.read_public_key()) raise DatacatsError(user_error_message)
def create(opts): """Create a new environment Usage: datacats create [-bin] [--interactive] [-s NAME] [--address=IP] [--syslog] [--ckan=CKAN_VERSION] [--no-datapusher] [--site-url SITE_URL] [--no-init-db] ENVIRONMENT_DIR [PORT] Options: --address=IP Address to listen on (Linux-only) --ckan=CKAN_VERSION Use CKAN version CKAN_VERSION [default: 2.3] -b --bare Bare CKAN site with no example extension -i --image-only Create the environment but don't start containers --interactive Doesn't detach from the web container --no-datapusher Don't install/enable ckanext-datapusher --no-init-db Don't initialize the database. Useful for importing CKANs. -n --no-sysadmin Don't prompt for an initial sysadmin user account -s --site=NAME Pick a site to create [default: primary] --site-url SITE_URL The site_url to use in API responses (e.g. http://example.org:{port}/) --syslog Log to the syslog ENVIRONMENT_DIR is a path for the new environment directory. The last part of this path will be used as the environment name. """ if opts['--address'] and is_boot2docker(): raise DatacatsError('Cannot specify address on boot2docker.') return create_environment( environment_dir=opts['ENVIRONMENT_DIR'], port=opts['PORT'], create_skin=not opts['--bare'], start_web=not opts['--image-only'], create_sysadmin=not opts['--no-sysadmin'], site_name=opts['--site'], ckan_version=opts['--ckan'], address=opts['--address'], log_syslog=opts['--syslog'], datapusher=not opts['--no-datapusher'], site_url=opts['--site-url'], interactive=opts['--interactive'], init_db=not opts['--no-init-db'], )
def deploy(environment, opts, profile): """Deploy environment to production DataCats.com cloud service Usage: datacats deploy [--create] [ENVIRONMENT [TARGET_NAME]] Options: --create Create a new environment on DataCats.com instead of updating an existing environment ENVIRONMENT may be an environment name or a path to a environment directory. Default: '.' TARGET_NAME is the name of the environment on DataCats.com. Defaults to the environment name. """ target_name = opts['TARGET_NAME'] if target_name is None: target_name = environment.name if not valid_deploy_name(target_name): raise DatacatsError( " `{target_name}` target name for deployment can't be accepted.\n" "Can't have http://{target_name}.datacats.io for your datcat URL\n" "Please choose a target name at least 5 characters long,\n" "and containing only lowercase letters and numbers\n".format( target_name=target_name)) if opts['--create']: profile.create(environment, target_name) profile.deploy(environment, target_name, stdout) print "Deployed source to http://{0}.datacats.io".format(target_name) if opts['--create']: try: pw = confirm_password() profile.admin_password(environment, target_name, pw) except KeyboardInterrupt: pass
def add_extra_container(self, container, error_on_exists=False): """ Add a container as a 'extra'. These are running containers which are not necessary for running default CKAN but are useful for certain extensions :param container: The container name to add :param error_on_exists: Raise a DatacatsError if the extra container already exists. """ if container in self.extra_containers: if error_on_exists: raise DatacatsError('{} is already added as an extra container.'.format(container)) else: return self.extra_containers.append(container) cp = SafeConfigParser() cp.read(self.target + '/.datacats-environment') cp.set('datacats', 'extra_containers', ' '.join(self.extra_containers)) with open(self.target + '/.datacats-environment', 'w') as f: cp.write(f)
def test_ssh_key(self, environment): """ Return True if this key is accepted by DataCats.com """ try: remote_server_command(["ssh", environment.deploy_target, 'test'], environment, self, clean_up=True) except WebCommandError as e: user_unrecognized_error_message = ( "Your ssh key " "(which is an equivalent of your password so" " that datacats.io could recognize you) " "does not seem to be recognized by the datacats.io server. \n \n" "Most likely it is because you need to go to" " www.datacats.com/account/key" " and add the following public key: \n \n {public_key} \n \n" "to your profile so that datacat's server could recognize you." " So maybe try that?\n" "If the problem persists, please contact the developer team." ).format(public_key=self.read_public_key()) network_unreachable_error_message = ( "Unable to connect to the hosting server: {0} \n" "Some of the reasons for that may be: \n" " 1) the internet connection is down,\n" " 2) the server is not up or functioning properly,\n" " 3) there is a firewall block for the datacats application\n" " or something of this sort.").format( environment.deploy_target) user_error_message = network_unreachable_error_message \ if "Network is unreachable" in e.logs \ else user_unrecognized_error_message raise DatacatsError(user_error_message, parent_exception=e)
def _retry_func(func, param, num, retry_notif, error_msg): """ A function which retries a given function num times and calls retry_notif each time the function is retried. :param func: The function to retry num times. :param num: The number of times to try before giving up. :param retry_notif: Will be called with the same parameter as func if we have to retry the function. Will also receive the number of retries so far as a second parameter. :param: error_msg: The message Throws DatacatsError if we run out of retries. Returns otherwise. """ for retry_num in range(num): if retry_num: retry_notif(param, retry_num) try: func(param) return except DatacatsError: pass raise DatacatsError(error_msg)
def purge_data(self, which_sites=None, never_delete=False): """ Remove uploaded files, postgres db, solr index, venv """ # Default to the set of all sites if not exists(self.datadir + '/.version'): format_version = 1 else: with open(self.datadir + '/.version') as f: format_version = int(f.read().strip()) if format_version == 1: print 'WARNING: Defaulting to old purge for version 1.' datadirs = ['files', 'solr'] if is_boot2docker(): remove_container('datacats_pgdata_{}'.format(self.name)) remove_container('datacats_venv_{}'.format(self.name)) else: datadirs += ['postgres', 'venv'] web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) shutil.rmtree(self.datadir) elif format_version == 2: if not which_sites: which_sites = self.sites datadirs = [] boot2docker = is_boot2docker() if which_sites: if self.target: cp = SafeConfigParser() cp.read([self.target + '/.datacats-environment']) for site in which_sites: if boot2docker: remove_container(self._get_container_name('pgdata')) else: datadirs += [site + '/postgres'] # Always rm the site dir & solr & files datadirs += [site, site + '/files', site + '/solr'] if self.target: cp.remove_section('site_' + site) self.sites.remove(site) if self.target: with open(self.target + '/.datacats-environment', 'w') as conf: cp.write(conf) datadirs = ['sites/' + datadir for datadir in datadirs] if not self.sites and not never_delete: datadirs.append('venv') web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) if not self.sites and not never_delete: shutil.rmtree(self.datadir) else: raise DatacatsError( 'Unknown format version {}'.format(format_version))
def _run_web_container(self, port, command, address, log_syslog=False, datapusher=True, interactive=False): """ Start web container on port with command """ if is_boot2docker(): ro = {} volumes_from = self._get_container_name('venv') else: ro = {self.datadir + '/venv': '/usr/lib/ckan'} volumes_from = None links = { self._get_container_name('solr'): 'solr', self._get_container_name('postgres'): 'db' } links.update({ self._get_container_name(container): container for container in self.extra_containers }) if datapusher: if 'datapusher' not in self.containers_running(): raise DatacatsError( container_logs(self._get_container_name('datapusher'), "all", False, False)) links[self._get_container_name('datapusher')] = 'datapusher' ro = dict( { self.target: '/project/', scripts.get_script_path('web.sh'): '/scripts/web.sh', scripts.get_script_path('adjust_devini.py'): '/scripts/adjust_devini.py' }, **ro) rw = { self.sitedir + '/files': '/var/www/storage', self.sitedir + '/run/development.ini': '/project/development.ini' } try: if not interactive: run_container(name=self._get_container_name('web'), image='datacats/web', rw=rw, ro=ro, links=links, volumes_from=volumes_from, command=command, port_bindings={ 5000: port if is_boot2docker() else (address, port) }, log_syslog=log_syslog) else: # FIXME: share more code with interactive_shell if is_boot2docker(): switches = [ '--volumes-from', self._get_container_name('pgdata'), '--volumes-from', self._get_container_name('venv') ] else: switches = [] switches += [ '--volume={}:{}:ro'.format(vol, ro[vol]) for vol in ro ] switches += [ '--volume={}:{}'.format(vol, rw[vol]) for vol in rw ] links = [ '--link={}:{}'.format(link, links[link]) for link in links ] args = ['docker', 'run', '-it', '--name', self._get_container_name('web'), '-p', '{}:5000'.format(port) if is_boot2docker() else '{}:{}:5000'.format(address, port)] + \ switches + links + ['datacats/web', ] + command subprocess.call(args) except APIError as e: if '409' in str(e): raise DatacatsError('Web container already running. ' 'Please stop_web before running.') else: raise