def install_postgis_sql(self): web_command( '/scripts/install_postgis.sh', image='datacats/postgres', ro={scripts.get_script_path('install_postgis.sh'): '/scripts/install_postgis.sh'}, links={self._get_container_name('postgres'): 'db'}, )
def deploy(self, project, target_name, stream_output=None): """ Return True if deployment was successful """ try: web_command( command=[ "rsync", "-lrv", "--safe-links", "--munge-links", "--delete", "--inplace", "--chmod=ugo=rwX", "--exclude=.datacats-environment", "--exclude=.git", "/project/.", _project_user_host(project) + ':' + target_name], ro={project.target: '/project', KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa'}, stream_output=stream_output, clean_up=True, ) web_command( command=[ "ssh", _project_user_host(project), "install", target_name, ], ro={KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa'}, stream_output=stream_output, clean_up=True, ) return True except WebCommandError: return False
def fix_storage_permissions(self): """ Set the owner of all apache storage files to www-data container user """ web_command( command='/bin/chown -R www-data: /var/www/storage', rw={self.sitedir + '/files': '/var/www/storage'})
def _two_to_one(datadir): """After this command, your environment will be converted to format version {} and will not work with Datacats versions beyond and including 1.0.0. This format version doesn't support multiple sites, and after this only your "primary" site will be usable, though other sites will be maintained if you wish to do a migration back to a version which supports multisite. Would you like to continue the migration? (y/n) [n]:""" _, env_name = _split_path(datadir) print 'Making sure that containers are stopped...' # New-style names remove_container('datacats_web_{}_primary'.format(env_name)) remove_container('datacats_postgres_{}_primary'.format(env_name)) remove_container('datacats_solr_{}_primary'.format(env_name)) print 'Doing conversion...' if exists(path_join(datadir, '.version')): os.remove(path_join(datadir, '.version')) to_move = (['files', 'passwords.ini', 'run', 'solr'] + (['postgres'] if not is_boot2docker() else [])) web_command( command=['/scripts/migrate.sh', '/project/data/sites/primary', '/project/data'] + to_move, ro={scripts.get_script_path('migrate.sh'): '/scripts/migrate.sh'}, rw={datadir: '/project/data'} ) pgdata_name = 'datacats_pgdata_{}_primary'.format(env_name) if is_boot2docker() and inspect_container(pgdata_name): rename_container(pgdata_name, 'datacats_pgdata_{}'.format(env_name)) print 'Doing cleanup...' with open(path_join(datadir, 'project-dir')) as pd: datacats_env_location = path_join(pd.read(), '.datacats-environment') cp = SafeConfigParser() cp.read(datacats_env_location) # We need to move the port OUT of site_primary section and INTO datacats cp.set('datacats', 'port', cp.get('site_primary', 'port')) cp.remove_section('site_primary') with open(datacats_env_location, 'w') as config: cp.write(config) cp = SafeConfigParser() cp.read(path_join(datadir, 'passwords.ini')) # This isn't needed in this version cp.remove_option('passwords', 'beaker_session_secret') with open(path_join(datadir, 'passwords.ini'), 'w') as config: cp.write(config)
def generate_ssh_key(self): """ Generate a new ssh private and public key """ web_command( command=["ssh-keygen", "-q", "-t", "rsa", "-N", "", "-C", "datacats generated {0}@{1}".format(getuser(), gethostname()), "-f", "/output/id_rsa"], rw={self.profiledir: '/output'}, )
def create_source(self): """ Populate ckan directory from preloaded image and copy who.ini and schema.xml info conf directory """ web_command(command='/bin/cp -a /project/ckan /project_target/ckan', rw={self.target: '/project_target'}, image=self._preload_image()) shutil.copy(self.target + '/ckan/ckan/config/who.ini', self.target) shutil.copy(self.target + '/ckan/ckan/config/solr/schema.xml', self.target)
def generate_ssh_key(self): """ Generate a new ssh private and public key """ web_command( command=[ "ssh-keygen", "-q", "-t", "rsa", "-N", "", "-C", "datacats generated {0}@{1}".format(getuser(), gethostname()), "-f", "/output/id_rsa" ], rw={self.profiledir: '/output'}, )
def create_source(self): """ Populate ckan directory from preloaded image and copy who.ini and schema.xml info conf directory """ web_command( command='/bin/cp -a /project/ckan /project_target/ckan', rw={self.target: '/project_target'}, image=self._preload_image()) shutil.copy( self.target + '/ckan/ckan/config/who.ini', self.target) shutil.copy( self.target + '/ckan/ckan/config/solr/schema.xml', self.target)
def run_command(self, command, db_links=False, rw_venv=False, rw_project=False, rw=None, ro=None, clean_up=False): rw = {} if rw is None else dict(rw) ro = {} if ro is None else dict(ro) ro.update(self._proxy_settings()) if is_boot2docker(): volumes_from = 'datacats_venv_' + self.name else: volumes_from = None venvmount = rw if rw_venv else ro venvmount[self.datadir + '/venv'] = '/usr/lib/ckan' projectmount = rw if rw_project else ro projectmount[self.target] = '/project' if db_links: self._create_run_ini(self.port, production=False, output='run.ini') links = { 'datacats_solr_' + self.name: 'solr', 'datacats_postgres_' + self.name: 'db', } ro[self.datadir + '/run/run.ini'] = '/project/development.ini' else: links = None return web_command(command=command, ro=ro, rw=rw, links=links, volumes_from=volumes_from, clean_up=clean_up)
def run_command(self, command, db_links=False, rw_venv=False, rw_project=False, rw=None, ro=None, clean_up=False, stream_output=None): rw = {} if rw is None else dict(rw) ro = {} if ro is None else dict(ro) ro.update(self._proxy_settings()) if is_boot2docker(): volumes_from = self._get_container_name('venv') else: volumes_from = None venvmount = rw if rw_venv else ro venvmount[self.datadir + '/venv'] = '/usr/lib/ckan' projectmount = rw if rw_project else ro projectmount[self.target] = '/project' if db_links: self._create_run_ini(self.port, production=False, output='run.ini') links = { self._get_container_name('solr'): 'solr', self._get_container_name('postgres'): 'db', } ro[self.sitedir + '/run/run.ini'] = '/project/development.ini' else: links = None return web_command(command=command, ro=ro, rw=rw, links=links, volumes_from=volumes_from, clean_up=clean_up, commit=True, stream_output=stream_output)
def test_ssh_key(self, project): """ Return True if this key is accepted by DataCats.com """ try: web_command(command=["ssh", _project_user_host(project), 'test'], ro={ KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa' }, clean_up=True) return True except WebCommandError: return False
def purge_data(self, which_sites=None, never_delete=False): """ Remove uploaded files, postgres db, solr index, venv """ # Default to the set of all sites if not which_sites: which_sites = self.sites datadirs = [] boot2docker = is_boot2docker() if which_sites: if self.target: cp = SafeConfigParser() cp.read([self.target + '/.datacats-environment']) for site in which_sites: if boot2docker: remove_container(self._get_container_name('pgdata')) else: datadirs += [site + '/postgres'] # Always rm the site dir & solr & files datadirs += [site, site + '/files', site + '/solr'] if self.target: cp.remove_section('site_' + site) self.sites.remove(site) if self.target: with open(self.target + '/.datacats-environment', 'w') as conf: cp.write(conf) datadirs = ['sites/' + datadir for datadir in datadirs] if not self.sites and not never_delete: datadirs.append('venv') web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) if not self.sites and not never_delete: shutil.rmtree(self.datadir)
def create_virtualenv(srcdir, datadir, preload_image, get_container_name): """ Populate venv from preloaded image """ try: if docker.is_boot2docker(): docker.data_only_container(get_container_name("venv"), ["/usr/lib/ckan"]) img_id = docker.web_command( "/bin/mv /usr/lib/ckan/ /usr/lib/ckan_original", image=preload_image, commit=True ) docker.web_command( command="/bin/cp -a /usr/lib/ckan_original/. /usr/lib/ckan/.", volumes_from=get_container_name("venv"), image=img_id, ) docker.remove_image(img_id) return docker.web_command( command="/bin/cp -a /usr/lib/ckan/. /usr/lib/ckan_target/.", rw={datadir + "/venv": "/usr/lib/ckan_target"}, image=preload_image, ) finally: rw = {datadir + "/venv": "/usr/lib/ckan"} if not docker.is_boot2docker() else {} volumes_from = get_container_name("venv") if docker.is_boot2docker() else None # fix venv permissions docker.web_command( command="/bin/chown -R --reference=/project /usr/lib/ckan", rw=rw, volumes_from=volumes_from, ro={srcdir: "/project"}, )
def admin_password(self, project, target_name, password): """ Return True if password was set successfully """ try: web_command( command=[ "ssh", _project_user_host(project), "admin_password", target_name, password, ], ro={KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa'}, clean_up=True, ) return True except WebCommandError: return False
def create(self, project, target_name, stream_output=None): """ Return True if project was created and now belongs to this user """ try: web_command( command=[ "ssh", _project_user_host(project), "create", target_name, ], ro={KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa'}, stream_output=stream_output, clean_up=True, ) return True except WebCommandError: return False
def purge_data(self): """ Remove uploaded files, postgres db, solr index, venv """ datadirs = ['files', 'solr'] if is_boot2docker(): remove_container('datacats_pgdata_' + self.name) remove_container('datacats_venv_' + self.name) else: datadirs += ['postgres', 'venv'] web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={PURGE: '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) shutil.rmtree(self.datadir)
def test_ssh_key(self, project): """ Return True if this key is accepted by DataCats.com """ try: web_command( command=["ssh", _project_user_host(project), 'test'], ro={ KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa'}, clean_up=True ) return True except WebCommandError: return False
def create_source(srcdir, preload_image, datapusher=False): """ Copy ckan source, datapusher source (optional), who.ini and schema.xml from preload image into srcdir """ try: docker.web_command( command='/bin/cp -a /project/ckan /project_target/ckan', rw={srcdir: '/project_target'}, image=preload_image) if datapusher: docker.web_command( command='/bin/cp -a /project/datapusher /project_target/datapusher', rw={srcdir: '/project_target'}, image=preload_image) shutil.copy( srcdir + '/ckan/ckan/config/who.ini', srcdir) shutil.copy( srcdir + '/ckan/ckan/config/solr/schema.xml', srcdir) finally: # fix srcdir permissions docker.web_command( command='/bin/chown -R --reference=/project /project', rw={srcdir: '/project'}, )
def deploy(self, project, target_name, stream_output=None): """ Return True if deployment was successful """ try: web_command( command=[ "rsync", "-lrv", "--safe-links", "--munge-links", "--delete", "--inplace", "--chmod=ugo=rwX", "--exclude=.datacats-environment", "--exclude=.git", "/project/.", _project_user_host(project) + ':' + target_name ], ro={ project.target: '/project', KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa' }, stream_output=stream_output, clean_up=True, ) web_command( command=[ "ssh", _project_user_host(project), "install", target_name, ], ro={ KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa' }, stream_output=stream_output, clean_up=True, ) return True except WebCommandError: return False
def create(self, project, target_name, stream_output=None): """ Return True if project was created and now belongs to this user """ try: web_command( command=[ "ssh", _project_user_host(project), "create", target_name, ], ro={ KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa' }, stream_output=stream_output, clean_up=True, ) return True except WebCommandError: return False
def create_virtualenv(self): """ Populate venv directory from preloaded image """ if is_boot2docker(): data_only_container('datacats_venv_' + self.name, ['/usr/lib/ckan']) img_id = web_command( '/bin/mv /usr/lib/ckan/ /usr/lib/ckan_original', image=self._preload_image(), commit=True, ) web_command( command='/bin/cp -a /usr/lib/ckan_original/. /usr/lib/ckan/.', volumes_from='datacats_venv_' + self.name, image=img_id, ) remove_image(img_id) else: web_command( command='/bin/cp -a /usr/lib/ckan/. /usr/lib/ckan_target/.', rw={self.datadir + '/venv': '/usr/lib/ckan_target'}, image=self._preload_image())
def admin_password(self, project, target_name, password): """ Return True if password was set successfully """ try: web_command( command=[ "ssh", _project_user_host(project), "admin_password", target_name, password, ], ro={ KNOWN_HOSTS: '/root/.ssh/known_hosts', SSH_CONFIG: '/etc/ssh/ssh_config', self.profiledir + '/id_rsa': '/root/.ssh/id_rsa' }, clean_up=True, ) return True except WebCommandError: return False
def create_virtualenv(srcdir, datadir, preload_image, get_container_name): """ Populate venv from preloaded image """ try: if docker.is_boot2docker(): docker.data_only_container( get_container_name('venv'), ['/usr/lib/ckan'], ) img_id = docker.web_command( '/bin/mv /usr/lib/ckan/ /usr/lib/ckan_original', image=preload_image, commit=True, ) docker.web_command( command='/bin/cp -a /usr/lib/ckan_original/. /usr/lib/ckan/.', volumes_from=get_container_name('venv'), image=img_id, ) docker.remove_image(img_id) return docker.web_command( command='/bin/cp -a /usr/lib/ckan/. /usr/lib/ckan_target/.', rw={datadir + '/venv': '/usr/lib/ckan_target'}, image=preload_image, ) finally: rw = { datadir + '/venv': '/usr/lib/ckan' } if not docker.is_boot2docker() else {} volumes_from = get_container_name( 'venv') if docker.is_boot2docker() else None # fix venv permissions docker.web_command( command='/bin/chown -R --reference=/project /usr/lib/ckan', rw=rw, volumes_from=volumes_from, ro={srcdir: '/project'}, )
def _one_to_two(datadir): """After this command, your environment will be converted to format version {}. and will only work with datacats version exceeding and including 1.0.0. This migration is necessary to support multiple sites within the same environment. Your current site will be kept and will be named "primary". Would you like to continue the migration? (y/n) [n]:""" new_site_name = 'primary' split = _split_path(datadir) print 'Making sure that containers are stopped...' env_name = split[1] # Old-style names on purpose! We need to stop old containers! remove_container('datacats_web_' + env_name) remove_container('datacats_solr_' + env_name) remove_container('datacats_postgres_' + env_name) print 'Doing conversion...' # Begin the actual conversion to_move = (['files', 'passwords.ini', 'run', 'solr'] + (['postgres'] if not is_boot2docker() else [])) # Make a primary site site_path = path_join(datadir, 'sites', new_site_name) if not exists(site_path): makedirs(site_path) web_command( command=['/scripts/migrate.sh', '/project/data', '/project/data/sites/' + new_site_name] + to_move, ro={scripts.get_script_path('migrate.sh'): '/scripts/migrate.sh'}, rw={datadir: '/project/data'}, clean_up=True ) if is_boot2docker(): rename_container('datacats_pgdata_' + env_name, 'datacats_pgdata_' + env_name + '_' + new_site_name) # Lastly, grab the project directory and update the ini file with open(path_join(datadir, 'project-dir')) as pd: project = pd.read() cp = SafeConfigParser() config_loc = path_join(project, '.datacats-environment') cp.read([config_loc]) new_section = 'site_' + new_site_name cp.add_section(new_section) # Ports need to be moved into the new section port = cp.get('datacats', 'port') cp.remove_option('datacats', 'port') cp.set(new_section, 'port', port) with open(config_loc, 'w') as config: cp.write(config) # Make a session secret for it (make it per-site) cp = SafeConfigParser() config_loc = path_join(site_path, 'passwords.ini') cp.read([config_loc]) # Generate a new secret cp.set('passwords', 'beaker_session_secret', generate_password()) with open(config_loc, 'w') as config: cp.write(config) with open(path_join(datadir, '.version'), 'w') as f: f.write('2')
def purge_data(self, which_sites=None, never_delete=False): """ Remove uploaded files, postgres db, solr index, venv """ # Default to the set of all sites if not exists(self.datadir + '/.version'): format_version = 1 else: with open(self.datadir + '/.version') as f: format_version = int(f.read().strip()) if format_version == 1: print 'WARNING: Defaulting to old purge for version 1.' datadirs = ['files', 'solr'] if is_boot2docker(): remove_container('datacats_pgdata_{}'.format(self.name)) remove_container('datacats_venv_{}'.format(self.name)) else: datadirs += ['postgres', 'venv'] web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) shutil.rmtree(self.datadir) elif format_version == 2: if not which_sites: which_sites = self.sites datadirs = [] boot2docker = is_boot2docker() if which_sites: if self.target: cp = SafeConfigParser() cp.read([self.target + '/.datacats-environment']) for site in which_sites: if boot2docker: remove_container(self._get_container_name('pgdata')) else: datadirs += [site + '/postgres'] # Always rm the site dir & solr & files datadirs += [site, site + '/files', site + '/solr'] if self.target: cp.remove_section('site_' + site) self.sites.remove(site) if self.target: with open(self.target + '/.datacats-environment', 'w') as conf: cp.write(conf) datadirs = ['sites/' + datadir for datadir in datadirs] if not self.sites and not never_delete: datadirs.append('venv') web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) if not self.sites and not never_delete: shutil.rmtree(self.datadir) else: raise DatacatsError( 'Unknown format version {}'.format(format_version))
def purge_data(self, which_sites=None, never_delete=False): """ Remove uploaded files, postgres db, solr index, venv """ # Default to the set of all sites if not exists(self.datadir + '/.version'): format_version = 1 else: with open(self.datadir + '/.version') as f: format_version = int(f.read().strip()) if format_version == 1: print 'WARNING: Defaulting to old purge for version 1.' datadirs = ['files', 'solr'] if is_boot2docker(): remove_container('datacats_pgdata_{}'.format(self.name)) remove_container('datacats_venv_{}'.format(self.name)) else: datadirs += ['postgres', 'venv'] web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) shutil.rmtree(self.datadir) elif format_version == 2: if not which_sites: which_sites = self.sites datadirs = [] boot2docker = is_boot2docker() if which_sites: if self.target: cp = SafeConfigParser() cp.read([self.target + '/.datacats-environment']) for site in which_sites: if boot2docker: remove_container(self._get_container_name('pgdata')) else: datadirs += [site + '/postgres'] # Always rm the site dir & solr & files datadirs += [site, site + '/files', site + '/solr'] if self.target: cp.remove_section('site_' + site) self.sites.remove(site) if self.target: with open(self.target + '/.datacats-environment', 'w') as conf: cp.write(conf) datadirs = ['sites/' + datadir for datadir in datadirs] if not self.sites and not never_delete: datadirs.append('venv') web_command( command=['/scripts/purge.sh'] + ['/project/data/' + d for d in datadirs], ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'}, rw={self.datadir: '/project/data'}, ) if not self.sites and not never_delete: shutil.rmtree(self.datadir) else: raise DatacatsError('Unknown format version {}'.format(format_version))