def compile_less(self):
c = run_container(
name=self._get_container_name('lessc'), image='datacats/lessc',
rw={self.target: '/project/target'},
ro={scripts.get_script_path('compile_less.sh'): '/project/compile_less.sh'})
for log in container_logs(c['Id'], "all", True, False):
yield log
remove_container(c)
def compile_less(self):
c = run_container(
name=self._get_container_name('lessc'), image='datacats/lessc',
rw={self.target: '/project/target'},
ro={scripts.get_script_path('compile_less.sh'): '/project/compile_less.sh'})
for log in container_logs(c['Id'], "all", True, False):
yield log
remove_container(c)
def stop_supporting_containers(get_container_name, extra_containers):
"""
Stop postgres and solr containers, along with any specified extra containers
"""
docker.remove_container(get_container_name('postgres'))
docker.remove_container(get_container_name('solr'))
docker.remove_container(get_container_name('redis'))
for container in extra_containers:
docker.remove_container(get_container_name(container))
def purge_data(self, which_sites=None, never_delete=False):
"""
Remove uploaded files, postgres db, solr index, venv
"""
# Default to the set of all sites
if not which_sites:
which_sites = self.sites
datadirs = []
boot2docker = is_boot2docker()
if which_sites:
if self.target:
cp = SafeConfigParser()
cp.read([self.target + '/.datacats-environment'])
for site in which_sites:
if boot2docker:
remove_container(self._get_container_name('pgdata'))
else:
datadirs += [site + '/postgres']
# Always rm the site dir & solr & files
datadirs += [site, site + '/files', site + '/solr']
if self.target:
cp.remove_section('site_' + site)
self.sites.remove(site)
if self.target:
with open(self.target + '/.datacats-environment', 'w') as conf:
cp.write(conf)
datadirs = ['sites/' + datadir for datadir in datadirs]
if not self.sites and not never_delete:
datadirs.append('venv')
web_command(
command=['/scripts/purge.sh']
+ ['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
if not self.sites and not never_delete:
shutil.rmtree(self.datadir)
def purge_data(self):
"""
Remove uploaded files, postgres db, solr index, venv
"""
datadirs = ['files', 'solr']
if is_boot2docker():
remove_container('datacats_pgdata_' + self.name)
remove_container('datacats_venv_' + self.name)
else:
datadirs += ['postgres', 'venv']
web_command(
command=['/scripts/purge.sh']
+ ['/project/data/' + d for d in datadirs],
ro={PURGE: '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
shutil.rmtree(self.datadir)
def purge_data(self):
"""
Remove uploaded files, postgres db, solr index, venv
"""
datadirs = ['files', 'solr']
if is_boot2docker():
remove_container('datacats_pgdata_' + self.name)
remove_container('datacats_venv_' + self.name)
else:
datadirs += ['postgres', 'venv']
web_command(
command=['/scripts/purge.sh'] +
['/project/data/' + d for d in datadirs],
ro={PURGE: '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
shutil.rmtree(self.datadir)
def stop_supporting_containers(get_container_name, extra_containers):
"""
Stop postgres and solr containers, along with any specified extra containers
"""
docker.remove_container(get_container_name('postgres'))
docker.remove_container(get_container_name('solr'))
for container in extra_containers:
docker.remove_container(get_container_name(container))
def _two_to_one(datadir):
"""After this command, your environment will be converted to format version {}
and will not work with Datacats versions beyond and including 1.0.0.
This format version doesn't support multiple sites, and after this only your
"primary" site will be usable, though other sites will be maintained if you
wish to do a migration back to a version which supports multisite.
Would you like to continue the migration? (y/n) [n]:"""
_, env_name = _split_path(datadir)
print 'Making sure that containers are stopped...'
# New-style names
remove_container('datacats_web_{}_primary'.format(env_name))
remove_container('datacats_postgres_{}_primary'.format(env_name))
remove_container('datacats_solr_{}_primary'.format(env_name))
print 'Doing conversion...'
if exists(path_join(datadir, '.version')):
os.remove(path_join(datadir, '.version'))
to_move = (['files', 'passwords.ini', 'run', 'solr'] +
(['postgres'] if not is_boot2docker() else []))
web_command(
command=['/scripts/migrate.sh',
'/project/data/sites/primary',
'/project/data'] + to_move,
ro={scripts.get_script_path('migrate.sh'): '/scripts/migrate.sh'},
rw={datadir: '/project/data'}
)
pgdata_name = 'datacats_pgdata_{}_primary'.format(env_name)
if is_boot2docker() and inspect_container(pgdata_name):
rename_container(pgdata_name, 'datacats_pgdata_{}'.format(env_name))
print 'Doing cleanup...'
with open(path_join(datadir, 'project-dir')) as pd:
datacats_env_location = path_join(pd.read(), '.datacats-environment')
cp = SafeConfigParser()
cp.read(datacats_env_location)
# We need to move the port OUT of site_primary section and INTO datacats
cp.set('datacats', 'port', cp.get('site_primary', 'port'))
cp.remove_section('site_primary')
with open(datacats_env_location, 'w') as config:
cp.write(config)
cp = SafeConfigParser()
cp.read(path_join(datadir, 'passwords.ini'))
# This isn't needed in this version
cp.remove_option('passwords', 'beaker_session_secret')
with open(path_join(datadir, 'passwords.ini'), 'w') as config:
cp.write(config)
def stop_ckan(self):
"""
Stop and remove the web container
"""
remove_container(self._get_container_name('web'), force=True)
remove_container(self._get_container_name('datapusher'), force=True)
def stop_web(self):
"""
Stop and remove the web container
"""
remove_container('datacats_web_' + self.name, force=True)
def purge_data(self, which_sites=None, never_delete=False):
"""
Remove uploaded files, postgres db, solr index, venv
"""
# Default to the set of all sites
if not exists(self.datadir + '/.version'):
format_version = 1
else:
with open(self.datadir + '/.version') as f:
format_version = int(f.read().strip())
if format_version == 1:
print 'WARNING: Defaulting to old purge for version 1.'
datadirs = ['files', 'solr']
if is_boot2docker():
remove_container('datacats_pgdata_{}'.format(self.name))
remove_container('datacats_venv_{}'.format(self.name))
else:
datadirs += ['postgres', 'venv']
web_command(
command=['/scripts/purge.sh']
+ ['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
shutil.rmtree(self.datadir)
elif format_version == 2:
if not which_sites:
which_sites = self.sites
datadirs = []
boot2docker = is_boot2docker()
if which_sites:
if self.target:
cp = SafeConfigParser()
cp.read([self.target + '/.datacats-environment'])
for site in which_sites:
if boot2docker:
remove_container(self._get_container_name('pgdata'))
else:
datadirs += [site + '/postgres']
# Always rm the site dir & solr & files
datadirs += [site, site + '/files', site + '/solr']
if self.target:
cp.remove_section('site_' + site)
self.sites.remove(site)
if self.target:
with open(self.target + '/.datacats-environment', 'w') as conf:
cp.write(conf)
datadirs = ['sites/' + datadir for datadir in datadirs]
if not self.sites and not never_delete:
datadirs.append('venv')
web_command(
command=['/scripts/purge.sh']
+ ['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
if not self.sites and not never_delete:
shutil.rmtree(self.datadir)
else:
raise DatacatsError('Unknown format version {}'.format(format_version))
def _one_to_two(datadir):
"""After this command, your environment will be converted to format version {}.
and will only work with datacats version exceeding and including 1.0.0.
This migration is necessary to support multiple sites within the same environment.
Your current site will be kept and will be named "primary".
Would you like to continue the migration? (y/n) [n]:"""
new_site_name = 'primary'
split = _split_path(datadir)
print 'Making sure that containers are stopped...'
env_name = split[1]
# Old-style names on purpose! We need to stop old containers!
remove_container('datacats_web_' + env_name)
remove_container('datacats_solr_' + env_name)
remove_container('datacats_postgres_' + env_name)
print 'Doing conversion...'
# Begin the actual conversion
to_move = (['files', 'passwords.ini', 'run', 'solr'] +
(['postgres'] if not is_boot2docker() else []))
# Make a primary site
site_path = path_join(datadir, 'sites', new_site_name)
if not exists(site_path):
makedirs(site_path)
web_command(
command=['/scripts/migrate.sh',
'/project/data',
'/project/data/sites/' + new_site_name] +
to_move,
ro={scripts.get_script_path('migrate.sh'): '/scripts/migrate.sh'},
rw={datadir: '/project/data'},
clean_up=True
)
if is_boot2docker():
rename_container('datacats_pgdata_' + env_name,
'datacats_pgdata_' + env_name + '_' + new_site_name)
# Lastly, grab the project directory and update the ini file
with open(path_join(datadir, 'project-dir')) as pd:
project = pd.read()
cp = SafeConfigParser()
config_loc = path_join(project, '.datacats-environment')
cp.read([config_loc])
new_section = 'site_' + new_site_name
cp.add_section(new_section)
# Ports need to be moved into the new section
port = cp.get('datacats', 'port')
cp.remove_option('datacats', 'port')
cp.set(new_section, 'port', port)
with open(config_loc, 'w') as config:
cp.write(config)
# Make a session secret for it (make it per-site)
cp = SafeConfigParser()
config_loc = path_join(site_path, 'passwords.ini')
cp.read([config_loc])
# Generate a new secret
cp.set('passwords', 'beaker_session_secret', generate_password())
with open(config_loc, 'w') as config:
cp.write(config)
with open(path_join(datadir, '.version'), 'w') as f:
f.write('2')
def stop_web(self):
"""
Stop and remove the web container
"""
remove_container('datacats_web_' + self.name, force=True)
def stop_postgres_and_solr(self):
"""
stop and remove postgres and solr containers
"""
remove_container('datacats_postgres_' + self.name)
remove_container('datacats_solr_' + self.name)
def stop_ckan(self):
"""
Stop and remove the web container
"""
remove_container(self._get_container_name('web'), force=True)
remove_container(self._get_container_name('datapusher'), force=True)
def purge_data(self, which_sites=None, never_delete=False):
"""
Remove uploaded files, postgres db, solr index, venv
"""
# Default to the set of all sites
if not exists(self.datadir + '/.version'):
format_version = 1
else:
with open(self.datadir + '/.version') as f:
format_version = int(f.read().strip())
if format_version == 1:
print 'WARNING: Defaulting to old purge for version 1.'
datadirs = ['files', 'solr']
if is_boot2docker():
remove_container('datacats_pgdata_{}'.format(self.name))
remove_container('datacats_venv_{}'.format(self.name))
else:
datadirs += ['postgres', 'venv']
web_command(
command=['/scripts/purge.sh'] +
['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
shutil.rmtree(self.datadir)
elif format_version == 2:
if not which_sites:
which_sites = self.sites
datadirs = []
boot2docker = is_boot2docker()
if which_sites:
if self.target:
cp = SafeConfigParser()
cp.read([self.target + '/.datacats-environment'])
for site in which_sites:
if boot2docker:
remove_container(self._get_container_name('pgdata'))
else:
datadirs += [site + '/postgres']
# Always rm the site dir & solr & files
datadirs += [site, site + '/files', site + '/solr']
if self.target:
cp.remove_section('site_' + site)
self.sites.remove(site)
if self.target:
with open(self.target + '/.datacats-environment',
'w') as conf:
cp.write(conf)
datadirs = ['sites/' + datadir for datadir in datadirs]
if not self.sites and not never_delete:
datadirs.append('venv')
web_command(
command=['/scripts/purge.sh'] +
['/project/data/' + d for d in datadirs],
ro={scripts.get_script_path('purge.sh'): '/scripts/purge.sh'},
rw={self.datadir: '/project/data'},
)
if not self.sites and not never_delete:
shutil.rmtree(self.datadir)
else:
raise DatacatsError(
'Unknown format version {}'.format(format_version))
def stop_postgres_and_solr(self):
"""
stop and remove postgres and solr containers
"""
remove_container('datacats_postgres_' + self.name)
remove_container('datacats_solr_' + self.name)
