def start_postgres_and_solr(self):
"""
run the DB and search containers
"""
# complicated because postgres needs hard links to
# work on its data volume. see issue #5
if is_boot2docker():
data_only_container('datacats_pgdata_' + self.name,
['/var/lib/postgresql/data'])
rw = {}
volumes_from = 'datacats_pgdata_' + self.name
else:
rw = {self.datadir + '/postgres': '/var/lib/postgresql/data'}
volumes_from = None
# users are created when data dir is blank so we must pass
# all the user passwords as environment vars
run_container(
name='datacats_postgres_' + self.name,
image='datacats/postgres',
environment=self.passwords,
rw=rw,
volumes_from=volumes_from)
run_container(
name='datacats_solr_' + self.name,
image='datacats/solr',
rw={self.datadir + '/solr': '/var/lib/solr'},
ro={self.target + '/schema.xml': '/etc/solr/conf/schema.xml'})
def _run_web_container(self, port, command):
"""
Start web comtainer on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = 'datacats_venv_' + self.name
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
run_container(
name='datacats_web_' + self.name,
image='datacats/web',
rw={self.datadir + '/files': '/var/www/storage'},
ro=dict({
self.target: '/project/',
self.datadir + '/run/development.ini':
'/project/development.ini',
WEB: '/scripts/web.sh'}, **ro),
links={'datacats_solr_' + self.name: 'solr',
'datacats_postgres_' + self.name: 'db'},
volumes_from=volumes_from,
command=command,
port_bindings={
5000: port if is_boot2docker() else ('127.0.0.1', port)},
)
def _run_web_container(self, port, command):
"""
Start web comtainer on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = 'datacats_venv_' + self.name
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
run_container(
name='datacats_web_' + self.name,
image='datacats/web',
rw={self.datadir + '/files': '/var/www/storage'},
ro=dict(
{
self.target: '/project/',
self.datadir + '/run/development.ini':
'/project/development.ini',
WEB: '/scripts/web.sh'
}, **ro),
links={
'datacats_solr_' + self.name: 'solr',
'datacats_postgres_' + self.name: 'db'
},
volumes_from=volumes_from,
command=command,
port_bindings={
5000: port if is_boot2docker() else ('127.0.0.1', port)
},
)
def start_postgres_and_solr(self):
"""
run the DB and search containers
"""
# complicated because postgres needs hard links to
# work on its data volume. see issue #5
if is_boot2docker():
data_only_container('datacats_pgdata_' + self.name,
['/var/lib/postgresql/data'])
rw = {}
volumes_from = 'datacats_pgdata_' + self.name
else:
rw = {self.datadir + '/postgres': '/var/lib/postgresql/data'}
volumes_from = None
# users are created when data dir is blank so we must pass
# all the user passwords as environment vars
run_container(name='datacats_postgres_' + self.name,
image='datacats/postgres',
environment=self.passwords,
rw=rw,
volumes_from=volumes_from)
run_container(
name='datacats_solr_' + self.name,
image='datacats/solr',
rw={self.datadir + '/solr': '/var/lib/solr'},
ro={self.target + '/schema.xml': '/etc/solr/conf/schema.xml'})
def start_ckan(self, production=False, log_syslog=False, paster_reload=True,
interactive=False):
"""
Start the apache server or paster serve
:param log_syslog: A flag to redirect all container logs to host's syslog
:param production: True for apache, False for paster serve + debug on
:param paster_reload: Instruct paster to watch for file changes
"""
self.stop_ckan()
address = self.address or '127.0.0.1'
port = self.port
# in prod we always use log_syslog driver
log_syslog = True if self.always_prod else log_syslog
production = production or self.always_prod
# We only override the site URL with the docker URL on three conditions
override_site_url = (self.address is None
and not is_boot2docker()
and not self.site_url)
command = ['/scripts/web.sh', str(production), str(override_site_url), str(paster_reload)]
# XXX nasty hack, remove this once we have a lessc command
# for users (not just for building our preload image)
if not production:
css = self.target + '/ckan/ckan/public/base/css'
if not exists(css + '/main.debug.css'):
from shutil import copyfile
copyfile(css + '/main.css', css + '/main.debug.css')
ro = {
self.target: '/project',
scripts.get_script_path('datapusher.sh'): '/scripts/datapusher.sh'
}
if not is_boot2docker():
ro[self.datadir + '/venv'] = '/usr/lib/ckan'
datapusher = self.needs_datapusher()
if datapusher:
run_container(
self._get_container_name('datapusher'),
'datacats/web',
'/scripts/datapusher.sh',
ro=ro,
volumes_from=(self._get_container_name('venv') if is_boot2docker() else None),
log_syslog=log_syslog)
while True:
self._create_run_ini(port, production)
try:
self._run_web_container(port, command, address, log_syslog=log_syslog,
datapusher=datapusher, interactive=interactive)
if not is_boot2docker():
self.address = address
except PortAllocatedError:
port = self._next_port(port)
continue
break
def start_ckan(self, production=False, log_syslog=False, paster_reload=True,
interactive=False):
"""
Start the apache server or paster serve
:param log_syslog: A flag to redirect all container logs to host's syslog
:param production: True for apache, False for paster serve + debug on
:param paster_reload: Instruct paster to watch for file changes
"""
self.stop_ckan()
address = self.address or '127.0.0.1'
port = self.port
# in prod we always use log_syslog driver
log_syslog = True if self.always_prod else log_syslog
production = production or self.always_prod
# We only override the site URL with the docker URL on three conditions
override_site_url = (self.address is None
and not is_boot2docker()
and not self.site_url)
command = ['/scripts/web.sh', str(production), str(override_site_url), str(paster_reload)]
# XXX nasty hack, remove this once we have a lessc command
# for users (not just for building our preload image)
if not production:
css = self.target + '/ckan/ckan/public/base/css'
if not exists(css + '/main.debug.css'):
from shutil import copyfile
copyfile(css + '/main.css', css + '/main.debug.css')
ro = {
self.target: '/project',
scripts.get_script_path('datapusher.sh'): '/scripts/datapusher.sh'
}
if not is_boot2docker():
ro[self.datadir + '/venv'] = '/usr/lib/ckan'
datapusher = self.needs_datapusher()
if datapusher:
run_container(
self._get_container_name('datapusher'),
'datacats/web',
'/scripts/datapusher.sh',
ro=ro,
volumes_from=(self._get_container_name('venv') if is_boot2docker() else None),
log_syslog=log_syslog)
while True:
self._create_run_ini(port, production)
try:
self._run_web_container(port, command, address, log_syslog=log_syslog,
datapusher=datapusher, interactive=interactive)
if not is_boot2docker():
self.address = address
except PortAllocatedError:
port = self._next_port(port)
continue
break
def _run_web_container(self, port, command, address='127.0.0.1', log_syslog=False,
datapusher=True):
"""
Start web container on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = self._get_container_name('venv')
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
links = {
self._get_container_name('solr'): 'solr',
self._get_container_name('postgres'): 'db'
}
links.update({self._get_container_name(container): container
for container in self.extra_containers})
if datapusher:
if 'datapusher' not in self.containers_running():
raise DatacatsError(container_logs(self._get_container_name('datapusher'), "all",
False, False))
links[self._get_container_name('datapusher')] = 'datapusher'
try:
run_container(
name=self._get_container_name('web'),
image='datacats/web',
rw={self.sitedir + '/files': '/var/www/storage',
self.sitedir + '/run/development.ini':
'/project/development.ini'},
ro=dict({
self.target: '/project/',
scripts.get_script_path('web.sh'): '/scripts/web.sh',
scripts.get_script_path('adjust_devini.py'): '/scripts/adjust_devini.py'},
**ro),
links=links,
volumes_from=volumes_from,
command=command,
port_bindings={
5000: port if is_boot2docker() else (address, port)},
log_syslog=log_syslog
)
except APIError as e:
if '409' in str(e):
raise DatacatsError('Web container already running. '
'Please stop_web before running.')
else:
raise
def compile_less(self):
c = run_container(
name=self._get_container_name('lessc'), image='datacats/lessc',
rw={self.target: '/project/target'},
ro={scripts.get_script_path('compile_less.sh'): '/project/compile_less.sh'})
for log in container_logs(c['Id'], "all", True, False):
yield log
remove_container(c)
def compile_less(self):
c = run_container(
name=self._get_container_name('lessc'), image='datacats/lessc',
rw={self.target: '/project/target'},
ro={scripts.get_script_path('compile_less.sh'): '/project/compile_less.sh'})
for log in container_logs(c['Id'], "all", True, False):
yield log
remove_container(c)
def start_supporting_containers(sitedir,
srcdir,
passwords,
get_container_name,
extra_containers,
log_syslog=False):
"""
Start all supporting containers (containers required for CKAN to
operate) if they aren't already running, along with some extra
containers specified by the user
"""
if docker.is_boot2docker():
docker.data_only_container(get_container_name('pgdata'),
['/var/lib/postgresql/data'])
rw = {}
volumes_from = get_container_name('pgdata')
else:
rw = {sitedir + '/postgres': '/var/lib/postgresql/data'}
volumes_from = None
running = set(containers_running(get_container_name))
needed = set(extra_containers).union({'postgres', 'solr'})
if not needed.issubset(running):
stop_supporting_containers(get_container_name, extra_containers)
# users are created when data dir is blank so we must pass
# all the user passwords as environment vars
# XXX: postgres entrypoint magic
docker.run_container(name=get_container_name('postgres'),
image='datacats/postgres',
environment=passwords,
rw=rw,
volumes_from=volumes_from,
log_syslog=log_syslog)
docker.run_container(
name=get_container_name('solr'),
image='datacats/solr',
rw={sitedir + '/solr': '/var/lib/solr'},
ro={srcdir + '/schema.xml': '/etc/solr/conf/schema.xml'},
log_syslog=log_syslog)
for container in extra_containers:
# We don't know a whole lot about the extra containers so we're just gonna have to
# mount /project and /datadir r/o even if they're not needed for ease of
# implementation.
docker.run_container(name=get_container_name(container),
image=EXTRA_IMAGE_MAPPING[container],
ro={
sitedir: '/datadir',
srcdir: '/project'
},
log_syslog=log_syslog)
def start_supporting_containers(sitedir, srcdir, passwords,
get_container_name, extra_containers, log_syslog=False):
"""
Start all supporting containers (containers required for CKAN to
operate) if they aren't already running, along with some extra
containers specified by the user
"""
if docker.is_boot2docker():
docker.data_only_container(get_container_name('pgdata'),
['/var/lib/postgresql/data'])
rw = {}
volumes_from = get_container_name('pgdata')
else:
rw = {sitedir + '/postgres': '/var/lib/postgresql/data'}
volumes_from = None
running = set(containers_running(get_container_name))
needed = set(extra_containers).union({'postgres', 'solr'})
if not needed.issubset(running):
stop_supporting_containers(get_container_name, extra_containers)
# users are created when data dir is blank so we must pass
# all the user passwords as environment vars
# XXX: postgres entrypoint magic
docker.run_container(
name=get_container_name('postgres'),
image='datacats/postgres',
environment=passwords,
rw=rw,
volumes_from=volumes_from,
log_syslog=log_syslog)
docker.run_container(
name=get_container_name('solr'),
image='datacats/solr',
rw={sitedir + '/solr': '/var/lib/solr'},
ro={srcdir + '/schema.xml': '/etc/solr/conf/schema.xml'},
log_syslog=log_syslog)
for container in extra_containers:
# We don't know a whole lot about the extra containers so we're just gonna have to
# mount /project and /datadir r/o even if they're not needed for ease of
# implementation.
docker.run_container(
name=get_container_name(container),
image=EXTRA_IMAGE_MAPPING[container],
ro={
sitedir: '/datadir',
srcdir: '/project'
},
log_syslog=log_syslog
)
def _run_web_container(self,
port,
command,
address,
log_syslog=False,
datapusher=True,
interactive=False):
"""
Start web container on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = self._get_container_name('venv')
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
links = {
self._get_container_name('solr'): 'solr',
self._get_container_name('postgres'): 'db'
}
links.update({
self._get_container_name(container): container
for container in self.extra_containers
})
if datapusher:
if 'datapusher' not in self.containers_running():
raise DatacatsError(
container_logs(self._get_container_name('datapusher'),
"all", False, False))
links[self._get_container_name('datapusher')] = 'datapusher'
ro = dict(
{
self.target:
'/project/',
scripts.get_script_path('web.sh'):
'/scripts/web.sh',
scripts.get_script_path('adjust_devini.py'):
'/scripts/adjust_devini.py'
}, **ro)
rw = {
self.sitedir + '/files': '/var/www/storage',
self.sitedir + '/run/development.ini': '/project/development.ini'
}
try:
if not interactive:
run_container(name=self._get_container_name('web'),
image='datacats/web',
rw=rw,
ro=ro,
links=links,
volumes_from=volumes_from,
command=command,
port_bindings={
5000:
port if is_boot2docker() else (address, port)
},
log_syslog=log_syslog)
else:
# FIXME: share more code with interactive_shell
if is_boot2docker():
switches = [
'--volumes-from',
self._get_container_name('pgdata'), '--volumes-from',
self._get_container_name('venv')
]
else:
switches = []
switches += [
'--volume={}:{}:ro'.format(vol, ro[vol]) for vol in ro
]
switches += [
'--volume={}:{}'.format(vol, rw[vol]) for vol in rw
]
links = [
'--link={}:{}'.format(link, links[link]) for link in links
]
args = ['docker', 'run', '-it', '--name', self._get_container_name('web'),
'-p', '{}:5000'.format(port) if is_boot2docker()
else '{}:{}:5000'.format(address, port)] + \
switches + links + ['datacats/web', ] + command
subprocess.call(args)
except APIError as e:
if '409' in str(e):
raise DatacatsError('Web container already running. '
'Please stop_web before running.')
else:
raise
def _run_web_container(self,
port,
command,
address='127.0.0.1',
log_syslog=False,
datapusher=True):
"""
Start web container on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = self._get_container_name('venv')
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
links = {
self._get_container_name('solr'): 'solr',
self._get_container_name('postgres'): 'db'
}
links.update({
self._get_container_name(container): container
for container in self.extra_containers
})
if datapusher:
if 'datapusher' not in self.containers_running():
raise DatacatsError(
container_logs(self._get_container_name('datapusher'),
"all", False, False))
links[self._get_container_name('datapusher')] = 'datapusher'
try:
run_container(name=self._get_container_name('web'),
image='datacats/web',
rw={
self.sitedir + '/files':
'/var/www/storage',
self.sitedir + '/run/development.ini':
'/project/development.ini'
},
ro=dict(
{
self.target:
'/project/',
scripts.get_script_path('web.sh'):
'/scripts/web.sh',
scripts.get_script_path('adjust_devini.py'):
'/scripts/adjust_devini.py'
}, **ro),
links=links,
volumes_from=volumes_from,
command=command,
port_bindings={
5000: port if is_boot2docker() else
(address, port)
},
log_syslog=log_syslog)
except APIError as e:
if '409' in str(e):
raise DatacatsError('Web container already running. '
'Please stop_web before running.')
else:
raise
def _run_web_container(self, port, command, address, log_syslog=False,
datapusher=True, interactive=False):
"""
Start web container on port with command
"""
if is_boot2docker():
ro = {}
volumes_from = self._get_container_name('venv')
else:
ro = {self.datadir + '/venv': '/usr/lib/ckan'}
volumes_from = None
links = {
self._get_container_name('solr'): 'solr',
self._get_container_name('postgres'): 'db'
}
links.update({self._get_container_name(container): container
for container in self.extra_containers})
if datapusher:
if 'datapusher' not in self.containers_running():
raise DatacatsError(container_logs(self._get_container_name('datapusher'), "all",
False, False))
links[self._get_container_name('datapusher')] = 'datapusher'
ro = dict({
self.target: '/project/',
scripts.get_script_path('web.sh'): '/scripts/web.sh',
scripts.get_script_path('adjust_devini.py'): '/scripts/adjust_devini.py'},
**ro)
rw = {
self.sitedir + '/files': '/var/www/storage',
self.sitedir + '/run/development.ini': '/project/development.ini'
}
try:
if not interactive:
run_container(
name=self._get_container_name('web'),
image='datacats/web',
rw=rw,
ro=ro,
links=links,
volumes_from=volumes_from,
command=command,
port_bindings={
5000: port if is_boot2docker() else (address, port)},
log_syslog=log_syslog
)
else:
# FIXME: share more code with interactive_shell
if is_boot2docker():
switches = ['--volumes-from',
self._get_container_name('pgdata'), '--volumes-from',
self._get_container_name('venv')]
else:
switches = []
switches += ['--volume={}:{}:ro'.format(vol, ro[vol]) for vol in ro]
switches += ['--volume={}:{}'.format(vol, rw[vol]) for vol in rw]
links = ['--link={}:{}'.format(link, links[link]) for link in links]
args = ['docker', 'run', '-it', '--name', self._get_container_name('web'),
'-p', '{}:5000'.format(port) if is_boot2docker()
else '{}:{}:5000'.format(address, port)] + \
switches + links + ['datacats/web', ] + command
subprocess.call(args)
except APIError as e:
if '409' in str(e):
raise DatacatsError('Web container already running. '
'Please stop_web before running.')
else:
raise