class Docker:
dockerconf = KOOPLEX.get('docker', {})
def __init__(self):
base_url = self.dockerconf.get('base_url', '')
self.client = Client(base_url=base_url)
logger.debug("Client init")
self.check = None
def list_imagenames(self):
logger.debug("Listing image names")
pattern_imagenamefilter = KOOPLEX.get('docker', {}).get(
'pattern_imagename_filter', r'^image-%(\w+):\w$')
for image in self.client.images(all=True):
if image['RepoTags'] is None:
continue
for tag in image['RepoTags']:
if re.match(pattern_imagenamefilter, tag):
_, imagename, _ = re.split(pattern_imagenamefilter, tag)
logger.debug("Found image: %s" % imagename)
yield imagename
def list_volumenames(self):
logger.debug("Listing volume names")
volumes = self.client.volumes()
for volume in volumes['Volumes']:
yield volume['Name']
def create_volume(self, volume):
volume_dir = None #self.dockerconf.get('volume_dir', '')
if volume_dir:
self.client.create_volume(name=volume.name,
driver='local',
driver_opts={
'device':
'%s/%s/' % (volume_dir, volume.name),
'o':
'bind',
'type':
'none'
})
else:
self.client.create_volume(name=volume.name, )
logger.debug("Volume %s created" % volume.name)
return True #self.get_container(container)
def delete_volume(self, volume):
self.client.remove_volume(name=volume.name)
logger.debug("Volume %s deleted" % volume.name)
def get_container(self, container):
for item in self.client.containers(all=True):
# docker API prepends '/' in front of container names
if '/' + container.name in item['Names']:
logger.debug("Get container %s" % container.name)
return item
return None
def create_container(self, container):
volumes = [] # the list of mount points in the container
binds = {} # a mapping dictionary of the container mounts
for volume in container.volumes:
logger.debug("container %s, volume %s" % (container, volume))
mp = volume.mountpoint
volumes.append(mp)
binds[volume.name] = {
'bind': mp,
'mode': volume.mode(container.user)
}
logger.debug("container %s binds %s" % (container, binds))
host_config = self.client.create_host_config(
binds=binds,
privileged=True,
mem_limit='2g',
memswap_limit='170m',
mem_swappiness=0,
# oom_kill_disable = True,
cpu_shares=2,
)
network = self.dockerconf.get('network', 'host')
networking_config = {'EndpointsConfig': {network: {}}}
ports = self.dockerconf.get('container_ports', [8000, 9000])
imagename = container.image.imagename if container.image else self.dockerconf.get(
'default_image', 'basic')
args = {
'name': container.name,
'image': imagename,
'detach': True,
'hostname': container.name,
'host_config': host_config,
'networking_config': networking_config,
'environment': container.environment,
'volumes': volumes,
'ports': ports,
}
self.client.create_container(**args)
logger.debug("Container created")
self.managemount(container) #FIXME: check if not called twice
return self.get_container(container)
def _writefile(self, container_name, path, filename, content):
import tarfile
import time
from io import BytesIO
tarstream = BytesIO()
tar = tarfile.TarFile(fileobj=tarstream, mode='w')
tarinfo = tarfile.TarInfo(name=filename)
tarinfo.size = len(content)
tarinfo.mtime = time.time()
tar.addfile(tarinfo, BytesIO(content))
tar.close()
tarstream.seek(0)
try:
status = self.client.put_archive(container=container_name,
path=path,
data=tarstream)
logger.info("container %s put_archive %s/%s returns %s" %
(container_name, path, filename, status))
except Exception as e:
logger.error("container %s put_archive %s/%s fails -- %s" %
(container_name, path, filename, e))
def managemount(self, container):
from kooplex.lib.fs_dirname import Dirname
path, filename = os.path.split(
self.dockerconf.get('mountconf', '/tmp/mount.conf'))
mapper = []
for v in container.volumes:
mapper.extend([
"%s:%s" % (v.volumetype, d)
for d in Dirname.containervolume_listfolders(container, v)
])
#NOTE: mounter uses read to process the mapper configuration, thus we need to make sure '\n' terminates the config mapper file
mapper.append('')
logger.debug("container %s map %s" % (container, mapper))
file_data = "\n".join(mapper).encode('utf8')
self._writefile(container.name, path, filename, file_data)
def trigger_impersonator(self, vcproject): #FIXME: dont call it 1-by-1
from kooplex.lib.fs_dirname import Dirname
container_name = self.dockerconf.get('impersonator', 'impersonator')
path, filename = os.path.split(
self.dockerconf.get('gitcommandconf', '/tmp/gitcommand.conf'))
cmdmaps = []
token = vcproject.token
fn_clonesh = os.path.join(Dirname.vcpcache(vcproject), "clone.sh")
fn_key = os.path.join(Dirname.userhome(vcproject.token.user), '.ssh',
token.fn_rsa)
cmdmaps.append(
"%s:%s:%s:%s" %
(token.user.username, fn_key, token.repository.domain, fn_clonesh))
cmdmaps.append('')
file_data = "\n".join(cmdmaps).encode('utf8')
self._writefile(container_name, path, filename, file_data)
def run_container(self, container):
docker_container_info = self.get_container(container)
if docker_container_info is None:
logger.debug("Container did not exist, Creating new one")
docker_container_info = self.create_container(container)
container_state = docker_container_info['Status']
if container_state == 'Created' or container_state.startswith(
'Exited'):
logger.debug("Starting container")
self.start_container(container)
def refresh_container_state(self, container):
docker_container_info = self.get_container(container)
container_state = docker_container_info['State']
logger.debug("Container state %s" % container_state)
container.last_message = str(container_state)
container.last_message_at = now()
container.save()
def start_container(self, container):
self.client.start(container.name)
# we need to retrieve the container state after starting it
docker_container_info = self.get_container(container)
container_state = docker_container_info['State']
logger.debug("Container state %s" % container_state)
container.last_message = str(container_state)
container.last_message_at = now()
assert container_state == 'running', "Container failed to start: %s" % docker_container_info
def stop_container(self, container):
try:
self.client.stop(container.name)
container.last_message = 'Container stopped'
except Exception as e:
logger.warn("docker container not found by API -- %s" % e)
container.last_message = str(e)
def remove_container(self, container):
try:
self.client.remove_container(container.name)
container.last_message = 'Container removed'
container.last_message_at = now()
except Exception as e:
logger.warn("docker container not found by API -- %s" % e)
container.last_message = str(e)
container.last_message_at = now()
logger.debug("Container removed %s" % container.name)
#FIXME: az execute2 lesz az igazi...
def execute(self, container, command):
logger.info("execution: %s in %s" % (command, container))
execution = self.client.exec_create(container=container.name,
cmd=shlex.split(command))
return self.client.exec_start(execution, detach=False)
def execute2(self, container, command):
logger.info("execution: %s in %s" % (command, container))
execution = self.client.exec_create(container=container.name,
cmd=shlex.split(command))
response = self.client.exec_start(exec_id=execution['Id'],
stream=False)
check = self.client.exec_inspect(exec_id=execution['Id'])
self.check = check
if check['ExitCode'] != 0:
logger.error('Execution %s in %s failed -- %s' %
(command, container, check))
return response.decode()
class Docker:
dockerconf = KOOPLEX.get('docker', {})
def __init__(self):
base_url = self.dockerconf.get('base_url', '')
self.client = Client(base_url = base_url)
logger.debug("Client init")
self.check = None
def list_imagenames(self):
logger.debug("Listing image names")
pattern_imagenamefilter = KOOPLEX.get('docker', {}).get('pattern_imagename_filter', r'^image-%(\w+):\w$')
for image in self.client.images(all = True):
if image['RepoTags'] is None:
continue
for tag in image['RepoTags']:
if re.match(pattern_imagenamefilter, tag):
_, imagename, _ = re.split(pattern_imagenamefilter, tag)
logger.debug("Found image: %s" % imagename)
yield imagename
def list_volumenames(self):
logger.debug("Listing volume names")
volumes = self.client.volumes()
for volume in volumes['Volumes']:
yield volume['Name']
def get_container(self, container):
for item in self.client.containers(all = True):
# docker API prepends '/' in front of container names
if '/' + container.name in item['Names']:
logger.debug("Get container %s" % container.name)
return item
return None
def create_container(self, container):
volumes = [] # the list of mount points in the container
binds = {} # a mapping dictionary of the container mounts
for volume in container.volumes:
logger.debug("container %s, volume %s" % (container, volume))
mp = volume.mountpoint
volumes.append(mp)
binds[volume.name] = { 'bind': mp, 'mode': volume.mode(container.user) }
logger.debug("container %s binds %s" % (container, binds))
host_config = self.client.create_host_config(
binds = binds,
privileged = True,
mem_limit = '2g',
memswap_limit = '170m',
mem_swappiness = 0,
# oom_kill_disable = True,
cpu_shares = 2,
)
network = self.dockerconf.get('network', 'host')
networking_config = { 'EndpointsConfig': { network: {} } }
ports = self.dockerconf.get('container_ports', [ 8000, 9000 ])
imagename = container.image.imagename if container.image else self.dockerconf.get('default_image', 'basic')
args = {
'name': container.name,
'image': imagename,
'detach': True,
'hostname': container.name,
'host_config': host_config,
'networking_config': networking_config,
'environment': container.environment,
'volumes': volumes,
'ports': ports,
}
self.client.create_container(**args)
logger.debug("Container created")
self.managemount(container) #FIXME: check if not called twice
return self.get_container(container)
def _writefile(self, container_name, path, filename, content):
import tarfile
import time
from io import BytesIO
tarstream = BytesIO()
tar = tarfile.TarFile(fileobj = tarstream, mode = 'w')
tarinfo = tarfile.TarInfo(name = filename)
tarinfo.size = len(content)
tarinfo.mtime = time.time()
tar.addfile(tarinfo, BytesIO(content))
tar.close()
tarstream.seek(0)
try:
status = self.client.put_archive(container = container_name, path = path, data = tarstream)
logger.info("container %s put_archive %s/%s returns %s" % (container_name, path, filename, status))
except Exception as e:
logger.error("container %s put_archive %s/%s fails -- %s" % (container_name, path, filename, e))
def managemount(self, container):
from kooplex.lib.fs_dirname import Dirname
path, filename = os.path.split(self.dockerconf.get('mountconf', '/tmp/mount.conf'))
mapper = []
for v in container.volumes:
mapper.extend([ "%s:%s" % (v.volumetype, d) for d in Dirname.containervolume_listfolders(container, v) ])
#NOTE: mounter uses read to process the mapper configuration, thus we need to make sure '\n' terminates the config mapper file
mapper.append('')
logger.debug("container %s map %s" % (container, mapper))
file_data = "\n".join(mapper).encode('utf8')
self._writefile(container.name, path, filename, file_data)
def trigger_impersonator(self, vcproject): #FIXME: dont call it 1-by-1
from kooplex.lib.fs_dirname import Dirname
container_name = self.dockerconf.get('impersonator', 'impersonator')
path, filename = os.path.split(self.dockerconf.get('gitcommandconf', '/tmp/gitcommand.conf'))
cmdmaps = []
token = vcproject.token
fn_clonesh = os.path.join(Dirname.vcpcache(vcproject), "clone.sh")
fn_key = os.path.join(Dirname.userhome(vcproject.token.user), '.ssh', token.fn_rsa)
cmdmaps.append("%s:%s:%s:%s" % (token.user.username, fn_key, token.repository.domain, fn_clonesh))
cmdmaps.append('')
file_data = "\n".join(cmdmaps).encode('utf8')
self._writefile(container_name, path, filename, file_data)
def run_container(self, container):
docker_container_info = self.get_container(container)
if docker_container_info is None:
logger.debug("Container did not exist, Creating new one")
docker_container_info = self.create_container(container)
container_state = docker_container_info['Status']
if container_state == 'Created' or container_state.startswith('Exited'):
logger.debug("Starting container")
self.start_container(container)
def refresh_container_state(self, container):
docker_container_info = self.get_container(container)
container_state = docker_container_info['State']
logger.debug("Container state %s" % container_state)
container.last_message = str(container_state)
container.last_message_at = now()
container.save()
def start_container(self, container):
self.client.start(container.name)
# we need to retrieve the container state after starting it
docker_container_info = self.get_container(container)
container_state = docker_container_info['State']
logger.debug("Container state %s" % container_state)
container.last_message = str(container_state)
container.last_message_at = now()
assert container_state == 'running', "Container failed to start: %s" % docker_container_info
def stop_container(self, container):
try:
self.client.stop(container.name)
container.last_message = 'Container stopped'
except Exception as e:
logger.warn("docker container not found by API -- %s" % e)
container.last_message = str(e)
def remove_container(self, container):
try:
self.client.remove_container(container.name)
container.last_message = 'Container removed'
container.last_message_at = now()
except Exception as e:
logger.warn("docker container not found by API -- %s" % e)
container.last_message = str(e)
container.last_message_at = now()
logger.debug("Container removed %s" % container.name)
#FIXME: az execute2 lesz az igazi...
def execute(self, container, command):
logger.info("execution: %s in %s" % (command, container))
execution = self.client.exec_create(container = container.name, cmd = shlex.split(command))
return self.client.exec_start(execution, detach = False)
def execute2(self, container, command):
logger.info("execution: %s in %s" % (command, container))
execution = self.client.exec_create(container = container.name, cmd = shlex.split(command))
response = self.client.exec_start(exec_id = execution['Id'], stream = False)
check = self.client.exec_inspect(exec_id = execution['Id'])
self.check = check
if check['ExitCode'] != 0:
logger.error('Execution %s in %s failed -- %s' % (command, container, check))
return response.decode()
