def __init__(self, config, persistence, messaging):
"""
Call super-class constructor for common configuration items and
then do the docker-specific setup
:return: -
"""
super(DockerWorker, self).__init__(config=config,
persistence=persistence,
messaging=messaging)
logging.debug('DockerWorker initialization')
self.docker = AutoVersionClient(
base_url=self.config['worker']['docker_url'],
tls=_get_tls(config['worker']))
self._image_ports = self._initialize_image()
self._get_all_allocated_ports()
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b',
'b',
image='ubuntu:14.04',
command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c',
'c',
image='ubuntu:14.04',
command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = os.urandom(10)
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(d))
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
class DockerWorker(Worker):
def __init__(self, config, persistence, messaging):
"""
Call super-class constructor for common configuration items and
then do the docker-specific setup
:return: -
"""
super(DockerWorker, self).__init__(config=config,
persistence=persistence,
messaging=messaging)
logging.debug('DockerWorker initialization')
self.docker = AutoVersionClient(
base_url=self.config['worker']['docker_url'],
tls=_get_tls(config['worker']))
self._image_ports = self._initialize_image()
self._get_all_allocated_ports()
def create_instance(self, message):
instance = message.copy()
logging.info('Creating instance id: %s', instance['id'])
environment = self._create_instance_env()
ports = self.port_manager.acquire_ports(len(self._image_ports))
environment = _check_instance_env_port_injection(
environment=environment,
ports=ports)
if ports:
port_mapping = dict(zip(self._image_ports, ports))
container = self.docker.create_container(self.worker['image'],
environment=environment,
ports=self._image_ports)
self.docker.start(container, port_bindings=port_mapping)
instance['container_id'] = container['Id']
instance['environment'] = environment
self._set_networking(instance=instance)
self.local_persistence.update_instance_status(
instance=instance,
status=INSTANCE_STATUS.STARTING)
else:
self.local_persistence.update_instance_status(
instance=instance,
status=INSTANCE_STATUS.FAILED)
def delete_instance(self, message):
msg = message.copy()
instance = self.local_persistence.get_instance(msg['id'])
if instance['status'] == INSTANCE_STATUS.RUNNING:
logging.info('Deleting instance id: %s', msg['id'])
container = self._get_container(
container_id=instance['container_id'])
if not container:
logging.debug('Container does not exist, not stopping it')
return
free_ports = self._get_container_ports(instance['container_id'])
self.docker.kill(container)
self.docker.remove_container(container)
self.port_manager.release_ports(free_ports)
self.local_persistence.update_instance_status(
instance=instance,
status=INSTANCE_STATUS.DELETED)
else:
self.local_persistence.update_instance_status(
instance=instance,
status=INSTANCE_STATUS.DELETED)
def _initialize_image(self):
"""
download a docker image and get the ports that we have to link
:return: list of ports(str)
"""
logging.info('Initializing image %s', self.config['worker']['image'])
self.worker['image'] = self.config['worker']['image']
tmp = self.worker['image'].split(':')
if len(tmp) == 2:
self.docker.import_image(image=tmp[0], tag=tmp[1])
else:
self.docker.import_image(image=tmp)
logging.debug('Extracting ports from image')
ports = []
docker_image = self.docker.inspect_image(self.worker['image'])
for port in docker_image[u'ContainerConfig'][u'ExposedPorts'].keys():
ports.append(port.split('/')[0])
return ports
def _get_all_allocated_ports(self):
"""
get all containers, that have not been stopped, they may have been
started from outside of the workers scope.
:return: array with ports to use, None if not enough ports available
"""
used_ports = set()
containers = self.docker.containers()
for container in containers:
for port in self._get_container_ports(container['Id']):
used_ports.add(port)
self.port_manager.update_used_ports(used_ports)
def _get_container(self, container_id):
"""
get docker-py s container description
:param container_id: string, id for the container
:return: dict, containing the container
"""
try:
return self.docker.inspect_container({'Id': container_id})
except docker.errors.APIError:
logging.debug('Not able to get container %s', container_id)
return None
def _get_container_networking(self, container_id):
"""
return a dict with the container networking, using the appropriate
worker ip
:param container_id: id of the container
:return: dict with the Ports section of the container representation
"""
try:
networking = self._get_container(
container_id)['NetworkSettings']['Ports']
if 'ip' in self.config['worker'].keys():
for port in networking:
for index, unused in enumerate(networking[port]):
networking[port][index][u'HostIp'] = \
unicode(self.config['worker']['ip'])
return networking
except TypeError:
logging.error('Cannot get ports for container_id %s', container_id)
return None
def _get_container_ports(self, container_id):
"""
return a list of the concrete container ports that are used
:param container_id: id of the container
:return: list of integers
"""
networking = self._get_container_networking(container_id)
ports = list()
if networking:
for port in networking.keys():
for index, unused in enumerate(networking[port]):
ports.append(int(networking[port][index][u'HostPort']))
return ports
def _publish_updates(self):
instances = self.local_persistence.get_instances()
for instance_id in instances.keys():
if instances[instance_id]['status'] is INSTANCE_STATUS.DELETED:
continue
elif instances[instance_id]['status'] is INSTANCE_STATUS.FAILED:
self.local_persistence.publish_instance(instance_id)
continue
container_id = instances[instance_id]['container_id']
container = self._get_container(container_id)
if not container_id or not container or not _is_running(container):
instances[instance_id].pop('connection', None)
instances[instance_id].pop('urls', None)
self.local_persistence.update_instance_status(
instances[instance_id],
INSTANCE_STATUS.STOPPED)
continue
elif _is_running(container):
self._set_networking(instances[instance_id])
self.local_persistence.update_instance_status(
instances[instance_id],
INSTANCE_STATUS.RUNNING)
else:
logging.error("error while publishing updates")
self._get_all_allocated_ports()
self._update_worker_status()
def _update_worker_status(self):
number_required_ports = len(self._image_ports)
if self.port_manager.enough_ports_left(number_required_ports):
self.worker['available'] = True
self.worker['status'] = 'Worker available'
else:
self.worker['available'] = False
self.worker['status'] = 'Worker unavailable, ' \
'to many resources in use'
def _set_networking(self, instance):
instance['connection'] = \
self._get_container_networking(instance['container_id'])
instance['urls'] = self.url_builder.build(instance['connection'])
def run(self):
# Replace any placeholder in the commandline with the proper path or
# dataURL, depending on the type of input/output it is
# In the case of fs-based i/o we replace the command-line with the path
# that the Drop will receive *inside* the docker container (see below)
def isFSBased(x):
return isinstance(x, (FileDROP, DirectoryContainer))
fsInputs = [i for i in self.inputs if isFSBased(i)]
fsOutputs = [o for o in self.outputs if isFSBased(o)]
dockerInputs = [DockerPath(i.uid, DFMS_ROOT + i.path) for i in fsInputs]
dockerOutputs = [DockerPath(o.uid, DFMS_ROOT + o.path) for o in fsOutputs]
dataURLInputs = [i for i in self.inputs if not isFSBased(i)]
dataURLOutputs = [o for o in self.outputs if not isFSBased(o)]
cmd = droputils.replace_path_placeholders(self._command, dockerInputs, dockerOutputs)
cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs)
# We bind the inputs and outputs inside the docker under the DFMS_ROOT
# directory, maintaining the rest of their original paths.
# Outputs are bound only up to their dirname (see class doc for details)
# Volume bindings are setup for FileDROPs and DirectoryContainers only
vols = [x.path for x in dockerInputs] + [os.path.dirname(x.path) for x in dockerOutputs]
binds = [i.path + ":" + dockerInputs[x].path for x, i in enumerate(fsInputs)]
binds += [
os.path.dirname(o.path) + ":" + os.path.dirname(dockerOutputs[x].path) for x, o in enumerate(fsOutputs)
]
binds += [host_path + ":" + container_path for host_path, container_path in self._additionalBindings.items()]
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Volume bindings: %r" % (binds))
# Wait until the DockerApps this application runtime depends on have
# started, and replace their IP placeholders by the real IPs
for waiter in self._waiters:
uid, ip = waiter.waitForIp()
cmd = cmd.replace("%containerIp[{0}]%".format(uid), ip)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Command after IP replacement is: %s" % (cmd))
# If a user has been given, we run the container as that user. It is
# useful to make sure that the USER environment variable is set in those
# cases (e.g., casapy requires this to correctly operate)
user = self._user
env = {}
if user is not None:
env = {"USER": user}
if self._ensureUserAndSwitch is True:
# Append commands that will make sure a user is present with the
# same UID of the current user, and that the command that was
# supplied for this container runs as that user.
# Also make sure that the output will belong to that user
uid = os.getuid()
createUserAndGo = "id -u {0} &> /dev/null || adduser --uid {0} r; ".format(uid)
for dirname in set([os.path.dirname(x.path) for x in dockerOutputs]):
createUserAndGo += 'chown -R {0}.{0} "{1}"; '.format(uid, dirname)
createUserAndGo += "cd; su -l $(getent passwd {0} | cut -f1 -d:) -c /bin/bash -c '{1}'".format(
uid, utils.escapeQuotes(cmd, doubleQuotes=False)
)
cmd = createUserAndGo
# Wrap everything inside bash
cmd = '/bin/bash -c "%s"' % (utils.escapeQuotes(cmd, singleQuotes=False))
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Command after user creation and wrapping is: %s" % (cmd))
extra_kwargs = self._kwargs_from_env()
c = AutoVersionClient(**extra_kwargs)
# Remove the container unless it's specified that we should keep it
# (used below)
def rm(container):
if self._removeContainer:
c.remove_container(container)
# Create container
host_config = c.create_host_config(binds=binds)
container = c.create_container(
self._image, cmd, volumes=vols, host_config=host_config, user=user, environment=env
)
self._containerId = cId = container["Id"]
if logger.isEnabledFor(logging.INFO):
logger.info("Created container %s for %r" % (cId, self))
# Start it
start = time.time()
c.start(container)
if logger.isEnabledFor(logging.INFO):
logger.info("Started container %s" % (cId))
# Figure out the container's IP and save it
# Setting self.containerIp will trigger an event being sent to the
# registered listeners
inspection = c.inspect_container(container)
self.containerIp = inspection["NetworkSettings"]["IPAddress"]
# Wait until it finishes
self._exitCode = c.wait(container)
end = time.time()
if logger.isEnabledFor(logging.INFO):
logger.info("Container %s finished in %.2f [s] with exit code %d" % (cId, (end - start), self._exitCode))
if self._exitCode == 0 and logger.isEnabledFor(logging.DEBUG):
msg = "Container %s finished successfully" % (cId,)
stdout = c.logs(container, stdout=True, stderr=False)
stderr = c.logs(container, stdout=False, stderr=True)
logger.debug(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr))
elif self._exitCode != 0:
stdout = c.logs(container, stdout=True, stderr=False)
stderr = c.logs(container, stdout=False, stderr=True)
msg = "Container %s didn't finish successfully (exit code %d)" % (cId, self._exitCode)
logger.error(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr))
rm(container)
raise Exception(msg)
rm(container)
def initialize(self, **kwargs):
BarrierAppDROP.initialize(self, **kwargs)
self._image = self._getArg(kwargs, "image", None)
if not self._image:
raise Exception("No docker image specified, cannot create DockerApp")
if ":" not in self._image:
logger.warn("%r: Image %s is too generic since it doesn't specify a tag" % (self, self._image))
self._command = self._getArg(kwargs, "command", None)
if not self._command:
raise Exception("No command specified, cannot create DockerApp")
# The user used to run the process in the docker container
# By default docker containers run as root, but we don't want to run
# a process using a different user because otherwise anything that that
# process writes to the filesystem
self._user = self._getArg(kwargs, "user", None)
# In some cases we want to make sure the command in the container runs
# as a certain user, so we wrap up the command line in a small script
# that will create the user if missing and switch to it
self._ensureUserAndSwitch = self._getArg(kwargs, "ensureUserAndSwitch", self._user is None)
# By default containers are removed from the filesystem, but people
# might want to preserve them.
# TODO: This might be something that the data lifecycle manager could
# handle, but for the time being we do it here
self._removeContainer = self._getArg(kwargs, "removeContainer", True)
# Additional volume bindings can be specified for existing files/dirs
# on the host system.
self._additionalBindings = {}
for binding in self._getArg(kwargs, "additionalBindings", []):
if binding.find(":") == -1:
host_path = container_path = binding
else:
host_path, container_path = binding.split(":")
if not os.path.exists(host_path):
raise ValueError("'Path %s doesn't exist, cannot use as additional volume binding" % (host_path,))
self._additionalBindings[host_path] = container_path
if logger.isEnabledFor(logging.INFO):
logger.info("%r with image '%s' and command '%s' created" % (self, self._image, self._command))
# Check if we have the image; otherwise pull it.
extra_kwargs = self._kwargs_from_env()
c = AutoVersionClient(**extra_kwargs)
found = reduce(lambda a, b: a or self._image in b["RepoTags"], c.images(), False)
if not found:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Image '%s' not found, pulling it" % (self._image))
start = time.time()
c.pull(self._image)
end = time.time()
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Took %.2f [s] to pull image '%s'" % ((end - start), self._image))
else:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Image '%s' found, no need to pull it" % (self._image))
self._containerIp = None
self._containerId = None
self._waiters = []
