def run(self):
def isFSBased(x):
return isinstance(x, (FileDROP, DirectoryContainer))
fsInputs = [i for i in self.inputs if isFSBased(i)]
fsOutputs = [o for o in self.outputs if isFSBased(o)]
dataURLInputs = [i for i in self.inputs if not isFSBased(i)]
dataURLOutputs = [o for o in self.outputs if not isFSBased(o)]
cmd = droputils.replace_path_placeholders(self._command, fsInputs, fsOutputs)
cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs)
# Wrap everything inside bash
cmd = '/bin/bash -c "{0}"'.format(utils.escapeQuotes(cmd, singleQuotes=False))
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("Command after user creation and wrapping is: {0}".format(cmd))
start = time.time()
# Wait until it finishes
process = subprocess.Popen(cmd, bufsize=1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy())
stdout, stderr = process.communicate()
self._exit_code = process.returncode
end = time.time()
if LOG.isEnabledFor(logging.INFO):
LOG.info("Finished in {0:.2f} [s] with exit code {1}".format(end-start, self._exit_code))
if self._exit_code == 0 and LOG.isEnabledFor(logging.DEBUG):
LOG.debug("Command finished successfully, output follows.\n==STDOUT==\n{0}==STDERR==\n{1}".format(stdout, stderr))
elif self._exit_code != 0:
message = "Command didn't finish successfully (exit code {0})".format(self._exit_code)
LOG.error(message + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr))
raise Exception(message)
def run_bash(cmd, inputs, outputs, stdin=None, stdout=subprocess.PIPE):
"""
Runs the given `cmd`. If any `inputs` and/or `outputs` are given
(dictionaries of uid:drop elements) they are used to replace any placeholder
value in `cmd` with either drop paths or dataURLs.
`stdin` indicates at file descriptor or file object to use as the standard
input of the bash process. If not given no stdin is given to the process.
Similarly, `stdout` is a file descriptor or file object where the standard
output of the process is piped to. If not given it is consumed by this
method and potentially logged.
"""
# Replace inputs/outputs in command line with paths or data URLs
fsInputs = {uid: i for uid,i in inputs.items() if isFSBased(i)}
fsOutputs = {uid: o for uid,o in outputs.items() if isFSBased(o)}
cmd = droputils.replace_path_placeholders(cmd, fsInputs, fsOutputs)
dataURLInputs = {uid: i for uid,i in inputs.items() if not isFSBased(i)}
dataURLOutputs = {uid: o for uid,o in outputs.items() if not isFSBased(o)}
cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs)
# Wrap everything inside bash
cmd = ('/bin/bash', '-c', cmd)
logger.debug("Command after user creation and wrapping is: %s", cmd)
start = time.time()
# Run and wait until it finishes
process = subprocess.Popen(cmd,
close_fds=True,
stdin=stdin,
stdout=stdout,
stderr=subprocess.PIPE,
env=os.environ.copy())
logger.debug("Process launched, waiting now...")
pstdout, pstderr = process.communicate()
if stdout != subprocess.PIPE:
pstdout = "<piped-out>"
pcode = process.returncode
end = time.time()
logger.info("Finished in %.3f [s] with exit code %d", (end-start), pcode)
if pcode == 0 and logger.isEnabledFor(logging.DEBUG):
logger.debug(mesage_stdouts("Command finished successfully", pstdout, pstderr))
elif pcode != 0:
message = "Command didn't finish successfully (exit code %d)" % (pcode,)
logger.error(mesage_stdouts(message, pstdout, pstderr))
raise Exception(message)
def run(self):
# Replace any placeholder in the commandline with the proper path or
# dataURL, depending on the type of input/output it is
# In the case of fs-based i/o we replace the command-line with the path
# that the Drop will receive *inside* the docker container (see below)
def isFSBased(x):
return isinstance(x, (FileDROP, DirectoryContainer))
fsInputs = [i for i in self.inputs if isFSBased(i)]
fsOutputs = [o for o in self.outputs if isFSBased(o)]
dockerInputs = [DockerPath(i.uid, DFMS_ROOT + i.path) for i in fsInputs]
dockerOutputs = [DockerPath(o.uid, DFMS_ROOT + o.path) for o in fsOutputs]
dataURLInputs = [i for i in self.inputs if not isFSBased(i)]
dataURLOutputs = [o for o in self.outputs if not isFSBased(o)]
cmd = droputils.replace_path_placeholders(self._command, dockerInputs, dockerOutputs)
cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs)
# We bind the inputs and outputs inside the docker under the DFMS_ROOT
# directory, maintaining the rest of their original paths.
# Outputs are bound only up to their dirname (see class doc for details)
# Volume bindings are setup for FileDROPs and DirectoryContainers only
vols = [x.path for x in dockerInputs] + [os.path.dirname(x.path) for x in dockerOutputs]
binds = [i.path + ":" + dockerInputs[x].path for x, i in enumerate(fsInputs)]
binds += [
os.path.dirname(o.path) + ":" + os.path.dirname(dockerOutputs[x].path) for x, o in enumerate(fsOutputs)
]
binds += [host_path + ":" + container_path for host_path, container_path in self._additionalBindings.items()]
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Volume bindings: %r" % (binds))
# Wait until the DockerApps this application runtime depends on have
# started, and replace their IP placeholders by the real IPs
for waiter in self._waiters:
uid, ip = waiter.waitForIp()
cmd = cmd.replace("%containerIp[{0}]%".format(uid), ip)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Command after IP replacement is: %s" % (cmd))
# If a user has been given, we run the container as that user. It is
# useful to make sure that the USER environment variable is set in those
# cases (e.g., casapy requires this to correctly operate)
user = self._user
env = {}
if user is not None:
env = {"USER": user}
if self._ensureUserAndSwitch is True:
# Append commands that will make sure a user is present with the
# same UID of the current user, and that the command that was
# supplied for this container runs as that user.
# Also make sure that the output will belong to that user
uid = os.getuid()
createUserAndGo = "id -u {0} &> /dev/null || adduser --uid {0} r; ".format(uid)
for dirname in set([os.path.dirname(x.path) for x in dockerOutputs]):
createUserAndGo += 'chown -R {0}.{0} "{1}"; '.format(uid, dirname)
createUserAndGo += "cd; su -l $(getent passwd {0} | cut -f1 -d:) -c /bin/bash -c '{1}'".format(
uid, utils.escapeQuotes(cmd, doubleQuotes=False)
)
cmd = createUserAndGo
# Wrap everything inside bash
cmd = '/bin/bash -c "%s"' % (utils.escapeQuotes(cmd, singleQuotes=False))
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Command after user creation and wrapping is: %s" % (cmd))
extra_kwargs = self._kwargs_from_env()
c = AutoVersionClient(**extra_kwargs)
# Remove the container unless it's specified that we should keep it
# (used below)
def rm(container):
if self._removeContainer:
c.remove_container(container)
# Create container
host_config = c.create_host_config(binds=binds)
container = c.create_container(
self._image, cmd, volumes=vols, host_config=host_config, user=user, environment=env
)
self._containerId = cId = container["Id"]
if logger.isEnabledFor(logging.INFO):
logger.info("Created container %s for %r" % (cId, self))
# Start it
start = time.time()
c.start(container)
if logger.isEnabledFor(logging.INFO):
logger.info("Started container %s" % (cId))
# Figure out the container's IP and save it
# Setting self.containerIp will trigger an event being sent to the
# registered listeners
inspection = c.inspect_container(container)
self.containerIp = inspection["NetworkSettings"]["IPAddress"]
# Wait until it finishes
self._exitCode = c.wait(container)
end = time.time()
if logger.isEnabledFor(logging.INFO):
logger.info("Container %s finished in %.2f [s] with exit code %d" % (cId, (end - start), self._exitCode))
if self._exitCode == 0 and logger.isEnabledFor(logging.DEBUG):
msg = "Container %s finished successfully" % (cId,)
stdout = c.logs(container, stdout=True, stderr=False)
stderr = c.logs(container, stdout=False, stderr=True)
logger.debug(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr))
elif self._exitCode != 0:
stdout = c.logs(container, stdout=True, stderr=False)
stderr = c.logs(container, stdout=False, stderr=True)
msg = "Container %s didn't finish successfully (exit code %d)" % (cId, self._exitCode)
logger.error(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr))
rm(container)
raise Exception(msg)
rm(container)
