def run(self): def isFSBased(x): return isinstance(x, (FileDROP, DirectoryContainer)) fsInputs = [i for i in self.inputs if isFSBased(i)] fsOutputs = [o for o in self.outputs if isFSBased(o)] dataURLInputs = [i for i in self.inputs if not isFSBased(i)] dataURLOutputs = [o for o in self.outputs if not isFSBased(o)] cmd = droputils.replace_path_placeholders(self._command, fsInputs, fsOutputs) cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs) # Wrap everything inside bash cmd = '/bin/bash -c "{0}"'.format(utils.escapeQuotes(cmd, singleQuotes=False)) if LOG.isEnabledFor(logging.DEBUG): LOG.debug("Command after user creation and wrapping is: {0}".format(cmd)) start = time.time() # Wait until it finishes process = subprocess.Popen(cmd, bufsize=1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy()) stdout, stderr = process.communicate() self._exit_code = process.returncode end = time.time() if LOG.isEnabledFor(logging.INFO): LOG.info("Finished in {0:.2f} [s] with exit code {1}".format(end-start, self._exit_code)) if self._exit_code == 0 and LOG.isEnabledFor(logging.DEBUG): LOG.debug("Command finished successfully, output follows.\n==STDOUT==\n{0}==STDERR==\n{1}".format(stdout, stderr)) elif self._exit_code != 0: message = "Command didn't finish successfully (exit code {0})".format(self._exit_code) LOG.error(message + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr)) raise Exception(message)
def run_bash(cmd, inputs, outputs, stdin=None, stdout=subprocess.PIPE): """ Runs the given `cmd`. If any `inputs` and/or `outputs` are given (dictionaries of uid:drop elements) they are used to replace any placeholder value in `cmd` with either drop paths or dataURLs. `stdin` indicates at file descriptor or file object to use as the standard input of the bash process. If not given no stdin is given to the process. Similarly, `stdout` is a file descriptor or file object where the standard output of the process is piped to. If not given it is consumed by this method and potentially logged. """ # Replace inputs/outputs in command line with paths or data URLs fsInputs = {uid: i for uid,i in inputs.items() if isFSBased(i)} fsOutputs = {uid: o for uid,o in outputs.items() if isFSBased(o)} cmd = droputils.replace_path_placeholders(cmd, fsInputs, fsOutputs) dataURLInputs = {uid: i for uid,i in inputs.items() if not isFSBased(i)} dataURLOutputs = {uid: o for uid,o in outputs.items() if not isFSBased(o)} cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs) # Wrap everything inside bash cmd = ('/bin/bash', '-c', cmd) logger.debug("Command after user creation and wrapping is: %s", cmd) start = time.time() # Run and wait until it finishes process = subprocess.Popen(cmd, close_fds=True, stdin=stdin, stdout=stdout, stderr=subprocess.PIPE, env=os.environ.copy()) logger.debug("Process launched, waiting now...") pstdout, pstderr = process.communicate() if stdout != subprocess.PIPE: pstdout = "<piped-out>" pcode = process.returncode end = time.time() logger.info("Finished in %.3f [s] with exit code %d", (end-start), pcode) if pcode == 0 and logger.isEnabledFor(logging.DEBUG): logger.debug(mesage_stdouts("Command finished successfully", pstdout, pstderr)) elif pcode != 0: message = "Command didn't finish successfully (exit code %d)" % (pcode,) logger.error(mesage_stdouts(message, pstdout, pstderr)) raise Exception(message)
def run(self): # Replace any placeholder in the commandline with the proper path or # dataURL, depending on the type of input/output it is # In the case of fs-based i/o we replace the command-line with the path # that the Drop will receive *inside* the docker container (see below) def isFSBased(x): return isinstance(x, (FileDROP, DirectoryContainer)) fsInputs = [i for i in self.inputs if isFSBased(i)] fsOutputs = [o for o in self.outputs if isFSBased(o)] dockerInputs = [DockerPath(i.uid, DFMS_ROOT + i.path) for i in fsInputs] dockerOutputs = [DockerPath(o.uid, DFMS_ROOT + o.path) for o in fsOutputs] dataURLInputs = [i for i in self.inputs if not isFSBased(i)] dataURLOutputs = [o for o in self.outputs if not isFSBased(o)] cmd = droputils.replace_path_placeholders(self._command, dockerInputs, dockerOutputs) cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs) # We bind the inputs and outputs inside the docker under the DFMS_ROOT # directory, maintaining the rest of their original paths. # Outputs are bound only up to their dirname (see class doc for details) # Volume bindings are setup for FileDROPs and DirectoryContainers only vols = [x.path for x in dockerInputs] + [os.path.dirname(x.path) for x in dockerOutputs] binds = [i.path + ":" + dockerInputs[x].path for x, i in enumerate(fsInputs)] binds += [ os.path.dirname(o.path) + ":" + os.path.dirname(dockerOutputs[x].path) for x, o in enumerate(fsOutputs) ] binds += [host_path + ":" + container_path for host_path, container_path in self._additionalBindings.items()] if logger.isEnabledFor(logging.DEBUG): logger.debug("Volume bindings: %r" % (binds)) # Wait until the DockerApps this application runtime depends on have # started, and replace their IP placeholders by the real IPs for waiter in self._waiters: uid, ip = waiter.waitForIp() cmd = cmd.replace("%containerIp[{0}]%".format(uid), ip) if logger.isEnabledFor(logging.DEBUG): logger.debug("Command after IP replacement is: %s" % (cmd)) # If a user has been given, we run the container as that user. It is # useful to make sure that the USER environment variable is set in those # cases (e.g., casapy requires this to correctly operate) user = self._user env = {} if user is not None: env = {"USER": user} if self._ensureUserAndSwitch is True: # Append commands that will make sure a user is present with the # same UID of the current user, and that the command that was # supplied for this container runs as that user. # Also make sure that the output will belong to that user uid = os.getuid() createUserAndGo = "id -u {0} &> /dev/null || adduser --uid {0} r; ".format(uid) for dirname in set([os.path.dirname(x.path) for x in dockerOutputs]): createUserAndGo += 'chown -R {0}.{0} "{1}"; '.format(uid, dirname) createUserAndGo += "cd; su -l $(getent passwd {0} | cut -f1 -d:) -c /bin/bash -c '{1}'".format( uid, utils.escapeQuotes(cmd, doubleQuotes=False) ) cmd = createUserAndGo # Wrap everything inside bash cmd = '/bin/bash -c "%s"' % (utils.escapeQuotes(cmd, singleQuotes=False)) if logger.isEnabledFor(logging.DEBUG): logger.debug("Command after user creation and wrapping is: %s" % (cmd)) extra_kwargs = self._kwargs_from_env() c = AutoVersionClient(**extra_kwargs) # Remove the container unless it's specified that we should keep it # (used below) def rm(container): if self._removeContainer: c.remove_container(container) # Create container host_config = c.create_host_config(binds=binds) container = c.create_container( self._image, cmd, volumes=vols, host_config=host_config, user=user, environment=env ) self._containerId = cId = container["Id"] if logger.isEnabledFor(logging.INFO): logger.info("Created container %s for %r" % (cId, self)) # Start it start = time.time() c.start(container) if logger.isEnabledFor(logging.INFO): logger.info("Started container %s" % (cId)) # Figure out the container's IP and save it # Setting self.containerIp will trigger an event being sent to the # registered listeners inspection = c.inspect_container(container) self.containerIp = inspection["NetworkSettings"]["IPAddress"] # Wait until it finishes self._exitCode = c.wait(container) end = time.time() if logger.isEnabledFor(logging.INFO): logger.info("Container %s finished in %.2f [s] with exit code %d" % (cId, (end - start), self._exitCode)) if self._exitCode == 0 and logger.isEnabledFor(logging.DEBUG): msg = "Container %s finished successfully" % (cId,) stdout = c.logs(container, stdout=True, stderr=False) stderr = c.logs(container, stdout=False, stderr=True) logger.debug(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr)) elif self._exitCode != 0: stdout = c.logs(container, stdout=True, stderr=False) stderr = c.logs(container, stdout=False, stderr=True) msg = "Container %s didn't finish successfully (exit code %d)" % (cId, self._exitCode) logger.error(msg + ", output follows.\n==STDOUT==\n%s==STDERR==\n%s" % (stdout, stderr)) rm(container) raise Exception(msg) rm(container)