def create_env(env_file, filetype="yaml"): # Copy env file to env_path (because they can be on # different volumes and singularity should only mount one). # In addition, this allows to immediately see what an # environment in .snakemake/conda contains. target_env_file = env_path + f".{filetype}" shutil.copy(env_file, target_env_file) logger.info( "Downloading and installing remote packages.") strict_priority = ([ "conda config --set channel_priority strict &&" ] if self._container_img else []) subcommand = [self.frontend] yes_flag = ["--yes"] if filetype == "yaml": subcommand.append("env") yes_flag = [] cmd = (strict_priority + subcommand + [ "create", "--quiet", '--file "{}"'.format(target_env_file), '--prefix "{}"'.format(env_path), ] + yes_flag) cmd = " ".join(cmd) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, args=self._singularity_args, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) # cleanup if requested if self._cleanup is CondaCleanupMode.tarballs: logger.info("Cleaning up conda package tarballs.") shell.check_output("conda clean -y --tarballs") elif self._cleanup is CondaCleanupMode.cache: logger.info( "Cleaning up conda package tarballs and package cache." ) shell.check_output( "conda clean -y --tarballs --packages") return out
def __new__(cls, cmd, *args, iterable=False, read=False, bench_record=None, **kwargs): if "stepout" in kwargs: raise KeyError("Argument stepout is not allowed in shell command.") cmd = format(cmd, *args, stepout=2, **kwargs) context = inspect.currentframe().f_back.f_locals # add kwargs to context (overwriting the locals of the caller) context.update(kwargs) stdout = sp.PIPE if iterable or read else STDOUT close_fds = sys.platform != "win32" jobid = context.get("jobid") if not context.get("is_shell"): logger.shellcmd(cmd) env_prefix = "" conda_env = context.get("conda_env", None) container_img = context.get("container_img", None) env_modules = context.get("env_modules", None) shadow_dir = context.get("shadow_dir", None) cmd = "{} {} {}".format(cls._process_prefix, cmd.strip(), cls._process_suffix).strip() if env_modules: cmd = env_modules.shellcmd(cmd) logger.info( "Activating environment modules: {}".format(env_modules)) if conda_env: cmd = Conda(container_img).shellcmd(conda_env, cmd) if container_img: args = context.get("singularity_args", "") cmd = singularity.shellcmd( container_img, cmd, args, shell_executable=cls._process_args["executable"], container_workdir=shadow_dir, ) logger.info( "Activating singularity image {}".format(container_img)) if conda_env: logger.info("Activating conda environment: {}".format(conda_env)) threads = str(context.get("threads", 1)) # environment variable lists for linear algebra libraries taken from: # https://stackoverflow.com/a/53224849/2352071 # https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables envvars = dict(os.environ) envvars["OMP_NUM_THREADS"] = threads envvars["GOTO_NUM_THREADS"] = threads envvars["OPENBLAS_NUM_THREADS"] = threads envvars["MKL_NUM_THREADS"] = threads envvars["VECLIB_MAXIMUM_THREADS"] = threads envvars["NUMEXPR_NUM_THREADS"] = threads use_shell = True if ON_WINDOWS and cls.get_executable(): # If executable is set on Windows shell mode can not be used # and the executable should be prepended the command together # with a command prefix (e.g. -c for bash). use_shell = False cmd = '"{}" {} {}'.format(cls.get_executable(), cls._win_command_prefix, argvquote(cmd)) proc = sp.Popen( cmd, bufsize=-1, shell=use_shell, stdout=stdout, universal_newlines=iterable or read or None, close_fds=close_fds, **cls._process_args, env=envvars, ) if jobid is not None: with cls._lock: cls._processes[jobid] = proc ret = None if iterable: return cls.iter_stdout(proc, cmd) if read: ret = proc.stdout.read() if bench_record is not None: from snakemake.benchmark import benchmarked with benchmarked(proc.pid, bench_record): retcode = proc.wait() else: retcode = proc.wait() if jobid is not None: with cls._lock: del cls._processes[jobid] if retcode: raise sp.CalledProcessError(retcode, cmd) return ret
def __new__(cls, cmd, *args, iterable=False, read=False, bench_record=None, **kwargs): if "stepout" in kwargs: raise KeyError("Argument stepout is not allowed in shell command.") cmd = format(cmd, *args, stepout=2, **kwargs) context = inspect.currentframe().f_back.f_locals # add kwargs to context (overwriting the locals of the caller) context.update(kwargs) stdout = sp.PIPE if iterable or read else STDOUT close_fds = sys.platform != "win32" jobid = context.get("jobid") if not context.get("is_shell"): logger.shellcmd(cmd) env_prefix = "" conda_env = context.get("conda_env", None) singularity_img = context.get("singularity_img", None) env_modules = context.get("env_modules", None) shadow_dir = context.get("shadow_dir", None) cmd = "{} {} {}".format(cls._process_prefix, cmd.strip(), cls._process_suffix).strip() if env_modules: cmd = env_modules.shellcmd(cmd) logger.info( "Activating environment modules: {}".format(env_modules)) if conda_env: cmd = Conda(singularity_img).shellcmd(conda_env, cmd) if singularity_img: args = context.get("singularity_args", "") cmd = singularity.shellcmd( singularity_img, cmd, args, shell_executable=cls._process_args["executable"], container_workdir=shadow_dir, ) logger.info( "Activating singularity image {}".format(singularity_img)) if conda_env: logger.info("Activating conda environment: {}".format(conda_env)) proc = sp.Popen(cmd, bufsize=-1, shell=True, stdout=stdout, universal_newlines=iterable or None, close_fds=close_fds, **cls._process_args) if jobid is not None: with cls._lock: cls._processes[jobid] = proc ret = None if iterable: return cls.iter_stdout(proc, cmd) if read: ret = proc.stdout.read() if bench_record is not None: from snakemake.benchmark import benchmarked with benchmarked(proc.pid, bench_record): retcode = proc.wait() else: retcode = proc.wait() if jobid is not None: with cls._lock: del cls._processes[jobid] if retcode: raise sp.CalledProcessError(retcode, cmd) return ret
def _get_cmd(self, cmd): if self.container_img: return singularity.shellcmd(self.container_img, cmd) return cmd
def create(self, dryrun=False): """ Create the conda enviroment.""" from snakemake.shell import shell # Read env file and create hash. env_file = self.file tmp_file = None url_scheme, *_ = urlparse(env_file) if (url_scheme and not url_scheme == "file") or ( not url_scheme and env_file.startswith("git+file:/")): with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as tmp: tmp.write(self.content) env_file = tmp.name tmp_file = tmp.name env_hash = self.hash env_path = self.path # Check for broken environment if os.path.exists(os.path.join( env_path, "env_setup_start")) and not os.path.exists( os.path.join(env_path, "env_setup_done")): if dryrun: logger.info( "Incomplete Conda environment {} will be recreated.". format(utils.simplify_path(self.file))) else: logger.info( "Removing incomplete Conda environment {}...".format( utils.simplify_path(self.file))) shutil.rmtree(env_path, ignore_errors=True) # Create environment if not already present. if not os.path.exists(env_path): if dryrun: logger.info("Conda environment {} will be created.".format( utils.simplify_path(self.file))) return env_path conda = Conda(self._container_img) logger.info("Creating conda environment {}...".format( utils.simplify_path(self.file))) # Check if env archive exists. Use that if present. env_archive = self.archive_file try: # Touch "start" flag file os.makedirs(env_path, exist_ok=True) with open(os.path.join(env_path, "env_setup_start"), "a") as f: pass if os.path.exists(env_archive): logger.info("Installing archived conda packages.") pkg_list = os.path.join(env_archive, "packages.txt") if os.path.exists(pkg_list): # read pacakges in correct order # this is for newer env archives where the package list # was stored packages = [ os.path.join(env_archive, pkg.rstrip()) for pkg in open(pkg_list) ] else: # guess order packages = glob(os.path.join(env_archive, "*.tar.bz2")) # install packages manually from env archive cmd = " ".join([ "conda", "create", "--copy", "--prefix '{}'".format( env_path) ] + packages) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT) else: # Copy env file to env_path (because they can be on # different volumes and singularity should only mount one). # In addition, this allows to immediately see what an # environment in .snakemake/conda contains. target_env_file = env_path + ".yaml" shutil.copy(env_file, target_env_file) logger.info("Downloading and installing remote packages.") cmd = " ".join([ "conda", "env", "create", "--file '{}'".format(target_env_file), "--prefix '{}'".format(env_path), ]) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT) # Touch "done" flag file with open(os.path.join(env_path, "env_setup_done"), "a") as f: pass logger.debug(out.decode()) logger.info("Environment for {} created (location: {})".format( os.path.relpath(env_file), os.path.relpath(env_path))) except subprocess.CalledProcessError as e: # remove potential partially installed environment shutil.rmtree(env_path, ignore_errors=True) raise CreateCondaEnvironmentException( "Could not create conda environment from {}:\n".format( env_file) + e.output.decode()) if tmp_file: # temporary file was created os.remove(tmp_file) return env_path
def create(self, dryrun=False): """Create the conda enviroment.""" from snakemake.shell import shell # Read env file and create hash. env_file = self.file tmp_file = None if not isinstance(env_file, LocalSourceFile) or isinstance( env_file, LocalGitFile): with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as tmp: # write to temp file such that conda can open it tmp.write(self.content) env_file = tmp.name tmp_file = tmp.name else: env_file = env_file.get_path_or_uri() env_hash = self.hash env_path = self.path if self.is_containerized: if not dryrun: try: shell.check_output( singularity.shellcmd( self._container_img.path, "[ -d '{}' ]".format(env_path), args=self._singularity_args, envvars=self.get_singularity_envvars(), quiet=True, ), stderr=subprocess.PIPE, ) except subprocess.CalledProcessError as e: raise WorkflowError( "Unable to find environment in container image. " "Maybe a conda environment was modified without containerizing again " "(see snakemake --containerize)?\nDetails:\n{}\n{}". format(e, e.stderr.decode())) return env_path else: # env should be present in the container return env_path # Check for broken environment if os.path.exists(os.path.join( env_path, "env_setup_start")) and not os.path.exists( os.path.join(env_path, "env_setup_done")): if dryrun: logger.info( "Incomplete Conda environment {} will be recreated.". format(self.file.simplify_path())) else: logger.info( "Removing incomplete Conda environment {}...".format( self.file.simplify_path())) shutil.rmtree(env_path, ignore_errors=True) # Create environment if not already present. if not os.path.exists(env_path): if dryrun: logger.info("Conda environment {} will be created.".format( self.file.simplify_path())) return env_path conda = Conda(self._container_img) logger.info("Creating conda environment {}...".format( self.file.simplify_path())) # Check if env archive exists. Use that if present. env_archive = self.archive_file try: # Touch "start" flag file os.makedirs(env_path, exist_ok=True) with open(os.path.join(env_path, "env_setup_start"), "a") as f: pass if os.path.exists(env_archive): logger.info("Installing archived conda packages.") pkg_list = os.path.join(env_archive, "packages.txt") if os.path.exists(pkg_list): # read pacakges in correct order # this is for newer env archives where the package list # was stored packages = [ os.path.join(env_archive, pkg.rstrip()) for pkg in open(pkg_list) ] else: # guess order packages = glob(os.path.join(env_archive, "*.tar.bz2")) # install packages manually from env archive cmd = " ".join([ "conda", "create", "--quiet", "--yes", "--prefix '{}'".format(env_path), ] + packages) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, args=self._singularity_args, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) else: # Copy env file to env_path (because they can be on # different volumes and singularity should only mount one). # In addition, this allows to immediately see what an # environment in .snakemake/conda contains. target_env_file = env_path + ".yaml" shutil.copy(env_file, target_env_file) logger.info("Downloading and installing remote packages.") cmd = " ".join([ self.frontend, "env", "create", "--quiet", '--file "{}"'.format(target_env_file), '--prefix "{}"'.format(env_path), ]) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, args=self._singularity_args, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) # cleanup if requested if self._cleanup is CondaCleanupMode.tarballs: logger.info("Cleaning up conda package tarballs.") shell.check_output("conda clean -y --tarballs") elif self._cleanup is CondaCleanupMode.cache: logger.info( "Cleaning up conda package tarballs and package cache." ) shell.check_output( "conda clean -y --tarballs --packages") # Touch "done" flag file with open(os.path.join(env_path, "env_setup_done"), "a") as f: pass logger.debug(out) logger.info("Environment for {} created (location: {})".format( os.path.relpath(env_file), os.path.relpath(env_path))) except subprocess.CalledProcessError as e: # remove potential partially installed environment shutil.rmtree(env_path, ignore_errors=True) raise CreateCondaEnvironmentException( "Could not create conda environment from {}:\n".format( env_file) + e.output) if tmp_file: # temporary file was created os.remove(tmp_file) return env_path
def create(self, dryrun=False): """Create the conda enviroment.""" from snakemake.shell import shell self.check_is_file_based() # Read env file and create hash. env_file = self.file deploy_file = None pin_file = None tmp_env_file = None tmp_deploy_file = None tmp_pin_file = None if not isinstance(env_file, LocalSourceFile) or isinstance( env_file, LocalGitFile): with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as tmp: # write to temp file such that conda can open it tmp.write(self.content) env_file = tmp.name tmp_env_file = tmp.name if self.post_deploy_file: with tempfile.NamedTemporaryFile( delete=False, suffix=".post-deploy.sh") as tmp: # write to temp file such that conda can open it tmp.write(self.content_deploy) deploy_file = tmp.name tmp_deploy_file = tmp.name if self.pin_file: with tempfile.NamedTemporaryFile(delete=False, suffix="pin.txt") as tmp: tmp.write(self.content_pin) pin_file = tmp.name tmp_pin_file = tmp.name else: env_file = env_file.get_path_or_uri() deploy_file = self.post_deploy_file pin_file = self.pin_file env_path = self.address if self.is_containerized: if not dryrun: try: shell.check_output( singularity.shellcmd( self._container_img.path, "[ -d '{}' ]".format(env_path), args=self._singularity_args, envvars=self.get_singularity_envvars(), quiet=True, ), stderr=subprocess.PIPE, ) except subprocess.CalledProcessError as e: raise WorkflowError( "Unable to find environment in container image. " "Maybe a conda environment was modified without containerizing again " "(see snakemake --containerize)?\nDetails:\n{}\n{}". format(e, e.stderr.decode())) return env_path else: # env should be present in the container return env_path # Check for broken environment if os.path.exists(os.path.join( env_path, "env_setup_start")) and not os.path.exists( os.path.join(env_path, "env_setup_done")): if dryrun: logger.info( "Incomplete Conda environment {} will be recreated.". format(self.file.simplify_path())) else: logger.info( "Removing incomplete Conda environment {}...".format( self.file.simplify_path())) shutil.rmtree(env_path, ignore_errors=True) # Create environment if not already present. if not os.path.exists(env_path): if dryrun: logger.info("Conda environment {} will be created.".format( self.file.simplify_path())) return env_path logger.info("Creating conda environment {}...".format( self.file.simplify_path())) env_archive = self.archive_file try: # Touch "start" flag file os.makedirs(env_path, exist_ok=True) with open(os.path.join(env_path, "env_setup_start"), "a") as f: pass # Check if env archive exists. Use that if present. if os.path.exists(env_archive): logger.info("Installing archived conda packages.") pkg_list = os.path.join(env_archive, "packages.txt") if os.path.exists(pkg_list): # read pacakges in correct order # this is for newer env archives where the package list # was stored packages = [ os.path.join(env_archive, pkg.rstrip()) for pkg in open(pkg_list) ] else: # guess order packages = glob(os.path.join(env_archive, "*.tar.bz2")) # install packages manually from env archive cmd = " ".join([ "conda", "create", "--quiet", "--yes", "--prefix '{}'".format(env_path), ] + packages) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, args=self._singularity_args, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) else: def create_env(env_file, filetype="yaml"): # Copy env file to env_path (because they can be on # different volumes and singularity should only mount one). # In addition, this allows to immediately see what an # environment in .snakemake/conda contains. target_env_file = env_path + f".{filetype}" shutil.copy(env_file, target_env_file) logger.info( "Downloading and installing remote packages.") strict_priority = ([ "conda config --set channel_priority strict &&" ] if self._container_img else []) subcommand = [self.frontend] yes_flag = ["--yes"] if filetype == "yaml": subcommand.append("env") yes_flag = [] cmd = (strict_priority + subcommand + [ "create", "--quiet", '--file "{}"'.format(target_env_file), '--prefix "{}"'.format(env_path), ] + yes_flag) cmd = " ".join(cmd) if self._container_img: cmd = singularity.shellcmd( self._container_img.path, cmd, args=self._singularity_args, envvars=self.get_singularity_envvars(), ) out = shell.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) # cleanup if requested if self._cleanup is CondaCleanupMode.tarballs: logger.info("Cleaning up conda package tarballs.") shell.check_output("conda clean -y --tarballs") elif self._cleanup is CondaCleanupMode.cache: logger.info( "Cleaning up conda package tarballs and package cache." ) shell.check_output( "conda clean -y --tarballs --packages") return out if pin_file is not None: try: logger.info( f"Using pinnings from {self.pin_file.get_path_or_uri()}." ) out = create_env(pin_file, filetype="pin.txt") except subprocess.CalledProcessError as e: # remove potential partially installed environment shutil.rmtree(env_path, ignore_errors=True) advice = "" if isinstance(self.file, LocalSourceFile): advice = ( " If that works, make sure to update the pin file with " f"'snakedeploy pin-conda-env {self.file.get_path_or_uri()}'." ) logger.warning( f"Failed to install conda environment from pin file ({self.pin_file.get_path_or_uri()}). " f"Trying regular environment definition file.{advice}" ) out = create_env(env_file, filetype="yaml") else: out = create_env(env_file, filetype="yaml") # Execute post-deplay script if present if deploy_file: target_deploy_file = env_path + ".post-deploy.sh" shutil.copy(deploy_file, target_deploy_file) self.execute_deployment_script(env_file, target_deploy_file) # Touch "done" flag file with open(os.path.join(env_path, "env_setup_done"), "a") as f: pass logger.debug(out) logger.info( f"Environment for {self.file.get_path_or_uri()} created (location: {os.path.relpath(env_path)})" ) except subprocess.CalledProcessError as e: # remove potential partially installed environment shutil.rmtree(env_path, ignore_errors=True) raise CreateCondaEnvironmentException( f"Could not create conda environment from {env_file}:\nCommand:\n{e.cmd}\nOutput:\n{e.output}" ) if tmp_env_file: # temporary file was created os.remove(tmp_env_file) if tmp_deploy_file: os.remove(tmp_deploy_file) return env_path
def __new__(cls, cmd, *args, iterable=False, read=False, bench_record=None, **kwargs): if "stepout" in kwargs: raise KeyError("Argument stepout is not allowed in shell command.") if ON_WINDOWS and not cls.get_executable(): # If bash is not used on Windows quoting must be handled in a special way kwargs["quote_func"] = cmd_exe_quote cmd = format(cmd, *args, stepout=2, **kwargs) stdout = sp.PIPE if iterable or read else STDOUT close_fds = sys.platform != "win32" func_context = inspect.currentframe().f_back.f_locals if func_context.get(RULEFUNC_CONTEXT_MARKER): # If this comes from a rule, we expect certain information to be passed # implicitly via the rule func context, which is added here. context = func_context else: # Otherwise, context is just filled via kwargs. context = dict() # add kwargs to context (overwriting the locals of the caller) context.update(kwargs) jobid = context.get("jobid") if not context.get("is_shell"): logger.shellcmd(cmd) conda_env = context.get("conda_env", None) conda_base_path = context.get("conda_base_path", None) container_img = context.get("container_img", None) env_modules = context.get("env_modules", None) shadow_dir = context.get("shadow_dir", None) resources = context.get("resources", {}) singularity_args = context.get("singularity_args", "") threads = context.get("threads", 1) cmd = " ".join((cls._process_prefix, cmd, cls._process_suffix)).strip() if env_modules: cmd = env_modules.shellcmd(cmd) logger.info( "Activating environment modules: {}".format(env_modules)) if conda_env: if ON_WINDOWS and not cls.get_executable(): # If we use cmd.exe directly on winodws we need to prepend batch activation script. cmd = Conda(container_img, prefix_path=conda_base_path).shellcmd_win( conda_env, cmd) else: cmd = Conda(container_img, prefix_path=conda_base_path).shellcmd( conda_env, cmd) tmpdir = None if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN: tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="shell_tmp.") script = os.path.join(os.path.abspath(tmpdir), "script.sh") with open(script, "w") as script_fd: print(cmd, file=script_fd) os.chmod(script, os.stat(script).st_mode | stat.S_IXUSR | stat.S_IRUSR) cmd = '"{}" "{}"'.format(cls.get_executable() or "/bin/sh", script) if container_img: cmd = singularity.shellcmd( container_img, cmd, singularity_args, envvars=None, shell_executable=cls._process_args["executable"], container_workdir=shadow_dir, is_python_script=context.get("is_python_script", False), ) logger.info( "Activating singularity image {}".format(container_img)) if conda_env: logger.info("Activating conda environment: {}".format( os.path.relpath(conda_env))) tmpdir_resource = resources.get("tmpdir", None) # environment variable lists for linear algebra libraries taken from: # https://stackoverflow.com/a/53224849/2352071 # https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables envvars = dict(os.environ) threads = str(threads) envvars["OMP_NUM_THREADS"] = threads envvars["GOTO_NUM_THREADS"] = threads envvars["OPENBLAS_NUM_THREADS"] = threads envvars["MKL_NUM_THREADS"] = threads envvars["VECLIB_MAXIMUM_THREADS"] = threads envvars["NUMEXPR_NUM_THREADS"] = threads if tmpdir_resource: envvars["TMPDIR"] = tmpdir_resource envvars["TMP"] = tmpdir_resource envvars["TEMPDIR"] = tmpdir_resource envvars["TEMP"] = tmpdir_resource if "additional_envvars" in kwargs: env = kwargs["additional_envvars"] if not isinstance(env, dict) or not all( isinstance(v, str) for v in env.values()): raise WorkflowError( "Given environment variables for shell command have to be a dict of strings, " "but the following was provided instead:\n{}".format(env)) envvars.update(env) if conda_env and cls.conda_block_conflicting_envvars: # remove envvars that conflict with conda for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]: try: del envvars[var] except KeyError: pass use_shell = True if ON_WINDOWS and cls.get_executable(): # If executable is set on Windows shell mode can not be used # and the executable should be prepended the command together # with a command prefix (e.g. -c for bash). use_shell = False cmd = '"{}" {} {}'.format(cls.get_executable(), cls._win_command_prefix, argvquote(cmd)) proc = sp.Popen( cmd, bufsize=-1, shell=use_shell, stdout=stdout, universal_newlines=iterable or read or None, close_fds=close_fds, **cls._process_args, env=envvars, ) if jobid is not None: with cls._lock: cls._processes[jobid] = proc ret = None if iterable: return cls.iter_stdout(proc, cmd, tmpdir) if read: ret = proc.stdout.read() if bench_record is not None: from snakemake.benchmark import benchmarked with benchmarked(proc.pid, bench_record): retcode = proc.wait() else: retcode = proc.wait() if tmpdir: shutil.rmtree(tmpdir) if jobid is not None: with cls._lock: del cls._processes[jobid] if retcode: raise sp.CalledProcessError(retcode, cmd) return ret
def __new__(cls, cmd, *args, iterable=False, read=False, bench_record=None, **kwargs): if "stepout" in kwargs: raise KeyError("Argument stepout is not allowed in shell command.") cmd = format(cmd, *args, stepout=2, **kwargs) context = inspect.currentframe().f_back.f_locals # add kwargs to context (overwriting the locals of the caller) context.update(kwargs) stdout = sp.PIPE if iterable or read else STDOUT close_fds = sys.platform != "win32" jobid = context.get("jobid") if not context.get("is_shell"): logger.shellcmd(cmd) conda_env = context.get("conda_env", None) container_img = context.get("container_img", None) env_modules = context.get("env_modules", None) shadow_dir = context.get("shadow_dir", None) cmd = " ".join((cls._process_prefix, cmd, cls._process_suffix)).strip() if env_modules: cmd = env_modules.shellcmd(cmd) logger.info( "Activating environment modules: {}".format(env_modules)) if conda_env: cmd = Conda(container_img).shellcmd(conda_env, cmd) tmpdir = None if len(cmd.replace("'", r"'\''")) + 2 > MAX_ARG_LEN: tmpdir = tempfile.mkdtemp(dir=".snakemake", prefix="shell_tmp.") script = os.path.join(os.path.abspath(tmpdir), "script.sh") with open(script, "w") as script_fd: print(cmd, file=script_fd) os.chmod(script, os.stat(script).st_mode | stat.S_IXUSR | stat.S_IRUSR) cmd = '"{}" "{}"'.format(cls.get_executable() or "/bin/sh", script) if container_img: args = context.get("singularity_args", "") cmd = singularity.shellcmd( container_img, cmd, args, envvars=None, shell_executable=cls._process_args["executable"], container_workdir=shadow_dir, ) logger.info( "Activating singularity image {}".format(container_img)) if conda_env: logger.info("Activating conda environment: {}".format(conda_env)) threads = str(context.get("threads", 1)) # environment variable lists for linear algebra libraries taken from: # https://stackoverflow.com/a/53224849/2352071 # https://github.com/xianyi/OpenBLAS/tree/59243d49ab8e958bb3872f16a7c0ef8c04067c0a#setting-the-number-of-threads-using-environment-variables envvars = dict(os.environ) envvars["OMP_NUM_THREADS"] = threads envvars["GOTO_NUM_THREADS"] = threads envvars["OPENBLAS_NUM_THREADS"] = threads envvars["MKL_NUM_THREADS"] = threads envvars["VECLIB_MAXIMUM_THREADS"] = threads envvars["NUMEXPR_NUM_THREADS"] = threads if conda_env and cls.conda_block_conflicting_envvars: # remove envvars that conflict with conda for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]: try: del envvars[var] except KeyError: pass use_shell = True if ON_WINDOWS and cls.get_executable(): # If executable is set on Windows shell mode can not be used # and the executable should be prepended the command together # with a command prefix (e.g. -c for bash). use_shell = False cmd = '"{}" {} {}'.format(cls.get_executable(), cls._win_command_prefix, argvquote(cmd)) proc = sp.Popen( cmd, bufsize=-1, shell=use_shell, stdout=stdout, universal_newlines=iterable or read or None, close_fds=close_fds, **cls._process_args, env=envvars, ) if jobid is not None: with cls._lock: cls._processes[jobid] = proc ret = None if iterable: return cls.iter_stdout(proc, cmd, tmpdir) if read: ret = proc.stdout.read() if bench_record is not None: from snakemake.benchmark import benchmarked with benchmarked(proc.pid, bench_record): retcode = proc.wait() else: retcode = proc.wait() if tmpdir: shutil.rmtree(tmpdir) if jobid is not None: with cls._lock: del cls._processes[jobid] if retcode: raise sp.CalledProcessError(retcode, cmd) return ret
def _get_cmd(self, cmd): if self.singularity_img: return singularity.shellcmd(self.singularity_img, cmd) return cmd