def serialize(self): """ Returns a string of the shell command """ if is_windows_platform(): return self.ARGV_SEPARATOR.join(map(double_quote, self)) return self.ARGV_SEPARATOR.join(map(quote, self))
def create(self): """ Create a new environment """ output = Argv( self.conda, "create", "--yes", "--mkdir", "--prefix", self.path, "python={}".format(self.python), ).get_output(stderr=DEVNULL) match = re.search( r"\W*(.*activate) ({})".format(re.escape(str(self.path))), output ) self.source = self.pip.source = ( tuple(match.group(1).split()) + (match.group(2),) if match else ("activate", self.path) ) conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh' if conda_env.is_file() and not is_windows_platform(): self.source = self.pip.source = CommandSequence(('source', conda_env.as_posix()), self.source) # install cuda toolkit try: cuda_version = float(int(self.session.config['agent.cuda_version'])) / 10.0 if cuda_version > 0: self._install('cudatoolkit={:.1f}'.format(cuda_version)) except Exception: pass return self
def __init__(self, source=None, *args, **kwargs): super(CondaPip, self).__init__( *args, interpreter=Path(kwargs.get('path'), "python.exe") if is_windows_platform() and kwargs.get('path') else None, **kwargs) self.source = source
def remove(self): """ Delete a conda environment. Use 'conda env remove', then 'rm_tree' to be safe. Conda seems to load "vcruntime140.dll" from all its environment on startup. This means environment have to be deleted using 'conda env remove'. If necessary, conda can be fooled into deleting a partially-deleted environment by creating an empty file in '<ENV>\conda-meta\history' (value found in 'conda.gateways.disk.test.PREFIX_MAGIC_FILE'). Otherwise, it complains that said directory is not a conda environment. See: https://github.com/conda/conda/issues/7682 """ try: self._run_command(("env", "remove", "-p", self.path)) except Exception: pass rm_tree(self.path) # if we failed removing the path, change it's name if is_windows_platform() and Path(self.path).exists(): try: Path(self.path).rename( Path(self.path).as_posix() + '_' + str(time())) except Exception: pass
def pretty(self): serialized = self.serialize() if is_windows_platform(): return " ".join(serialized) return serialized
def normalize(command): return list(command) if is_windows_platform() else command.serialize()
def get_cuda_version(config): # type: (ConfigTree) -> (Text, Text) # we assume os.environ already updated the config['agent.cuda_version'] & config['agent.cudnn_version'] cuda_version = config['agent.cuda_version'] cudnn_version = config['agent.cudnn_version'] if cuda_version and cudnn_version: return normalize_cuda_version( cuda_version), normalize_cuda_version(cudnn_version) if not cuda_version and is_windows_platform(): try: cuda_vers = [ int(k.replace('CUDA_PATH_V', '').replace('_', '')) for k in os.environ.keys() if k.startswith('CUDA_PATH_V') ] cuda_vers = max(cuda_vers) if cuda_vers > 40: cuda_version = cuda_vers except: pass if not cuda_version: try: try: nvcc = 'nvcc.exe' if is_windows_platform() else 'nvcc' if is_windows_platform() and 'CUDA_PATH' in os.environ: nvcc = os.path.join(os.environ['CUDA_PATH'], nvcc) output = Argv(nvcc, '--version').get_output() except OSError: raise CudaNotFound('nvcc not found') match = re.search(r'release (.{3})', output).group(1) cuda_version = Text(int(float(match) * 10)) except: pass if not cuda_version: try: try: output = Argv('nvidia-smi', ).get_output() except OSError: raise CudaNotFound('nvcc not found') match = re.search(r'CUDA Version: ([0-9]+).([0-9]+)', output) match = match.group(1) + '.' + match.group(2) cuda_version = Text(int(float(match) * 10)) except: pass if not cudnn_version: try: cuda_lib = which('nvcc') if is_windows_platform: cudnn_h = path.sep.join( cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h']) else: cudnn_h = path.join( path.sep, *(cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h'])) cudnn_major, cudnn_minor = None, None try: include_file = open(cudnn_h) except OSError: raise CudaNotFound('Could not read cudnn.h') with include_file: for line in include_file: if 'CUDNN_MAJOR' in line: cudnn_major = line.split()[-1] if 'CUDNN_MINOR' in line: cudnn_minor = line.split()[-1] if cudnn_major and cudnn_minor: break cudnn_version = cudnn_major + (cudnn_minor or '0') except: pass return (normalize_cuda_version(cuda_version or 0), normalize_cuda_version(cudnn_version or 0))
def __iter__(self): if is_windows_platform(): return (word.as_posix().replace('/', '\\') if isinstance( word, Path) else six.text_type(word) for word in self.argv) return (six.text_type(word) for word in self.argv)
def create(self): """ Create a new environment """ if self.conda_env_as_base_docker and self.conda_pre_build_env_path: if Path(self.conda_pre_build_env_path).is_dir(): print("Using pre-existing Conda environment from {}".format( self.conda_pre_build_env_path)) self.path = Path(self.conda_pre_build_env_path) self.source = ("conda", "activate", self.path.as_posix()) self.pip = CondaPip( session=self.session, source=self.source, python=self.python, requirements_manager=self.requirements_manager, path=self.path, ) conda_env = Path( self.conda ).parent.parent / 'etc' / 'profile.d' / 'conda.sh' self.source = self.pip.source = CommandSequence( ('source', conda_env.as_posix()), self.source) self.env_read_only = True return self elif Path(self.conda_pre_build_env_path).is_file(): print("Restoring Conda environment from {}".format( self.conda_pre_build_env_path)) tar_path = find_executable("tar") self.path.mkdir(parents=True, exist_ok=True) output = Argv( tar_path, "-xzf", self.conda_pre_build_env_path, "-C", self.path, ).get_output() self.source = self.pip.source = ("conda", "activate", self.path.as_posix()) conda_env = Path( self.conda ).parent.parent / 'etc' / 'profile.d' / 'conda.sh' self.source = self.pip.source = CommandSequence( ('source', conda_env.as_posix()), self.source) # unpack cleanup print("Fixing prefix in Conda environment {}".format( self.path)) CommandSequence(('source', conda_env.as_posix()), ((self.path / 'bin' / 'conda-unpack').as_posix(), )).get_output() return self else: raise ValueError( "Could not restore Conda environment, cannot find {}". format(self.conda_pre_build_env_path)) output = Argv( self.conda, "create", "--yes", "--mkdir", "--prefix", self.path, "python={}".format(self.python), ).get_output(stderr=DEVNULL) match = re.search( r"\W*(.*activate) ({})".format(re.escape(str(self.path))), output) self.source = self.pip.source = (tuple(match.group(1).split()) + (match.group(2), ) if match else ("conda", "activate", self.path.as_posix())) conda_env = Path( self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh' if conda_env.is_file() and not is_windows_platform(): self.source = self.pip.source = CommandSequence( ('source', conda_env.as_posix()), self.source) # install cuda toolkit # noinspection PyBroadException try: cuda_version = float(int( self.session.config['agent.cuda_version'])) / 10.0 if cuda_version > 0: self._install('cudatoolkit={:.1f}'.format(cuda_version)) except Exception: pass return self