def serialize(self): """ Returns a string of the shell command """ if is_windows_platform(): return self.ARGV_SEPARATOR.join(map(double_quote, self)) return self.ARGV_SEPARATOR.join(map(quote, self))
def __init__(self, source=None, *args, **kwargs): super(CondaPip, self).__init__( *args, interpreter=Path(kwargs.get('path'), "python.exe") if is_windows_platform() and kwargs.get('path') else None, **kwargs) self.source = source
def remove(self): """ Delete a conda environment. Use 'conda env remove', then 'rm_tree' to be safe. Conda seems to load "vcruntime140.dll" from all its environment on startup. This means environment have to be deleted using 'conda env remove'. If necessary, conda can be fooled into deleting a partially-deleted environment by creating an empty file in '<ENV>\conda-meta\history' (value found in 'conda.gateways.disk.test.PREFIX_MAGIC_FILE'). Otherwise, it complains that said directory is not a conda environment. See: https://github.com/conda/conda/issues/7682 """ try: self._run_command(("env", "remove", "-p", self.path)) except Exception: pass rm_tree(self.path) # if we failed removing the path, change it's name if is_windows_platform() and Path(self.path).exists(): try: Path(self.path).rename( Path(self.path).as_posix() + '_' + str(time())) except Exception: pass
def __iter__(self): if is_windows_platform(): return (word.as_posix().replace('/', '\\') if isinstance(word, Path) else six.text_type(word) for word in self.argv) return (six.text_type(word) for word in self.argv)
def create(self): """ Create a new environment """ if self.conda_env_as_base_docker and self.conda_pre_build_env_path: if Path(self.conda_pre_build_env_path).is_dir(): self._init_existing_environment(self.conda_pre_build_env_path) return self elif Path(self.conda_pre_build_env_path).is_file(): print("Restoring Conda environment from {}".format( self.conda_pre_build_env_path)) tar_path = find_executable("tar") self.path.mkdir(parents=True, exist_ok=True) output = Argv( tar_path, "-xzf", self.conda_pre_build_env_path, "-C", self.path, ).get_output() self.source = self.pip.source = ("conda", "activate", self.path.as_posix()) conda_env = self._get_conda_sh() self.source = self.pip.source = CommandSequence( ('source', conda_env.as_posix()), self.source) # unpack cleanup print("Fixing prefix in Conda environment {}".format( self.path)) CommandSequence(('source', conda_env.as_posix()), ((self.path / 'bin' / 'conda-unpack').as_posix(), )).get_output() return self else: raise ValueError( "Could not restore Conda environment, cannot find {}". format(self.conda_pre_build_env_path)) output = Argv( self.conda, "create", "--yes", "--mkdir", "--prefix", self.path, "python={}".format(self.python), ).get_output(stderr=DEVNULL) match = re.search( r"\W*(.*activate) ({})".format(re.escape(str(self.path))), output) self.source = self.pip.source = (tuple(match.group(1).split()) + (match.group(2), ) if match else ("conda", "activate", self.path.as_posix())) conda_env = self._get_conda_sh() if conda_env.is_file() and not is_windows_platform(): self.source = self.pip.source = CommandSequence( ('source', conda_env.as_posix()), self.source) # install cuda toolkit # noinspection PyBroadException try: cuda_version = float(int( self.session.config['agent.cuda_version'])) / 10.0 if cuda_version > 0: self._install('cudatoolkit={:.1f}'.format(cuda_version)) except Exception: pass return self
def _register_instance(cls, unique_worker_id=None, worker_name=None, api_client=None, allow_double=False): if cls.worker_id and cls.instance_slot is not None: return cls.worker_id, cls.instance_slot # make sure we have a unique name instance_num = 0 slots = {} for pid, uid, slot, file in cls.get_running_pids(): worker = None if api_client and ENV_DOCKER_HOST_MOUNT.get() and uid: try: worker = [ w for w in api_client.workers.get_all() if w.id == uid ] except Exception: worker = None # count active instances and delete dead files if not worker and pid < 0: # delete the file try: os.remove(os.path.join(file)) except Exception: pass continue instance_num += 1 if slot is None: continue if uid == unique_worker_id: if allow_double: warning( 'Instance with the same WORKER_ID [{}] was found on this machine. ' 'We are ignoring it, make sure this not a mistake.'. format(unique_worker_id)) else: return None, None slots[slot] = uid # get a new slot if not slots: cls.instance_slot = 0 else: # guarantee we have the minimal slot possible for i in range(max(slots.keys()) + 2): if i not in slots: cls.instance_slot = i break # build worker id based on slot if not unique_worker_id: unique_worker_id = worker_name + cls.worker_name_sep + str( cls.instance_slot) # create lock cls._pid = str(os.getpid()) cls._pid_file = NamedTemporaryFile( dir=cls._get_temp_folder(), prefix=cls.prefix + cls.sep + cls._pid + cls.sep, suffix=cls.ext, delete=False if is_windows_platform() else True) cls._pid_file.write(('{}\n{}'.format(unique_worker_id, cls.instance_slot)).encode()) cls._pid_file.flush() cls.worker_id = unique_worker_id return cls.worker_id, cls.instance_slot
def get_cuda_version(config): # type: (ConfigTree) -> (Text, Text) # we assume os.environ already updated the config['agent.cuda_version'] & config['agent.cudnn_version'] cuda_version = config['agent.cuda_version'] cudnn_version = config['agent.cudnn_version'] if cuda_version and cudnn_version: return normalize_cuda_version( cuda_version), normalize_cuda_version(cudnn_version) if not cuda_version and is_windows_platform(): try: cuda_vers = [ int(k.replace('CUDA_PATH_V', '').replace('_', '')) for k in os.environ.keys() if k.startswith('CUDA_PATH_V') ] cuda_vers = max(cuda_vers) if cuda_vers > 40: cuda_version = cuda_vers except: pass if not cuda_version: try: try: nvcc = 'nvcc.exe' if is_windows_platform() else 'nvcc' if is_windows_platform() and 'CUDA_PATH' in os.environ: nvcc = os.path.join(os.environ['CUDA_PATH'], nvcc) output = Argv(nvcc, '--version').get_output() except OSError: raise CudaNotFound('nvcc not found') match = re.search(r'release (.{3})', output).group(1) cuda_version = Text(int(float(match) * 10)) except: pass if not cuda_version: try: try: output = Argv('nvidia-smi', ).get_output() except OSError: raise CudaNotFound('nvcc not found') match = re.search(r'CUDA Version: ([0-9]+).([0-9]+)', output) match = match.group(1) + '.' + match.group(2) cuda_version = Text(int(float(match) * 10)) except: pass if not cudnn_version: try: cuda_lib = which('nvcc') if is_windows_platform: cudnn_h = path.sep.join( cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h']) else: cudnn_h = path.join( path.sep, *(cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h'])) cudnn_major, cudnn_minor = None, None try: include_file = open(cudnn_h) except OSError: raise CudaNotFound('Could not read cudnn.h') with include_file: for line in include_file: if 'CUDNN_MAJOR' in line: cudnn_major = line.split()[-1] if 'CUDNN_MINOR' in line: cudnn_minor = line.split()[-1] if cudnn_major and cudnn_minor: break cudnn_version = cudnn_major + (cudnn_minor or '0') except: pass return (normalize_cuda_version(cuda_version or 0), normalize_cuda_version(cudnn_version or 0))