def update_conda(self): """ Update the version of conda, if possible. The defaults channel is used because that is the default for a normal miniconda installation. Parameters ---------- None """ command_list = [self.conda_exe, 'update', '-n', 'base', '-c', 'defaults', '-y', 'conda'] try: output = check_output(command_list, env=self.env) except Exception: print(""" ******************************************************************************* There was a failure in updating your base conda installaion. To update manually, try running conda update -n base -c defaults conda If you are using conda from a different channel, replace "defaults" with that channel ******************************************************************************* """) else: print(output)
def install_miniconda(self, prefix=None): """ Download a miniconda installer and install. The default installation location is at the same directory level as the "modules" directory. Parameters ---------- prefix: str The installation directory for miniconda Returns ------- conda_base: str The location of the "base" conda environment """ if prefix is None: prefix = self.root_dir # construct Miniconda3 filename os_names = { 'Darwin': 'MacOSX', 'Linux': 'Linux', 'Windows': 'Windows', } filename = 'Miniconda3-latest-{platform}-x86_64'.format( platform=os_names[self.system]) if self.system == 'Windows': filename += '.exe' else: filename += '.sh' url_base = 'https://repo.anaconda.com/miniconda/' url = urljoin(url_base, filename) filename = os.path.join(prefix, filename) # Download from public repository if not os.path.isfile(filename): print('Downloading {url}'.format(url=url), file=self.log) download_file(url, filename) print('Downloaded file to {filename}'.format(filename=filename), file=self.log) else: print('Using local copy at {filename}'.format(filename=filename), file=self.log) # run the installer install_dir = os.path.join(prefix, 'mc3') if self.system == 'Windows': flags = '/InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /S /D={install_dir}'.\ format(install_dir=install_dir) command_list = ['"' + filename + '"', flags] else: flags = '-b -u -p "{install_dir}"'.format(install_dir=install_dir) command_list = ['/bin/sh', filename, flags] print('Installing miniconda to "{install_dir}"'.format( install_dir=install_dir), file=self.log) output = check_output(command_list, env=self.env) if self.verbose: print(output, file=self.log) return install_dir
def create_environment(self, builder='cctbx', filename=None, python=None, copy=False, offline=False): """ Create the environment based on the builder and file. The environment name is "conda_base". Parameters ---------- builder: str The builder from bootstrap.py. The default environment is defined by the env_locations class variable filename: str If filename is not None, the argument overrides the file defined in the env_locations dictionary. The filename should be a relative path to the "modules" directory. python: str If set, the specific Python version of the environment for the builder is used instead of the default. Current options are '27' and '36' for Python 2.7 and 3.6, respectively. copy: bool If set to True, the --copy flag is passed to conda offline: bool If set to True, the --offline flag is passed to conda """ # handles check for choices in case parser is not available if builder not in self.env_locations: raise RuntimeError(""" The builder, {builder}, is not recognized. The available builders are, {builders} """.\ format(builder=builder, builders=', '.join(sorted(self.env_locations.keys())))) if self.conda_base is None: raise RuntimeError("""A conda installation is not available.""") if filename is None: filename = os.path.join( self.root_dir, 'modules', self.env_locations[builder]) if python is not None: if python not in ['27', '36']: raise RuntimeError( """Only Python 2.7 and 3.6 are currently supported.""") filename = filename.replace('PYTHON_VERSION', python) else: filename = os.path.abspath(filename) if not os.path.isfile(filename): raise RuntimeError("""The file, {filename}, is not available""".\ format(filename=filename)) yaml_format = False if filename.endswith('yml') or filename.endswith('yaml'): yaml_format = True # make a new environment directory if self.conda_env is None: name = 'conda_base' prefix = os.path.join(self.root_dir, name) # or use the existing one else: prefix = os.path.abspath(self.conda_env) # install a new environment or update and existing one if prefix in self.environments: command = 'install' if yaml_format: command = 'update' text_messages = ['Updating', 'update of'] else: command = 'create' text_messages = ['Installing', 'installation into'] command_list = [self.conda_exe, command, '--prefix', prefix, '--file', filename] if yaml_format: command_list.insert(1, 'env') if self.system == 'Windows': command_list = [os.path.join(self.conda_base, 'Scripts', 'activate'), 'base', '&&'] + command_list if copy and not yaml_format: command_list.append('--copy') if offline and not yaml_format: command_list.append('--offline') if builder == "dials": command_list.append("-y") # RuntimeError is raised on failure print('{text} {builder} environment with:\n {filename}'.format( text=text_messages[0], builder=builder, filename=filename), file=self.log) for retry in range(self.max_retries): retry += 1 try: output = check_output(command_list, env=self.env) except Exception: print(""" ******************************************************************************* There was a failure in constructing the conda environment. Attempt {retry} of {max_retries} will start {retry} minute(s) from {t}. ******************************************************************************* """.format(retry=retry, max_retries=self.max_retries, t=time.asctime())) time.sleep(retry*60) else: break if retry == self.max_retries: raise RuntimeError(""" The conda environment could not be constructed. Please check that there is a working network connection for downloading conda packages. """) if self.verbose: print(output, file=self.log) print('Completed {text}:\n {prefix}'.format(text=text_messages[1], prefix=prefix), file=self.log) # on Windows, also download the Visual C++ 2008 Redistributable # use the same version as conda-forge # https://github.com/conda-forge/vs2008_runtime-feedstock if self.system == 'Windows' and prefix.endswith('conda_base'): download_file( url='https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe', filename=os.path.join(prefix, 'vcredist_x64.exe')) # check that environment file is updated self.environments = self.update_environments() if prefix not in self.environments: raise RuntimeError(""" The newly installed environment cannot be found in ${HOME}/.conda/environments.txt. """)
def __init__(self, root_dir=root_dir, conda_base=None, conda_env=None, check_file=True, max_retries=5, verbose=False, log=sys.stdout): """ Constructor that performs a basic check for the conda installation. If an installation is not found, the latest version can be downloaded and installed. Parameters ---------- root_dir: str Required argument for specifying the root level directory for CCTBX builds. This is where the "modules" and "build" directories reside. If a new conda installation is required, it will be placed here under the "mc3" directory. The conda environment for building will also be placed in this directory conda_base: str Required argument for specifying location of a conda installation. If this is None, miniconda will be installed unless conda_env is set to a valid path. conda_env: str Optional argument for specifying location of a conda environment. Since this is assumed to be a working conda environment, a new installation of conda will not be created. This is for developers that want to manage their own environments. check_file: bool Flag for checking if a file exists. A RuntimeError is raised if a this flag is set and a file does not exist. Used in get_conda_exe and get_conda_python. max_retries: int When downloading conda packages, there may be network issues that prevent the environment from being constructed. This parameter controls the number of retry attempts for constructing the conda environment. The first retry is attempted 1 minute after the initial failure. The second retry is attempted 2 minutes, etc. verbose: bool Flag for showing conda output log: file For storing log output """ self.system = platform.system() self.root_dir = root_dir self.conda_base = None if conda_base is not None: self.conda_base = os.path.normpath(conda_base) self.conda_env = None if conda_env is not None: self.conda_env = os.path.normpath(conda_env) self.check_file = check_file self.max_retries = max_retries self.verbose = verbose self.log = log self.conda_exe = None if self.conda_base is not None: self.conda_exe = self.get_conda_exe(self.conda_base, check_file=False) # default environment file for users self.environment_file = os.path.join( os.path.expanduser('~'), '.conda', 'environments.txt') self.environments = self.update_environments() # Clean environment for external Python processes self.env = os.environ.copy() self.env['PYTHONPATH'] = '' # error messages self.conda_exe_not_found = """ The conda executable cannot be found. Please make sure the correct directory for the base conda installation was provided. The directory can be found by running "conda info" and looking the "base environment" value.""" # try to determine base environment from $PATH if self.conda_base is None: paths = os.environ.get('PATH') if paths is not None: if self.system == 'Windows': paths = paths.split(';') else: paths = paths.split(':') for path in paths: conda_base = os.path.abspath(os.path.join(path, '..')) conda_exe = self.get_conda_exe(conda_base, check_file=False) if os.path.isfile(conda_exe): self.conda_base = conda_base self.conda_exe = conda_exe break # try to determine base environment from .conda/environments.txt if self.conda_base is None: for environment in self.environments: conda_exe = self.get_conda_exe(environment, check_file=False) if os.path.isfile(conda_exe): self.conda_base = environment self.conda_exe = conda_exe break # ------------------------------------------------------------------------- # helper function for searching for conda_exe def walk_up_check(new_conda_base, new_conda_exe): if new_conda_base is None: new_conda_base = '' if new_conda_exe is None: new_conda_exe = '' while not os.path.isfile(new_conda_exe): # move up directory and recheck new_conda_base = os.path.abspath(os.path.join(new_conda_base, '..')) new_conda_exe = self.get_conda_exe(new_conda_base, check_file=False) if os.path.isfile(new_conda_exe): return new_conda_base, new_conda_exe # moved to root directory if new_conda_base == \ os.path.abspath(os.path.join(new_conda_base, '..')): return '', '' # ------------------------------------------------------------------------- # try to determine base evironment from conda_env if (self.conda_base is None) and (self.conda_env is not None): conda_base, conda_exe = walk_up_check(self.conda_env, self.conda_exe) if os.path.isfile(conda_exe): self.conda_base = conda_base self.conda_exe = conda_exe # install conda if necessary if (self.conda_base is None) and (self.conda_env is None): install_dir = os.path.join(self.root_dir, 'mc3') if os.path.isdir(install_dir): print('Using default conda installation', file=self.log) self.conda_base = install_dir else: print('Location of conda installation not provided', file=self.log) print('Proceeding with a fresh installation', file=self.log) self.conda_base = self.install_miniconda(prefix=self.root_dir) self.conda_exe = self.get_conda_exe(self.conda_base, check_file=False) self.environments = self.update_environments() # verify consistency and check conda version if self.conda_base is not None: # maybe a conda evironment was provided instead of the base environment if not os.path.isfile(self.conda_exe): self.conda_base, self.conda_exe = \ walk_up_check(self.conda_base, self.conda_exe) self.environments = self.update_environments() if not os.path.isfile(self.conda_exe): raise RuntimeError(self.conda_exe_not_found) conda_info = json.loads(check_output([self.conda_exe, 'info', '--json'], env=self.env)) consistency_check = [self.conda_base == conda_info['root_prefix']] for env in self.environments: consistency_check.append(env in conda_info['envs']) if False in consistency_check: message = """ There is a mismatch between the conda settings in your home directory and what "conda info" is reporting. This is not a fatal error, but if an error is encountered, please check that your conda installation and environments exist and are working. """ warnings.warn(message, RuntimeWarning) if conda_info['conda_version'] < '4.4': raise RuntimeError(""" CCTBX programs require conda version 4.4 and greater to make use of the common compilers provided by conda. Please update your version with "conda update conda". """) print('Base conda installation:\n {base}'.format(base=self.conda_base), file=self.log) if self.verbose: output = check_output([self.conda_exe, 'info'], env=self.env) print(output, file=self.log) if self.conda_env is not None: print('Build environment:\n {conda_env}'.format( conda_env=self.conda_env), file=self.log)
def create_environment(self, builder='cctbx', filename=None, copy=False, offline=False): """ Create the environment based on the builder and file. The environment name is "conda_base". Parameters ---------- builder: str The builder from bootstrap.py. The default environment is defined by the env_locations class variable filename: str If filename is not None, the argument overrides the file defined in the env_locations dictionary. The filename should be a relative path to the "modules" directory. copy: bool If set to True, the --copy flag is passed to conda offline: bool If set to True, the --offline flag is passed to conda """ # handles check for choices in case parser is not available if builder not in self.env_locations: raise RuntimeError(""" The builder, {builder}, is not recognized. The available builders are, {builders} """.\ format(builder=builder, builders=', '.join(sorted(self.env_locations.keys())))) if self.conda_base is None: raise RuntimeError("""A conda installation is not available.""") if filename is None: filename = os.path.join(self.root_dir, 'modules', self.env_locations[builder]) else: filename = os.path.abspath(filename) if not os.path.isfile(filename): raise RuntimeError("""The file, {filename}, is not available""".\ format(filename=filename)) # make a new environment directory if self.conda_env is None: name = 'conda_base' prefix = os.path.join(self.root_dir, name) # or use the existing one else: prefix = os.path.abspath(self.conda_env) # install a new environment or update and existing one if prefix in self.environments: command = 'install' text_messages = ['Updating', 'update of'] else: command = 'create' text_messages = ['Installing', 'installation into'] command_list = [ self.conda_exe, command, '--prefix', prefix, '--file', filename ] if self.system == 'Windows': command_list = [ os.path.join(self.conda_base, 'Scripts', 'activate'), 'base', '&&' ] + command_list if copy: command_list.append('--copy') if offline: command_list.append('--offline') # RuntimeError is raised on failure print('{text} {builder} environment with:\n {filename}'.format( text=text_messages[0], builder=builder, filename=filename), file=self.log) output = check_output(command_list, env=self.env) if self.verbose: print(output, file=self.log) print('Completed {text}:\n {prefix}'.format(text=text_messages[1], prefix=prefix), file=self.log) # on Windows, also download the Visual C++ 2008 Redistributable # use the same version as conda-forge # https://github.com/conda-forge/vs2008_runtime-feedstock if self.system == 'Windows' and prefix.endswith('conda_base'): download_file( url= 'https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe', filename=os.path.join(prefix, 'vcredist_x64.exe')) # check that environment file is updated self.environments = self.update_environments() if prefix not in self.environments: raise RuntimeError(""" The newly installed environment cannot be found in ${HOME}/.conda/environments.txt. """)