def update_conda(self):
"""
Update the version of conda, if possible. The defaults channel is
used because that is the default for a normal miniconda installation.
Parameters
----------
None
"""
command_list = [self.conda_exe, 'update', '-n', 'base', '-c', 'defaults',
'-y', 'conda']
try:
output = check_output(command_list, env=self.env)
except Exception:
print("""
*******************************************************************************
There was a failure in updating your base conda installaion. To update
manually, try running
conda update -n base -c defaults conda
If you are using conda from a different channel, replace "defaults" with that
channel
*******************************************************************************
""")
else:
print(output)
def install_miniconda(self, prefix=None):
"""
Download a miniconda installer and install. The default installation
location is at the same directory level as the "modules" directory.
Parameters
----------
prefix: str
The installation directory for miniconda
Returns
-------
conda_base: str
The location of the "base" conda environment
"""
if prefix is None:
prefix = self.root_dir
# construct Miniconda3 filename
os_names = {
'Darwin': 'MacOSX',
'Linux': 'Linux',
'Windows': 'Windows',
}
filename = 'Miniconda3-latest-{platform}-x86_64'.format(
platform=os_names[self.system])
if self.system == 'Windows':
filename += '.exe'
else:
filename += '.sh'
url_base = 'https://repo.anaconda.com/miniconda/'
url = urljoin(url_base, filename)
filename = os.path.join(prefix, filename)
# Download from public repository
if not os.path.isfile(filename):
print('Downloading {url}'.format(url=url), file=self.log)
download_file(url, filename)
print('Downloaded file to {filename}'.format(filename=filename),
file=self.log)
else:
print('Using local copy at {filename}'.format(filename=filename),
file=self.log)
# run the installer
install_dir = os.path.join(prefix, 'mc3')
if self.system == 'Windows':
flags = '/InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /S /D={install_dir}'.\
format(install_dir=install_dir)
command_list = ['"' + filename + '"', flags]
else:
flags = '-b -u -p "{install_dir}"'.format(install_dir=install_dir)
command_list = ['/bin/sh', filename, flags]
print('Installing miniconda to "{install_dir}"'.format(
install_dir=install_dir), file=self.log)
output = check_output(command_list, env=self.env)
if self.verbose:
print(output, file=self.log)
return install_dir
def create_environment(self, builder='cctbx', filename=None, python=None,
copy=False, offline=False):
"""
Create the environment based on the builder and file. The
environment name is "conda_base".
Parameters
----------
builder: str
The builder from bootstrap.py. The default environment is defined
by the env_locations class variable
filename: str
If filename is not None, the argument overrides the file defined
in the env_locations dictionary. The filename should be a
relative path to the "modules" directory.
python: str
If set, the specific Python version of the environment for the
builder is used instead of the default. Current options are
'27' and '36' for Python 2.7 and 3.6, respectively.
copy: bool
If set to True, the --copy flag is passed to conda
offline: bool
If set to True, the --offline flag is passed to conda
"""
# handles check for choices in case parser is not available
if builder not in self.env_locations:
raise RuntimeError("""
The builder, {builder}, is not recognized. The available builders are,
{builders}
""".\
format(builder=builder, builders=', '.join(sorted(self.env_locations.keys()))))
if self.conda_base is None:
raise RuntimeError("""A conda installation is not available.""")
if filename is None:
filename = os.path.join(
self.root_dir, 'modules', self.env_locations[builder])
if python is not None:
if python not in ['27', '36']:
raise RuntimeError(
"""Only Python 2.7 and 3.6 are currently supported.""")
filename = filename.replace('PYTHON_VERSION', python)
else:
filename = os.path.abspath(filename)
if not os.path.isfile(filename):
raise RuntimeError("""The file, {filename}, is not available""".\
format(filename=filename))
yaml_format = False
if filename.endswith('yml') or filename.endswith('yaml'):
yaml_format = True
# make a new environment directory
if self.conda_env is None:
name = 'conda_base'
prefix = os.path.join(self.root_dir, name)
# or use the existing one
else:
prefix = os.path.abspath(self.conda_env)
# install a new environment or update and existing one
if prefix in self.environments:
command = 'install'
if yaml_format:
command = 'update'
text_messages = ['Updating', 'update of']
else:
command = 'create'
text_messages = ['Installing', 'installation into']
command_list = [self.conda_exe, command, '--prefix', prefix,
'--file', filename]
if yaml_format:
command_list.insert(1, 'env')
if self.system == 'Windows':
command_list = [os.path.join(self.conda_base, 'Scripts', 'activate'),
'base', '&&'] + command_list
if copy and not yaml_format:
command_list.append('--copy')
if offline and not yaml_format:
command_list.append('--offline')
if builder == "dials":
command_list.append("-y")
# RuntimeError is raised on failure
print('{text} {builder} environment with:\n {filename}'.format(
text=text_messages[0], builder=builder, filename=filename),
file=self.log)
for retry in range(self.max_retries):
retry += 1
try:
output = check_output(command_list, env=self.env)
except Exception:
print("""
*******************************************************************************
There was a failure in constructing the conda environment.
Attempt {retry} of {max_retries} will start {retry} minute(s) from {t}.
*******************************************************************************
""".format(retry=retry, max_retries=self.max_retries, t=time.asctime()))
time.sleep(retry*60)
else:
break
if retry == self.max_retries:
raise RuntimeError("""
The conda environment could not be constructed. Please check that there is a
working network connection for downloading conda packages.
""")
if self.verbose:
print(output, file=self.log)
print('Completed {text}:\n {prefix}'.format(text=text_messages[1],
prefix=prefix), file=self.log)
# on Windows, also download the Visual C++ 2008 Redistributable
# use the same version as conda-forge
# https://github.com/conda-forge/vs2008_runtime-feedstock
if self.system == 'Windows' and prefix.endswith('conda_base'):
download_file(
url='https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe',
filename=os.path.join(prefix, 'vcredist_x64.exe'))
# check that environment file is updated
self.environments = self.update_environments()
if prefix not in self.environments:
raise RuntimeError("""
The newly installed environment cannot be found in
${HOME}/.conda/environments.txt.
""")
def __init__(self, root_dir=root_dir, conda_base=None, conda_env=None,
check_file=True, max_retries=5, verbose=False, log=sys.stdout):
"""
Constructor that performs a basic check for the conda installation.
If an installation is not found, the latest version can be downloaded
and installed.
Parameters
----------
root_dir: str
Required argument for specifying the root level directory for
CCTBX builds. This is where the "modules" and "build" directories
reside. If a new conda installation is required, it will be placed
here under the "mc3" directory. The conda environment for building
will also be placed in this directory
conda_base: str
Required argument for specifying location of a conda installation.
If this is None, miniconda will be installed unless conda_env is
set to a valid path.
conda_env: str
Optional argument for specifying location of a conda environment.
Since this is assumed to be a working conda environment, a new
installation of conda will not be created. This is for developers
that want to manage their own environments.
check_file: bool
Flag for checking if a file exists. A RuntimeError is raised if a
this flag is set and a file does not exist. Used in get_conda_exe
and get_conda_python.
max_retries: int
When downloading conda packages, there may be network issues that
prevent the environment from being constructed. This parameter
controls the number of retry attempts for constructing the conda
environment. The first retry is attempted 1 minute after the
initial failure. The second retry is attempted 2 minutes, etc.
verbose: bool
Flag for showing conda output
log: file
For storing log output
"""
self.system = platform.system()
self.root_dir = root_dir
self.conda_base = None
if conda_base is not None:
self.conda_base = os.path.normpath(conda_base)
self.conda_env = None
if conda_env is not None:
self.conda_env = os.path.normpath(conda_env)
self.check_file = check_file
self.max_retries = max_retries
self.verbose = verbose
self.log = log
self.conda_exe = None
if self.conda_base is not None:
self.conda_exe = self.get_conda_exe(self.conda_base, check_file=False)
# default environment file for users
self.environment_file = os.path.join(
os.path.expanduser('~'), '.conda', 'environments.txt')
self.environments = self.update_environments()
# Clean environment for external Python processes
self.env = os.environ.copy()
self.env['PYTHONPATH'] = ''
# error messages
self.conda_exe_not_found = """
The conda executable cannot be found. Please make sure the correct
directory for the base conda installation was provided. The directory
can be found by running "conda info" and looking the "base environment"
value."""
# try to determine base environment from $PATH
if self.conda_base is None:
paths = os.environ.get('PATH')
if paths is not None:
if self.system == 'Windows':
paths = paths.split(';')
else:
paths = paths.split(':')
for path in paths:
conda_base = os.path.abspath(os.path.join(path, '..'))
conda_exe = self.get_conda_exe(conda_base, check_file=False)
if os.path.isfile(conda_exe):
self.conda_base = conda_base
self.conda_exe = conda_exe
break
# try to determine base environment from .conda/environments.txt
if self.conda_base is None:
for environment in self.environments:
conda_exe = self.get_conda_exe(environment, check_file=False)
if os.path.isfile(conda_exe):
self.conda_base = environment
self.conda_exe = conda_exe
break
# -------------------------------------------------------------------------
# helper function for searching for conda_exe
def walk_up_check(new_conda_base, new_conda_exe):
if new_conda_base is None:
new_conda_base = ''
if new_conda_exe is None:
new_conda_exe = ''
while not os.path.isfile(new_conda_exe):
# move up directory and recheck
new_conda_base = os.path.abspath(os.path.join(new_conda_base, '..'))
new_conda_exe = self.get_conda_exe(new_conda_base, check_file=False)
if os.path.isfile(new_conda_exe):
return new_conda_base, new_conda_exe
# moved to root directory
if new_conda_base == \
os.path.abspath(os.path.join(new_conda_base, '..')):
return '', ''
# -------------------------------------------------------------------------
# try to determine base evironment from conda_env
if (self.conda_base is None) and (self.conda_env is not None):
conda_base, conda_exe = walk_up_check(self.conda_env, self.conda_exe)
if os.path.isfile(conda_exe):
self.conda_base = conda_base
self.conda_exe = conda_exe
# install conda if necessary
if (self.conda_base is None) and (self.conda_env is None):
install_dir = os.path.join(self.root_dir, 'mc3')
if os.path.isdir(install_dir):
print('Using default conda installation', file=self.log)
self.conda_base = install_dir
else:
print('Location of conda installation not provided', file=self.log)
print('Proceeding with a fresh installation', file=self.log)
self.conda_base = self.install_miniconda(prefix=self.root_dir)
self.conda_exe = self.get_conda_exe(self.conda_base, check_file=False)
self.environments = self.update_environments()
# verify consistency and check conda version
if self.conda_base is not None:
# maybe a conda evironment was provided instead of the base environment
if not os.path.isfile(self.conda_exe):
self.conda_base, self.conda_exe = \
walk_up_check(self.conda_base, self.conda_exe)
self.environments = self.update_environments()
if not os.path.isfile(self.conda_exe):
raise RuntimeError(self.conda_exe_not_found)
conda_info = json.loads(check_output([self.conda_exe, 'info', '--json'],
env=self.env))
consistency_check = [self.conda_base == conda_info['root_prefix']]
for env in self.environments:
consistency_check.append(env in conda_info['envs'])
if False in consistency_check:
message = """
There is a mismatch between the conda settings in your home directory
and what "conda info" is reporting. This is not a fatal error, but if
an error is encountered, please check that your conda installation and
environments exist and are working.
"""
warnings.warn(message, RuntimeWarning)
if conda_info['conda_version'] < '4.4':
raise RuntimeError("""
CCTBX programs require conda version 4.4 and greater to make use of the
common compilers provided by conda. Please update your version with
"conda update conda".
""")
print('Base conda installation:\n {base}'.format(base=self.conda_base),
file=self.log)
if self.verbose:
output = check_output([self.conda_exe, 'info'], env=self.env)
print(output, file=self.log)
if self.conda_env is not None:
print('Build environment:\n {conda_env}'.format(
conda_env=self.conda_env), file=self.log)
def create_environment(self,
builder='cctbx',
filename=None,
copy=False,
offline=False):
"""
Create the environment based on the builder and file. The
environment name is "conda_base".
Parameters
----------
builder: str
The builder from bootstrap.py. The default environment is defined
by the env_locations class variable
filename: str
If filename is not None, the argument overrides the file defined
in the env_locations dictionary. The filename should be a
relative path to the "modules" directory.
copy: bool
If set to True, the --copy flag is passed to conda
offline: bool
If set to True, the --offline flag is passed to conda
"""
# handles check for choices in case parser is not available
if builder not in self.env_locations:
raise RuntimeError("""
The builder, {builder}, is not recognized. The available builders are,
{builders}
""".\
format(builder=builder, builders=', '.join(sorted(self.env_locations.keys()))))
if self.conda_base is None:
raise RuntimeError("""A conda installation is not available.""")
if filename is None:
filename = os.path.join(self.root_dir, 'modules',
self.env_locations[builder])
else:
filename = os.path.abspath(filename)
if not os.path.isfile(filename):
raise RuntimeError("""The file, {filename}, is not available""".\
format(filename=filename))
# make a new environment directory
if self.conda_env is None:
name = 'conda_base'
prefix = os.path.join(self.root_dir, name)
# or use the existing one
else:
prefix = os.path.abspath(self.conda_env)
# install a new environment or update and existing one
if prefix in self.environments:
command = 'install'
text_messages = ['Updating', 'update of']
else:
command = 'create'
text_messages = ['Installing', 'installation into']
command_list = [
self.conda_exe, command, '--prefix', prefix, '--file', filename
]
if self.system == 'Windows':
command_list = [
os.path.join(self.conda_base, 'Scripts', 'activate'), 'base',
'&&'
] + command_list
if copy:
command_list.append('--copy')
if offline:
command_list.append('--offline')
# RuntimeError is raised on failure
print('{text} {builder} environment with:\n {filename}'.format(
text=text_messages[0], builder=builder, filename=filename),
file=self.log)
output = check_output(command_list, env=self.env)
if self.verbose:
print(output, file=self.log)
print('Completed {text}:\n {prefix}'.format(text=text_messages[1],
prefix=prefix),
file=self.log)
# on Windows, also download the Visual C++ 2008 Redistributable
# use the same version as conda-forge
# https://github.com/conda-forge/vs2008_runtime-feedstock
if self.system == 'Windows' and prefix.endswith('conda_base'):
download_file(
url=
'https://download.microsoft.com/download/5/D/8/5D8C65CB-C849-4025-8E95-C3966CAFD8AE/vcredist_x64.exe',
filename=os.path.join(prefix, 'vcredist_x64.exe'))
# check that environment file is updated
self.environments = self.update_environments()
if prefix not in self.environments:
raise RuntimeError("""
The newly installed environment cannot be found in
${HOME}/.conda/environments.txt.
""")
