def test_command(command, env, package):
try:
check_call(command, env=env)
except subprocess.CalledProcessError as e:
print(' {} failed'.format(package))
raise e
print(' {} passes'.format(package))
def remove_install_env(activate_base):
print('Removing conda environment for installing compass')
commands = '{}; ' \
'conda remove -y --all -n ' \
'temp_compass_install'.format(activate_base)
check_call(commands)
def setup_install_env(activate_base):
print('Setting up a conda environment for installing compass')
commands = '{}; ' \
'mamba create -y -n temp_compass_install ' \
'progressbar2 jinja2 "mache>=1.1.4"'.format(activate_base)
check_call(commands)
def install_miniconda(conda_base, activate_base):
if not os.path.exists(conda_base):
print('Installing Miniconda3')
if platform.system() == 'Linux':
system = 'Linux'
elif platform.system() == 'Darwin':
system = 'MacOSX'
else:
system = 'Linux'
miniconda = 'Miniconda3-latest-{}-x86_64.sh'.format(system)
url = 'https://repo.continuum.io/miniconda/{}'.format(miniconda)
print(url)
req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
f = urlopen(req)
html = f.read()
with open(miniconda, 'wb') as outfile:
outfile.write(html)
f.close()
command = '/bin/bash {} -b -p {}'.format(miniconda, conda_base)
check_call(command)
os.remove(miniconda)
print('Doing initial setup')
commands = '{}; ' \
'conda config --add channels conda-forge; ' \
'conda config --set channel_priority strict; ' \
'conda install -y mamba boa; ' \
'conda update -y --all'.format(activate_base)
check_call(commands)
def bootstrap(activate_install_env, source_path):
print('Creating the compass conda environment')
bootstrap_command = '{}/conda/bootstrap.py'.format(source_path)
command = '{}; ' \
'{} {}'.format(activate_install_env, bootstrap_command,
' '.join(sys.argv[1:]))
check_call(command)
sys.exit(0)
def main():
args = parse_args()
source_path = os.getcwd()
template_path = '{}/conda/compass_env'.format(source_path)
version = get_version()
machine = None
machine_info = None
if not args.env_only:
if args.machine is None:
machine = discover_machine()
else:
machine = args.machine
if machine is not None:
machine_info = MachineInfo(machine=machine)
config = get_config(args.config_file, machine)
is_test = not config.getboolean('deploy', 'release')
conda_base = get_conda_base(args.conda_base, is_test, config)
base_activation_script = os.path.abspath(
'{}/etc/profile.d/conda.sh'.format(conda_base))
activate_base = 'source {}; conda activate'.format(base_activation_script)
python, recreate, compiler, mpi, conda_mpi, activ_suffix, env_suffix, \
activ_path = get_env_setup(args, config, machine, is_test, source_path,
conda_base)
if machine is None and not args.env_only and args.mpi is None:
raise ValueError('Your machine wasn\'t recognized by compass but you '
'didn\'t specify the MPI version. Please provide '
'either the --mpi or --env_only flag.')
if machine is None:
if args.env_only:
compiler = None
elif platform.system() == 'Linux':
compiler = 'gnu'
elif platform.system() == 'Darwin':
compiler = 'clang'
else:
compiler = 'gnu'
if machine_info is not None:
mpicc, mpicxx, mpifc, mod_commands, env_vars = \
machine_info.get_modules_and_mpi_compilers(compiler, mpi)
else:
# using conda-forge compilers
mpicc = 'mpicc'
mpicxx = 'mpicxx'
mpifc = 'mpifort'
mod_commands = list()
env_vars = dict()
env_path, env_name, activate_env = build_env(
is_test, recreate, machine, compiler, mpi, conda_mpi, version, python,
source_path, template_path, conda_base, activ_suffix, args.env_name,
env_suffix, activate_base, args.use_local)
if compiler is not None:
sys_info, system_libs = build_system_libraries(
config, machine, compiler, mpi, version, template_path, env_path,
env_name, activate_base, activate_env, mpicc, mpicxx, mpifc,
mod_commands, env_vars)
else:
sys_info = dict(modules=[], env_vars=[], mpas_netcdf_paths='')
system_libs = None
if is_test:
if args.env_name is not None:
prefix = 'load_{}'.format(args.env_name)
else:
prefix = 'load_dev_compass_{}'.format(version)
else:
prefix = 'load_compass_{}'.format(version)
script_filename = write_load_compass(template_path, activ_path, conda_base,
is_test, version, activ_suffix,
prefix, env_name, machine, sys_info,
args.env_only)
if args.check:
check_env(script_filename, env_name)
commands = '{}; conda clean -y -p -t'.format(activate_base)
check_call(commands)
if machine is not None:
update_permissions(config, is_test, activ_path, conda_base,
system_libs)
def build_system_libraries(config, machine, compiler, mpi, version,
template_path, env_path, env_name, activate_base,
activate_env, mpicc, mpicxx, mpifc, mod_commands,
env_vars):
if machine is not None:
esmf = config.get('deploy', 'esmf')
scorpio = config.get('deploy', 'scorpio')
else:
# stick with the conda-forge ESMF and e3sm/label/compass SCORPIO
esmf = 'None'
scorpio = 'None'
if esmf != 'None':
# remove conda-forge esmf because we will use the system build
commands = '{}; conda remove -y --force -n {} esmf'.format(
activate_base, env_name)
check_call(commands)
force_build = False
if machine is not None:
system_libs = config.get('deploy', 'system_libs')
compiler_path = os.path.join(system_libs, 'compass_{}'.format(version),
compiler, mpi)
scorpio_path = os.path.join(compiler_path,
'scorpio_{}'.format(scorpio))
esmf_path = os.path.join(compiler_path, 'esmf_{}'.format(esmf))
else:
# using conda-forge compilers
system_libs = None
scorpio_path = env_path
esmf_path = env_path
force_build = True
sys_info = get_sys_info(machine, compiler, mpi, mpicc, mpicxx, mpifc,
mod_commands, env_vars)
if esmf != 'None':
sys_info['env_vars'].append('export PATH="{}:$PATH"'.format(
os.path.join(esmf_path, 'bin')))
sys_info['env_vars'].append(
'export LD_LIBRARY_PATH={}:$LD_LIBRARY_PATH'.format(
os.path.join(esmf_path, 'lib')))
sys_info['env_vars'].append('export PIO={}'.format(scorpio_path))
build_esmf = 'False'
if esmf == 'None':
esmf_branch = 'None'
else:
esmf_branch = 'ESMF_{}'.format(esmf.replace('.', '_'))
if not os.path.exists(esmf_path) or force_build:
build_esmf = 'True'
build_scorpio = 'False'
if scorpio != 'None' and (not os.path.exists(scorpio_path) or force_build):
build_scorpio = 'True'
script_filename = 'build.bash'
with open('{}/build.template'.format(template_path), 'r') as f:
template = Template(f.read())
modules = '\n'.join(sys_info['modules'])
if machine is None:
# need to activate the conda environment because that's where the
# libraries are
modules = '{}\n{}'.format(activate_env.replace('; ', '\n'), modules)
script = template.render(sys_info=sys_info,
modules=modules,
scorpio=scorpio,
scorpio_path=scorpio_path,
build_scorpio=build_scorpio,
esmf_path=esmf_path,
esmf_branch=esmf_branch,
build_esmf=build_esmf)
print('Writing {}'.format(script_filename))
with open(script_filename, 'w') as handle:
handle.write(script)
command = '/bin/bash build.bash'
check_call(command)
return sys_info, system_libs
def build_env(is_test, recreate, machine, compiler, mpi, conda_mpi, version,
python, source_path, template_path, conda_base, activ_suffix,
env_name, env_suffix, activate_base, use_local):
if compiler is not None or is_test:
build_dir = f'conda/build{activ_suffix}'
try:
shutil.rmtree(build_dir)
except OSError:
pass
try:
os.makedirs(build_dir)
except FileExistsError:
pass
os.chdir(build_dir)
if is_test:
if env_name is None:
env_name = f'dev_compass_{version}{env_suffix}'
else:
env_name = f'compass_{version}{env_suffix}'
env_path = os.path.join(conda_base, 'envs', env_name)
if conda_mpi == 'nompi':
mpi_prefix = 'nompi'
else:
mpi_prefix = f'mpi_{mpi}'
channels = ['-c conda-forge', '-c defaults']
if use_local:
channels = ['--use-local'] + channels
if machine is None or not is_test:
# we need libpnetcdf and scorpio from the e3sm channel, compass label
channels = channels + ['-c e3sm/label/compass']
channels = f'--override-channels {" ".join(channels)}'
packages = f'python={python}'
base_activation_script = os.path.abspath(
f'{conda_base}/etc/profile.d/conda.sh')
activate_env = \
f'source {base_activation_script}; conda activate {env_name}'
with open(f'{template_path}/spec-file.template', 'r') as f:
template = Template(f.read())
if is_test:
spec_file = template.render(mpi=conda_mpi, mpi_prefix=mpi_prefix)
spec_filename = f'spec-file-{conda_mpi}.txt'
with open(spec_filename, 'w') as handle:
handle.write(spec_file)
else:
spec_filename = None
if not os.path.exists(env_path) or recreate:
print(f'creating {env_name}')
if is_test:
# install dev dependencies and compass itself
commands = \
f'{activate_base}; ' \
f'mamba create -y -n {env_name} {channels} ' \
f'--file {spec_filename} {packages}'
check_call(commands)
commands = \
f'{activate_env}; ' \
f'cd {source_path}; ' \
f'python -m pip install -e .'
check_call(commands)
else:
packages = f'{packages} "compass={version}={mpi_prefix}_*"'
commands = f'{activate_base}; ' \
f'mamba create -y -n {env_name} {channels} {packages}'
check_call(commands)
else:
if is_test:
print(f'updating {env_name}')
# install dev dependencies and compass itself
commands = \
f'{activate_base}; ' \
f'mamba install -y -n {env_name} {channels} ' \
f'--file {spec_filename} {packages}'
check_call(commands)
commands = \
f'{activate_env}; ' \
f'cd {source_path}; ' \
f'python -m pip install -e .'
check_call(commands)
else:
print(f'{env_name} already exists')
return env_path, env_name, activate_env
def build_env(is_test, recreate, machine, compiler, mpi, conda_mpi, version,
python, source_path, template_path, conda_base, activ_suffix,
env_name, env_suffix, activate_base):
if compiler is not None or is_test:
build_dir = 'conda/build{}'.format(activ_suffix)
try:
shutil.rmtree(build_dir)
except OSError:
pass
try:
os.makedirs(build_dir)
except FileExistsError:
pass
os.chdir(build_dir)
if is_test:
if env_name is None:
env_name = 'dev_compass_{}{}'.format(version, env_suffix)
else:
env_name = 'compass_{}{}'.format(version, env_suffix)
env_path = os.path.join(conda_base, 'envs', env_name)
if conda_mpi == 'nompi':
mpi_prefix = 'nompi'
else:
mpi_prefix = 'mpi_{}'.format(mpi)
channels = '--override-channels -c conda-forge -c defaults'
if machine is None or not is_test:
# we need libpnetcdf and scorpio from the e3sm channel, compass label
channels = '{} -c e3sm/label/compass'.format(channels)
packages = 'python={}'.format(python)
base_activation_script = os.path.abspath(
'{}/etc/profile.d/conda.sh'.format(conda_base))
activate_env = \
'source {}; conda activate {}'.format(base_activation_script, env_name)
with open('{}/spec-file.template'.format(template_path), 'r') as f:
template = Template(f.read())
if is_test:
spec_file = template.render(mpi=conda_mpi, mpi_prefix=mpi_prefix)
spec_filename = 'spec-file-{}.txt'.format(conda_mpi)
with open(spec_filename, 'w') as handle:
handle.write(spec_file)
else:
spec_filename = None
if not os.path.exists(env_path) or recreate:
print('creating {}'.format(env_name))
if is_test:
# install dev dependencies and compass itself
commands = \
'{}; ' \
'mamba create -y -n {} {} ' \
'--file {} {}'.format(activate_base, env_name, channels,
spec_filename, packages)
check_call(commands)
commands = \
'{}; ' \
'cd {}; ' \
'python -m pip install -e .'.format(activate_env, source_path)
check_call(commands)
else:
packages = '{} "compass={}={}_*"'.format(packages, version,
mpi_prefix)
commands = '{}; mamba create -y -n {} {} {}'.format(
activate_base, env_name, channels, packages)
check_call(commands)
else:
if is_test:
print('updating {}'.format(env_name))
# install dev dependencies and compass itself
commands = \
'{}; ' \
'mamba install -y -n {} {} ' \
'--file {} {}'.format(activate_base, env_name, channels,
spec_filename, packages)
check_call(commands)
commands = \
'{}; ' \
'cd {}; ' \
'python -m pip install -e .'.format(activate_env, source_path)
check_call(commands)
else:
print('{} already exists'.format(env_name))
return env_path, env_name, activate_env
