def pack_venv_in_conda(env_path: str, reqs: Dict[str, str],
new_env_diff: bool) -> str:
if not new_env_diff:
conda_pack.pack(output=env_path)
return env_path
else:
return create_and_pack_conda_env(env_path, reqs)
def test_run_with_packed_env():
import conda_pack
temp_dir = os.environ.get("MARS_YARN_TEST_DIR")
clean_after_test = False
if temp_dir is None:
clean_after_test = True
temp_dir = tempfile.mkdtemp(prefix="test-mars-yarn-")
else:
os.makedirs(temp_dir, exist_ok=True)
packed_env_file = os.path.join(temp_dir, "mars-test-env.tar.gz")
if not os.path.exists(packed_env_file):
try:
conda_pack.pack(output=packed_env_file,
ignore_editable_packages=True)
except conda_pack.CondaPackException:
logger.exception(
"Failed to pack environment, this test will be skipped")
return
try:
_run_yarn_test_with_env(packed_env_file, 1200)
finally:
if clean_after_test:
shutil.rmtree(temp_dir)
def _pack_environment(env: dict, file_path: str) -> str:
with NamedTemporaryFile(mode="w", suffix=".yml") as file:
# TODO: Save copy of environment.yaml alongside tarball
yaml.safe_dump(env, file)
# Create env
tmp_env_path = file.name
tmp_env_name = f"tempo-{uuid.uuid4()}"
cmd = f"conda env create --name {tmp_env_name} --file {tmp_env_path}"
run(cmd, shell=True, check=True)
try:
# Pack environment
conda_pack.pack(
name=tmp_env_name,
output=file_path,
force=True,
ignore_editable_packages=True,
ignore_missing_files=True,
)
finally:
# Remove environment
cmd = f"conda remove --name {tmp_env_name} --all --yes"
run(cmd, shell=True, check=True)
return file_path
def _pack_conda_main(self, args):
import sys
import traceback
from conda_pack.cli import fail, PARSER, context
import conda_pack
from conda_pack import pack, CondaPackException
args = PARSER.parse_args(args=args)
# Manually handle version printing to output to stdout in python < 3.4
if args.version:
print('conda-pack %s' % conda_pack.__version__)
sys.exit(0)
try:
with context.set_cli():
pack(name=args.name,
prefix=args.prefix,
output=args.output,
format=args.format,
force=args.force,
compress_level=args.compress_level,
n_threads=args.n_threads,
zip_symlinks=args.zip_symlinks,
zip_64=not args.no_zip_64,
arcroot=args.arcroot,
dest_prefix=args.dest_prefix,
verbose=not args.quiet,
filters=args.filters)
except CondaPackException as e:
fail("CondaPackError: %s" % e)
except KeyboardInterrupt:
fail("Interrupted")
except Exception:
fail(traceback.format_exc())
def testRunWithPackedEnv(self):
import conda_pack
temp_dir = os.environ.get('MARS_YARN_TEST_DIR')
clean_after_test = False
if temp_dir is None:
clean_after_test = True
temp_dir = tempfile.mkdtemp(prefix='test-mars-yarn-')
else:
os.makedirs(temp_dir, exist_ok=True)
packed_env_file = os.path.join(temp_dir, 'mars-test-env.tar.gz')
if not os.path.exists(packed_env_file):
try:
conda_pack.pack(output=packed_env_file,
ignore_editable_packages=True)
except conda_pack.CondaPackException:
logger.exception(
'Failed to pack environment, this test will be skipped')
return
try:
self._runYarnTestWithEnv(packed_env_file, 1200)
finally:
if clean_after_test:
shutil.rmtree(temp_dir)
def test_pack_exceptions(py36_env):
# Can't pass both prefix and name
with pytest.raises(CondaPackException):
pack(prefix=py36_path, name='py36')
# Unknown filter type
with pytest.raises(CondaPackException):
pack(prefix=py36_path, filters=[("exclude", "*.py"), ("foo", "*.pyc")])
def conda_pack(prefix, output, ignore_missing_files=True):
from conda_pack import pack
pack(
prefix=str(prefix),
output=str(output),
ignore_missing_files=ignore_missing_files,
)
def pack_venv_in_conda(output: str, reqs: Dict[str, str],
additional_packages: Dict[str, str],
ignored_packages: Collection[str]) -> str:
if len(additional_packages) == 0 and len(ignored_packages) == 0:
conda_pack.pack(output=output)
return output
else:
return create_and_pack_conda_env(output, reqs)
def _pack_environment(env_name: str, file_path: str):
# Pack environment
conda_pack.pack(
name=env_name,
output=file_path,
force=True,
verbose=True,
ignore_editable_packages=False,
ignore_missing_files=True,
)
def _pack_environment(env_name: str, file_path: str):
logger.info(f"packing conda environment from {env_name} to {file_path}")
# Pack environment
conda_pack.pack(
name=env_name,
output=file_path,
force=True,
verbose=True,
ignore_editable_packages=False,
ignore_missing_files=True,
)
def create_and_pack_conda_env(spec_file: str = None,
reqs: List[str] = None,
output: str = None) -> str:
"""
Create a new conda virtual environment and zip it
:param spec_file: conda yaml spec file to use
:param reqs: dependencies to install
:param output: a dedicated output path
:return: destination of the archive
"""
project_env_name = get_conda_env_name(spec_file=spec_file, reqs=reqs)
_logger.info(f"Found project env name {project_env_name}")
env_path = get_or_create_conda_env(project_env_name, spec_file)
if reqs:
env_python_bin = os.path.join(env_path, "bin", "python")
if not os.path.exists(env_python_bin):
raise RuntimeError("Failed to create Python binary at " +
env_python_bin)
_logger.info("Installing packages into " + env_path)
process.call([env_python_bin, "-m", "pip", "install"] + reqs)
return conda_pack.pack(prefix=env_path, output=output, force=True)
def test_pack(tmpdir, py36_env):
out_path = os.path.join(str(tmpdir), 'py36.tar')
exclude1 = "*.py"
exclude2 = "*.pyc"
include = "lib/python3.6/site-packages/conda_pack_test_lib1/*"
res = pack(prefix=py36_path,
output=out_path,
filters=[("exclude", exclude1), ("exclude", exclude2),
("include", include)])
assert res == out_path
assert os.path.exists(out_path)
assert tarfile.is_tarfile(out_path)
with tarfile.open(out_path) as fil:
paths = fil.getnames()
filtered = (py36_env.exclude(exclude1).exclude(exclude2).include(include))
# Files line up with filtering, with extra conda-unpack command
sol = set(f.target for f in filtered.files)
res = set(paths)
diff = res.difference(sol)
assert len(diff) == 1
extra = list(diff)[0]
assert 'conda-unpack' in extra
def test_pack(tmpdir, py36_env):
out_path = os.path.join(str(tmpdir), 'py36.tar')
exclude1 = "*.py"
exclude2 = "*.pyc"
include = os.path.join(SP_36, 'conda_pack_test_lib1', '*')
res = pack(prefix=py36_path,
output=out_path,
filters=[("exclude", exclude1), ("exclude", exclude2),
("include", include)])
assert res == out_path
assert os.path.exists(out_path)
assert tarfile.is_tarfile(out_path)
with tarfile.open(out_path) as fil:
paths = fil.getnames()
filtered = (py36_env.exclude(exclude1).exclude(exclude2).include(include))
# Files line up with filtering, with extra conda-unpack command
sol = set(os.path.normcase(f.target) for f in filtered.files)
res = set(os.path.normcase(p) for p in paths)
diff = res.difference(sol)
if on_win:
fnames = ('conda-unpack.exe', 'conda-unpack-script.py', 'activate.bat',
'deactivate.bat')
else:
fnames = ('conda-unpack', 'activate', 'deactivate')
assert diff == set(os.path.join(BIN_DIR_L, f) for f in fnames)
def test_dest_prefix(tmpdir, py36_env):
out_path = os.path.join(str(tmpdir), 'py36.tar')
dest = r'C:\foo\bar\baz\biz' if on_win else '/foo/bar/baz/biz'
res = pack(prefix=py36_path, dest_prefix=dest, output=out_path)
assert res == out_path
assert os.path.exists(out_path)
assert tarfile.is_tarfile(out_path)
with tarfile.open(out_path) as fil:
paths = fil.getnames()
# No conda-unpack generated
assert 'conda-unpack' not in paths
if on_win:
test_files = ['Scripts/conda-pack-test-lib1', 'Scripts/pytest.exe']
else:
test_files = ['bin/conda-pack-test-lib1', 'bin/pytest', 'bin/clear']
orig_bytes = py36_env.prefix.encode()
new_bytes = dest.encode()
# all paths, including shebangs, are rewritten using the prefix
with tarfile.open(out_path) as fil:
for test_file in test_files:
orig_path = os.path.join(py36_env.prefix, test_file)
with open(orig_path, 'rb') as fil2:
orig_data = fil2.read()
if orig_bytes in orig_data:
data = fil.extractfile(test_file).read()
assert orig_bytes not in data, test_file
assert new_bytes in data, test_file
def test_pack_exceptions(tmpdir, py36_env):
already_exists = os.path.join(str(tmpdir), 'py36.tar')
with open(already_exists, 'wb'):
pass
# file already exists
with pytest.raises(CondaPackException):
py36_env.pack(output=already_exists)
# Can't pass both prefix and name
with pytest.raises(CondaPackException):
pack(prefix=py36_path, name='py36')
# Unknown filter type
with pytest.raises(CondaPackException):
pack(prefix=py36_path,
filters=[("exclude", "*.py"),
("foo", "*.pyc")])
def test_dest_prefix(tmpdir, py36_env):
out_path = os.path.join(str(tmpdir), 'py36.tar')
dest = r'c:\foo\bar\baz\biz' if on_win else '/foo/bar/baz/biz'
res = pack(prefix=py36_path, dest_prefix=dest, output=out_path)
assert res == out_path
assert os.path.exists(out_path)
assert tarfile.is_tarfile(out_path)
_test_dest_prefix(py36_env.prefix, dest, '', out_path, 'r')
def create_and_pack_conda_env(
env_path: str,
reqs: Dict[str, str],
) -> str:
try:
_call(["conda"])
except CalledProcessError:
raise RuntimeError("conda is not available in $PATH")
env_path_split = env_path.split('.', 1)
env_name = env_path_split[0]
compression_format = env_path_split[1] if len(
env_path_split) > 1 else ".zip"
archive_path = f"{env_name}.{compression_format}"
if os.path.exists(env_name):
shutil.rmtree(env_name)
_logger.info("Creating new env " + env_name)
python_version = sys.version_info
_call([
"conda", "create", "-p", env_name, "-y", "-q", "--copy",
f"python={python_version.major}.{python_version.minor}.{python_version.micro}"
],
env=dict(os.environ))
env_python_bin = os.path.join(env_name, "bin", "python")
if not os.path.exists(env_python_bin):
raise RuntimeError("Failed to create Python binary at " +
env_python_bin)
_logger.info("Installing packages into " + env_name)
_call([env_python_bin, "-m", "pip", "install"] + format_requirements(reqs))
if os.path.exists(archive_path):
os.remove(archive_path)
conda_pack.pack(prefix=env_name, output=archive_path)
return archive_path
def pack_venv_in_conda(reqs: List[str],
changed_reqs: bool = False,
output: str = None) -> str:
"""
Pack the current virtual environment
:param reqs: directory to zip
:param changed_reqs:
we prefer zipping the current virtual env as much as possible,
if it has been changed we need to create a new one with 'conda create -n env ..'
and reinstall the dependencies inside
:param output: a dedicated output path
:return: destination of the archive
"""
if not changed_reqs:
return conda_pack.pack(output=output)
else:
return create_and_pack_conda_env(reqs=reqs, output=output)
def test_dest_prefix(tmpdir, py36_env):
out_path = os.path.join(str(tmpdir), 'py36.tar')
dest = '/foo/bar/baz/biz'
res = pack(prefix=py36_path,
dest_prefix=dest,
output=out_path)
assert res == out_path
assert os.path.exists(out_path)
assert tarfile.is_tarfile(out_path)
with tarfile.open(out_path) as fil:
paths = fil.getnames()
# No conda-unpack generated
assert 'conda-unpack' not in paths
dest_bytes = dest.encode()
# shebangs are rewritten using env
with tarfile.open(out_path) as fil:
text_from_conda = fil.extractfile('/'.join([BIN_DIR, 'conda-pack-test-lib1'])).read()
text_from_pip = fil.extractfile('scripts/pytest.exe' if on_win else 'bin/pytest').read()
assert dest_bytes not in text_from_conda
assert dest_bytes not in text_from_pip
assert b'env python' in text_from_conda
if not on_win:
# pip entrypoint on Windows is complicated...
assert b'env python' in text_from_pip
with tarfile.open(out_path) as fil:
binary_from_conda = fil.extractfile('bin/clear').read()
# Other files are rewritten to use specified prefix This is only checked if
# the original file did include the prefix, which is true at least on osx.
orig_path = os.path.join(py36_env.prefix, 'bin/clear')
with open(orig_path, 'rb') as fil:
orig_bytes = fil.read()
if py36_env.prefix.encode() in orig_bytes:
assert py36_env.prefix.encode() not in binary_from_conda
assert dest_bytes in binary_from_conda
def pack(**conda_pack_kwargs):
"""
Calls conda_pack.pack.
If the packed output file already exists, this will not repackage
it unless conda_pack_kwargs["force"] == True.
Returns the path to the output packed env file.
Arguments:
* `conda_pack_kwargs` args to pass to conda_pack.pack().
"""
kwargs = conda_pack_defaults.copy()
kwargs.update(conda_pack_kwargs)
# Make sure output is set to something, so we can return it if it already exists.
if "output" not in kwargs:
conda_env_name = "env"
if "prefix" in kwargs:
conda_env_name = os.path.basename(kwargs["prefix"])
elif "name" in kwargs:
conda_env_name = kwargs["name"]
elif is_active():
conda_env_name = active_name()
kwargs["output"] = "conda-{}.{}".format(conda_env_name,
kwargs["format"])
conda_packed_file = kwargs["output"]
if os.path.isfile(conda_packed_file) and not kwargs["force"]:
print_err(
f"A conda environment is already packed at {conda_packed_file}. "
"If you have recently installed new packages into your conda env, set "
"force=True in conda_pack_kwargs and it will be repacked for you.")
return conda_packed_file
else:
# Isolate the import here so that we don"t get import errors
# if conda_pack is not installed (e.g. in a virtualenv).
import conda_pack
# NOTE: If no conda env is currently active, and kwargs
# doesn"t contain information about what env to pack (i.e. no name or prefix)
# then this raise an error.
return conda_pack.pack(**kwargs)
def test_parcel(tmpdir, py36_env):
if on_win:
pytest.skip("Not parcel tests on Windows")
arcroot = 'py36-1234.56'
out_path = os.path.join(str(tmpdir), arcroot + '-el7.parcel')
pdir = os.getcwd()
try:
os.chdir(str(tmpdir))
res = pack(prefix=py36_path, format='parcel', parcel_version='1234.56')
finally:
os.chdir(pdir)
assert os.path.join(str(tmpdir), res) == out_path
assert os.path.exists(out_path)
# Verify that only the parcel files were added
with tarfile.open(out_path, 'r:gz') as fil:
paths = fil.getnames()
sol = set(os.path.join(arcroot, f.target) for f in py36_env.files)
diff = set(paths).difference(sol)
fnames = ('conda_env.sh', 'parcel.json')
assert diff == set(os.path.join(arcroot, 'meta', f) for f in fnames)
# Verify correct metadata in parcel.json
with tarfile.open(out_path) as fil:
fpath = os.path.join(arcroot, 'meta', 'parcel.json')
data = fil.extractfile(fpath).read()
data = json.loads(data)
assert data['name'] == 'py36' and data['components'][0]['name'] == 'py36'
assert data['version'] == '1234.56' and data['components'][0][
'version'] == '1234.56'
# Verify the correct dest_prefix substitution
dest = os.path.join('/opt/cloudera/parcels', arcroot)
_test_dest_prefix(py36_env.prefix, dest, arcroot, out_path, 'r:gz')
import argparse
parser = argparse.ArgumentParser(description='AGS release package tool.')
parser.add_argument('--skip_packing',
action='store_true',
help="skip packaging to dist folder")
parser.add_argument('--rhi',
action='store_true',
help="pack into rhi installer")
parser.add_argument('--version', default="v0.0.0", help="version number")
args = parser.parse_args()
start = time.time()
conda_pack.pack(output="env.zip", verbose=True, n_threads=-1, force=True)
print('unpacking to ui/Rhino/AGS/dev/env')
shutil.unpack_archive("env.zip", "ui/Rhino/AGS/dev/env")
print('removing unnecessary files')
for root, dirs, files in os.walk("ui/Rhino/AGS/dev/env"):
for d in dirs:
if d.find("node_modules") >= 0:
shutil.rmtree(os.path.join(root, d))
for f in files:
if f.find("electron.zip") >= 0:
os.remove(os.path.join(root, f))
def conda_pack(prefix, output):
import conda_pack
conda_pack.pack(prefix=str(prefix), output=str(output))
def create_and_pack_conda_env(
name: str,
python: str,
pip_packages: typing.List[str],
root: typing.Optional[str] = tempfile.tempdir
) -> str:
"""Create a Conda environment.
The environment is created via ``conda``. However, all the
packages other than the Python interpreter are installed via
``pip`` to allow for more flexibility.
Parameters
----------
name : str
A human-readable name of the environment.
python : str
Python version in the MAJOR.MINOR.MICRO format.
pip_packages : list
PyPI packages to install in the environment.
root : list
Path to the root directory with Conda environments. If ``None``,
system temporary directory will be used with a fallback to the
current directory.
Returns
-------
env_path : str
Path to the packed environment.
"""
try:
_call(["conda"])
except CalledProcessError:
raise RuntimeError("conda is not available in $PATH")
env_path = os.path.join(root or os.getcwd(), name)
if not os.path.exists(env_path):
logger.info("Creating new env " + name)
_call([
"conda", "create", "-p", env_path, "-y", "-q", "--copy",
"python=" + python,
# TensorFlow enforces an upper bound on setuptools which
# conflicts with the version installed by Conda by default.
"setuptools=" + setuptools.__version__
], env=dict(os.environ))
env_python_bin = os.path.join(env_path, "bin", "python")
if not os.path.exists(env_python_bin):
raise RuntimeError(
"Failed to create Python binary at " + env_python_bin)
if pip_packages:
logger.info("Installing packages into " + name)
_call([env_python_bin, "-m", "pip", "install"] +
pip_packages)
requirements_path = os.path.join(env_path, "requirements.txt")
with open(requirements_path, "w") as f:
print(*pip_packages, sep=os.linesep, file=f)
env_zip_path = env_path + ".zip"
if not os.path.exists(env_zip_path):
import conda_pack
conda_pack.pack(prefix=env_path, output=env_zip_path)
return env_zip_path
def build_parcel():
jinja_env = Environment(loader=FileSystemLoader('templates'))
(jpy_version, parcel_version) = get_version()
# Create the JupyterHub Environment
render_environment_yaml(jinja_env, jpy_version=jpy_version)
with open('environment.yml') as f:
conda_env = yaml.load(f, Loader=yaml.Loader)
check_call(
'conda env create --force -p {name}'.format(**conda_env).split(' '))
# Get the complete list of installed packages
components = json.loads(
run_command(Commands.LIST, '--json', '-p', conda_env['name'])[0])
jpy_version = [
x['version'] for x in components if x['name'] == conda_env['name']
][0]
# Create the meta folder under the conda-environment
meta_dir = os.path.join(conda_env['name'], 'meta')
os.makedirs(meta_dir, exist_ok=True)
# Create the meta/parcel.json file
parcel_template = jinja_env.get_template('parcel.yaml')
parcel_rendered = parcel_template.render(conda_env=conda_env,
jpy_version=jpy_version,
parcel_version=parcel_version,
components=components)
parcel_dict = yaml.load(parcel_rendered, Loader=yaml.Loader)
parcel_json = os.path.join(meta_dir, 'parcel.json')
with open(parcel_json, 'w') as f:
json.dump(parcel_dict, f)
print(check_output(['validator.sh', '-p', parcel_json]).decode('utf-8'))
# Create the scripts.defines file (a.k.a. activate script)
shutil.copy('env.sh',
os.path.join(meta_dir, parcel_dict['scripts']['defines']))
# Use conda-pack to create the .parcel file
if os.path.exists('parcels'):
shutil.rmtree('parcels')
os.makedirs('parcels', exist_ok=True)
parcel_fqn = '{name}-{version}'.format(**parcel_dict)
parcel_file = conda_pack.pack(prefix=conda_env['name'],
output='%s-distro.parcel' % parcel_fqn,
arcroot=parcel_fqn,
dest_prefix='/opt/cloudera/parcels/%s' %
parcel_fqn,
format="tar.gz",
force=True)
# Link generated file to selected distros
distros = ['el6', 'el7']
for distro in distros:
dest_name = 'parcels/{}-{}.parcel'.format(parcel_fqn, distro)
os.symlink('../%s-distro.parcel' % parcel_fqn, dest_name)
check_call(['validator.sh', '-f', dest_name])
# Generate the parcel manifest file
print(check_output(['make_manifest.py', 'parcels']).decode('utf-8'))
def build():
import conda_pack
version = open("simnibs/_version.py").readlines()[-1].split()[-1].strip(
"\"'")
# Create a new environment
if os.path.dirname(__file__):
os.chdir(os.path.dirname(__file__))
if not os.path.isdir('pack'):
os.mkdir('pack')
env = os.path.join(os.path.dirname(__file__), 'environment_{}.yml')
if sys.platform == 'linux':
env = env.format('linux')
out_pack = os.path.join('pack', 'simnibs_env.tar.gz')
os_name = 'linux'
elif sys.platform == 'darwin':
env = env.format('macOS')
out_pack = os.path.join('pack', 'simnibs_env.tar.gz')
os_name = 'macOS'
elif sys.platform == 'win32':
env = env.format('win')
out_pack = os.path.join('pack', 'simnibs_env.zip')
os_name = 'win'
else:
raise OSError('OS not supported!')
# Create temporary environment
subprocess.run(f'conda env create -n simnibs_env_tmp -f {env}',
check=True,
shell=True)
# Pack
if os.path.isfile(out_pack):
os.remove(out_pack)
conda_pack.pack(
name='simnibs_env_tmp',
dest_prefix='simnibs_env',
output=out_pack,
)
# Remove temporary env
subprocess.run('conda env remove -y --name simnibs_env_tmp',
check=True,
shell=True)
# Copy wheel
wheels = glob.glob(f'dist/simnibs-{version}*.whl')
if len(wheels) == 0:
raise FileNotFoundError(
f'Did not find any wheels for version {version}')
for f in wheels:
shutil.copy(f, 'pack')
# Create bash or bat file for installation
if sys.platform == 'win32':
with open(os.path.join('pack/install.cmd'), 'w') as f:
f.write(f'SET INSTALL_DIR=%LOCALAPPDATA%\SimNIBS\n')
f.write('mkdir "%INSTALL_DIR%\simnibs_env"\n')
f.write(
'powershell.exe -nologo -noprofile -command "& '
'{ Add-Type -A \'System.IO.Compression.FileSystem\'; '
'[IO.Compression.ZipFile]::ExtractToDirectory(\'%~dp0simnibs_env.zip\', \'%INSTALL_DIR%\simnibs_env\'); }"\n'
)
f.write('call "%INSTALL_DIR%\simnibs_env\\Scripts\\activate"\n')
f.write(
'python -m pip install simnibs --no-cache-dir --no-index --upgrade --find-links=./\n'
)
f.write(
'postinstall_simnibs -d "%INSTALL_DIR%" --copy-matlab --setup-links --no-extra-coils'
)
else:
if sys.platform == 'darwin':
fn_script = os.path.join('pack', 'install')
install_dir = "$HOME/Applications/SimNIBS"
else:
fn_script = os.path.join('pack', 'install')
install_dir = "$HOME/SimNIBS"
with open(fn_script, 'w') as f:
# Some problem with Qt
f.write('#! /bin/bash -e\n')
f.write(f'INSTALL_DIR={install_dir}\n')
f.write(
'CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)"\n'
)
f.write('mkdir -p "$INSTALL_DIR/simnibs_env"\n')
f.write(
'tar -zxf "$CUR_DIR/simnibs_env.tar.gz" -C "$INSTALL_DIR/simnibs_env"\n'
)
f.write('source "$INSTALL_DIR/simnibs_env/bin/activate"\n')
f.write(
'python -m pip install simnibs --no-cache-dir --no-index --upgrade --find-links="$CUR_DIR"\n'
)
f.write(
'python -m pip install pyqt5 --force-reinstall --no-cache-dir\n'
) # I need to re-install pyqt, this requires internet
f.write(
'postinstall_simnibs -d "$INSTALL_DIR" --copy-matlab --setup-links --no-extra-coils'
)
os.chmod(
fn_script,
os.stat(fn_script).st_mode | stat.S_IXUSR | stat.S_IXGRP
| stat.S_IXOTH)
# zip the whole thing
shutil.make_archive(f'simnibs-{version}-{os_name}', 'zip', 'pack')
def _archive_project(project, filename, pack_envs=False):
"""Make an archive of the non-ignored files in the project.
Args:
project (``Project``): the project
filename (str): name for the new zip or tar.gz archive file
Returns:
a ``Status``, if failed has ``errors``
"""
failed = project.problems_status()
if failed is not None:
for error in failed.errors:
project.frontend.error(error)
return failed
frontend = _new_error_recorder(project.frontend)
if not os.path.exists(project.project_file.filename):
frontend.error("%s does not exist." % project.project_file.basename)
return SimpleStatus(success=False,
description="Can't create an archive.",
errors=frontend.pop_errors())
# this would most likely happen in a GUI editor, if it reloaded
# the project from memory but hadn't saved yet.
if project.project_file.has_unsaved_changes:
frontend.error("%s has been modified but not saved." %
project.project_file.basename)
return SimpleStatus(success=False,
description="Can't create an archive.",
errors=frontend.pop_errors())
envs_path = os.path.join(project.project_file.project_dir, 'envs')
packed_envs = []
if pack_envs and os.path.isdir(envs_path):
conda_pack_dir = tempfile.mkdtemp()
import conda_pack
for env in os.listdir(envs_path):
ext = 'zip' if filename.lower().endswith(".zip") else 'tar'
pack = os.path.join(
conda_pack_dir, '{}_envs_{}.{}'.format(current_platform(), env,
ext))
zip_symlinks = True if ext == 'zip' else False
fn = conda_pack.pack(prefix=os.path.join(envs_path, env),
arcroot=os.path.join(project.name, 'envs',
env),
output=pack,
zip_symlinks=zip_symlinks,
verbose=True,
force=True)
packed_envs.append(fn)
infos = _enumerate_archive_files(
project.directory_path,
frontend,
requirements=project.union_of_requirements_for_all_envs)
if infos is None:
return SimpleStatus(success=False,
description="Failed to list files in the project.",
errors=frontend.pop_errors())
# don't put the destination zip into itself, since it's fairly natural to
# create a archive right in the project directory
relative_dest_file = subdirectory_relative_to_directory(
filename, project.directory_path)
if not os.path.isabs(relative_dest_file):
infos = [
info for info in infos if info.relative_path != relative_dest_file
]
tmp_filename = filename + ".tmp-" + str(uuid.uuid4())
try:
if filename.lower().endswith(".zip"):
_write_zip(project.name,
infos,
tmp_filename,
packed_envs=packed_envs,
frontend=frontend)
elif filename.lower().endswith(".tar.gz"):
_write_tar(project.name,
infos,
tmp_filename,
compression="gz",
packed_envs=packed_envs,
frontend=frontend)
elif filename.lower().endswith(".tar.bz2"):
_write_tar(project.name,
infos,
tmp_filename,
compression="bz2",
packed_envs=packed_envs,
frontend=frontend)
elif filename.lower().endswith(".tar"):
_write_tar(project.name,
infos,
tmp_filename,
compression=None,
packed_envs=packed_envs,
frontend=frontend)
else:
frontend.error("Unsupported archive filename %s." % (filename))
return SimpleStatus(
success=False,
description=
"Project archive filename must be a .zip, .tar.gz, or .tar.bz2.",
errors=frontend.pop_errors())
rename_over_existing(tmp_filename, filename)
except IOError as e:
frontend.error(str(e))
return SimpleStatus(
success=False,
description=("Failed to write project archive %s." % (filename)),
errors=frontend.pop_errors())
finally:
try:
os.remove(tmp_filename)
if pack_envs:
os.remove(conda_pack_dir)
except (IOError, OSError):
pass
unlocked = []
for env_spec in project.env_specs.values():
if env_spec.lock_set.disabled:
unlocked.append(env_spec.name)
if len(unlocked) > 0:
frontend.info(
"Warning: env specs are not locked, which means they may not "
"work consistently for others or when deployed.")
frontend.info(
" Consider using the 'anaconda-project lock' command to lock the project."
)
if len(unlocked) != len(project.env_specs):
frontend.info(" Unlocked env specs are: " +
(", ".join(sorted(unlocked))))
return SimpleStatus(success=True,
description=("Created project archive %s" % filename))
pack: typing.Callable[[str, typing.Dict[str, str]], str]
def get_env_name(env_var_name) -> str:
"""
Return default virtual env
"""
virtual_env_path = os.environ.get(env_var_name)
if not virtual_env_path:
return 'default'
else:
return os.path.basename(virtual_env_path)
CONDA_PACKER = Packer(get_env_name(CONDA_DEFAULT_ENV), 'zip',
lambda output, reqs: conda_pack.pack(output=output))
PEX_PACKER = Packer(get_env_name('VIRTUAL_ENV'), 'pex',
pack_current_venv_in_pex)
def get_editable_requirements(executable: str = sys.executable):
def _get(name):
pkg = __import__(name.replace("-", "_"))
return os.path.dirname(pkg.__file__)
return [
_get(package["name"]) for package in _get_packages(True, executable)
]
def get_non_editable_requirements(executable: str = sys.executable):
def build(simnibs_dist_dir, include_spyder=False, developer_id=None):
simnibs_root_dir = os.path.normpath(
os.path.join(os.path.abspath(os.path.dirname(__file__)), '..'))
version = open(
os.path.join(simnibs_root_dir, "simnibs",
"_version.py")).readlines()[-1].split()[-1].strip("\"'")
pack_dir = os.path.abspath('simnibs_installer')
env_prefix = os.path.join(pack_dir, 'simnibs_env_tmp')
simnibs_dist_dir = os.path.abspath(simnibs_dist_dir)
# Create a new environment
if os.path.isdir(pack_dir):
shutil.rmtree(pack_dir)
if sys.platform == 'linux':
os_name = 'linux'
elif sys.platform == 'darwin':
os_name = 'macOS'
elif sys.platform == 'win32':
os_name = 'win'
else:
raise OSError('OS not supported!')
# Create temporary environment
env = os.path.join(simnibs_root_dir, f'environment_{os_name}.yml')
# Install requirements
subprocess.run(f'conda env create -p {env_prefix} -f {env} --force',
check=True,
shell=True)
# Install SimNIBS
wheels = glob.glob(
os.path.join(simnibs_dist_dir, f'simnibs-{version}*.whl'))
if len(wheels) == 0:
raise FileNotFoundError(
f'Did not find any wheels for version {version}')
if sys.platform == 'win32':
env_pip = os.path.join(env_prefix, 'Scripts', 'pip.exe')
else:
env_pip = os.path.join(env_prefix, 'bin', 'pip')
subprocess.run(
f'{env_pip} install simnibs --no-cache-dir --no-index --upgrade --find-links={simnibs_dist_dir}',
check=True,
shell=True)
if include_spyder:
subprocess.run(
f'{env_pip} install --upgrade '
'pyqt5==5.12 '
'pyqtwebengine==5.12 '
'pyflakes==2.2 '
'spyder==4.1',
check=True,
shell=True)
# Spyder can be started with simnibs_python -m spyder.app.start
# Pack
# I use .tar because MacOS erases the execute permission in .zip
conda_pack.pack(prefix=env_prefix,
dest_prefix='simnibs_env',
output=os.path.join(pack_dir, 'simnibs_env.tar'),
compress_level=0,
force=True)
shutil.unpack_archive(
os.path.join(pack_dir, 'simnibs_env.tar'),
os.path.join(pack_dir, 'simnibs_env'),
)
os.remove(os.path.join(pack_dir, 'simnibs_env.tar'))
# Remove temporary env
subprocess.run(f'conda env remove -y -p {env_prefix}',
check=True,
shell=True)
# Copy documentation
shutil.copytree(os.path.join(simnibs_root_dir, 'docs', 'build', 'html'),
os.path.join(pack_dir, 'documentation'))
# Copy the fix_entrypoints script and the postinstall script
shutil.copy(
os.path.join(simnibs_root_dir, 'packing', 'fix_entrypoints.py'),
os.path.join(pack_dir, 'simnibs_env'))
# Create OS-specific installer
if sys.platform == 'win32':
# Move the sitecustomize.py file to the site-packages directory
# This should allow for using the python interpreter without activating the environment
shutil.copy(
os.path.join(simnibs_root_dir, 'simnibs', 'utils',
'sitecustomize.py'),
os.path.join(pack_dir, 'simnibs_env', 'Lib', 'site-packages'))
#Use the installer.nsi template to create an NSIS installer
shutil.copy(
os.path.join(simnibs_root_dir, 'simnibs',
'resources', 'gui_icon.ico'),
os.path.join(pack_dir, 'gui_icon.ico'))
fn_script = os.path.join(pack_dir, 'installer.nsi')
with open(os.path.join(simnibs_root_dir, 'packing', 'installer.nsi'),
'r') as f:
install_script = Template(f.read()).render(version='.'.join(
version.split('.')[:2]),
full_version=version)
with open(fn_script, 'w') as f:
f.write(install_script)
print('Creating NSIS installer')
subprocess.run(fr'"%programfiles(x86)%\NSIS\makensis.exe" {fn_script}',
check=True,
shell=True)
shutil.move(os.path.join(pack_dir, 'simnibs_installer_windows.exe'),
'simnibs_installer_windows.exe')
if sys.platform == 'darwin':
with tempfile.TemporaryDirectory() as tmpdir:
for fn in glob.glob(
os.path.join(simnibs_root_dir, 'packing',
'macOS_installer', '*')):
fn_out = os.path.join(tmpdir, os.path.basename(fn))
with open(fn, 'r') as f:
template = Template(f.read()).render(version='.'.join(
version.split('.')[:2]),
full_version=version)
with open(fn_out, 'w') as f:
f.write(template)
os.chmod(fn_out, os.stat(fn).st_mode)
# Workaroud for Notarization
# Instead of signing all binaries, I zip the enironment with a password
# The postinstall script will unzip it in the user's computer
orig_folder = os.path.abspath(os.curdir)
os.chdir(pack_dir)
subprocess.run([
'zip', '-P', 'password', '-r', 'simnibs_env.zip', 'simnibs_env'
])
os.chdir(orig_folder)
shutil.rmtree(os.path.join(pack_dir, 'simnibs_env'))
print('Running pkgbuild')
subprocess.run(
[
'pkgbuild', '--root', pack_dir, '--identifier',
f'org.SimNIBS.{version}', '--version', version,
'--scripts', tmpdir, '--install-location',
'/Applications/SimNIBS-' +
'.'.join(version.split('.')[:2]),
os.path.join(tmpdir, 'simnibs_installer_macos.pkg')
],
check=True,
)
print('Running productbuid')
if developer_id is not None:
sign = ['--sign', developer_id]
else:
sign = []
subprocess.run([
'productbuild', '--distribution',
os.path.join(tmpdir, 'Distribution'), '--package-path', tmpdir,
'--resources', tmpdir, 'simnibs_installer_macos.pkg'
] + sign,
check=True)
elif sys.platform == 'linux':
# Write the install script
fn_script = os.path.join(pack_dir, 'install')
with open(os.path.join(simnibs_root_dir, 'packing', 'install'),
'r') as f:
install_script = Template(f.read()).render(version='.'.join(
version.split('.')[:2]),
full_version=version)
with open(fn_script, 'w') as f:
f.write(install_script)
os.chmod(
fn_script,
os.stat(fn_script).st_mode | stat.S_IXUSR | stat.S_IXGRP
| stat.S_IXOTH)
print('compressing')
shutil.make_archive(
'simnibs_installer_linux',
'gztar',
# I use root_dir and base_dir so that it decompresses into a folder called
# simnibs_installer
root_dir='.',
base_dir=os.path.relpath(pack_dir))
def conda_env():
envpath = 'dask-yarn-py%d%d.tar.gz' % sys.version_info[:2]
if not os.path.exists(envpath):
conda_pack.pack(output=envpath, verbose=True)
return envpath
