def pack_venv_in_conda(env_path: str, reqs: Dict[str, str], new_env_diff: bool) -> str: if not new_env_diff: conda_pack.pack(output=env_path) return env_path else: return create_and_pack_conda_env(env_path, reqs)
def test_run_with_packed_env(): import conda_pack temp_dir = os.environ.get("MARS_YARN_TEST_DIR") clean_after_test = False if temp_dir is None: clean_after_test = True temp_dir = tempfile.mkdtemp(prefix="test-mars-yarn-") else: os.makedirs(temp_dir, exist_ok=True) packed_env_file = os.path.join(temp_dir, "mars-test-env.tar.gz") if not os.path.exists(packed_env_file): try: conda_pack.pack(output=packed_env_file, ignore_editable_packages=True) except conda_pack.CondaPackException: logger.exception( "Failed to pack environment, this test will be skipped") return try: _run_yarn_test_with_env(packed_env_file, 1200) finally: if clean_after_test: shutil.rmtree(temp_dir)
def _pack_environment(env: dict, file_path: str) -> str: with NamedTemporaryFile(mode="w", suffix=".yml") as file: # TODO: Save copy of environment.yaml alongside tarball yaml.safe_dump(env, file) # Create env tmp_env_path = file.name tmp_env_name = f"tempo-{uuid.uuid4()}" cmd = f"conda env create --name {tmp_env_name} --file {tmp_env_path}" run(cmd, shell=True, check=True) try: # Pack environment conda_pack.pack( name=tmp_env_name, output=file_path, force=True, ignore_editable_packages=True, ignore_missing_files=True, ) finally: # Remove environment cmd = f"conda remove --name {tmp_env_name} --all --yes" run(cmd, shell=True, check=True) return file_path
def _pack_conda_main(self, args): import sys import traceback from conda_pack.cli import fail, PARSER, context import conda_pack from conda_pack import pack, CondaPackException args = PARSER.parse_args(args=args) # Manually handle version printing to output to stdout in python < 3.4 if args.version: print('conda-pack %s' % conda_pack.__version__) sys.exit(0) try: with context.set_cli(): pack(name=args.name, prefix=args.prefix, output=args.output, format=args.format, force=args.force, compress_level=args.compress_level, n_threads=args.n_threads, zip_symlinks=args.zip_symlinks, zip_64=not args.no_zip_64, arcroot=args.arcroot, dest_prefix=args.dest_prefix, verbose=not args.quiet, filters=args.filters) except CondaPackException as e: fail("CondaPackError: %s" % e) except KeyboardInterrupt: fail("Interrupted") except Exception: fail(traceback.format_exc())
def testRunWithPackedEnv(self): import conda_pack temp_dir = os.environ.get('MARS_YARN_TEST_DIR') clean_after_test = False if temp_dir is None: clean_after_test = True temp_dir = tempfile.mkdtemp(prefix='test-mars-yarn-') else: os.makedirs(temp_dir, exist_ok=True) packed_env_file = os.path.join(temp_dir, 'mars-test-env.tar.gz') if not os.path.exists(packed_env_file): try: conda_pack.pack(output=packed_env_file, ignore_editable_packages=True) except conda_pack.CondaPackException: logger.exception( 'Failed to pack environment, this test will be skipped') return try: self._runYarnTestWithEnv(packed_env_file, 1200) finally: if clean_after_test: shutil.rmtree(temp_dir)
def test_pack_exceptions(py36_env): # Can't pass both prefix and name with pytest.raises(CondaPackException): pack(prefix=py36_path, name='py36') # Unknown filter type with pytest.raises(CondaPackException): pack(prefix=py36_path, filters=[("exclude", "*.py"), ("foo", "*.pyc")])
def conda_pack(prefix, output, ignore_missing_files=True): from conda_pack import pack pack( prefix=str(prefix), output=str(output), ignore_missing_files=ignore_missing_files, )
def pack_venv_in_conda(output: str, reqs: Dict[str, str], additional_packages: Dict[str, str], ignored_packages: Collection[str]) -> str: if len(additional_packages) == 0 and len(ignored_packages) == 0: conda_pack.pack(output=output) return output else: return create_and_pack_conda_env(output, reqs)
def _pack_environment(env_name: str, file_path: str): # Pack environment conda_pack.pack( name=env_name, output=file_path, force=True, verbose=True, ignore_editable_packages=False, ignore_missing_files=True, )
def _pack_environment(env_name: str, file_path: str): logger.info(f"packing conda environment from {env_name} to {file_path}") # Pack environment conda_pack.pack( name=env_name, output=file_path, force=True, verbose=True, ignore_editable_packages=False, ignore_missing_files=True, )
def create_and_pack_conda_env(spec_file: str = None, reqs: List[str] = None, output: str = None) -> str: """ Create a new conda virtual environment and zip it :param spec_file: conda yaml spec file to use :param reqs: dependencies to install :param output: a dedicated output path :return: destination of the archive """ project_env_name = get_conda_env_name(spec_file=spec_file, reqs=reqs) _logger.info(f"Found project env name {project_env_name}") env_path = get_or_create_conda_env(project_env_name, spec_file) if reqs: env_python_bin = os.path.join(env_path, "bin", "python") if not os.path.exists(env_python_bin): raise RuntimeError("Failed to create Python binary at " + env_python_bin) _logger.info("Installing packages into " + env_path) process.call([env_python_bin, "-m", "pip", "install"] + reqs) return conda_pack.pack(prefix=env_path, output=output, force=True)
def test_pack(tmpdir, py36_env): out_path = os.path.join(str(tmpdir), 'py36.tar') exclude1 = "*.py" exclude2 = "*.pyc" include = "lib/python3.6/site-packages/conda_pack_test_lib1/*" res = pack(prefix=py36_path, output=out_path, filters=[("exclude", exclude1), ("exclude", exclude2), ("include", include)]) assert res == out_path assert os.path.exists(out_path) assert tarfile.is_tarfile(out_path) with tarfile.open(out_path) as fil: paths = fil.getnames() filtered = (py36_env.exclude(exclude1).exclude(exclude2).include(include)) # Files line up with filtering, with extra conda-unpack command sol = set(f.target for f in filtered.files) res = set(paths) diff = res.difference(sol) assert len(diff) == 1 extra = list(diff)[0] assert 'conda-unpack' in extra
def test_pack(tmpdir, py36_env): out_path = os.path.join(str(tmpdir), 'py36.tar') exclude1 = "*.py" exclude2 = "*.pyc" include = os.path.join(SP_36, 'conda_pack_test_lib1', '*') res = pack(prefix=py36_path, output=out_path, filters=[("exclude", exclude1), ("exclude", exclude2), ("include", include)]) assert res == out_path assert os.path.exists(out_path) assert tarfile.is_tarfile(out_path) with tarfile.open(out_path) as fil: paths = fil.getnames() filtered = (py36_env.exclude(exclude1).exclude(exclude2).include(include)) # Files line up with filtering, with extra conda-unpack command sol = set(os.path.normcase(f.target) for f in filtered.files) res = set(os.path.normcase(p) for p in paths) diff = res.difference(sol) if on_win: fnames = ('conda-unpack.exe', 'conda-unpack-script.py', 'activate.bat', 'deactivate.bat') else: fnames = ('conda-unpack', 'activate', 'deactivate') assert diff == set(os.path.join(BIN_DIR_L, f) for f in fnames)
def test_dest_prefix(tmpdir, py36_env): out_path = os.path.join(str(tmpdir), 'py36.tar') dest = r'C:\foo\bar\baz\biz' if on_win else '/foo/bar/baz/biz' res = pack(prefix=py36_path, dest_prefix=dest, output=out_path) assert res == out_path assert os.path.exists(out_path) assert tarfile.is_tarfile(out_path) with tarfile.open(out_path) as fil: paths = fil.getnames() # No conda-unpack generated assert 'conda-unpack' not in paths if on_win: test_files = ['Scripts/conda-pack-test-lib1', 'Scripts/pytest.exe'] else: test_files = ['bin/conda-pack-test-lib1', 'bin/pytest', 'bin/clear'] orig_bytes = py36_env.prefix.encode() new_bytes = dest.encode() # all paths, including shebangs, are rewritten using the prefix with tarfile.open(out_path) as fil: for test_file in test_files: orig_path = os.path.join(py36_env.prefix, test_file) with open(orig_path, 'rb') as fil2: orig_data = fil2.read() if orig_bytes in orig_data: data = fil.extractfile(test_file).read() assert orig_bytes not in data, test_file assert new_bytes in data, test_file
def test_pack_exceptions(tmpdir, py36_env): already_exists = os.path.join(str(tmpdir), 'py36.tar') with open(already_exists, 'wb'): pass # file already exists with pytest.raises(CondaPackException): py36_env.pack(output=already_exists) # Can't pass both prefix and name with pytest.raises(CondaPackException): pack(prefix=py36_path, name='py36') # Unknown filter type with pytest.raises(CondaPackException): pack(prefix=py36_path, filters=[("exclude", "*.py"), ("foo", "*.pyc")])
def test_dest_prefix(tmpdir, py36_env): out_path = os.path.join(str(tmpdir), 'py36.tar') dest = r'c:\foo\bar\baz\biz' if on_win else '/foo/bar/baz/biz' res = pack(prefix=py36_path, dest_prefix=dest, output=out_path) assert res == out_path assert os.path.exists(out_path) assert tarfile.is_tarfile(out_path) _test_dest_prefix(py36_env.prefix, dest, '', out_path, 'r')
def create_and_pack_conda_env( env_path: str, reqs: Dict[str, str], ) -> str: try: _call(["conda"]) except CalledProcessError: raise RuntimeError("conda is not available in $PATH") env_path_split = env_path.split('.', 1) env_name = env_path_split[0] compression_format = env_path_split[1] if len( env_path_split) > 1 else ".zip" archive_path = f"{env_name}.{compression_format}" if os.path.exists(env_name): shutil.rmtree(env_name) _logger.info("Creating new env " + env_name) python_version = sys.version_info _call([ "conda", "create", "-p", env_name, "-y", "-q", "--copy", f"python={python_version.major}.{python_version.minor}.{python_version.micro}" ], env=dict(os.environ)) env_python_bin = os.path.join(env_name, "bin", "python") if not os.path.exists(env_python_bin): raise RuntimeError("Failed to create Python binary at " + env_python_bin) _logger.info("Installing packages into " + env_name) _call([env_python_bin, "-m", "pip", "install"] + format_requirements(reqs)) if os.path.exists(archive_path): os.remove(archive_path) conda_pack.pack(prefix=env_name, output=archive_path) return archive_path
def pack_venv_in_conda(reqs: List[str], changed_reqs: bool = False, output: str = None) -> str: """ Pack the current virtual environment :param reqs: directory to zip :param changed_reqs: we prefer zipping the current virtual env as much as possible, if it has been changed we need to create a new one with 'conda create -n env ..' and reinstall the dependencies inside :param output: a dedicated output path :return: destination of the archive """ if not changed_reqs: return conda_pack.pack(output=output) else: return create_and_pack_conda_env(reqs=reqs, output=output)
def test_dest_prefix(tmpdir, py36_env): out_path = os.path.join(str(tmpdir), 'py36.tar') dest = '/foo/bar/baz/biz' res = pack(prefix=py36_path, dest_prefix=dest, output=out_path) assert res == out_path assert os.path.exists(out_path) assert tarfile.is_tarfile(out_path) with tarfile.open(out_path) as fil: paths = fil.getnames() # No conda-unpack generated assert 'conda-unpack' not in paths dest_bytes = dest.encode() # shebangs are rewritten using env with tarfile.open(out_path) as fil: text_from_conda = fil.extractfile('/'.join([BIN_DIR, 'conda-pack-test-lib1'])).read() text_from_pip = fil.extractfile('scripts/pytest.exe' if on_win else 'bin/pytest').read() assert dest_bytes not in text_from_conda assert dest_bytes not in text_from_pip assert b'env python' in text_from_conda if not on_win: # pip entrypoint on Windows is complicated... assert b'env python' in text_from_pip with tarfile.open(out_path) as fil: binary_from_conda = fil.extractfile('bin/clear').read() # Other files are rewritten to use specified prefix This is only checked if # the original file did include the prefix, which is true at least on osx. orig_path = os.path.join(py36_env.prefix, 'bin/clear') with open(orig_path, 'rb') as fil: orig_bytes = fil.read() if py36_env.prefix.encode() in orig_bytes: assert py36_env.prefix.encode() not in binary_from_conda assert dest_bytes in binary_from_conda
def pack(**conda_pack_kwargs): """ Calls conda_pack.pack. If the packed output file already exists, this will not repackage it unless conda_pack_kwargs["force"] == True. Returns the path to the output packed env file. Arguments: * `conda_pack_kwargs` args to pass to conda_pack.pack(). """ kwargs = conda_pack_defaults.copy() kwargs.update(conda_pack_kwargs) # Make sure output is set to something, so we can return it if it already exists. if "output" not in kwargs: conda_env_name = "env" if "prefix" in kwargs: conda_env_name = os.path.basename(kwargs["prefix"]) elif "name" in kwargs: conda_env_name = kwargs["name"] elif is_active(): conda_env_name = active_name() kwargs["output"] = "conda-{}.{}".format(conda_env_name, kwargs["format"]) conda_packed_file = kwargs["output"] if os.path.isfile(conda_packed_file) and not kwargs["force"]: print_err( f"A conda environment is already packed at {conda_packed_file}. " "If you have recently installed new packages into your conda env, set " "force=True in conda_pack_kwargs and it will be repacked for you.") return conda_packed_file else: # Isolate the import here so that we don"t get import errors # if conda_pack is not installed (e.g. in a virtualenv). import conda_pack # NOTE: If no conda env is currently active, and kwargs # doesn"t contain information about what env to pack (i.e. no name or prefix) # then this raise an error. return conda_pack.pack(**kwargs)
def test_parcel(tmpdir, py36_env): if on_win: pytest.skip("Not parcel tests on Windows") arcroot = 'py36-1234.56' out_path = os.path.join(str(tmpdir), arcroot + '-el7.parcel') pdir = os.getcwd() try: os.chdir(str(tmpdir)) res = pack(prefix=py36_path, format='parcel', parcel_version='1234.56') finally: os.chdir(pdir) assert os.path.join(str(tmpdir), res) == out_path assert os.path.exists(out_path) # Verify that only the parcel files were added with tarfile.open(out_path, 'r:gz') as fil: paths = fil.getnames() sol = set(os.path.join(arcroot, f.target) for f in py36_env.files) diff = set(paths).difference(sol) fnames = ('conda_env.sh', 'parcel.json') assert diff == set(os.path.join(arcroot, 'meta', f) for f in fnames) # Verify correct metadata in parcel.json with tarfile.open(out_path) as fil: fpath = os.path.join(arcroot, 'meta', 'parcel.json') data = fil.extractfile(fpath).read() data = json.loads(data) assert data['name'] == 'py36' and data['components'][0]['name'] == 'py36' assert data['version'] == '1234.56' and data['components'][0][ 'version'] == '1234.56' # Verify the correct dest_prefix substitution dest = os.path.join('/opt/cloudera/parcels', arcroot) _test_dest_prefix(py36_env.prefix, dest, arcroot, out_path, 'r:gz')
import argparse parser = argparse.ArgumentParser(description='AGS release package tool.') parser.add_argument('--skip_packing', action='store_true', help="skip packaging to dist folder") parser.add_argument('--rhi', action='store_true', help="pack into rhi installer") parser.add_argument('--version', default="v0.0.0", help="version number") args = parser.parse_args() start = time.time() conda_pack.pack(output="env.zip", verbose=True, n_threads=-1, force=True) print('unpacking to ui/Rhino/AGS/dev/env') shutil.unpack_archive("env.zip", "ui/Rhino/AGS/dev/env") print('removing unnecessary files') for root, dirs, files in os.walk("ui/Rhino/AGS/dev/env"): for d in dirs: if d.find("node_modules") >= 0: shutil.rmtree(os.path.join(root, d)) for f in files: if f.find("electron.zip") >= 0: os.remove(os.path.join(root, f))
def conda_pack(prefix, output): import conda_pack conda_pack.pack(prefix=str(prefix), output=str(output))
def create_and_pack_conda_env( name: str, python: str, pip_packages: typing.List[str], root: typing.Optional[str] = tempfile.tempdir ) -> str: """Create a Conda environment. The environment is created via ``conda``. However, all the packages other than the Python interpreter are installed via ``pip`` to allow for more flexibility. Parameters ---------- name : str A human-readable name of the environment. python : str Python version in the MAJOR.MINOR.MICRO format. pip_packages : list PyPI packages to install in the environment. root : list Path to the root directory with Conda environments. If ``None``, system temporary directory will be used with a fallback to the current directory. Returns ------- env_path : str Path to the packed environment. """ try: _call(["conda"]) except CalledProcessError: raise RuntimeError("conda is not available in $PATH") env_path = os.path.join(root or os.getcwd(), name) if not os.path.exists(env_path): logger.info("Creating new env " + name) _call([ "conda", "create", "-p", env_path, "-y", "-q", "--copy", "python=" + python, # TensorFlow enforces an upper bound on setuptools which # conflicts with the version installed by Conda by default. "setuptools=" + setuptools.__version__ ], env=dict(os.environ)) env_python_bin = os.path.join(env_path, "bin", "python") if not os.path.exists(env_python_bin): raise RuntimeError( "Failed to create Python binary at " + env_python_bin) if pip_packages: logger.info("Installing packages into " + name) _call([env_python_bin, "-m", "pip", "install"] + pip_packages) requirements_path = os.path.join(env_path, "requirements.txt") with open(requirements_path, "w") as f: print(*pip_packages, sep=os.linesep, file=f) env_zip_path = env_path + ".zip" if not os.path.exists(env_zip_path): import conda_pack conda_pack.pack(prefix=env_path, output=env_zip_path) return env_zip_path
def build_parcel(): jinja_env = Environment(loader=FileSystemLoader('templates')) (jpy_version, parcel_version) = get_version() # Create the JupyterHub Environment render_environment_yaml(jinja_env, jpy_version=jpy_version) with open('environment.yml') as f: conda_env = yaml.load(f, Loader=yaml.Loader) check_call( 'conda env create --force -p {name}'.format(**conda_env).split(' ')) # Get the complete list of installed packages components = json.loads( run_command(Commands.LIST, '--json', '-p', conda_env['name'])[0]) jpy_version = [ x['version'] for x in components if x['name'] == conda_env['name'] ][0] # Create the meta folder under the conda-environment meta_dir = os.path.join(conda_env['name'], 'meta') os.makedirs(meta_dir, exist_ok=True) # Create the meta/parcel.json file parcel_template = jinja_env.get_template('parcel.yaml') parcel_rendered = parcel_template.render(conda_env=conda_env, jpy_version=jpy_version, parcel_version=parcel_version, components=components) parcel_dict = yaml.load(parcel_rendered, Loader=yaml.Loader) parcel_json = os.path.join(meta_dir, 'parcel.json') with open(parcel_json, 'w') as f: json.dump(parcel_dict, f) print(check_output(['validator.sh', '-p', parcel_json]).decode('utf-8')) # Create the scripts.defines file (a.k.a. activate script) shutil.copy('env.sh', os.path.join(meta_dir, parcel_dict['scripts']['defines'])) # Use conda-pack to create the .parcel file if os.path.exists('parcels'): shutil.rmtree('parcels') os.makedirs('parcels', exist_ok=True) parcel_fqn = '{name}-{version}'.format(**parcel_dict) parcel_file = conda_pack.pack(prefix=conda_env['name'], output='%s-distro.parcel' % parcel_fqn, arcroot=parcel_fqn, dest_prefix='/opt/cloudera/parcels/%s' % parcel_fqn, format="tar.gz", force=True) # Link generated file to selected distros distros = ['el6', 'el7'] for distro in distros: dest_name = 'parcels/{}-{}.parcel'.format(parcel_fqn, distro) os.symlink('../%s-distro.parcel' % parcel_fqn, dest_name) check_call(['validator.sh', '-f', dest_name]) # Generate the parcel manifest file print(check_output(['make_manifest.py', 'parcels']).decode('utf-8'))
def build(): import conda_pack version = open("simnibs/_version.py").readlines()[-1].split()[-1].strip( "\"'") # Create a new environment if os.path.dirname(__file__): os.chdir(os.path.dirname(__file__)) if not os.path.isdir('pack'): os.mkdir('pack') env = os.path.join(os.path.dirname(__file__), 'environment_{}.yml') if sys.platform == 'linux': env = env.format('linux') out_pack = os.path.join('pack', 'simnibs_env.tar.gz') os_name = 'linux' elif sys.platform == 'darwin': env = env.format('macOS') out_pack = os.path.join('pack', 'simnibs_env.tar.gz') os_name = 'macOS' elif sys.platform == 'win32': env = env.format('win') out_pack = os.path.join('pack', 'simnibs_env.zip') os_name = 'win' else: raise OSError('OS not supported!') # Create temporary environment subprocess.run(f'conda env create -n simnibs_env_tmp -f {env}', check=True, shell=True) # Pack if os.path.isfile(out_pack): os.remove(out_pack) conda_pack.pack( name='simnibs_env_tmp', dest_prefix='simnibs_env', output=out_pack, ) # Remove temporary env subprocess.run('conda env remove -y --name simnibs_env_tmp', check=True, shell=True) # Copy wheel wheels = glob.glob(f'dist/simnibs-{version}*.whl') if len(wheels) == 0: raise FileNotFoundError( f'Did not find any wheels for version {version}') for f in wheels: shutil.copy(f, 'pack') # Create bash or bat file for installation if sys.platform == 'win32': with open(os.path.join('pack/install.cmd'), 'w') as f: f.write(f'SET INSTALL_DIR=%LOCALAPPDATA%\SimNIBS\n') f.write('mkdir "%INSTALL_DIR%\simnibs_env"\n') f.write( 'powershell.exe -nologo -noprofile -command "& ' '{ Add-Type -A \'System.IO.Compression.FileSystem\'; ' '[IO.Compression.ZipFile]::ExtractToDirectory(\'%~dp0simnibs_env.zip\', \'%INSTALL_DIR%\simnibs_env\'); }"\n' ) f.write('call "%INSTALL_DIR%\simnibs_env\\Scripts\\activate"\n') f.write( 'python -m pip install simnibs --no-cache-dir --no-index --upgrade --find-links=./\n' ) f.write( 'postinstall_simnibs -d "%INSTALL_DIR%" --copy-matlab --setup-links --no-extra-coils' ) else: if sys.platform == 'darwin': fn_script = os.path.join('pack', 'install') install_dir = "$HOME/Applications/SimNIBS" else: fn_script = os.path.join('pack', 'install') install_dir = "$HOME/SimNIBS" with open(fn_script, 'w') as f: # Some problem with Qt f.write('#! /bin/bash -e\n') f.write(f'INSTALL_DIR={install_dir}\n') f.write( 'CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd)"\n' ) f.write('mkdir -p "$INSTALL_DIR/simnibs_env"\n') f.write( 'tar -zxf "$CUR_DIR/simnibs_env.tar.gz" -C "$INSTALL_DIR/simnibs_env"\n' ) f.write('source "$INSTALL_DIR/simnibs_env/bin/activate"\n') f.write( 'python -m pip install simnibs --no-cache-dir --no-index --upgrade --find-links="$CUR_DIR"\n' ) f.write( 'python -m pip install pyqt5 --force-reinstall --no-cache-dir\n' ) # I need to re-install pyqt, this requires internet f.write( 'postinstall_simnibs -d "$INSTALL_DIR" --copy-matlab --setup-links --no-extra-coils' ) os.chmod( fn_script, os.stat(fn_script).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) # zip the whole thing shutil.make_archive(f'simnibs-{version}-{os_name}', 'zip', 'pack')
def _archive_project(project, filename, pack_envs=False): """Make an archive of the non-ignored files in the project. Args: project (``Project``): the project filename (str): name for the new zip or tar.gz archive file Returns: a ``Status``, if failed has ``errors`` """ failed = project.problems_status() if failed is not None: for error in failed.errors: project.frontend.error(error) return failed frontend = _new_error_recorder(project.frontend) if not os.path.exists(project.project_file.filename): frontend.error("%s does not exist." % project.project_file.basename) return SimpleStatus(success=False, description="Can't create an archive.", errors=frontend.pop_errors()) # this would most likely happen in a GUI editor, if it reloaded # the project from memory but hadn't saved yet. if project.project_file.has_unsaved_changes: frontend.error("%s has been modified but not saved." % project.project_file.basename) return SimpleStatus(success=False, description="Can't create an archive.", errors=frontend.pop_errors()) envs_path = os.path.join(project.project_file.project_dir, 'envs') packed_envs = [] if pack_envs and os.path.isdir(envs_path): conda_pack_dir = tempfile.mkdtemp() import conda_pack for env in os.listdir(envs_path): ext = 'zip' if filename.lower().endswith(".zip") else 'tar' pack = os.path.join( conda_pack_dir, '{}_envs_{}.{}'.format(current_platform(), env, ext)) zip_symlinks = True if ext == 'zip' else False fn = conda_pack.pack(prefix=os.path.join(envs_path, env), arcroot=os.path.join(project.name, 'envs', env), output=pack, zip_symlinks=zip_symlinks, verbose=True, force=True) packed_envs.append(fn) infos = _enumerate_archive_files( project.directory_path, frontend, requirements=project.union_of_requirements_for_all_envs) if infos is None: return SimpleStatus(success=False, description="Failed to list files in the project.", errors=frontend.pop_errors()) # don't put the destination zip into itself, since it's fairly natural to # create a archive right in the project directory relative_dest_file = subdirectory_relative_to_directory( filename, project.directory_path) if not os.path.isabs(relative_dest_file): infos = [ info for info in infos if info.relative_path != relative_dest_file ] tmp_filename = filename + ".tmp-" + str(uuid.uuid4()) try: if filename.lower().endswith(".zip"): _write_zip(project.name, infos, tmp_filename, packed_envs=packed_envs, frontend=frontend) elif filename.lower().endswith(".tar.gz"): _write_tar(project.name, infos, tmp_filename, compression="gz", packed_envs=packed_envs, frontend=frontend) elif filename.lower().endswith(".tar.bz2"): _write_tar(project.name, infos, tmp_filename, compression="bz2", packed_envs=packed_envs, frontend=frontend) elif filename.lower().endswith(".tar"): _write_tar(project.name, infos, tmp_filename, compression=None, packed_envs=packed_envs, frontend=frontend) else: frontend.error("Unsupported archive filename %s." % (filename)) return SimpleStatus( success=False, description= "Project archive filename must be a .zip, .tar.gz, or .tar.bz2.", errors=frontend.pop_errors()) rename_over_existing(tmp_filename, filename) except IOError as e: frontend.error(str(e)) return SimpleStatus( success=False, description=("Failed to write project archive %s." % (filename)), errors=frontend.pop_errors()) finally: try: os.remove(tmp_filename) if pack_envs: os.remove(conda_pack_dir) except (IOError, OSError): pass unlocked = [] for env_spec in project.env_specs.values(): if env_spec.lock_set.disabled: unlocked.append(env_spec.name) if len(unlocked) > 0: frontend.info( "Warning: env specs are not locked, which means they may not " "work consistently for others or when deployed.") frontend.info( " Consider using the 'anaconda-project lock' command to lock the project." ) if len(unlocked) != len(project.env_specs): frontend.info(" Unlocked env specs are: " + (", ".join(sorted(unlocked)))) return SimpleStatus(success=True, description=("Created project archive %s" % filename))
pack: typing.Callable[[str, typing.Dict[str, str]], str] def get_env_name(env_var_name) -> str: """ Return default virtual env """ virtual_env_path = os.environ.get(env_var_name) if not virtual_env_path: return 'default' else: return os.path.basename(virtual_env_path) CONDA_PACKER = Packer(get_env_name(CONDA_DEFAULT_ENV), 'zip', lambda output, reqs: conda_pack.pack(output=output)) PEX_PACKER = Packer(get_env_name('VIRTUAL_ENV'), 'pex', pack_current_venv_in_pex) def get_editable_requirements(executable: str = sys.executable): def _get(name): pkg = __import__(name.replace("-", "_")) return os.path.dirname(pkg.__file__) return [ _get(package["name"]) for package in _get_packages(True, executable) ] def get_non_editable_requirements(executable: str = sys.executable):
def build(simnibs_dist_dir, include_spyder=False, developer_id=None): simnibs_root_dir = os.path.normpath( os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')) version = open( os.path.join(simnibs_root_dir, "simnibs", "_version.py")).readlines()[-1].split()[-1].strip("\"'") pack_dir = os.path.abspath('simnibs_installer') env_prefix = os.path.join(pack_dir, 'simnibs_env_tmp') simnibs_dist_dir = os.path.abspath(simnibs_dist_dir) # Create a new environment if os.path.isdir(pack_dir): shutil.rmtree(pack_dir) if sys.platform == 'linux': os_name = 'linux' elif sys.platform == 'darwin': os_name = 'macOS' elif sys.platform == 'win32': os_name = 'win' else: raise OSError('OS not supported!') # Create temporary environment env = os.path.join(simnibs_root_dir, f'environment_{os_name}.yml') # Install requirements subprocess.run(f'conda env create -p {env_prefix} -f {env} --force', check=True, shell=True) # Install SimNIBS wheels = glob.glob( os.path.join(simnibs_dist_dir, f'simnibs-{version}*.whl')) if len(wheels) == 0: raise FileNotFoundError( f'Did not find any wheels for version {version}') if sys.platform == 'win32': env_pip = os.path.join(env_prefix, 'Scripts', 'pip.exe') else: env_pip = os.path.join(env_prefix, 'bin', 'pip') subprocess.run( f'{env_pip} install simnibs --no-cache-dir --no-index --upgrade --find-links={simnibs_dist_dir}', check=True, shell=True) if include_spyder: subprocess.run( f'{env_pip} install --upgrade ' 'pyqt5==5.12 ' 'pyqtwebengine==5.12 ' 'pyflakes==2.2 ' 'spyder==4.1', check=True, shell=True) # Spyder can be started with simnibs_python -m spyder.app.start # Pack # I use .tar because MacOS erases the execute permission in .zip conda_pack.pack(prefix=env_prefix, dest_prefix='simnibs_env', output=os.path.join(pack_dir, 'simnibs_env.tar'), compress_level=0, force=True) shutil.unpack_archive( os.path.join(pack_dir, 'simnibs_env.tar'), os.path.join(pack_dir, 'simnibs_env'), ) os.remove(os.path.join(pack_dir, 'simnibs_env.tar')) # Remove temporary env subprocess.run(f'conda env remove -y -p {env_prefix}', check=True, shell=True) # Copy documentation shutil.copytree(os.path.join(simnibs_root_dir, 'docs', 'build', 'html'), os.path.join(pack_dir, 'documentation')) # Copy the fix_entrypoints script and the postinstall script shutil.copy( os.path.join(simnibs_root_dir, 'packing', 'fix_entrypoints.py'), os.path.join(pack_dir, 'simnibs_env')) # Create OS-specific installer if sys.platform == 'win32': # Move the sitecustomize.py file to the site-packages directory # This should allow for using the python interpreter without activating the environment shutil.copy( os.path.join(simnibs_root_dir, 'simnibs', 'utils', 'sitecustomize.py'), os.path.join(pack_dir, 'simnibs_env', 'Lib', 'site-packages')) #Use the installer.nsi template to create an NSIS installer shutil.copy( os.path.join(simnibs_root_dir, 'simnibs', 'resources', 'gui_icon.ico'), os.path.join(pack_dir, 'gui_icon.ico')) fn_script = os.path.join(pack_dir, 'installer.nsi') with open(os.path.join(simnibs_root_dir, 'packing', 'installer.nsi'), 'r') as f: install_script = Template(f.read()).render(version='.'.join( version.split('.')[:2]), full_version=version) with open(fn_script, 'w') as f: f.write(install_script) print('Creating NSIS installer') subprocess.run(fr'"%programfiles(x86)%\NSIS\makensis.exe" {fn_script}', check=True, shell=True) shutil.move(os.path.join(pack_dir, 'simnibs_installer_windows.exe'), 'simnibs_installer_windows.exe') if sys.platform == 'darwin': with tempfile.TemporaryDirectory() as tmpdir: for fn in glob.glob( os.path.join(simnibs_root_dir, 'packing', 'macOS_installer', '*')): fn_out = os.path.join(tmpdir, os.path.basename(fn)) with open(fn, 'r') as f: template = Template(f.read()).render(version='.'.join( version.split('.')[:2]), full_version=version) with open(fn_out, 'w') as f: f.write(template) os.chmod(fn_out, os.stat(fn).st_mode) # Workaroud for Notarization # Instead of signing all binaries, I zip the enironment with a password # The postinstall script will unzip it in the user's computer orig_folder = os.path.abspath(os.curdir) os.chdir(pack_dir) subprocess.run([ 'zip', '-P', 'password', '-r', 'simnibs_env.zip', 'simnibs_env' ]) os.chdir(orig_folder) shutil.rmtree(os.path.join(pack_dir, 'simnibs_env')) print('Running pkgbuild') subprocess.run( [ 'pkgbuild', '--root', pack_dir, '--identifier', f'org.SimNIBS.{version}', '--version', version, '--scripts', tmpdir, '--install-location', '/Applications/SimNIBS-' + '.'.join(version.split('.')[:2]), os.path.join(tmpdir, 'simnibs_installer_macos.pkg') ], check=True, ) print('Running productbuid') if developer_id is not None: sign = ['--sign', developer_id] else: sign = [] subprocess.run([ 'productbuild', '--distribution', os.path.join(tmpdir, 'Distribution'), '--package-path', tmpdir, '--resources', tmpdir, 'simnibs_installer_macos.pkg' ] + sign, check=True) elif sys.platform == 'linux': # Write the install script fn_script = os.path.join(pack_dir, 'install') with open(os.path.join(simnibs_root_dir, 'packing', 'install'), 'r') as f: install_script = Template(f.read()).render(version='.'.join( version.split('.')[:2]), full_version=version) with open(fn_script, 'w') as f: f.write(install_script) os.chmod( fn_script, os.stat(fn_script).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) print('compressing') shutil.make_archive( 'simnibs_installer_linux', 'gztar', # I use root_dir and base_dir so that it decompresses into a folder called # simnibs_installer root_dir='.', base_dir=os.path.relpath(pack_dir))
def conda_env(): envpath = 'dask-yarn-py%d%d.tar.gz' % sys.version_info[:2] if not os.path.exists(envpath): conda_pack.pack(output=envpath, verbose=True) return envpath