def test_lock_set_properties(monkeypatch):
lock_set = CondaLockSet(
{
'all': ["something=0.5=2", "bokeh=0.12.4=1"],
'linux-64': ["linux-thing=1.0=0"],
'unix': ["unix-thing=5=1"],
'win': ["windows-cross-bit-thing=3.2"],
'win-32': ["windows-thing=2.0=3", "bokeh=2.3=7"]
},
platforms=['linux-64', 'win-32'])
# it is part of the API definition that we need to APPEND the
# per-platform stuff, so it overrides.
assert lock_set.package_specs_for_platform('win-32') == (
"something=0.5=2", "windows-cross-bit-thing=3.2",
"windows-thing=2.0=3", "bokeh=2.3=7")
# on Linux-64, test that it works without monkeypatch
if conda_api.current_platform() != 'linux-64':
monkeypatch.setattr(
'anaconda_project.internal.conda_api.current_platform',
lambda: 'linux-64')
assert lock_set.package_specs_for_current_platform == ("something=0.5=2",
"bokeh=0.12.4=1",
"unix-thing=5=1",
"linux-thing=1.0=0")
assert lock_set.platforms == ('linux-64', 'win-32')
def resolve_dependencies(self, package_specs, channels, platforms):
by_platform = {}
current = conda_api.current_platform()
resolve_for_platforms = list(platforms)
# always resolve "current" first because it's confusing if
# an error says resolution failed on another platform when
# the real issue is that resolution will fail on all platforms.
if current in resolve_for_platforms:
resolve_for_platforms.remove(current)
resolve_for_platforms = [current] + resolve_for_platforms
for conda_platform in resolve_for_platforms:
try:
self._log_info("Resolving conda packages for %s" %
conda_platform)
deps = conda_api.resolve_dependencies(pkgs=package_specs,
platform=conda_platform,
channels=channels)
except conda_api.CondaError as e:
raise CondaManagerError("Error resolving for {}: {}".format(
conda_platform, str(e)))
locked_specs = ["%s=%s=%s" % dep for dep in deps]
by_platform[conda_platform] = sorted(locked_specs)
by_platform = _extract_common(by_platform)
lock_set = CondaLockSet(package_specs_by_platform=by_platform,
platforms=resolve_for_platforms)
return lock_set
def test_resolve_dependencies_with_actual_conda():
manager = DefaultCondaManager(frontend=NullFrontend())
lock_set = manager.resolve_dependencies(['bokeh'], channels=(), platforms=(conda_api.current_platform(), ))
specs = lock_set.package_specs_for_current_platform
pprint(specs)
names = [conda_api.parse_spec(spec).name for spec in specs]
assert 'bokeh' in names
assert len(specs) > 5 # 5 is an arbitrary number of deps that surely bokeh has
def test_resolve_dependencies_with_conda_api_mock(monkeypatch):
def mock_resolve_dependencies(pkgs, platform, channels):
return [('bokeh', '0.12.4', '0'), ('thing', '1.0', '1')]
monkeypatch.setattr('anaconda_project.internal.conda_api.resolve_dependencies', mock_resolve_dependencies)
manager = DefaultCondaManager(frontend=NullFrontend())
lock_set = manager.resolve_dependencies(['bokeh'], channels=(), platforms=(conda_api.current_platform(), ))
assert lock_set.package_specs_for_current_platform == ('bokeh=0.12.4=0', 'thing=1.0=1')
def test_resolve_dependencies_with_conda_api_mock_raises_error(monkeypatch):
def mock_resolve_dependencies(pkgs, platform, channels):
raise conda_api.CondaError("nope")
monkeypatch.setattr('anaconda_project.internal.conda_api.resolve_dependencies', mock_resolve_dependencies)
manager = DefaultCondaManager(frontend=NullFrontend())
with pytest.raises(CondaManagerError) as excinfo:
manager.resolve_dependencies(['bokeh'], channels=(), platforms=(conda_api.current_platform(), ))
assert 'Error resolving for' in str(excinfo.value)
def _broken_lock_set_error(self, spec):
error = None
if spec.lock_set is not None and spec.lock_set.enabled:
# We have to check this first, because getting our package list
# is not valid if we don't have platform support.
current_platform = conda_api.current_platform()
if current_platform not in spec.platforms:
error = "Env spec '%s' does not support current platform %s (it supports: %s)" % \
(spec.name, current_platform, ", ".join(spec.platforms))
elif not spec.lock_set.supports_current_platform:
error = "Env spec '%s' does not have the current platform %s in the lock file" % \
(spec.name, current_platform)
return error
def do_test(dirname):
envdir = os.path.join(dirname, spec.name)
manager = DefaultCondaManager(frontend=NullFrontend())
deviations = manager.find_environment_deviations(envdir, spec)
error = "Env spec 'myenv' does not support current platform %s (it supports: apple-2, commodore-64)" % \
conda_api.current_platform()
assert error == deviations.summary
with pytest.raises(CondaManagerError) as excinfo:
manager.fix_environment_deviations(envdir, spec, deviations=deviations)
assert str(excinfo.value).startswith("Unable to update environment at ")
def check(dirname):
project = Project(dirname)
environ = minimal_environ()
result = prepare_without_interaction(project, environ=environ)
assert result
# now mimmick an unpacked project
packed_file = os.path.join(dirname, 'envs', 'default', 'conda-meta',
'.packed')
with open(packed_file, 'wt') as f:
f.write(conda_api.current_platform())
# without a functional conda-unpack script it will rebuild the env
result = prepare_without_interaction(project, environ=environ)
assert result
assert not os.path.exists(packed_file)
def _write_zip(archive_root_name, infos, filename, packed_envs, frontend):
with zipfile.ZipFile(filename, 'w') as zf:
for info in _leaf_infos(infos):
arcname = os.path.join(archive_root_name, info.relative_path)
frontend.info(" added %s" % arcname)
zf.write(info.full_path, arcname=arcname)
for pack in packed_envs:
env_name = os.path.basename(pack)
print('Joining packed env {}'.format(env_name))
with zipfile.ZipFile(pack, mode='r') as env:
with progressbar(env.infolist()) as infolist:
for file in infolist:
data = env.read(file)
zf.writestr(file, data)
env_spec = env_name.split('.')[0].split('_')[-1]
dot_packed = os.path.join(archive_root_name, 'envs',
env_spec, 'conda-meta',
'.packed')
zf.writestr(dot_packed, '{}\n'.format(current_platform()))
def test_resolve_dependencies_with_actual_conda_other_platforms():
for p in conda_api.default_platforms_plus_32_bit:
if p == conda_api.current_platform():
print(
"Skipping dependency resolution test on current platform %s" %
p)
continue
try:
result = conda_api.resolve_dependencies(['bokeh=0.12.4'],
platform=p)
except conda_api.CondaError as e:
print("*** Dependency resolution failed on %s" % p)
pprint(e.json)
raise e
names = [pkg[0] for pkg in result]
assert 'bokeh' in names
names_and_versions = [(pkg[0], pkg[1]) for pkg in result]
assert ('bokeh', '0.12.4') in names_and_versions
assert len(result) > 1 # bokeh has some dependencies so should be >1
print("Dependency resolution test OK on %s" % p)
def _write_tar(archive_root_name, infos, filename, compression, packed_envs,
frontend):
if compression is None:
compression = ""
else:
compression = ":" + compression
with tarfile.open(filename, ('w%s' % compression)) as tf:
for info in _leaf_infos(infos):
arcname = os.path.join(archive_root_name, info.relative_path)
frontend.info(" added %s" % arcname)
tf.add(info.full_path, arcname=arcname)
for pack in packed_envs:
env_name = os.path.basename(pack)
print('Joining packed env {}'.format(env_name))
with tarfile.open(pack, mode='r', dereference=False) as env:
with progressbar(env.getmembers()) as env_p:
for file in env_p:
try:
data = env.extractfile(file)
tf.addfile(file, data)
except KeyError: # pragma: no cover
tf.addfile(file)
env_spec = env_name.split('.')[0].split('_')[-1]
dot_packed = os.path.join(archive_root_name, 'envs',
env_spec, 'conda-meta',
'.packed')
platform = '{}\n'.format(current_platform())
f = BytesIO()
f.write(platform.encode())
tinfo = tarfile.TarInfo(dot_packed)
tinfo.size = f.tell()
f.seek(0)
tf.addfile(tinfo, fileobj=f)
code = _parse_args_and_run_subcommand(['anaconda-project', 'remove-service', 'redis'])
assert code == 1
out, err = capsys.readouterr()
assert '' == out
expected_err = ("Conflicting results, found 2 matches, use list-services"
" to identify which service you want to remove\n")
assert expected_err == err
with_directory_contents_completing_project_file(
{DEFAULT_PROJECT_FILENAME: 'services:\n ABC: redis\n TEST: redis'}, check)
@pytest.mark.skipif(platform.system() == 'Windows', reason='Windows has a hard time with read-only directories')
@pytest.mark.skipif(conda_api.current_platform() == 'osx-arm64', reason='We cannot install redis server on osx-arm64')
def test_remove_service_running_redis(monkeypatch):
from anaconda_project.requirements_registry.network_util import can_connect_to_socket as real_can_connect_to_socket
from anaconda_project.requirements_registry.providers.test import test_redis_provider
can_connect_args_list = test_redis_provider._monkeypatch_can_connect_to_socket_on_nonstandard_port_only(
monkeypatch, real_can_connect_to_socket)
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = test_redis_provider._prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
def test_current_platform_is_in_default():
assert conda_api.current_platform() in conda_api.default_platforms
def test_current_platform_non_x86_linux(monkeypatch):
monkeypatch.setattr('platform.machine', lambda: 'armv7l')
assert conda_api.current_platform() == 'linux-armv7l'
def fix_environment_deviations(self,
prefix,
spec,
deviations=None,
create=True):
if deviations is None:
deviations = self.find_environment_deviations(prefix, spec)
if deviations.unfixable:
raise CondaManagerError("Unable to update environment at %s" %
prefix)
conda_meta = os.path.join(prefix, 'conda-meta')
packed = os.path.join(conda_meta, '.packed')
install_pip = True
if os.path.isdir(conda_meta) and os.path.exists(packed):
with open(packed) as f:
packed_arch = f.read().strip()
matched = packed_arch == conda_api.current_platform()
if matched:
if 'win' in conda_api.current_platform():
unpack_script = [
'python',
os.path.join(prefix, 'Scripts',
'conda-unpack-script.py')
]
else:
unpack_script = os.path.join(prefix, 'bin', 'conda-unpack')
try:
subprocess.check_call(unpack_script)
os.remove(packed)
install_pip = False
except (subprocess.CalledProcessError, OSError) as e:
self._log_info(
'Warning: conda-unpack could not be run: \n{}\n'
'The environment will be recreated.'.format(str(e)))
create = True
shutil.rmtree(prefix)
else:
self._log_info(
'Warning: The unpacked env does not match the current architecture. '
'It will be recreated.')
create = True
shutil.rmtree(prefix)
if os.path.isdir(conda_meta):
to_update = list(
set(deviations.missing_packages +
deviations.wrong_version_packages))
if len(to_update) > 0:
specs = spec.specs_for_conda_package_names(to_update)
assert len(specs) == len(to_update)
spec.apply_pins(prefix, specs)
try:
conda_api.install(prefix=prefix,
pkgs=specs,
channels=spec.channels,
stdout_callback=self._on_stdout,
stderr_callback=self._on_stderr)
except conda_api.CondaError as e:
raise CondaManagerError(
"Failed to install packages: {}: {}".format(
", ".join(specs), str(e)))
finally:
spec.remove_pins(prefix)
elif create:
# Create environment from scratch
command_line_packages = set(spec.conda_packages_for_create)
try:
conda_api.create(prefix=prefix,
pkgs=list(command_line_packages),
channels=spec.channels,
stdout_callback=self._on_stdout,
stderr_callback=self._on_stderr)
except conda_api.CondaError as e:
raise CondaManagerError(
"Failed to create environment at %s: %s" %
(prefix, str(e)))
else:
raise CondaManagerError("Conda environment at %s does not exist" %
(prefix))
# now add pip if needed
missing = list(deviations.missing_pip_packages)
if (len(missing) > 0) and install_pip:
specs = spec.specs_for_pip_package_names(missing)
assert len(specs) == len(missing)
try:
pip_api.install(prefix=prefix,
pkgs=specs,
stdout_callback=self._on_stdout,
stderr_callback=self._on_stderr)
except pip_api.PipError as e:
raise CondaManagerError(
"Failed to install missing pip packages: {}: {}".format(
", ".join(missing), str(e)))
# write a file to tell us we can short-circuit next time
self._write_timestamp_file(prefix, spec)
def test_current_platform_non_x86_linux(monkeypatch):
monkeypatch.setenv('CONDA_SUBDIR', 'linux-armv7l')
assert conda_api.current_platform() == 'linux-armv7l'
def test_current_platform_non_x86_mac(monkeypatch):
monkeypatch.setenv('CONDA_SUBDIR', 'osx-arm64')
assert conda_api.current_platform() == 'osx-arm64'
def supports_current_platform(self):
"""Whether we have locked deps for the current platform."""
return self.enabled and conda_api.current_platform() in self.platforms
def package_specs_for_current_platform(self):
"""Sequence of package spec strings for the current platform."""
assert self.supports_current_platform
return self.package_specs_for_platform(
platform=conda_api.current_platform())
if port == 6379:
return False
else:
return real_can_connect_to_socket(host, port, timeout_seconds)
monkeypatch.setattr(
"anaconda_project.requirements_registry.network_util.can_connect_to_socket",
mock_can_connect_to_socket)
return can_connect_args_list
@pytest.mark.skipif(platform.system() == 'Windows',
reason='Windows has a hard time with read-only directories'
)
@pytest.mark.skipif(conda_api.current_platform() == 'osx-arm64',
reason='We cannot install redis server on osx-arm64')
def test_prepare_and_unprepare_local_redis_server(monkeypatch):
from anaconda_project.requirements_registry.network_util import can_connect_to_socket as real_can_connect_to_socket
can_connect_args_list = _monkeypatch_can_connect_to_socket_on_nonstandard_port_only(
monkeypatch, real_can_connect_to_socket)
def start_local_redis(dirname):
project = project_no_dedicated_env(dirname)
result = _prepare_printing_errors(project, environ=minimal_environ())
assert result
local_state_file = LocalStateFile.load_for_directory(dirname)
state = local_state_file.get_service_run_state('REDIS_URL')
assert 'port' in state
def _archive_project(project, filename, pack_envs=False):
"""Make an archive of the non-ignored files in the project.
Args:
project (``Project``): the project
filename (str): name for the new zip or tar.gz archive file
Returns:
a ``Status``, if failed has ``errors``
"""
failed = project.problems_status()
if failed is not None:
for error in failed.errors:
project.frontend.error(error)
return failed
frontend = _new_error_recorder(project.frontend)
if not os.path.exists(project.project_file.filename):
frontend.error("%s does not exist." % project.project_file.basename)
return SimpleStatus(success=False,
description="Can't create an archive.",
errors=frontend.pop_errors())
# this would most likely happen in a GUI editor, if it reloaded
# the project from memory but hadn't saved yet.
if project.project_file.has_unsaved_changes:
frontend.error("%s has been modified but not saved." %
project.project_file.basename)
return SimpleStatus(success=False,
description="Can't create an archive.",
errors=frontend.pop_errors())
envs_path = os.path.join(project.project_file.project_dir, 'envs')
packed_envs = []
if pack_envs and os.path.isdir(envs_path):
conda_pack_dir = tempfile.mkdtemp()
import conda_pack
for env in os.listdir(envs_path):
ext = 'zip' if filename.lower().endswith(".zip") else 'tar'
pack = os.path.join(
conda_pack_dir, '{}_envs_{}.{}'.format(current_platform(), env,
ext))
zip_symlinks = True if ext == 'zip' else False
fn = conda_pack.pack(prefix=os.path.join(envs_path, env),
arcroot=os.path.join(project.name, 'envs',
env),
output=pack,
zip_symlinks=zip_symlinks,
verbose=True,
force=True)
packed_envs.append(fn)
infos = _enumerate_archive_files(
project.directory_path,
frontend,
requirements=project.union_of_requirements_for_all_envs)
if infos is None:
return SimpleStatus(success=False,
description="Failed to list files in the project.",
errors=frontend.pop_errors())
# don't put the destination zip into itself, since it's fairly natural to
# create a archive right in the project directory
relative_dest_file = subdirectory_relative_to_directory(
filename, project.directory_path)
if not os.path.isabs(relative_dest_file):
infos = [
info for info in infos if info.relative_path != relative_dest_file
]
tmp_filename = filename + ".tmp-" + str(uuid.uuid4())
try:
if filename.lower().endswith(".zip"):
_write_zip(project.name,
infos,
tmp_filename,
packed_envs=packed_envs,
frontend=frontend)
elif filename.lower().endswith(".tar.gz"):
_write_tar(project.name,
infos,
tmp_filename,
compression="gz",
packed_envs=packed_envs,
frontend=frontend)
elif filename.lower().endswith(".tar.bz2"):
_write_tar(project.name,
infos,
tmp_filename,
compression="bz2",
packed_envs=packed_envs,
frontend=frontend)
elif filename.lower().endswith(".tar"):
_write_tar(project.name,
infos,
tmp_filename,
compression=None,
packed_envs=packed_envs,
frontend=frontend)
else:
frontend.error("Unsupported archive filename %s." % (filename))
return SimpleStatus(
success=False,
description=
"Project archive filename must be a .zip, .tar.gz, or .tar.bz2.",
errors=frontend.pop_errors())
rename_over_existing(tmp_filename, filename)
except IOError as e:
frontend.error(str(e))
return SimpleStatus(
success=False,
description=("Failed to write project archive %s." % (filename)),
errors=frontend.pop_errors())
finally:
try:
os.remove(tmp_filename)
if pack_envs:
os.remove(conda_pack_dir)
except (IOError, OSError):
pass
unlocked = []
for env_spec in project.env_specs.values():
if env_spec.lock_set.disabled:
unlocked.append(env_spec.name)
if len(unlocked) > 0:
frontend.info(
"Warning: env specs are not locked, which means they may not "
"work consistently for others or when deployed.")
frontend.info(
" Consider using the 'anaconda-project lock' command to lock the project."
)
if len(unlocked) != len(project.env_specs):
frontend.info(" Unlocked env specs are: " +
(", ".join(sorted(unlocked))))
return SimpleStatus(success=True,
description=("Created project archive %s" % filename))
