def test_lock_set_properties(monkeypatch): lock_set = CondaLockSet( { 'all': ["something=0.5=2", "bokeh=0.12.4=1"], 'linux-64': ["linux-thing=1.0=0"], 'unix': ["unix-thing=5=1"], 'win': ["windows-cross-bit-thing=3.2"], 'win-32': ["windows-thing=2.0=3", "bokeh=2.3=7"] }, platforms=['linux-64', 'win-32']) # it is part of the API definition that we need to APPEND the # per-platform stuff, so it overrides. assert lock_set.package_specs_for_platform('win-32') == ( "something=0.5=2", "windows-cross-bit-thing=3.2", "windows-thing=2.0=3", "bokeh=2.3=7") # on Linux-64, test that it works without monkeypatch if conda_api.current_platform() != 'linux-64': monkeypatch.setattr( 'anaconda_project.internal.conda_api.current_platform', lambda: 'linux-64') assert lock_set.package_specs_for_current_platform == ("something=0.5=2", "bokeh=0.12.4=1", "unix-thing=5=1", "linux-thing=1.0=0") assert lock_set.platforms == ('linux-64', 'win-32')
def resolve_dependencies(self, package_specs, channels, platforms): by_platform = {} current = conda_api.current_platform() resolve_for_platforms = list(platforms) # always resolve "current" first because it's confusing if # an error says resolution failed on another platform when # the real issue is that resolution will fail on all platforms. if current in resolve_for_platforms: resolve_for_platforms.remove(current) resolve_for_platforms = [current] + resolve_for_platforms for conda_platform in resolve_for_platforms: try: self._log_info("Resolving conda packages for %s" % conda_platform) deps = conda_api.resolve_dependencies(pkgs=package_specs, platform=conda_platform, channels=channels) except conda_api.CondaError as e: raise CondaManagerError("Error resolving for {}: {}".format( conda_platform, str(e))) locked_specs = ["%s=%s=%s" % dep for dep in deps] by_platform[conda_platform] = sorted(locked_specs) by_platform = _extract_common(by_platform) lock_set = CondaLockSet(package_specs_by_platform=by_platform, platforms=resolve_for_platforms) return lock_set
def test_resolve_dependencies_with_actual_conda(): manager = DefaultCondaManager(frontend=NullFrontend()) lock_set = manager.resolve_dependencies(['bokeh'], channels=(), platforms=(conda_api.current_platform(), )) specs = lock_set.package_specs_for_current_platform pprint(specs) names = [conda_api.parse_spec(spec).name for spec in specs] assert 'bokeh' in names assert len(specs) > 5 # 5 is an arbitrary number of deps that surely bokeh has
def test_resolve_dependencies_with_conda_api_mock(monkeypatch): def mock_resolve_dependencies(pkgs, platform, channels): return [('bokeh', '0.12.4', '0'), ('thing', '1.0', '1')] monkeypatch.setattr('anaconda_project.internal.conda_api.resolve_dependencies', mock_resolve_dependencies) manager = DefaultCondaManager(frontend=NullFrontend()) lock_set = manager.resolve_dependencies(['bokeh'], channels=(), platforms=(conda_api.current_platform(), )) assert lock_set.package_specs_for_current_platform == ('bokeh=0.12.4=0', 'thing=1.0=1')
def test_resolve_dependencies_with_conda_api_mock_raises_error(monkeypatch): def mock_resolve_dependencies(pkgs, platform, channels): raise conda_api.CondaError("nope") monkeypatch.setattr('anaconda_project.internal.conda_api.resolve_dependencies', mock_resolve_dependencies) manager = DefaultCondaManager(frontend=NullFrontend()) with pytest.raises(CondaManagerError) as excinfo: manager.resolve_dependencies(['bokeh'], channels=(), platforms=(conda_api.current_platform(), )) assert 'Error resolving for' in str(excinfo.value)
def _broken_lock_set_error(self, spec): error = None if spec.lock_set is not None and spec.lock_set.enabled: # We have to check this first, because getting our package list # is not valid if we don't have platform support. current_platform = conda_api.current_platform() if current_platform not in spec.platforms: error = "Env spec '%s' does not support current platform %s (it supports: %s)" % \ (spec.name, current_platform, ", ".join(spec.platforms)) elif not spec.lock_set.supports_current_platform: error = "Env spec '%s' does not have the current platform %s in the lock file" % \ (spec.name, current_platform) return error
def do_test(dirname): envdir = os.path.join(dirname, spec.name) manager = DefaultCondaManager(frontend=NullFrontend()) deviations = manager.find_environment_deviations(envdir, spec) error = "Env spec 'myenv' does not support current platform %s (it supports: apple-2, commodore-64)" % \ conda_api.current_platform() assert error == deviations.summary with pytest.raises(CondaManagerError) as excinfo: manager.fix_environment_deviations(envdir, spec, deviations=deviations) assert str(excinfo.value).startswith("Unable to update environment at ")
def check(dirname): project = Project(dirname) environ = minimal_environ() result = prepare_without_interaction(project, environ=environ) assert result # now mimmick an unpacked project packed_file = os.path.join(dirname, 'envs', 'default', 'conda-meta', '.packed') with open(packed_file, 'wt') as f: f.write(conda_api.current_platform()) # without a functional conda-unpack script it will rebuild the env result = prepare_without_interaction(project, environ=environ) assert result assert not os.path.exists(packed_file)
def _write_zip(archive_root_name, infos, filename, packed_envs, frontend): with zipfile.ZipFile(filename, 'w') as zf: for info in _leaf_infos(infos): arcname = os.path.join(archive_root_name, info.relative_path) frontend.info(" added %s" % arcname) zf.write(info.full_path, arcname=arcname) for pack in packed_envs: env_name = os.path.basename(pack) print('Joining packed env {}'.format(env_name)) with zipfile.ZipFile(pack, mode='r') as env: with progressbar(env.infolist()) as infolist: for file in infolist: data = env.read(file) zf.writestr(file, data) env_spec = env_name.split('.')[0].split('_')[-1] dot_packed = os.path.join(archive_root_name, 'envs', env_spec, 'conda-meta', '.packed') zf.writestr(dot_packed, '{}\n'.format(current_platform()))
def test_resolve_dependencies_with_actual_conda_other_platforms(): for p in conda_api.default_platforms_plus_32_bit: if p == conda_api.current_platform(): print( "Skipping dependency resolution test on current platform %s" % p) continue try: result = conda_api.resolve_dependencies(['bokeh=0.12.4'], platform=p) except conda_api.CondaError as e: print("*** Dependency resolution failed on %s" % p) pprint(e.json) raise e names = [pkg[0] for pkg in result] assert 'bokeh' in names names_and_versions = [(pkg[0], pkg[1]) for pkg in result] assert ('bokeh', '0.12.4') in names_and_versions assert len(result) > 1 # bokeh has some dependencies so should be >1 print("Dependency resolution test OK on %s" % p)
def _write_tar(archive_root_name, infos, filename, compression, packed_envs, frontend): if compression is None: compression = "" else: compression = ":" + compression with tarfile.open(filename, ('w%s' % compression)) as tf: for info in _leaf_infos(infos): arcname = os.path.join(archive_root_name, info.relative_path) frontend.info(" added %s" % arcname) tf.add(info.full_path, arcname=arcname) for pack in packed_envs: env_name = os.path.basename(pack) print('Joining packed env {}'.format(env_name)) with tarfile.open(pack, mode='r', dereference=False) as env: with progressbar(env.getmembers()) as env_p: for file in env_p: try: data = env.extractfile(file) tf.addfile(file, data) except KeyError: # pragma: no cover tf.addfile(file) env_spec = env_name.split('.')[0].split('_')[-1] dot_packed = os.path.join(archive_root_name, 'envs', env_spec, 'conda-meta', '.packed') platform = '{}\n'.format(current_platform()) f = BytesIO() f.write(platform.encode()) tinfo = tarfile.TarInfo(dot_packed) tinfo.size = f.tell() f.seek(0) tf.addfile(tinfo, fileobj=f)
code = _parse_args_and_run_subcommand(['anaconda-project', 'remove-service', 'redis']) assert code == 1 out, err = capsys.readouterr() assert '' == out expected_err = ("Conflicting results, found 2 matches, use list-services" " to identify which service you want to remove\n") assert expected_err == err with_directory_contents_completing_project_file( {DEFAULT_PROJECT_FILENAME: 'services:\n ABC: redis\n TEST: redis'}, check) @pytest.mark.skipif(platform.system() == 'Windows', reason='Windows has a hard time with read-only directories') @pytest.mark.skipif(conda_api.current_platform() == 'osx-arm64', reason='We cannot install redis server on osx-arm64') def test_remove_service_running_redis(monkeypatch): from anaconda_project.requirements_registry.network_util import can_connect_to_socket as real_can_connect_to_socket from anaconda_project.requirements_registry.providers.test import test_redis_provider can_connect_args_list = test_redis_provider._monkeypatch_can_connect_to_socket_on_nonstandard_port_only( monkeypatch, real_can_connect_to_socket) def start_local_redis(dirname): project = project_no_dedicated_env(dirname) result = test_redis_provider._prepare_printing_errors(project, environ=minimal_environ()) assert result local_state_file = LocalStateFile.load_for_directory(dirname) state = local_state_file.get_service_run_state('REDIS_URL') assert 'port' in state
def test_current_platform_is_in_default(): assert conda_api.current_platform() in conda_api.default_platforms
def test_current_platform_non_x86_linux(monkeypatch): monkeypatch.setattr('platform.machine', lambda: 'armv7l') assert conda_api.current_platform() == 'linux-armv7l'
def fix_environment_deviations(self, prefix, spec, deviations=None, create=True): if deviations is None: deviations = self.find_environment_deviations(prefix, spec) if deviations.unfixable: raise CondaManagerError("Unable to update environment at %s" % prefix) conda_meta = os.path.join(prefix, 'conda-meta') packed = os.path.join(conda_meta, '.packed') install_pip = True if os.path.isdir(conda_meta) and os.path.exists(packed): with open(packed) as f: packed_arch = f.read().strip() matched = packed_arch == conda_api.current_platform() if matched: if 'win' in conda_api.current_platform(): unpack_script = [ 'python', os.path.join(prefix, 'Scripts', 'conda-unpack-script.py') ] else: unpack_script = os.path.join(prefix, 'bin', 'conda-unpack') try: subprocess.check_call(unpack_script) os.remove(packed) install_pip = False except (subprocess.CalledProcessError, OSError) as e: self._log_info( 'Warning: conda-unpack could not be run: \n{}\n' 'The environment will be recreated.'.format(str(e))) create = True shutil.rmtree(prefix) else: self._log_info( 'Warning: The unpacked env does not match the current architecture. ' 'It will be recreated.') create = True shutil.rmtree(prefix) if os.path.isdir(conda_meta): to_update = list( set(deviations.missing_packages + deviations.wrong_version_packages)) if len(to_update) > 0: specs = spec.specs_for_conda_package_names(to_update) assert len(specs) == len(to_update) spec.apply_pins(prefix, specs) try: conda_api.install(prefix=prefix, pkgs=specs, channels=spec.channels, stdout_callback=self._on_stdout, stderr_callback=self._on_stderr) except conda_api.CondaError as e: raise CondaManagerError( "Failed to install packages: {}: {}".format( ", ".join(specs), str(e))) finally: spec.remove_pins(prefix) elif create: # Create environment from scratch command_line_packages = set(spec.conda_packages_for_create) try: conda_api.create(prefix=prefix, pkgs=list(command_line_packages), channels=spec.channels, stdout_callback=self._on_stdout, stderr_callback=self._on_stderr) except conda_api.CondaError as e: raise CondaManagerError( "Failed to create environment at %s: %s" % (prefix, str(e))) else: raise CondaManagerError("Conda environment at %s does not exist" % (prefix)) # now add pip if needed missing = list(deviations.missing_pip_packages) if (len(missing) > 0) and install_pip: specs = spec.specs_for_pip_package_names(missing) assert len(specs) == len(missing) try: pip_api.install(prefix=prefix, pkgs=specs, stdout_callback=self._on_stdout, stderr_callback=self._on_stderr) except pip_api.PipError as e: raise CondaManagerError( "Failed to install missing pip packages: {}: {}".format( ", ".join(missing), str(e))) # write a file to tell us we can short-circuit next time self._write_timestamp_file(prefix, spec)
def test_current_platform_non_x86_linux(monkeypatch): monkeypatch.setenv('CONDA_SUBDIR', 'linux-armv7l') assert conda_api.current_platform() == 'linux-armv7l'
def test_current_platform_non_x86_mac(monkeypatch): monkeypatch.setenv('CONDA_SUBDIR', 'osx-arm64') assert conda_api.current_platform() == 'osx-arm64'
def supports_current_platform(self): """Whether we have locked deps for the current platform.""" return self.enabled and conda_api.current_platform() in self.platforms
def package_specs_for_current_platform(self): """Sequence of package spec strings for the current platform.""" assert self.supports_current_platform return self.package_specs_for_platform( platform=conda_api.current_platform())
if port == 6379: return False else: return real_can_connect_to_socket(host, port, timeout_seconds) monkeypatch.setattr( "anaconda_project.requirements_registry.network_util.can_connect_to_socket", mock_can_connect_to_socket) return can_connect_args_list @pytest.mark.skipif(platform.system() == 'Windows', reason='Windows has a hard time with read-only directories' ) @pytest.mark.skipif(conda_api.current_platform() == 'osx-arm64', reason='We cannot install redis server on osx-arm64') def test_prepare_and_unprepare_local_redis_server(monkeypatch): from anaconda_project.requirements_registry.network_util import can_connect_to_socket as real_can_connect_to_socket can_connect_args_list = _monkeypatch_can_connect_to_socket_on_nonstandard_port_only( monkeypatch, real_can_connect_to_socket) def start_local_redis(dirname): project = project_no_dedicated_env(dirname) result = _prepare_printing_errors(project, environ=minimal_environ()) assert result local_state_file = LocalStateFile.load_for_directory(dirname) state = local_state_file.get_service_run_state('REDIS_URL') assert 'port' in state
def _archive_project(project, filename, pack_envs=False): """Make an archive of the non-ignored files in the project. Args: project (``Project``): the project filename (str): name for the new zip or tar.gz archive file Returns: a ``Status``, if failed has ``errors`` """ failed = project.problems_status() if failed is not None: for error in failed.errors: project.frontend.error(error) return failed frontend = _new_error_recorder(project.frontend) if not os.path.exists(project.project_file.filename): frontend.error("%s does not exist." % project.project_file.basename) return SimpleStatus(success=False, description="Can't create an archive.", errors=frontend.pop_errors()) # this would most likely happen in a GUI editor, if it reloaded # the project from memory but hadn't saved yet. if project.project_file.has_unsaved_changes: frontend.error("%s has been modified but not saved." % project.project_file.basename) return SimpleStatus(success=False, description="Can't create an archive.", errors=frontend.pop_errors()) envs_path = os.path.join(project.project_file.project_dir, 'envs') packed_envs = [] if pack_envs and os.path.isdir(envs_path): conda_pack_dir = tempfile.mkdtemp() import conda_pack for env in os.listdir(envs_path): ext = 'zip' if filename.lower().endswith(".zip") else 'tar' pack = os.path.join( conda_pack_dir, '{}_envs_{}.{}'.format(current_platform(), env, ext)) zip_symlinks = True if ext == 'zip' else False fn = conda_pack.pack(prefix=os.path.join(envs_path, env), arcroot=os.path.join(project.name, 'envs', env), output=pack, zip_symlinks=zip_symlinks, verbose=True, force=True) packed_envs.append(fn) infos = _enumerate_archive_files( project.directory_path, frontend, requirements=project.union_of_requirements_for_all_envs) if infos is None: return SimpleStatus(success=False, description="Failed to list files in the project.", errors=frontend.pop_errors()) # don't put the destination zip into itself, since it's fairly natural to # create a archive right in the project directory relative_dest_file = subdirectory_relative_to_directory( filename, project.directory_path) if not os.path.isabs(relative_dest_file): infos = [ info for info in infos if info.relative_path != relative_dest_file ] tmp_filename = filename + ".tmp-" + str(uuid.uuid4()) try: if filename.lower().endswith(".zip"): _write_zip(project.name, infos, tmp_filename, packed_envs=packed_envs, frontend=frontend) elif filename.lower().endswith(".tar.gz"): _write_tar(project.name, infos, tmp_filename, compression="gz", packed_envs=packed_envs, frontend=frontend) elif filename.lower().endswith(".tar.bz2"): _write_tar(project.name, infos, tmp_filename, compression="bz2", packed_envs=packed_envs, frontend=frontend) elif filename.lower().endswith(".tar"): _write_tar(project.name, infos, tmp_filename, compression=None, packed_envs=packed_envs, frontend=frontend) else: frontend.error("Unsupported archive filename %s." % (filename)) return SimpleStatus( success=False, description= "Project archive filename must be a .zip, .tar.gz, or .tar.bz2.", errors=frontend.pop_errors()) rename_over_existing(tmp_filename, filename) except IOError as e: frontend.error(str(e)) return SimpleStatus( success=False, description=("Failed to write project archive %s." % (filename)), errors=frontend.pop_errors()) finally: try: os.remove(tmp_filename) if pack_envs: os.remove(conda_pack_dir) except (IOError, OSError): pass unlocked = [] for env_spec in project.env_specs.values(): if env_spec.lock_set.disabled: unlocked.append(env_spec.name) if len(unlocked) > 0: frontend.info( "Warning: env specs are not locked, which means they may not " "work consistently for others or when deployed.") frontend.info( " Consider using the 'anaconda-project lock' command to lock the project." ) if len(unlocked) != len(project.env_specs): frontend.info(" Unlocked env specs are: " + (", ".join(sorted(unlocked)))) return SimpleStatus(success=True, description=("Created project archive %s" % filename))