def install(prefix, specs, args, env, *_, **kwargs): # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults new_specs = [] channel_urls = set() for elem in specs: if "::" in elem: channel_urls.add(elem.split("::")[0]) new_specs.append(elem.split("::")[-1]) else: new_specs.append(elem) specs = new_specs channel_urls = list(channel_urls) # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults channel_urls = channel_urls + [chan for chan in env.channels if chan != 'nodefaults'] if 'nodefaults' not in env.channels: channel_urls.extend(context.channels) _channel_priority_map = prioritize_channels(channel_urls) channels = IndexedSet(Channel(url) for url in _channel_priority_map) subdirs = IndexedSet(basename(url) for url in _channel_priority_map) solver = Solver(prefix, channels, subdirs, specs_to_add=specs) unlink_link_transaction = solver.solve_for_transaction(prune=getattr(args, 'prune', False)) pfe = unlink_link_transaction.get_pfe() pfe.execute() unlink_link_transaction.execute()
def install(prefix, specs, args, env, *_, **kwargs): # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults new_specs = [] channel_urls = set() for elem in specs: if "::" in elem: channel_urls.add(elem.split("::")[0]) new_specs.append(elem.split("::")[-1]) else: new_specs.append(elem) specs = new_specs channel_urls = list(channel_urls) # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults channel_urls = channel_urls + [chan for chan in env.channels if chan != 'nodefaults'] if 'nodefaults' not in env.channels: channel_urls.extend(context.channels) _channel_priority_map = prioritize_channels(channel_urls) channels = IndexedSet(Channel(url) for url in _channel_priority_map) subdirs = IndexedSet(basename(url) for url in _channel_priority_map) solver = Solver(prefix, channels, subdirs, specs_to_add=specs) unlink_link_transaction = solver.solve_for_transaction(prune=getattr(args, 'prune', False)) pfe = unlink_link_transaction._get_pfe() pfe.execute() unlink_link_transaction.execute()
def install(prefix, specs, args, env, *_, **kwargs): # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults channel_urls = [chan for chan in env.channels if chan != 'nodefaults'] if 'nodefaults' not in env.channels: channel_urls.extend(context.channels) _channel_priority_map = prioritize_channels(channel_urls) channels = IndexedSet(Channel(url) for url in _channel_priority_map) subdirs = IndexedSet(basename(url) for url in _channel_priority_map) solver = Solver(prefix, channels, subdirs, specs_to_add=specs) try: unlink_link_transaction = solver.solve_for_transaction( prune=getattr(args, 'prune', False), update_modifier=UpdateModifier.FREEZE_INSTALLED) except (UnsatisfiableError, SystemExit): unlink_link_transaction = solver.solve_for_transaction( prune=getattr(args, 'prune', False), update_modifier=NULL) if unlink_link_transaction.nothing_to_do: return None unlink_link_transaction.download_and_extract() unlink_link_transaction.execute() return unlink_link_transaction._make_legacy_action_groups()[0]
def _create_env_conda_44(prefix, full_list_of_packages): assert CONDA_VERSION_MAJOR_MINOR >= (4, 4) from conda.models.match_spec import MatchSpec from conda.core.solve import Solver matched_list_of_packages = (MatchSpec(d) for d in full_list_of_packages) m = Solver(prefix, (), specs_to_add=matched_list_of_packages) txn = m.solve_for_transaction() txn.execute()
def solve_for_packages(packages: List[str]) -> Tuple[IndexedSet, IndexedSet]: """ given a list of conda packages solve the environment for dependencies :param packages: list of conda packages to be installed :return: tuple of packages that will be removed and will be added to the current conda environment """ specs_to_add = [MatchSpec(p) for p in packages] info = json.loads(conda.run_command(conda.Commands.INFO, '--json')[0]) prefix = info['conda_prefix'] if info['active_prefix'] == 'null' else info[ 'active_prefix'] solver = Solver(prefix, info['channels'], specs_to_add=specs_to_add) return solver.solve_for_diff()
def _solve(prefix, specs, args, env, *_, **kwargs): # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults channel_urls = [chan for chan in env.channels if chan != 'nodefaults'] if 'nodefaults' not in env.channels: channel_urls.extend(context.channels) _channel_priority_map = prioritize_channels(channel_urls) channels = IndexedSet(Channel(url) for url in _channel_priority_map) subdirs = IndexedSet(basename(url) for url in _channel_priority_map) solver = Solver(prefix, channels, subdirs, specs_to_add=specs) return solver
def get_conda_solver(filepath=None, prefix=None, channels=None, subdirs=None, specs_to_add=None, existing_solver=None): ''' Initialize or update a conda environment solver from a file or spec Parameters ---------- filepath : str, optional Path to a yaml file to parse and solve. If not provided, ``prefix`` is required and any transactions must be defined in ``channels``, ``subdirs``, and ``specs_to_add``. prefix : str, optional Name or path of the environment to be created/modified. Ignored if ``filepath`` is provided; required if omitted. channels: list, optional Conda channels. Ignored if ``filepath`` is provided. subdirs: tuple, optional Architecture subdirs. Ignored if ``filepath`` is provided. specs_to_add: list, optional List of strings of package specs to add to the environment. Ignored if ``filepath`` is provided. existing_solver : conda.core.solve.Solver, optional Optionally, upgrade a provided spec with the new dependencies provided in an environment file. Default is to create a new environment. Returns ------- solver: conda.core.solve.Solver Examples -------- Initializing with an environment file: .. code-block:: python >>> get_conda_solver('notebook/environment.yml') <conda.core.solve.Solver at ...> Initializing with a spec .. code-block:: python >>> s = get_conda_solver( ... prefix='vis', ... channels=['pyviz', 'conda-forge'], ... specs_to_add=['python=3.7', 'holoviews', 'bokeh>=1.2']) ... Testing for unsatisfiable environments: .. code-block:: python >>> s = get_conda_solver( ... prefix='base', ... specs_to_add=['xarray>=0.13.0', 'python<3.0']) ... >>> s.solve_final_state() # doctest: +ELLIPSIS ... Traceback (most recent calls last): ... UnsatisfiableError: The following specifications were found to be incompatible with the existing python installation in your environment: - xarray[version='>=0.13.0'] -> python[version='>=3.5.3'] If python is on the left-most side of the chain, that's the version you've asked for. When python appears to the right, that indicates that the thing on the left is somehow not available for the python version you've asked for. Your current python version is (python[version='<3.0']). The following specifications were found to be incompatible with each other: - python[version='<3.0'] TODO ---- * figure out how to solve for/align pip dependencies. currently these are ignored ''' if filepath is not None: with open(filepath, 'r') as f: spec = yaml.safe_load(f) else: spec = { 'prefix': prefix, 'channels': channels if channels is not None else [], 'subdirs': subdirs if subdirs is not None else [], 'dependencies': specs_to_add if specs_to_add is not None else [] } # exclude pip dependencies - these trip up the Solver conda_packages = [ d for d in spec.get('dependencies', []) if not isinstance(d, dict) ] if existing_solver is None: solver = Solver(spec.get('name', 'base'), channels=spec.get('channels', []), specs_to_add=conda_packages) else: solver = Solver( prefix=existing_solver.prefix, channels=spec.get('channels', existing_solver.channels), subdirs=spec.get('subdirs', existing_solver.subdirs), specs_to_add=(list(existing_solver.specs_to_add) + conda_packages), specs_to_remove=spec.get('specs_to_remove', existing_solver.specs_to_remove)) return solver
def pkg_env(environment_file: Path, coex_path: Path, cache_dir: Path) -> None: """Resolve, fetch, and repackage conda env into coex /pkgs directory. Resolve conda environment file to a specific package list via conda solver, then fetch and unpack target packages. Repack into .coex package data in cache_dir or reuse if pre-packed, and assemble into /pkgs under coex_path. Args: environment_file: Standard conda env file, can not contain pip deps. coex_path: Output coex build path. cache_dir: Coex build cache directory. """ # Resolve environment file to dependencies # Logic culled from conda-env spec = YamlFileSpec(filename=str(environment_file)) env = spec.environment logging.info(env.dependencies) assert set(env.dependencies) == { "conda" }, f"coex environments do not support pip dependencies: {env}" channel_urls = [chan for chan in env.channels if chan != "nodefaults"] if "nodefaults" not in env.channels: channel_urls.extend(context.channels) _channel_priority_map = prioritize_channels(channel_urls) # Setup an dummpy environment resolution for install into /dev/null # Execute fetch-and-extract operations for required conda packages prefix = "/dev/null" channels = IndexedSet(Channel(url) for url in _channel_priority_map) subdirs = IndexedSet( os.path.basename(url) for url in _channel_priority_map) solver = Solver(prefix, channels, subdirs, specs_to_add=env.dependencies["conda"]) transaction: UnlinkLinkTransaction = solver.solve_for_transaction() logging.info(transaction) transaction.download_and_extract() # Resolve all the, now extracted, target packages in the filesystem fetcher: ProgressiveFetchExtract = transaction._pfe target_records: Set[PackageRecord] = set(fetcher.link_precs) logging.debug("target_records=%s", target_records) extracted: Set[PackageCacheRecord] = { next( (pcrec for pcrec in chain(*(PackageCacheData(pkgs_dir).query(precord) for pkgs_dir in context.pkgs_dirs)) if pcrec.is_extracted), None, ) for precord in target_records } logging.debug("extracted=%s", extracted) # Repackage into a single-file .zst in the cache, then copy into the output # package. output_path = coex_path / "pkgs" for e in extracted: extracted_dir = Path(e.extracted_package_dir) pkgname = extracted_dir.name + ".tar.zst" cache_dir.mkdir(parents=True, exist_ok=True) if not (cache_dir / pkgname).exists(): pkg_cmd = ( # tar filtered through zstd # Seeing errors on macos 10.13 image when using --use-compress-program # with arguments, consider (a) installing conda-forge tar or (b) using # a wrapper script if zstd arguments are needed [ "tar", "--use-compress-program", "zstd -T0" if platform.system() != "Darwin" else "zstd", ] # write to archive file + ["-f", str(cache_dir / pkgname)] # chdir to extracted package directory + ["-C", str(extracted_dir)] # and add all package dirs + (["-c"] + [f.name for f in extracted_dir.iterdir()])) logging.info("packaging: %s", pkg_cmd) subprocess.check_call(pkg_cmd) output_path.mkdir(parents=True, exist_ok=True) shutil.copyfile(cache_dir / pkgname, output_path / pkgname)
def execute(args, parser): from .common import (confirm_yn, ensure_override_channels_requires_channel, ensure_use_local, specs_from_args, stdout_json) from ..base.context import context from ..common.compat import iteritems, iterkeys from ..core.index import get_index from ..exceptions import CondaEnvironmentError, CondaValueError from ..gateways.disk.delete import delete_trash from ..resolve import MatchSpec from ..core.envs_manager import EnvsDirectory from ..core.linked_data import linked_data from ..gateways.disk.delete import rm_rf from ..instructions import PREFIX from ..plan import (add_unlink) if not (args.all or args.package_names): raise CondaValueError('no package names supplied,\n' ' try "conda remove -h" for more details') prefix = context.target_prefix if args.all and prefix == context.default_prefix: msg = "cannot remove current environment. deactivate and run conda remove again" raise CondaEnvironmentError(msg) if args.all and not isdir(prefix): # full environment removal was requested, but environment doesn't exist anyway return 0 if not EnvsDirectory.is_conda_environment(prefix): from ..exceptions import EnvironmentLocationNotFound raise EnvironmentLocationNotFound(prefix) ensure_use_local(args) ensure_override_channels_requires_channel(args) if not args.features and args.all: index = linked_data(prefix) index = {dist: info for dist, info in iteritems(index)} else: index = get_index(channel_urls=context.channels, prepend=not args.override_channels, use_local=args.use_local, use_cache=args.use_index_cache, prefix=prefix) delete_trash() if args.all: if prefix == context.root_prefix: raise CondaEnvironmentError( 'cannot remove root environment,\n' ' add -n NAME or -p PREFIX option') print("\nRemove all packages in environment %s:\n" % prefix, file=sys.stderr) actions = defaultdict(list) actions[PREFIX] = prefix for dist in sorted(iterkeys(index)): add_unlink(actions, dist) actions['ACTION'] = 'REMOVE_ALL' action_groups = (actions, index), if not context.json: confirm_yn() rm_rf(prefix) if context.json: stdout_json({ 'success': True, 'actions': tuple(x[0] for x in action_groups) }) return else: if args.features: specs = tuple( MatchSpec(track_features=f) for f in set(args.package_names)) channel_urls = context.channels subdirs = context.subdirs else: specs = specs_from_args(args.package_names) channel_urls = () subdirs = () solver = Solver(prefix, channel_urls, subdirs, specs_to_remove=specs) txn = solver.solve_for_transaction(force_remove=args.force) pfe = txn.get_pfe() handle_txn(pfe, txn, prefix, args, False, True)
def bypass_satsolver_on_install( pkg_names, conda_channel="ggd-genomics", debug=False, prefix=None ): """Method to bypass the sat solver used by conda when a cached recipe is being installed bypass_satsolver_on_install ============================ This method is used to run the conda install steps to install a ggd aws cahced reicpe. The intsallation will skip the sat solver step, ignore packages that may be additionaly installed or uninstalled, and performs other steps in order to install the data package without using the sat solver. The majority of the work is still done by conda through the use of the conda module. This method should only be used when a cached recipe is being installed. Parameters: ----------- #1) pkg_name: The name of the ggd package to install. (Example: hg19-gaps) 1) pkg_names: A list of the names of the ggd packages to install. (Example: [hg19-gaps]) 2) conda_channel: The ggd conda channel that package is being installed from. (Example: ggd-genomics) """ # ------------------------------------------------------------------------- # import statments # ------------------------------------------------------------------------- from conda.base.context import context from conda.cli import common from conda.cli import install from conda.core.solve import Solver from conda.core.solve import SolverStateContainer from conda.common.io import Spinner from conda.core.link import PrefixSetup from conda.core.link import UnlinkLinkTransaction from argparse import Namespace from conda._vendor.boltons.setutils import IndexedSet from conda.models.prefix_graph import PrefixGraph from conda.core.solve import diff_for_unlink_link_precs from conda.common.compat import iteritems, itervalues, odict, text_type from conda._vendor.toolz import concat, concatv from conda.resolve import Resolve from conda.models.match_spec import MatchSpec from conda.base.constants import UpdateModifier from conda.common.io import ProgressBar from conda.gateways.logging import set_all_logger_level, set_conda_log_level from conda.gateways.logging import VERBOSITY_LEVELS from conda.gateways.logging import log from logging import ( DEBUG, ERROR, Filter, Formatter, INFO, StreamHandler, WARN, getLogger, ) import sys print( "\n:ggd:utils:bypass: Installing %s from the %s conda channel\n" % (", ".join(pkg_names), conda_channel) ) # ------------------------------------------------------------------------- # Nested functions # ------------------------------------------------------------------------- # def bypass_sat(package_name,ssc_object): ## Package_name will be used as a key def bypass_sat(package_names, ssc_object): ## Package_name will be used as a key """Method used to extract information during sat solving, but to bypass the sat solving step bypass_sat ========== This method is used to extract and process information that would have been done during the sat solvering step, (Solving Enviroment), bypass the sat solver, and return a filtered set of packages to install. Parameters: ----------- #1) package_name: The name of the package to extract. (This is the package that will be installed) 1) package_names: A list of package names of the packages to extract. (This is the package that will be installed) 2) ssc_object: A processed conda SolverStateContainer object. Returns: +++++++ 1) The updated ssc object based off the sat bypass and package filtering. """ ## From Solver.run_sat specs_map_set = set(itervalues(ssc_object.specs_map)) ## Get the specs from ssc filtered by the package name new_odict = odict( [(p_name, ssc_object.specs_map[p_name]) for p_name in package_names] ) final_environment_specs = IndexedSet( concatv( itervalues(new_odict), ssc_object.track_features_specs, ssc_object.pinned_specs, ) ) ## Run the resolve process and get info for desired package ssc_object.solution_precs = ssc_object.r.solve(tuple(final_environment_specs)) wanted_indices = [] for i, info in enumerate(ssc_object.solution_precs): for p_name in package_names: if p_name in ssc_object.solution_precs[i].namekey: wanted_indices.append(i) filtered_ssc_solution_precs = [ ssc_object.solution_precs[x] for x in wanted_indices ] ssc_object.solution_precs = filtered_ssc_solution_precs ## Add the final environment specs to ssc ssc_object.final_environment_specs = final_environment_specs return ssc_object # ------------------------------------------------------------------------- # Run install # ------------------------------------------------------------------------- ## Set the context.always_yes to True to bypass user input context.always_yes = True target_prefix = context.target_prefix if prefix == None else prefix # Setup solver object # solve = Solver(target_prefix, (conda_channel,u'default'), context.subdirs, [pkg_name]) solve = Solver( target_prefix, (conda_channel, u"default"), context.subdirs, pkg_names ) ## Create a solver state container ### Make sure to Freeze those packages already installed in the current env in order to bypass update checking. ssc = SolverStateContainer( prefix=target_prefix, update_modifier=UpdateModifier.FREEZE_INSTALLED, deps_modifier=context.deps_modifier, prune=True, ignore_pinned=context.ignore_pinned, force_remove=context.force_remove, should_retry_solve=False, ) ## Get channel metadata with Spinner( "Collecting package metadata", not context.verbosity and not context.quiet, context.json, ): ssc = solve._collect_all_metadata(ssc) ## Set specs map to an empty map. (No need to check other specs) add_spec = [] for p_name, spec in iteritems(ssc.specs_map): for pkg_name in pkg_names: if str(p_name) in pkg_name: add_spec.append((pkg_name, MatchSpec(pkg_name))) ssc.specs_map = odict(add_spec) ## Process the data in the solver state container with Spinner( "Processing data", not context.verbosity and not context.quiet, context.json ): ssc = solve._add_specs(ssc) ssc = bypass_sat(pkg_names, ssc) ssc = solve._post_sat_handling(ssc) ## create an IndexedSet from ssc.solution_precs ssc.solution_precs = IndexedSet(PrefixGraph(ssc.solution_precs).graph) ## Get linked and unlinked unlink_precs, link_precs = diff_for_unlink_link_precs( target_prefix, ssc.solution_precs, solve.specs_to_add ) # set unlinked to empty indexed set so we do not unlink/remove any pacakges unlink_precs = IndexedSet() ## Create a PrefixSetup stp = PrefixSetup( solve.prefix, unlink_precs, link_precs, solve.specs_to_remove, solve.specs_to_add, solve.neutered_specs, ) ## Create an UnlinkLinkTransaction with stp unlink_link_transaction = UnlinkLinkTransaction(stp) # create Namespace args = Namespace( channel=None, cmd="install", deps_modifier=context.deps_modifier, json=False, packages=pkg_names, ) ## Set logger level if debug: WARN, INFO, DEBUG, TRACE = VERBOSITY_LEVELS set_all_logger_level(DEBUG) ## Install package install.handle_txn(unlink_link_transaction, solve.prefix, args, False) ## Retrun True if finished return True