def get_env_dependencies(m, env, variant, exclude_pattern=None, permit_unsatisfiable_variants=False, merge_build_host_on_same_platform=True): specs = m.get_depends_top_and_out(env) # replace x.x with our variant's numpy version, or else conda tries to literally go get x.x if env in ('build', 'host'): no_xx_specs = [] for spec in specs: if ' x.x' in spec: pkg_name = spec.split()[0] no_xx_specs.append(' '.join( (pkg_name, variant.get(pkg_name, "")))) else: no_xx_specs.append(spec) specs = no_xx_specs subpackages, dependencies, pass_through_deps = _categorize_deps( m, specs, exclude_pattern, variant) dependencies = set(dependencies) unsat = None random_string = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir: try: actions = environ.get_install_actions( tmpdir, tuple(dependencies), env, subdir=getattr(m.config, '{}_subdir'.format(env)), debug=m.config.debug, verbose=m.config.verbose, locking=m.config.locking, bldpkgs_dirs=tuple(m.config.bldpkgs_dirs), timeout=m.config.timeout, disable_pip=m.config.disable_pip, max_env_retry=m.config.max_env_retry, output_folder=m.config.output_folder, channel_urls=tuple(m.config.channel_urls)) except (UnsatisfiableError, DependencyNeedsBuildingError) as e: # we'll get here if the environment is unsatisfiable if hasattr(e, 'packages'): unsat = ', '.join(e.packages) else: unsat = e.message if permit_unsatisfiable_variants: actions = {} else: raise specs = actions_to_pins(actions) return (utils.ensure_list( (specs + subpackages + pass_through_deps) or m.meta.get('requirements', {}).get(env, [])), actions, unsat)
def get_env_dependencies(m, env, variant, exclude_pattern=None): dash_or_under = re.compile("[-_]") index, index_ts = get_build_index( m.config, getattr(m.config, "{}_subdir".format(env))) specs = [ms.spec for ms in m.ms_depends(env)] # replace x.x with our variant's numpy version, or else conda tries to literally go get x.x if env == 'build': specs = [ spec.replace(' x.x', ' {}'.format(variant.get('numpy', ""))) for spec in specs ] subpackages = [] dependencies = [] pass_through_deps = [] # ones that get filtered from actual versioning, to exclude them from the hash calculation for spec in specs: if not exclude_pattern or not exclude_pattern.match(spec): is_subpackage = False spec_name = spec.split()[0] for entry in m.get_section('outputs'): name = entry.get('name') if name == spec_name: subpackages.append(' '.join((name, m.version()))) is_subpackage = True if not is_subpackage: dependencies.append(spec) for key, value in variant.items(): if dash_or_under.sub("", key) == dash_or_under.sub("", spec_name): dependencies.append(" ".join((spec_name, value))) elif exclude_pattern.match(spec): pass_through_deps.append(spec.split(' ')[0]) random_string = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) dependencies = list(set(dependencies)) with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir: try: actions = environ.get_install_actions(tmpdir, index, dependencies, m.config, timestamp=index_ts) except UnsatisfiableError as e: # we'll get here if the environment is unsatisfiable raise DependencyNeedsBuildingError(e) specs = actions_to_pins(actions) return specs + subpackages + pass_through_deps, actions
def get_env_dependencies(m, env, variant, exclude_pattern=None, permit_unsatisfiable_variants=False, merge_build_host_on_same_platform=True): specs = m.get_depends_top_and_out(env) # replace x.x with our variant's numpy version, or else conda tries to literally go get x.x if env in ('build', 'host'): no_xx_specs = [] for spec in specs: if ' x.x' in spec: pkg_name = spec.split()[0] no_xx_specs.append(' '.join((pkg_name, variant.get(pkg_name, "")))) else: no_xx_specs.append(spec) specs = no_xx_specs subpackages, dependencies, pass_through_deps = _categorize_deps(m, specs, exclude_pattern, variant) dependencies = set(dependencies) unsat = None random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir: try: actions = environ.get_install_actions(tmpdir, tuple(dependencies), env, subdir=getattr(m.config, '{}_subdir'.format(env)), debug=m.config.debug, verbose=m.config.verbose, locking=m.config.locking, bldpkgs_dirs=tuple(m.config.bldpkgs_dirs), timeout=m.config.timeout, disable_pip=m.config.disable_pip, max_env_retry=m.config.max_env_retry, output_folder=m.config.output_folder, channel_urls=tuple(m.config.channel_urls)) except (UnsatisfiableError, DependencyNeedsBuildingError) as e: # we'll get here if the environment is unsatisfiable if hasattr(e, 'packages'): unsat = ', '.join(e.packages) else: unsat = e.message if permit_unsatisfiable_variants: actions = {} else: raise specs = actions_to_pins(actions) return (utils.ensure_list((specs + subpackages + pass_through_deps) or m.meta.get('requirements', {}).get(env, [])), actions, unsat)
def get_env_dependencies(m, env, variant, index=None, exclude_pattern=None): dash_or_under = re.compile("[-_]") if not index: index = get_build_index(m.config, getattr(m.config, "{}_subdir".format(env))) specs = [ms.spec for ms in m.ms_depends(env)] # replace x.x with our variant's numpy version, or else conda tries to literally go get x.x if env == 'build': specs = [ spec.replace(' x.x', ' {}'.format(variant.get('numpy', ""))) for spec in specs ] subpackages = [] dependencies = [] # ones that get filtered from actual versioning, to exclude them from the hash calculation append_specs = [] for spec in specs: if not exclude_pattern or not exclude_pattern.match(spec): is_subpackage = False spec_name = spec.split()[0] for entry in m.get_section('outputs'): name = entry.get('name') if name == spec_name: subpackages.append(' '.join((name, m.version()))) is_subpackage = True if not is_subpackage: dependencies.append(spec) for key, value in variant.items(): if dash_or_under.sub("", key) == dash_or_under.sub("", spec_name): dependencies.append(" ".join((spec_name, value))) else: append_specs.append(spec) prefix = m.config.host_prefix if env == 'host' else m.config.build_prefix try: actions = environ.get_install_actions(prefix, index, dependencies, m.config) except UnsatisfiableError as e: # we'll get here if the environment is unsatisfiable raise DependencyNeedsBuildingError(e) specs = actions_to_pins(actions) return specs + subpackages + append_specs
def get_upstream_pins(m, dependencies, index): """Download packages from specs, then inspect each downloaded package for additional downstream dependency specs. Return these additional specs.""" dependencies = [strip_channel(dep) for dep in dependencies] random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) with TemporaryDirectory(suffix=random_string) as tmpdir: actions = environ.get_install_actions(tmpdir, index, dependencies, m.config) additional_specs = [] linked_packages = actions['LINK'] # edit the plan to download all necessary packages for key in ('LINK', 'EXTRACT', 'UNLINK'): if key in actions: del actions[key] # this should be just downloading packages. We don't need to extract them - # we read contents directly if actions: plan.execute_actions(actions, index, verbose=m.config.debug) pkgs_dirs = cc.pkgs_dirs + list(m.config.bldpkgs_dirs) for pkg in linked_packages: for pkgs_dir in pkgs_dirs: if hasattr(pkg, 'dist_name'): pkg_dist = pkg.dist_name else: pkg = strip_channel(pkg) pkg_dist = pkg.split(' ')[0] pkg_dir = os.path.join(pkgs_dir, pkg_dist) pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2') if os.path.isdir(pkg_dir): downstream_file = os.path.join(pkg_dir, 'info/pin_downstream') if os.path.isfile(downstream_file): additional_specs.extend(open(downstream_file).read().splitlines()) break elif os.path.isfile(pkg_file): extra_specs = utils.package_has_file(pkg_file, 'info/pin_downstream') if extra_specs: additional_specs.extend(extra_specs.splitlines()) break elif utils.conda_43(): # TODO: this is a vile hack reaching into conda's internals. Replace with # proper conda API when available. try: pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index) pfe.execute() for pkgs_dir in pkgs_dirs: pkg_file = os.path.join(pkgs_dir, pkg.dist_name + '.tar.bz2') if os.path.isfile(pkg_file): extra_specs = utils.package_has_file(pkg_file, 'info/pin_downstream') if extra_specs: additional_specs.extend(extra_specs.splitlines()) break break except KeyError: raise DependencyNeedsBuildingError(packages=[pkg.name]) else: raise RuntimeError("Didn't find expected package {} in package cache ({})" .format(pkg_dist, pkgs_dirs)) return additional_specs
def get_upstream_pins(m, dependencies, index): """Download packages from specs, then inspect each downloaded package for additional downstream dependency specs. Return these additional specs.""" dependencies = [strip_channel(dep) for dep in dependencies] # Add _tmp here to prevent creating the build_prefix too early. This is because, when # dirty is set, we skip calling create_env if the folder already exists. actions = environ.get_install_actions(m.config.build_prefix[:-4] + "_tmp", index, dependencies, m.config) additional_specs = [] linked_packages = actions['LINK'] # edit the plan to download all necessary packages if 'LINK' in actions: del actions['LINK'] if 'EXTRACT' in actions: del actions['EXTRACT'] # this should be just downloading packages. We don't need to extract them - # we read contents directly if actions: plan.execute_actions(actions, index, verbose=m.config.debug) pkgs_dirs = cc.pkgs_dirs + list(m.config.bldpkgs_dirs) for pkg in linked_packages: for pkgs_dir in pkgs_dirs: if hasattr(pkg, 'dist_name'): pkg_dist = pkg.dist_name else: pkg = strip_channel(pkg) pkg_dist = pkg.split(' ')[0] pkg_dir = os.path.join(pkgs_dir, pkg_dist) pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2') if os.path.isdir(pkg_dir): downstream_file = os.path.join(pkg_dir, 'info/pin_downstream') if os.path.isfile(downstream_file): additional_specs.extend( open(downstream_file).read().splitlines()) break elif os.path.isfile(pkg_file): extra_specs = utils.package_has_file( pkg_file, 'info/pin_downstream') if extra_specs: additional_specs.extend(extra_specs.splitlines()) break elif utils.conda_43(): # TODO: this is a vile hack reaching into conda's internals. Replace with proper # conda API when available. try: pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index) pfe.execute() for pkgs_dir in pkgs_dirs: pkg_file = os.path.join(pkgs_dir, pkg.dist_name + '.tar.bz2') if os.path.isfile(pkg_file): extra_specs = utils.package_has_file( pkg_file, 'info/pin_downstream') if extra_specs: additional_specs.extend( extra_specs.splitlines()) break break except KeyError: raise DependencyNeedsBuildingError(packages=[pkg.name]) else: raise RuntimeError( "Didn't find expected package {} in package cache ({})". format(pkg_dist, pkgs_dirs)) return additional_specs
def get_env_dependencies(m, env, variant, exclude_pattern=None, permit_unsatisfiable_variants=False): dash_or_under = re.compile("[-_]") specs = [ms.spec for ms in m.ms_depends(env)] if env == 'build' and m.is_cross and m.config.build_subdir == m.config.host_subdir: specs.extend([ms.spec for ms in m.ms_depends('host')]) # replace x.x with our variant's numpy version, or else conda tries to literally go get x.x if env in ('build', 'host'): no_xx_specs = [] for spec in specs: if ' x.x' in spec: pkg_name = spec.split()[0] no_xx_specs.append(' '.join( (pkg_name, variant.get(pkg_name, "")))) else: no_xx_specs.append(spec) specs = no_xx_specs subpackages = [] dependencies = [] pass_through_deps = [] # ones that get filtered from actual versioning, to exclude them from the hash calculation for spec in specs: if not exclude_pattern or not exclude_pattern.match(spec): is_subpackage = False spec_name = spec.split()[0] for entry in m.get_section('outputs'): name = entry.get('name') if name == spec_name: subpackages.append(' '.join((name, m.version()))) is_subpackage = True if not is_subpackage: dependencies.append(spec) # fill in variant version iff no version at all is provided for key, value in variant.items(): # for sake of comparison, ignore dashes and underscores if (dash_or_under.sub("", key) == dash_or_under.sub( "", spec_name) and not re.search( r'%s\s+[0-9a-zA-Z\_\.\<\>\=\*]' % spec_name, spec)): dependencies.append(" ".join((spec_name, value))) elif exclude_pattern.match(spec): pass_through_deps.append(spec) random_string = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) dependencies = set(dependencies) unsat = None with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir: try: actions = environ.get_install_actions( tmpdir, tuple(dependencies), env, subdir=getattr(m.config, '{}_subdir'.format(env)), debug=m.config.debug, verbose=m.config.verbose, locking=m.config.locking, bldpkgs_dirs=tuple(m.config.bldpkgs_dirs), timeout=m.config.timeout, disable_pip=m.config.disable_pip, max_env_retry=m.config.max_env_retry, output_folder=m.config.output_folder, channel_urls=tuple(m.config.channel_urls)) except (UnsatisfiableError, DependencyNeedsBuildingError) as e: # we'll get here if the environment is unsatisfiable if hasattr(e, 'packages'): unsat = ', '.join(e.packages) else: unsat = e.message if permit_unsatisfiable_variants: actions = {} else: raise specs = actions_to_pins(actions) return specs + subpackages + pass_through_deps, actions, unsat
def get_env_dependencies(m, env, variant, exclude_pattern=None, permit_unsatisfiable_variants=False, merge_build_host_on_same_platform=True): dash_or_under = re.compile("[-_]") specs = m.get_depends_top_and_out(env) # replace x.x with our variant's numpy version, or else conda tries to literally go get x.x if env in ('build', 'host'): no_xx_specs = [] for spec in specs: if ' x.x' in spec: pkg_name = spec.split()[0] no_xx_specs.append(' '.join((pkg_name, variant.get(pkg_name, "")))) else: no_xx_specs.append(spec) specs = no_xx_specs subpackages = [] dependencies = [] pass_through_deps = [] # ones that get filtered from actual versioning, to exclude them from the hash calculation for spec in specs: if not exclude_pattern or not exclude_pattern.match(spec): is_subpackage = False spec_name = spec.split()[0] for entry in m.get_section('outputs'): name = entry.get('name') if name == spec_name: subpackages.append(' '.join((name, m.version()))) is_subpackage = True if not is_subpackage: dependencies.append(spec) # fill in variant version iff no version at all is provided for key, value in variant.items(): # for sake of comparison, ignore dashes and underscores if (dash_or_under.sub("", key) == dash_or_under.sub("", spec_name) and not re.search(r'%s\s+[0-9a-zA-Z\_\.\<\>\=\*]' % spec_name, spec)): dependencies.append(" ".join((spec_name, value))) elif exclude_pattern.match(spec): pass_through_deps.append(spec) random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) dependencies = set(dependencies) unsat = None with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir: try: actions = environ.get_install_actions(tmpdir, tuple(dependencies), env, subdir=getattr(m.config, '{}_subdir'.format(env)), debug=m.config.debug, verbose=m.config.verbose, locking=m.config.locking, bldpkgs_dirs=tuple(m.config.bldpkgs_dirs), timeout=m.config.timeout, disable_pip=m.config.disable_pip, max_env_retry=m.config.max_env_retry, output_folder=m.config.output_folder, channel_urls=tuple(m.config.channel_urls)) except (UnsatisfiableError, DependencyNeedsBuildingError) as e: # we'll get here if the environment is unsatisfiable if hasattr(e, 'packages'): unsat = ', '.join(e.packages) else: unsat = e.message if permit_unsatisfiable_variants: actions = {} else: raise specs = actions_to_pins(actions) return (utils.ensure_list((specs + subpackages + pass_through_deps) or m.meta.get('requirements', {}).get(env, [])), actions, unsat)