def get_install_actions(prefix, index, specs, config, retries=0):
log = utils.get_logger(__name__)
if config.verbose:
capture = contextlib.contextmanager(lambda: (yield))
else:
capture = utils.capture
actions = {'LINK': []}
specs = [_ensure_valid_spec(spec) for spec in specs]
if specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with capture():
try:
actions = plan.install_actions(prefix, index, specs)
except NoPackagesFoundError as exc:
raise DependencyNeedsBuildingError(exc)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
AssertionError) as exc:
if 'lock' in str(exc):
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)
or isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(config)
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < config.max_env_retry:
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(prefix,
index,
specs,
config,
retries=retries + 1)
else:
log.error(
"Failed to get install actions, max retries exceeded.")
raise
if config.disable_pip:
actions['LINK'] = [
spec for spec in actions['LINK'] if not spec.startswith('pip-')
and not spec.startswith('setuptools-')
]
return actions
def distribute_variants(metadata, variants, index, permit_unsatisfiable_variants=False):
rendered_metadata = {}
need_reparse_in_env = False
unsatisfiable_variants = []
packages_needing_building = set()
for variant in variants:
mv = metadata.copy()
# deep copy the sensitive parts to decouple metadata objects
mv.config = metadata.config.copy()
mv.config.variant = combine_variants(variant, mv.config.variant)
mv.final = False
# TODO: may need to compute new build id, or at least remove any envs before building
# another variant
if 'target_platform' in variant:
mv.config.host_subdir = variant['target_platform']
if not need_reparse_in_env:
try:
mv.parse_until_resolved()
need_source_download = (bool(mv.meta.get('source')) and
not mv.needs_source_for_render and
not os.listdir(mv.config.work_dir))
# this is a bit wasteful. We don't store the output here - we'll have to recompute
# it later. We don't store it, so that we can have per-subpackage exclusions
# from the hash. Since finalizing brings in *all* build-time packages, notest
# just the ones from the recipe, it is impossible to remove them in the general
# case. Instead, we just leave the recipe unfinalized until then, so that by
# excluding one higher-level package (e.g. python), we also won't include its
# deps in the hash
finalize_metadata(mv, index)
except DependencyNeedsBuildingError as e:
unsatisfiable_variants.append(variant)
packages_needing_building.update(set(e.packages))
if permit_unsatisfiable_variants:
rendered_metadata[mv.dist()] = (mv, need_source_download,
need_reparse_in_env)
continue
except exceptions.UnableToParseMissingSetuptoolsDependencies:
need_reparse_in_env = True
except:
raise
# computes hashes based on whatever the current specs are - not the final specs
# This is a deduplication step. Any variants that end up identical because a
# given variant is not used in a recipe are effectively ignored, though we still pay
# the price to parse for that variant.
rendered_metadata[mv.build_id()] = (mv, need_source_download, need_reparse_in_env)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
if unsatisfiable_variants and not permit_unsatisfiable_variants:
raise DependencyNeedsBuildingError(packages=packages_needing_building)
return list(rendered_metadata.values())
def distribute_variants(metadata,
variants,
index,
permit_unsatisfiable_variants=False):
rendered_metadata = {}
need_reparse_in_env = False
unsatisfiable_variants = []
packages_needing_building = set()
for variant in variants:
mv = metadata.copy()
# deep copy the sensitive parts to decouple metadata objects
mv.config = metadata.config.copy()
mv.config.variant = combine_variants(variant, mv.config.variant)
mv.final = False
# TODO: may need to compute new build id, or at least remove any envs before building
# another variant
if 'target_platform' in variant:
mv.config.host_subdir = variant['target_platform']
if not need_reparse_in_env:
try:
mv.parse_until_resolved()
need_source_download = (bool(mv.meta.get('source'))
and not mv.needs_source_for_render
and not os.listdir(mv.config.work_dir))
mv = finalize_metadata(mv, index)
except DependencyNeedsBuildingError as e:
unsatisfiable_variants.append(variant)
packages_needing_building.update(set(e.packages))
if permit_unsatisfiable_variants:
rendered_metadata[mv.build_id()] = (mv,
need_source_download,
need_reparse_in_env)
continue
except exceptions.UnableToParseMissingSetuptoolsDependencies:
need_reparse_in_env = True
except:
raise
# computes hashes based on whatever the current specs are - not the final specs
# This is a deduplication step. Any variants that end up identical because a
# given variant is not used in a recipe are effectively ignored, though we still pay
# the price to parse for that variant.
rendered_metadata[mv.build_id()] = (mv, need_source_download,
need_reparse_in_env)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
if unsatisfiable_variants and not permit_unsatisfiable_variants:
raise DependencyNeedsBuildingError(packages=packages_needing_building)
return list(rendered_metadata.values())
def get_env_dependencies(m, env, variant, exclude_pattern=None):
dash_or_under = re.compile("[-_]")
index, index_ts = get_build_index(
m.config, getattr(m.config, "{}_subdir".format(env)))
specs = [ms.spec for ms in m.ms_depends(env)]
# replace x.x with our variant's numpy version, or else conda tries to literally go get x.x
if env == 'build':
specs = [
spec.replace(' x.x', ' {}'.format(variant.get('numpy', "")))
for spec in specs
]
subpackages = []
dependencies = []
pass_through_deps = []
# ones that get filtered from actual versioning, to exclude them from the hash calculation
for spec in specs:
if not exclude_pattern or not exclude_pattern.match(spec):
is_subpackage = False
spec_name = spec.split()[0]
for entry in m.get_section('outputs'):
name = entry.get('name')
if name == spec_name:
subpackages.append(' '.join((name, m.version())))
is_subpackage = True
if not is_subpackage:
dependencies.append(spec)
for key, value in variant.items():
if dash_or_under.sub("",
key) == dash_or_under.sub("", spec_name):
dependencies.append(" ".join((spec_name, value)))
elif exclude_pattern.match(spec):
pass_through_deps.append(spec.split(' ')[0])
random_string = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
dependencies = list(set(dependencies))
with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir:
try:
actions = environ.get_install_actions(tmpdir,
index,
dependencies,
m.config,
timestamp=index_ts)
except UnsatisfiableError as e:
# we'll get here if the environment is unsatisfiable
raise DependencyNeedsBuildingError(e)
specs = actions_to_pins(actions)
return specs + subpackages + pass_through_deps, actions
def get_env_dependencies(m, env, variant, index=None, exclude_pattern=None):
dash_or_under = re.compile("[-_]")
if not index:
index = get_build_index(m.config,
getattr(m.config, "{}_subdir".format(env)))
specs = [ms.spec for ms in m.ms_depends(env)]
# replace x.x with our variant's numpy version, or else conda tries to literally go get x.x
if env == 'build':
specs = [
spec.replace(' x.x', ' {}'.format(variant.get('numpy', "")))
for spec in specs
]
subpackages = []
dependencies = []
# ones that get filtered from actual versioning, to exclude them from the hash calculation
append_specs = []
for spec in specs:
if not exclude_pattern or not exclude_pattern.match(spec):
is_subpackage = False
spec_name = spec.split()[0]
for entry in m.get_section('outputs'):
name = entry.get('name')
if name == spec_name:
subpackages.append(' '.join((name, m.version())))
is_subpackage = True
if not is_subpackage:
dependencies.append(spec)
for key, value in variant.items():
if dash_or_under.sub("",
key) == dash_or_under.sub("", spec_name):
dependencies.append(" ".join((spec_name, value)))
else:
append_specs.append(spec)
prefix = m.config.host_prefix if env == 'host' else m.config.build_prefix
try:
actions = environ.get_install_actions(prefix, index, dependencies,
m.config)
except UnsatisfiableError as e:
# we'll get here if the environment is unsatisfiable
raise DependencyNeedsBuildingError(e)
specs = actions_to_pins(actions)
return specs + subpackages + append_specs
def get_upstream_pins(m, dependencies, index):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
dependencies = [strip_channel(dep) for dep in dependencies]
random_string = ''.join(random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
with TemporaryDirectory(suffix=random_string) as tmpdir:
actions = environ.get_install_actions(tmpdir, index, dependencies,
m.config)
additional_specs = []
linked_packages = actions['LINK']
# edit the plan to download all necessary packages
for key in ('LINK', 'EXTRACT', 'UNLINK'):
if key in actions:
del actions[key]
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
if actions:
plan.execute_actions(actions, index, verbose=m.config.debug)
pkgs_dirs = cc.pkgs_dirs + list(m.config.bldpkgs_dirs)
for pkg in linked_packages:
for pkgs_dir in pkgs_dirs:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_dir = os.path.join(pkgs_dir, pkg_dist)
pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
if os.path.isdir(pkg_dir):
downstream_file = os.path.join(pkg_dir, 'info/pin_downstream')
if os.path.isfile(downstream_file):
additional_specs.extend(open(downstream_file).read().splitlines())
break
elif os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(pkg_file, 'info/pin_downstream')
if extra_specs:
additional_specs.extend(extra_specs.splitlines())
break
elif utils.conda_43():
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
try:
pfe = ProgressiveFetchExtract(link_dists=[pkg],
index=index)
pfe.execute()
for pkgs_dir in pkgs_dirs:
pkg_file = os.path.join(pkgs_dir, pkg.dist_name + '.tar.bz2')
if os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(pkg_file,
'info/pin_downstream')
if extra_specs:
additional_specs.extend(extra_specs.splitlines())
break
break
except KeyError:
raise DependencyNeedsBuildingError(packages=[pkg.name])
else:
raise RuntimeError("Didn't find expected package {} in package cache ({})"
.format(pkg_dist, pkgs_dirs))
return additional_specs
def get_install_actions(prefix,
specs,
env,
retries=0,
subdir=None,
verbose=True,
debug=False,
locking=True,
bldpkgs_dirs=None,
timeout=900,
disable_pip=False,
max_env_retry=3,
output_folder=None,
channel_urls=None):
global cached_actions
global last_index_ts
actions = {}
log = utils.get_logger(__name__)
conda_log_level = logging.WARN
specs = list(specs)
if specs:
specs.extend(create_default_packages)
if verbose or debug:
capture = contextlib.contextmanager(lambda: (yield))
if debug:
conda_log_level = logging.DEBUG
else:
capture = utils.capture
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
bldpkgs_dirs = ensure_list(bldpkgs_dirs)
index, index_ts, _ = get_build_index(subdir,
list(bldpkgs_dirs)[0],
output_folder=output_folder,
channel_urls=channel_urls,
debug=debug,
verbose=verbose,
locking=locking,
timeout=timeout)
specs = tuple(
utils.ensure_valid_spec(spec) for spec in specs
if not str(spec).endswith('@'))
if ((specs, env, subdir, channel_urls, disable_pip) in cached_actions
and last_index_ts >= index_ts):
actions = cached_actions[(specs, env, subdir, channel_urls,
disable_pip)].copy()
if "PREFIX" in actions:
actions['PREFIX'] = prefix
elif specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with utils.LoggingContext(conda_log_level):
with capture():
try:
actions = install_actions(prefix, index, specs, force=True)
except (NoPackagesFoundError, UnsatisfiableError) as exc:
raise DependencyNeedsBuildingError(exc, subdir=subdir)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
AssertionError, BuildLockError) as exc:
if 'lock' in str(exc):
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)
or isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(
locking, bldpkgs_dirs, timeout)
with utils.try_acquire_locks(locks, timeout=timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < max_env_retry:
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(
prefix,
tuple(specs),
env,
retries=retries + 1,
subdir=subdir,
verbose=verbose,
debug=debug,
locking=locking,
bldpkgs_dirs=tuple(bldpkgs_dirs),
timeout=timeout,
disable_pip=disable_pip,
max_env_retry=max_env_retry,
output_folder=output_folder,
channel_urls=tuple(channel_urls))
else:
log.error(
"Failed to get install actions, max retries exceeded."
)
raise
if disable_pip:
for pkg in ('pip', 'setuptools', 'wheel'):
# specs are the raw specifications, not the conda-derived actual specs
# We're testing that pip etc. are manually specified
if not any(
re.match(r'^%s(?:$|[\s=].*)' % pkg, str(dep))
for dep in specs):
actions['LINK'] = [
spec for spec in actions['LINK'] if spec.name != pkg
]
utils.trim_empty_keys(actions)
cached_actions[(specs, env, subdir, channel_urls,
disable_pip)] = actions.copy()
last_index_ts = index_ts
return actions
def get_upstream_pins(m, dependencies, index):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
dependencies = [strip_channel(dep) for dep in dependencies]
# Add _tmp here to prevent creating the build_prefix too early. This is because, when
# dirty is set, we skip calling create_env if the folder already exists.
actions = environ.get_install_actions(m.config.build_prefix[:-4] + "_tmp",
index, dependencies, m.config)
additional_specs = []
linked_packages = actions['LINK']
# edit the plan to download all necessary packages
if 'LINK' in actions:
del actions['LINK']
if 'EXTRACT' in actions:
del actions['EXTRACT']
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
if actions:
plan.execute_actions(actions, index, verbose=m.config.debug)
pkgs_dirs = cc.pkgs_dirs + list(m.config.bldpkgs_dirs)
for pkg in linked_packages:
for pkgs_dir in pkgs_dirs:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_dir = os.path.join(pkgs_dir, pkg_dist)
pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
if os.path.isdir(pkg_dir):
downstream_file = os.path.join(pkg_dir,
'info/pin_downstream')
if os.path.isfile(downstream_file):
additional_specs.extend(
open(downstream_file).read().splitlines())
break
elif os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/pin_downstream')
if extra_specs:
additional_specs.extend(extra_specs.splitlines())
break
elif utils.conda_43():
# TODO: this is a vile hack reaching into conda's internals. Replace with proper
# conda API when available.
try:
pfe = ProgressiveFetchExtract(link_dists=[pkg],
index=index)
pfe.execute()
for pkgs_dir in pkgs_dirs:
pkg_file = os.path.join(pkgs_dir,
pkg.dist_name + '.tar.bz2')
if os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/pin_downstream')
if extra_specs:
additional_specs.extend(
extra_specs.splitlines())
break
break
except KeyError:
raise DependencyNeedsBuildingError(packages=[pkg.name])
else:
raise RuntimeError(
"Didn't find expected package {} in package cache ({})".
format(pkg_dist, pkgs_dirs))
return additional_specs
def get_upstream_pins(m, actions, env):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
additional_specs = []
linked_packages = actions.get('LINK', [])
# edit the plan to download all necessary packages
for key in ('LINK', 'EXTRACT', 'UNLINK'):
if key in actions:
del actions[key]
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
index, index_ts = get_build_index(getattr(m.config,
'{}_subdir'.format(env)),
bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder,
channel_urls=m.config.channel_urls,
debug=m.config.debug,
verbose=m.config.verbose,
locking=m.config.locking,
timeout=m.config.timeout)
if actions:
execute_actions(actions, index, verbose=m.config.debug)
ignore_list = utils.ensure_list(
m.get_value('build/ignore_run_exports'))
_pkgs_dirs = pkgs_dirs + list(m.config.bldpkgs_dirs)
for pkg in linked_packages:
for pkgs_dir in _pkgs_dirs:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_dir = os.path.join(pkgs_dir, pkg_dist)
pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
if os.path.isdir(pkg_dir):
downstream_file = os.path.join(pkg_dir, 'info/run_exports')
if os.path.isfile(downstream_file):
specs = open(downstream_file).read().splitlines()
additional_specs.extend(
_filter_run_exports(specs, ignore_list))
break
elif os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/run_exports')
if extra_specs:
# exclude packages pinning themselves (makes no sense)
extra_specs = [
spec for spec in extra_specs.splitlines()
if not spec.startswith(pkg_dist.rsplit('-', 2)[0])
]
additional_specs.extend(
_filter_run_exports(extra_specs, ignore_list))
break
elif utils.conda_43():
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
try:
try:
# the conda 4.4 API uses a single `link_prefs` kwarg
# whereas conda 4.3 used `index` and `link_dists` kwargs
pfe = ProgressiveFetchExtract(
link_prefs=(index[pkg], ))
except TypeError:
# TypeError: __init__() got an unexpected keyword argument 'link_prefs'
pfe = ProgressiveFetchExtract(link_dists=[pkg],
index=index)
with utils.LoggingContext():
pfe.execute()
for pkgs_dir in _pkgs_dirs:
pkg_file = os.path.join(pkgs_dir,
pkg.dist_name + '.tar.bz2')
if os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/run_exports')
if extra_specs:
specs = extra_specs.splitlines()
additional_specs.extend(
_filter_run_exports(
specs, ignore_list))
break
break
except KeyError:
raise DependencyNeedsBuildingError(packages=[pkg.name])
else:
raise RuntimeError(
"Didn't find expected package {} in package cache ({})".
format(pkg_dist, _pkgs_dirs))
return additional_specs
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
stub_subpackages=False):
rendered_metadata = {}
need_reparse_in_env = False
need_source_download = True
unsatisfiable_variants = []
packages_needing_building = set()
# don't bother distributing python if it's a noarch package
if metadata.noarch or metadata.noarch_python:
conform_dict = {'python': variants[0]['python']}
variants = conform_variants_to_value(variants, conform_dict)
# store these for reference later
metadata.config.variants = variants
if variants:
recipe_requirements = metadata.extract_requirements_text()
for variant in variants:
mv = metadata.copy()
# this determines which variants were used, and thus which ones should be locked for
# future rendering
mv.final = False
mv.config.variant = {}
mv.parse_again(permit_undefined_jinja=True, stub_subpackages=True)
vars_in_recipe = set(mv.undefined_jinja_vars)
mv.config.variant = variant
conform_dict = {}
for key in vars_in_recipe:
if PY3 and hasattr(recipe_requirements, 'decode'):
recipe_requirements = recipe_requirements.decode()
elif not PY3 and hasattr(recipe_requirements, 'encode'):
recipe_requirements = recipe_requirements.encode()
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
if re.search(r"\s+\{\{\s*%s\s*(?:.*?)?\}\}" % key,
recipe_requirements):
conform_dict[key] = variant[key]
compiler_matches = re.findall(r"compiler\([\'\"](.*)[\'\"].*\)",
recipe_requirements)
if compiler_matches:
from conda_build.jinja_context import native_compiler
for match in compiler_matches:
compiler_key = '{}_compiler'.format(match)
conform_dict[compiler_key] = variant.get(
compiler_key, native_compiler(match, mv.config))
conform_dict['target_platform'] = variant[
'target_platform']
build_reqs = mv.meta.get('requirements', {}).get('build', [])
if 'python' in build_reqs:
conform_dict['python'] = variant['python']
mv.config.variants = conform_variants_to_value(
mv.config.variants, conform_dict)
# reset this to our current variant to go ahead
mv.config.variant = variant
if 'target_platform' in variant:
mv.config.host_subdir = variant['target_platform']
if not need_reparse_in_env:
try:
mv.parse_until_resolved(stub_subpackages=stub_subpackages)
need_source_download = (bool(mv.meta.get('source'))
and not mv.needs_source_for_render
and
not os.listdir(mv.config.work_dir))
# if python is in the build specs, but doesn't have a specific associated
# version, make sure to add one to newly parsed 'requirements/build'.
if build_reqs and 'python' in build_reqs:
python_version = 'python {}'.format(
mv.config.variant['python'])
mv.meta['requirements']['build'] = [
python_version
if re.match('^python(?:$| .*)', pkg) else pkg
for pkg in mv.meta['requirements']['build']
]
fm = finalize_metadata(mv)
rendered_metadata[fm.dist()] = (fm, need_source_download,
need_reparse_in_env)
except DependencyNeedsBuildingError as e:
unsatisfiable_variants.append(variant)
packages_needing_building.update(set(e.packages))
if permit_unsatisfiable_variants:
rendered_metadata[mv.dist()] = (mv,
need_source_download,
need_reparse_in_env)
continue
except exceptions.UnableToParseMissingSetuptoolsDependencies:
need_reparse_in_env = True
except:
raise
else:
# computes hashes based on whatever the current specs are - not the final specs
# This is a deduplication step. Any variants that end up identical because a
# given variant is not used in a recipe are effectively ignored, though we still
# pay the price to parse for that variant.
rendered_metadata[mv.build_id()] = (mv, need_source_download,
need_reparse_in_env)
else:
rendered_metadata['base_recipe'] = (metadata, need_source_download,
need_reparse_in_env)
if unsatisfiable_variants and not permit_unsatisfiable_variants:
raise DependencyNeedsBuildingError(packages=packages_needing_building)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def get_install_actions(prefix,
index,
specs,
config,
retries=0,
timestamp=0,
subdir=None):
global _cached_install_actions
global _last_timestamp
log = utils.get_logger(__name__)
if config.verbose:
capture = contextlib.contextmanager(lambda: (yield))
else:
capture = utils.capture
actions = {'LINK': []}
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
specs = tuple(_ensure_valid_spec(spec) for spec in specs)
if (specs, subdir, timestamp
) in _cached_install_actions and timestamp > _last_timestamp:
actions = _cached_install_actions[(specs, subdir, timestamp)]
else:
if specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with capture():
try:
actions = install_actions(prefix, index, specs, force=True)
# Experimenting with getting conda to create fewer Resolve objects
# Experiment failed, seemingly due to conda's statefulness. Packages could
# not be found.
# index_timestamp=timestamp)
except NoPackagesFoundError as exc:
# Attempt to skeleton packages it can't find
packages = [x.split(" ")[0] for x in exc.pkgs]
for pkg in packages:
if pkg.startswith("r-"):
api.skeletonize([pkg], "cran")
else:
api.skeletonize([pkg], "pypi")
raise DependencyNeedsBuildingError(exc, subdir=subdir)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
AssertionError) as exc:
if 'lock' in str(exc):
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)
or isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(config)
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < config.max_env_retry:
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(prefix,
index,
specs,
config,
retries=retries + 1,
timestamp=timestamp)
else:
log.error(
"Failed to get install actions, max retries exceeded."
)
raise
if config.disable_pip:
actions['LINK'] = [
spec for spec in actions['LINK']
if not spec.startswith('pip-')
and not spec.startswith('setuptools-')
]
utils.trim_empty_keys(actions)
_cached_install_actions[(specs, subdir, timestamp)] = actions
_last_timestamp = timestamp
return actions
def get_upstream_pins(m, actions, index):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
additional_specs = []
linked_packages = actions.get('LINK', [])
# edit the plan to download all necessary packages
for key in ('LINK', 'EXTRACT', 'UNLINK'):
if key in actions:
del actions[key]
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
if actions:
execute_actions(actions, index, verbose=m.config.debug)
_pkgs_dirs = pkgs_dirs + list(m.config.bldpkgs_dirs)
for pkg in linked_packages:
for pkgs_dir in _pkgs_dirs:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_dir = os.path.join(pkgs_dir, pkg_dist)
pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
if os.path.isdir(pkg_dir):
downstream_file = os.path.join(pkg_dir, 'info/run_exports')
if os.path.isfile(downstream_file):
additional_specs.extend(
open(downstream_file).read().splitlines())
break
elif os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/run_exports')
if extra_specs:
# exclude packages pinning themselves (makes no sense)
extra_specs = [
spec for spec in extra_specs
if not spec.startswith(pkg_dist.rsplit('-', 2)[0])
]
additional_specs.extend(extra_specs.splitlines())
break
elif utils.conda_43():
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
try:
pfe = ProgressiveFetchExtract(link_dists=[pkg],
index=index)
pfe.execute()
for pkgs_dir in _pkgs_dirs:
pkg_file = os.path.join(pkgs_dir,
pkg.dist_name + '.tar.bz2')
if os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/run_exports')
if extra_specs:
additional_specs.extend(
extra_specs.splitlines())
break
break
except KeyError:
raise DependencyNeedsBuildingError(packages=[pkg.name])
else:
raise RuntimeError(
"Didn't find expected package {} in package cache ({})".
format(pkg_dist, _pkgs_dirs))
return additional_specs