def render_recipe(recipe_path, config, no_download_source=False, variants=None,
permit_unsatisfiable_variants=True, reset_build_id=True, expand_output=False):
arg = recipe_path
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
elif arg.endswith('.yaml'):
recipe_dir = os.path.dirname(arg)
need_cleanup = False
else:
print("Ignoring non-recipe: %s" % arg)
return None, None
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir, config=config)
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
if config.set_build_id:
m.config.compute_build_id(m.name(), reset=reset_build_id)
if m.needs_source_for_render and (not os.path.isdir(m.config.work_dir) or
len(os.listdir(m.config.work_dir)) == 0):
try_download(m, no_download_source=no_download_source)
rendered_metadata = {}
if m.final:
rendered_metadata = [(m, False, False), ]
index = None
else:
variants = (dict_of_lists_to_list_of_dicts(variants, m.config.platform)
if variants else get_package_variants(m, m.config))
index = get_build_index(m.config, m.config.build_subdir)
rendered_metadata = distribute_variants(m, variants, index,
permit_unsatisfiable_variants=permit_unsatisfiable_variants)
if not rendered_metadata:
raise ValueError("No variants were satisfiable - no valid recipes could be rendered.")
if expand_output:
rendered_metadata = expand_outputs(rendered_metadata, index)
if need_cleanup:
utils.rm_rf(recipe_dir)
return rendered_metadata, index
def determine_package_nature(pkg, prefix, subdir, bldpkgs_dir, output_folder, channel_urls):
dsos = []
run_exports = None
lib_prefix = pkg.name.startswith('lib')
codefiles = get_package_obj_files(pkg, prefix)
dsos = [f for f in codefiles for ext in ('.dylib', '.so', '.dll') if ext in f]
# we don't care about the actual run_exports value, just whether or not run_exports are present. We can use channeldata
# and it'll be a more reliable source (no disk race condition nonsense)
_, _, channeldata = get_build_index(subdir=subdir,
bldpkgs_dir=bldpkgs_dir,
output_folder=output_folder,
channel_urls=channel_urls,
debug=False,
verbose=False,
clear_cache=False)
channel_used = pkg.channel
channeldata = channeldata.get(channel_used)
if channeldata and pkg.name in channeldata['packages']:
run_exports = channeldata['packages'][pkg.name].get('run_exports', {})
else:
for pkgs_dir in pkgs_dirs:
test_folder = os.path.join(pkgs_dir, pkg.dist_name)
test_filename = os.path.join(pkgs_dir, pkg.fn)
if os.path.exists(test_folder):
run_exports = get_run_exports(test_folder)
break
elif os.path.isfile(test_filename):
run_exports = get_run_exports(test_filename)
break
return (dsos, run_exports, lib_prefix)
def get_upstream_pins(m, actions, env):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
env_specs = m.meta.get('requirements', {}).get(env, [])
explicit_specs = [req.split(' ')[0] for req in env_specs] if env_specs else []
linked_packages = actions.get('LINK', [])
linked_packages = [pkg for pkg in linked_packages if pkg.name in explicit_specs]
# edit the plan to download all necessary packages
for key in ('LINK', 'EXTRACT', 'UNLINK'):
if key in actions:
del actions[key]
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
index, index_ts = get_build_index(getattr(m.config, '{}_subdir'.format(env)),
bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder,
channel_urls=m.config.channel_urls,
debug=m.config.debug, verbose=m.config.verbose,
locking=m.config.locking, timeout=m.config.timeout)
if 'FETCH' in actions or 'EXTRACT' in actions:
# this is to force the download
execute_actions(actions, index, verbose=m.config.debug)
ignore_list = utils.ensure_list(m.get_value('build/ignore_run_exports'))
additional_specs = {}
for pkg in linked_packages:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs(pkg_dist, m)
# ran through all pkgs_dirs, and did not find package or folder. Download it.
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
if not pkg_loc and conda_43:
try:
# the conda 4.4 API uses a single `link_prefs` kwarg
# whereas conda 4.3 used `index` and `link_dists` kwargs
pfe = ProgressiveFetchExtract(link_prefs=(index[pkg],))
except TypeError:
# TypeError: __init__() got an unexpected keyword argument 'link_prefs'
pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index)
with utils.LoggingContext():
pfe.execute()
for pkg_dir in pkgs_dirs:
_loc = os.path.join(pkg_dir, index[pkg].fn)
if os.path.isfile(_loc):
pkg_loc = _loc
break
specs = _read_specs_from_package(pkg_loc, pkg_dist)
additional_specs = utils.merge_dicts_of_lists(additional_specs,
_filter_run_exports(specs, ignore_list))
return additional_specs
def execute_download_actions(m, actions, env, package_subset=None, require_files=False):
index, _, _ = get_build_index(getattr(m.config, '{}_subdir'.format(env)), bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder, channel_urls=m.config.channel_urls,
debug=m.config.debug, verbose=m.config.verbose, locking=m.config.locking,
timeout=m.config.timeout)
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
if 'FETCH' in actions or 'EXTRACT' in actions:
# this is to force the download
execute_actions(actions, index, verbose=m.config.debug)
pkg_files = {}
packages = actions.get('LINK', [])
package_subset = utils.ensure_list(package_subset)
selected_packages = set()
if package_subset:
for pkg in package_subset:
if hasattr(pkg, 'name'):
if pkg in packages:
selected_packages.add(pkg)
else:
pkg_name = pkg.split()[0]
for link_pkg in packages:
if pkg_name == link_pkg.name:
selected_packages.add(link_pkg)
break
packages = selected_packages
for pkg in packages:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs(pkg_dist, m, files_only=require_files)
# ran through all pkgs_dirs, and did not find package or folder. Download it.
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
if not pkg_loc and conda_43:
try:
pkg_record = [_ for _ in index if _.dist_name == pkg_dist][0]
# the conda 4.4 API uses a single `link_prefs` kwarg
# whereas conda 4.3 used `index` and `link_dists` kwargs
pfe = ProgressiveFetchExtract(link_prefs=(index[pkg_record],))
except TypeError:
# TypeError: __init__() got an unexpected keyword argument 'link_prefs'
pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index)
with utils.LoggingContext():
pfe.execute()
for pkg_dir in pkgs_dirs:
_loc = os.path.join(pkg_dir, index[pkg].fn)
if os.path.isfile(_loc):
pkg_loc = _loc
break
pkg_files[pkg] = pkg_loc, pkg_dist
return pkg_files
def testing_index(request):
with capture():
index, index_ts = get_build_index(config=Config(debug=False,
verbose=False),
subdir=subdir,
clear_cache=True)
return index
def collect_tasks(path,
folders,
matrix_base_dir,
channels=None,
steps=0,
test=False,
max_downstream=5,
variant_config_files=None,
platform_filters=None):
# runs = ['test']
# not testing means build and test
# if not test:
# runs.insert(0, 'build')
runs = ['build']
task_graph = nx.DiGraph()
config = conda_build.api.Config()
platform_filters = ensure_list(platform_filters) if platform_filters else [
'*'
]
for run in runs:
platforms = parse_platforms(matrix_base_dir, run, platform_filters)
# loop over platforms here because each platform may have different dependencies
# each platform will be submitted with a different label
for platform in platforms:
index_key = '-'.join([platform['platform'], str(platform['arch'])])
config.channel_urls = channels or []
config.variant_config_files = variant_config_files or []
conda_resolve = Resolve(
get_build_index(subdir=index_key,
bldpkgs_dir=config.bldpkgs_dir)[0])
# this graph is potentially different for platform and for build or test mode ("run")
g = construct_graph(path,
worker=platform,
folders=folders,
run=run,
matrix_base_dir=matrix_base_dir,
conda_resolve=conda_resolve,
config=config)
# Apply the build label to any nodes that need (re)building or testing
expand_run(g,
conda_resolve=conda_resolve,
worker=platform,
run=run,
steps=steps,
max_downstream=max_downstream,
recipes_dir=path,
matrix_base_dir=matrix_base_dir)
# merge this graph with the main one
task_graph = nx.compose(task_graph, g)
return task_graph
def get_env_dependencies(m, env, variant, exclude_pattern=None):
dash_or_under = re.compile("[-_]")
index, index_ts = get_build_index(
m.config, getattr(m.config, "{}_subdir".format(env)))
specs = [ms.spec for ms in m.ms_depends(env)]
# replace x.x with our variant's numpy version, or else conda tries to literally go get x.x
if env == 'build':
specs = [
spec.replace(' x.x', ' {}'.format(variant.get('numpy', "")))
for spec in specs
]
subpackages = []
dependencies = []
pass_through_deps = []
# ones that get filtered from actual versioning, to exclude them from the hash calculation
for spec in specs:
if not exclude_pattern or not exclude_pattern.match(spec):
is_subpackage = False
spec_name = spec.split()[0]
for entry in m.get_section('outputs'):
name = entry.get('name')
if name == spec_name:
subpackages.append(' '.join((name, m.version())))
is_subpackage = True
if not is_subpackage:
dependencies.append(spec)
for key, value in variant.items():
if dash_or_under.sub("",
key) == dash_or_under.sub("", spec_name):
dependencies.append(" ".join((spec_name, value)))
elif exclude_pattern.match(spec):
pass_through_deps.append(spec.split(' ')[0])
random_string = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
dependencies = list(set(dependencies))
with TemporaryDirectory(prefix="_", suffix=random_string) as tmpdir:
try:
actions = environ.get_install_actions(tmpdir,
index,
dependencies,
m.config,
timestamp=index_ts)
except UnsatisfiableError as e:
# we'll get here if the environment is unsatisfiable
raise DependencyNeedsBuildingError(e)
specs = actions_to_pins(actions)
return specs + subpackages + pass_through_deps, actions
def get_env_dependencies(m, env, variant, index=None, exclude_pattern=None):
dash_or_under = re.compile("[-_]")
if not index:
index = get_build_index(m.config,
getattr(m.config, "{}_subdir".format(env)))
specs = [ms.spec for ms in m.ms_depends(env)]
# replace x.x with our variant's numpy version, or else conda tries to literally go get x.x
if env == 'build':
specs = [
spec.replace(' x.x', ' {}'.format(variant.get('numpy', "")))
for spec in specs
]
subpackages = []
dependencies = []
# ones that get filtered from actual versioning, to exclude them from the hash calculation
append_specs = []
for spec in specs:
if not exclude_pattern or not exclude_pattern.match(spec):
is_subpackage = False
spec_name = spec.split()[0]
for entry in m.get_section('outputs'):
name = entry.get('name')
if name == spec_name:
subpackages.append(' '.join((name, m.version())))
is_subpackage = True
if not is_subpackage:
dependencies.append(spec)
for key, value in variant.items():
if dash_or_under.sub("",
key) == dash_or_under.sub("", spec_name):
dependencies.append(" ".join((spec_name, value)))
else:
append_specs.append(spec)
prefix = m.config.host_prefix if env == 'host' else m.config.build_prefix
try:
actions = environ.get_install_actions(prefix, index, dependencies,
m.config)
except UnsatisfiableError as e:
# we'll get here if the environment is unsatisfiable
raise DependencyNeedsBuildingError(e)
specs = actions_to_pins(actions)
return specs + subpackages + append_specs
def get_install_actions(prefix,
specs,
env,
retries=0,
subdir=None,
verbose=True,
debug=False,
locking=True,
bldpkgs_dirs=None,
timeout=900,
disable_pip=False,
max_env_retry=3,
output_folder=None,
channel_urls=None):
global cached_actions
global last_index_ts
actions = {}
log = utils.get_logger(__name__)
conda_log_level = logging.WARN
specs = list(specs)
if specs:
specs.extend(create_default_packages)
if verbose or debug:
capture = contextlib.contextmanager(lambda: (yield))
if debug:
conda_log_level = logging.DEBUG
else:
capture = utils.capture
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
bldpkgs_dirs = ensure_list(bldpkgs_dirs)
index, index_ts, _ = get_build_index(subdir,
list(bldpkgs_dirs)[0],
output_folder=output_folder,
channel_urls=channel_urls,
debug=debug,
verbose=verbose,
locking=locking,
timeout=timeout)
specs = tuple(
utils.ensure_valid_spec(spec) for spec in specs
if not str(spec).endswith('@'))
if ((specs, env, subdir, channel_urls, disable_pip) in cached_actions
and last_index_ts >= index_ts):
actions = cached_actions[(specs, env, subdir, channel_urls,
disable_pip)].copy()
if "PREFIX" in actions:
actions['PREFIX'] = prefix
elif specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with utils.LoggingContext(conda_log_level):
with capture():
try:
actions = install_actions(prefix, index, specs, force=True)
except (NoPackagesFoundError, UnsatisfiableError) as exc:
raise DependencyNeedsBuildingError(exc, subdir=subdir)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
AssertionError, BuildLockError) as exc:
if 'lock' in str(exc):
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)
or isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(
locking, bldpkgs_dirs, timeout)
with utils.try_acquire_locks(locks, timeout=timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < max_env_retry:
log.warn(
"failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(
prefix,
tuple(specs),
env,
retries=retries + 1,
subdir=subdir,
verbose=verbose,
debug=debug,
locking=locking,
bldpkgs_dirs=tuple(bldpkgs_dirs),
timeout=timeout,
disable_pip=disable_pip,
max_env_retry=max_env_retry,
output_folder=output_folder,
channel_urls=tuple(channel_urls))
else:
log.error(
"Failed to get install actions, max retries exceeded."
)
raise
if disable_pip:
for pkg in ('pip', 'setuptools', 'wheel'):
# specs are the raw specifications, not the conda-derived actual specs
# We're testing that pip etc. are manually specified
if not any(
re.match(r'^%s(?:$|[\s=].*)' % pkg, str(dep))
for dep in specs):
actions['LINK'] = [
spec for spec in actions['LINK'] if spec.name != pkg
]
utils.trim_empty_keys(actions)
cached_actions[(specs, env, subdir, channel_urls,
disable_pip)] = actions.copy()
last_index_ts = index_ts
return actions
def create_env(prefix,
specs,
config,
subdir,
clear_cache=True,
retry=0,
index=None,
locks=None):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
utils.get_logger("conda_build").setLevel(logging.DEBUG)
external_logger_context = utils.LoggingContext(logging.DEBUG)
else:
utils.get_logger("conda_build").setLevel(logging.INFO)
external_logger_context = utils.LoggingContext(logging.ERROR)
with external_logger_context:
log = utils.get_logger(__name__)
if os.path.isdir(prefix):
utils.rm_rf(prefix)
specs = list(set(specs))
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
if specs: # Don't waste time if there is nothing to do
log.debug("Creating environment in %s", prefix)
log.debug(str(specs))
with utils.path_prepended(prefix):
if not locks:
locks = utils.get_conda_operation_locks(config)
try:
with utils.try_acquire_locks(locks,
timeout=config.timeout):
if not index:
index = get_build_index(config=config,
subdir=subdir)
actions = get_install_actions(prefix, index, specs,
config)
plan.display_actions(actions, index)
if utils.on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
plan.execute_actions(actions,
index,
verbose=config.debug)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError) as exc:
if (("too short in" in str(exc) or re.search(
'post-link failed for: (?:[a-zA-Z]*::)?openssl',
str(exc)) or isinstance(exc, PaddingError))
and config.prefix_length > 80):
if config.prefix_length_fallback:
log.warn(
"Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.build_prefix
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache)
else:
raise
elif 'lock' in str(exc):
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache,
retry=retry + 1)
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache,
retry=retry + 1)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
else:
raise
# HACK: some of the time, conda screws up somehow and incomplete packages result.
# Just retry.
except (AssertionError, IOError, ValueError, RuntimeError,
LockError) as exc:
if isinstance(exc, AssertionError):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = os.path.dirname(os.path.dirname(
str(exc)))
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs,
config=config,
subdir=subdir,
clear_cache=clear_cache,
retry=retry + 1)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
if utils.on_win:
shell = "cmd.exe"
else:
shell = "bash"
symlink_conda(prefix, sys.prefix, shell)
def render_recipe(recipe_path,
config,
no_download_source=False,
variants=None,
permit_unsatisfiable_variants=True,
reset_build_id=True,
bypass_env_check=False):
"""Returns a list of tuples, each consisting of
(metadata-object, needs_download, needs_render_in_env)
You get one tuple per variant. Outputs are not factored in here (subpackages won't affect these
results returned here.)
"""
arg = recipe_path
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
elif arg.endswith('.yaml'):
recipe_dir = os.path.dirname(arg)
need_cleanup = False
else:
print("Ignoring non-recipe: %s" % arg)
return None, None
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir, config=config)
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
rendered_metadata = {}
# important: set build id *before* downloading source. Otherwise source goes into a different
# build folder.
if config.set_build_id:
m.config.compute_build_id(m.name(), reset=reset_build_id)
# this source may go into a folder that doesn't match the eventual build folder.
# There's no way around it AFAICT. We must download the source to be able to render
# the recipe (from anything like GIT_FULL_HASH), but we can't know the final build
# folder until rendering is complete, because package names can have variant jinja2 in them.
if m.needs_source_for_render and (not os.path.isdir(m.config.work_dir) or
len(os.listdir(m.config.work_dir)) == 0):
try_download(m, no_download_source=no_download_source)
if m.final:
if not hasattr(m.config, 'variants'):
m.config.variants = [m.config.variant]
rendered_metadata = [
(m, False, False),
]
else:
index, index_ts = get_build_index(m.config.build_subdir,
bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder,
channel_urls=m.config.channel_urls,
debug=m.config.debug,
verbose=m.config.verbose,
locking=m.config.locking,
timeout=m.config.timeout)
# when building, we don't want to fully expand all outputs into metadata, only expand
# whatever variants we have.
variants = (dict_of_lists_to_list_of_dicts(variants)
if variants else get_package_variants(m))
rendered_metadata = distribute_variants(
m,
variants,
permit_unsatisfiable_variants=permit_unsatisfiable_variants,
allow_no_other_outputs=True,
bypass_env_check=bypass_env_check)
if need_cleanup:
utils.rm_rf(recipe_dir)
return rendered_metadata
def get_upstream_pins(m, actions, env):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
additional_specs = []
linked_packages = actions.get('LINK', [])
# edit the plan to download all necessary packages
for key in ('LINK', 'EXTRACT', 'UNLINK'):
if key in actions:
del actions[key]
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
index, index_ts = get_build_index(getattr(m.config,
'{}_subdir'.format(env)),
bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder,
channel_urls=m.config.channel_urls,
debug=m.config.debug,
verbose=m.config.verbose,
locking=m.config.locking,
timeout=m.config.timeout)
if actions:
execute_actions(actions, index, verbose=m.config.debug)
ignore_list = utils.ensure_list(
m.get_value('build/ignore_run_exports'))
_pkgs_dirs = pkgs_dirs + list(m.config.bldpkgs_dirs)
for pkg in linked_packages:
for pkgs_dir in _pkgs_dirs:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_dir = os.path.join(pkgs_dir, pkg_dist)
pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
if os.path.isdir(pkg_dir):
downstream_file = os.path.join(pkg_dir, 'info/run_exports')
if os.path.isfile(downstream_file):
specs = open(downstream_file).read().splitlines()
additional_specs.extend(
_filter_run_exports(specs, ignore_list))
break
elif os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/run_exports')
if extra_specs:
# exclude packages pinning themselves (makes no sense)
extra_specs = [
spec for spec in extra_specs.splitlines()
if not spec.startswith(pkg_dist.rsplit('-', 2)[0])
]
additional_specs.extend(
_filter_run_exports(extra_specs, ignore_list))
break
elif utils.conda_43():
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
try:
try:
# the conda 4.4 API uses a single `link_prefs` kwarg
# whereas conda 4.3 used `index` and `link_dists` kwargs
pfe = ProgressiveFetchExtract(
link_prefs=(index[pkg], ))
except TypeError:
# TypeError: __init__() got an unexpected keyword argument 'link_prefs'
pfe = ProgressiveFetchExtract(link_dists=[pkg],
index=index)
with utils.LoggingContext():
pfe.execute()
for pkgs_dir in _pkgs_dirs:
pkg_file = os.path.join(pkgs_dir,
pkg.dist_name + '.tar.bz2')
if os.path.isfile(pkg_file):
extra_specs = utils.package_has_file(
pkg_file, 'info/run_exports')
if extra_specs:
specs = extra_specs.splitlines()
additional_specs.extend(
_filter_run_exports(
specs, ignore_list))
break
break
except KeyError:
raise DependencyNeedsBuildingError(packages=[pkg.name])
else:
raise RuntimeError(
"Didn't find expected package {} in package cache ({})".
format(pkg_dist, _pkgs_dirs))
return additional_specs
def get_upstream_pins(m, actions, env):
"""Download packages from specs, then inspect each downloaded package for additional
downstream dependency specs. Return these additional specs."""
# this attribute is added in the first pass of finalize_outputs_pass
extract_pattern = r'(.*)package:'
template_string = '\n'.join((
m.get_recipe_text(extract_pattern=extract_pattern,
force_top_level=True),
# second item: the requirements text for this particular metadata
# object (might be output)
m.extract_requirements_text())).rstrip()
raw_specs = {}
if template_string:
raw_specs = yaml.safe_load(
m._get_contents(permit_undefined_jinja=False,
template_string=template_string)) or {}
env_specs = utils.expand_reqs(raw_specs.get('requirements',
{})).get(env, [])
explicit_specs = [req.split(' ')[0]
for req in env_specs] if env_specs else []
linked_packages = actions.get('LINK', [])
linked_packages = [
pkg for pkg in linked_packages if pkg.name in explicit_specs
]
# edit the plan to download all necessary packages
for key in ('LINK', 'EXTRACT', 'UNLINK'):
if key in actions:
del actions[key]
# this should be just downloading packages. We don't need to extract them -
# we read contents directly
index, index_ts = get_build_index(getattr(m.config,
'{}_subdir'.format(env)),
bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder,
channel_urls=m.config.channel_urls,
debug=m.config.debug,
verbose=m.config.verbose,
locking=m.config.locking,
timeout=m.config.timeout)
if 'FETCH' in actions or 'EXTRACT' in actions:
# this is to force the download
execute_actions(actions, index, verbose=m.config.debug)
ignore_list = utils.ensure_list(m.get_value('build/ignore_run_exports'))
_pkgs_dirs = pkgs_dirs + list(m.config.bldpkgs_dirs)
additional_specs = {}
for pkg in linked_packages:
pkg_loc = None
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
for pkgs_dir in _pkgs_dirs:
pkg_dir = os.path.join(pkgs_dir, pkg_dist)
pkg_file = os.path.join(pkgs_dir, pkg_dist + '.tar.bz2')
if os.path.isdir(pkg_dir):
pkg_loc = pkg_dir
break
elif os.path.isfile(pkg_file):
pkg_loc = pkg_file
break
# ran through all pkgs_dirs, and did not find package or folder. Download it.
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
if not pkg_loc and conda_43:
try:
# the conda 4.4 API uses a single `link_prefs` kwarg
# whereas conda 4.3 used `index` and `link_dists` kwargs
pfe = ProgressiveFetchExtract(link_prefs=(index[pkg], ))
except TypeError:
# TypeError: __init__() got an unexpected keyword argument 'link_prefs'
pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index)
with utils.LoggingContext():
pfe.execute()
for pkg_dir in pkgs_dirs:
_loc = os.path.join(pkg_dir, index[pkg].fn)
if os.path.isfile(_loc):
pkg_loc = _loc
break
specs = {}
if os.path.isdir(pkg_loc):
downstream_file = os.path.join(pkg_dir, 'info/run_exports')
if os.path.isfile(downstream_file):
with open(downstream_file) as f:
specs = {'weak': [spec.rstrip() for spec in f.readlines()]}
# a later attempt: record more info in the yaml file, to support "strong" run exports
elif os.path.isfile(downstream_file + '.yaml'):
with open(downstream_file + '.yaml') as f:
specs = yaml.safe_load(f)
elif os.path.isfile(pkg_file):
legacy_specs = utils.package_has_file(pkg_file, 'info/run_exports')
specs_yaml = utils.package_has_file(pkg_file,
'info/run_exports.yaml')
if specs:
# exclude packages pinning themselves (makes no sense)
specs = {
'weak': [
spec.rstrip() for spec in legacy_specs.splitlines()
if not spec.startswith(pkg_dist.rsplit('-', 2)[0])
]
}
elif specs_yaml:
specs = yaml.safe_load(specs_yaml)
additional_specs = utils.merge_dicts_of_lists(
additional_specs, _filter_run_exports(specs, ignore_list))
return additional_specs
def finalize_metadata(m):
"""Fully render a recipe. Fill in versions for build/host dependencies."""
index, index_ts = get_build_index(m.config, m.config.build_subdir)
exclude_pattern = None
excludes = set(m.config.variant.get('ignore_version', []))
for key in m.config.variant.get('pin_run_as_build', {}).keys():
if key in excludes:
excludes.remove(key)
output_excludes = set()
if hasattr(m, 'other_outputs'):
output_excludes = set(name
for (name, variant) in m.other_outputs.keys())
if excludes or output_excludes:
exclude_pattern = re.compile('|'.join('(?:^{}(?:\s|$|\Z))'.format(exc)
for exc in excludes
| output_excludes))
build_reqs = m.meta.get('requirements', {}).get('build', [])
# if python is in the build specs, but doesn't have a specific associated
# version, make sure to add one
if build_reqs and 'python' in build_reqs:
build_reqs.append('python {}'.format(m.config.variant['python']))
m.meta['requirements']['build'] = build_reqs
# if we have host deps, they're more important than the build deps.
build_deps, build_actions = get_env_dependencies(m, 'build',
m.config.variant,
exclude_pattern)
# optimization: we don't need the index after here, and copying them takes a lot of time.
rendered_metadata = m.copy()
extra_run_specs = get_upstream_pins(m, build_actions, index)
reset_index = False
if m.is_cross:
host_reqs = m.get_value('requirements/host')
# if python is in the build specs, but doesn't have a specific associated
# version, make sure to add one
if host_reqs and 'python' in host_reqs:
host_reqs.append('python {}'.format(m.config.variant['python']))
m.meta['requirements']['host'] = host_reqs
host_deps, host_actions = get_env_dependencies(m, 'host',
m.config.variant,
exclude_pattern)
extra_run_specs += get_upstream_pins(m, host_actions, index)
else:
host_deps = []
# IMPORTANT: due to the statefulness of conda's index, this index invalidates the earlier one!
# To avoid confusion, any index passed around is always the native build platform.
if reset_index:
index = None
# here's where we pin run dependencies to their build time versions. This happens based
# on the keys in the 'pin_run_as_build' key in the variant, which is a list of package
# names to have this behavior.
requirements = rendered_metadata.meta.get('requirements', {})
run_deps = requirements.get('run', [])
if output_excludes:
exclude_pattern = re.compile('|'.join('(?:^{}(?:\s|$|\Z))'.format(exc)
for exc in output_excludes))
pinning_env = 'host' if m.is_cross else 'build'
full_build_deps, _ = get_env_dependencies(m,
pinning_env,
m.config.variant,
exclude_pattern=exclude_pattern)
full_build_dep_versions = {
dep.split()[0]: " ".join(dep.split()[1:])
for dep in full_build_deps
}
versioned_run_deps = [
get_pin_from_build(m, dep, full_build_dep_versions) for dep in run_deps
]
versioned_run_deps.extend(extra_run_specs)
for _env, values in (('build', build_deps), ('host', host_deps),
('run', versioned_run_deps)):
if values:
requirements[_env] = list({strip_channel(dep) for dep in values})
rendered_metadata.meta['requirements'] = requirements
test_deps = rendered_metadata.get_value('test/requires')
if test_deps:
versioned_test_deps = list({
get_pin_from_build(m, dep, full_build_dep_versions)
for dep in test_deps
})
rendered_metadata.meta['test']['requires'] = versioned_test_deps
# if source/path is relative, then the output package makes no sense at all. The next
# best thing is to hard-code the absolute path. This probably won't exist on any
# system other than the original build machine, but at least it will work there.
if m.meta.get('source'):
if 'path' in m.meta['source'] and not os.path.isabs(
m.meta['source']['path']):
rendered_metadata.meta['source']['path'] = os.path.normpath(
os.path.join(m.path, m.meta['source']['path']))
elif ('git_url' in m.meta['source'] and not (
# absolute paths are not relative paths
os.path.isabs(m.meta['source']['git_url']) or
# real urls are not relative paths
":" in m.meta['source']['git_url'])):
rendered_metadata.meta['source']['git_url'] = os.path.normpath(
os.path.join(m.path, m.meta['source']['git_url']))
if not rendered_metadata.meta.get('build'):
rendered_metadata.meta['build'] = {}
# hard-code build string so that any future "renderings" can't go wrong based on user env
rendered_metadata.meta['build']['string'] = rendered_metadata.build_id()
rendered_metadata.final = True
rendered_metadata.config.index = index
return rendered_metadata
def get_install_actions(prefix, specs, env, retries=0, subdir=None,
verbose=True, debug=False, locking=True,
bldpkgs_dirs=None, timeout=90, disable_pip=False,
max_env_retry=3, output_folder=None, channel_urls=None):
global cached_actions
global last_index_ts
actions = {}
log = utils.get_logger(__name__)
conda_log_level = logging.WARN
if verbose:
capture = contextlib.contextmanager(lambda: (yield))
elif debug:
capture = contextlib.contextmanager(lambda: (yield))
conda_log_level = logging.DEBUG
else:
capture = utils.capture
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
bldpkgs_dirs = ensure_list(bldpkgs_dirs)
index, index_ts = get_build_index(subdir, list(bldpkgs_dirs)[0], output_folder=output_folder,
channel_urls=channel_urls, debug=debug, verbose=verbose,
locking=locking, timeout=timeout)
specs = tuple(_ensure_valid_spec(spec) for spec in specs)
if (specs, env, subdir, channel_urls) in cached_actions and last_index_ts >= index_ts:
actions = cached_actions[(specs, env, subdir, channel_urls)].copy()
if "PREFIX" in actions:
actions['PREFIX'] = prefix
elif specs:
# this is hiding output like:
# Fetching package metadata ...........
# Solving package specifications: ..........
with utils.LoggingContext(conda_log_level):
with capture():
try:
actions = install_actions(prefix, index, specs, force=True)
except NoPackagesFoundError as exc:
raise DependencyNeedsBuildingError(exc, subdir=subdir)
except (SystemExit, PaddingError, LinkError, DependencyNeedsBuildingError,
CondaError, AssertionError) as exc:
if 'lock' in str(exc):
log.warn("failed to get install actions, retrying. exception was: %s",
str(exc))
elif ('requires a minimum conda version' in str(exc) or
'link a source that does not' in str(exc) or
isinstance(exc, AssertionError)):
locks = utils.get_conda_operation_locks(locking, bldpkgs_dirs, timeout)
with utils.try_acquire_locks(locks, timeout=timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn("I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if pkg_dir in pkgs_dirs and os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retries < max_env_retry:
log.warn("failed to get install actions, retrying. exception was: %s",
str(exc))
actions = get_install_actions(prefix, tuple(specs), env,
retries=retries + 1,
subdir=subdir,
verbose=verbose,
debug=debug,
locking=locking,
bldpkgs_dirs=tuple(bldpkgs_dirs),
timeout=timeout,
disable_pip=disable_pip,
max_env_retry=max_env_retry,
output_folder=output_folder,
channel_urls=tuple(channel_urls))
else:
log.error("Failed to get install actions, max retries exceeded.")
raise
if disable_pip:
actions['LINK'] = [spec for spec in actions['LINK']
if not spec.startswith('pip-') and
not spec.startswith('setuptools-')]
utils.trim_empty_keys(actions)
cached_actions[(specs, env, subdir, channel_urls)] = actions.copy()
last_index_ts = index_ts
return actions
def create_env(prefix,
specs_or_actions,
env,
config,
subdir,
clear_cache=True,
retry=0,
locks=None,
is_cross=False,
is_conda=False):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
external_logger_context = utils.LoggingContext(logging.DEBUG)
else:
external_logger_context = utils.LoggingContext(logging.WARN)
if os.path.exists(prefix):
for entry in glob(os.path.join(prefix, "*")):
utils.rm_rf(entry)
with external_logger_context:
log = utils.get_logger(__name__)
# if os.path.isdir(prefix):
# utils.rm_rf(prefix)
if specs_or_actions: # Don't waste time if there is nothing to do
log.debug("Creating environment in %s", prefix)
log.debug(str(specs_or_actions))
if not locks:
locks = utils.get_conda_operation_locks(config)
try:
with utils.try_acquire_locks(locks, timeout=config.timeout):
# input is a list - it's specs in MatchSpec format
if not hasattr(specs_or_actions, 'keys'):
specs = list(set(specs_or_actions))
actions = get_install_actions(
prefix,
tuple(specs),
env,
subdir=subdir,
verbose=config.verbose,
debug=config.debug,
locking=config.locking,
bldpkgs_dirs=tuple(config.bldpkgs_dirs),
timeout=config.timeout,
disable_pip=config.disable_pip,
max_env_retry=config.max_env_retry,
output_folder=config.output_folder,
channel_urls=tuple(config.channel_urls))
else:
actions = specs_or_actions
index, _, _ = get_build_index(
subdir=subdir,
bldpkgs_dir=config.bldpkgs_dir,
output_folder=config.output_folder,
channel_urls=config.channel_urls,
debug=config.debug,
verbose=config.verbose,
locking=config.locking,
timeout=config.timeout)
utils.trim_empty_keys(actions)
display_actions(actions, index)
if utils.on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
with env_var('CONDA_QUIET', not config.verbose,
reset_context):
with env_var('CONDA_JSON', not config.verbose,
reset_context):
execute_actions(actions, index)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError,
BuildLockError) as exc:
if (("too short in" in str(exc) or re.search(
'post-link failed for: (?:[a-zA-Z]*::)?openssl',
str(exc)) or isinstance(exc, PaddingError))
and config.prefix_length > 80):
if config.prefix_length_fallback:
log.warn("Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
host = '_h_env' in prefix
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.host_prefix if host else config.build_prefix
actions['PREFIX'] = prefix
create_env(prefix,
actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
is_cross=is_cross)
else:
raise
elif 'lock' in str(exc):
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs_or_actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs_or_actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
else:
raise
# HACK: some of the time, conda screws up somehow and incomplete packages result.
# Just retry.
except (AssertionError, IOError, ValueError, RuntimeError,
LockError) as exc:
if isinstance(exc, AssertionError):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = os.path.dirname(os.path.dirname(str(exc)))
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
specs_or_actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
else:
log.error("Failed to create env, max retries exceeded.")
raise
def render_recipe(recipe_path,
config,
no_download_source=False,
variants=None,
permit_unsatisfiable_variants=True,
reset_build_id=True):
"""Returns a list of tuples, each consisting of
(metadata-object, needs_download, needs_render_in_env)
You get one tuple per variant. Outputs are not factored in here (subpackages won't affect these
results returned here.)
"""
arg = recipe_path
# Don't use byte literals for paths in Python 2
if not PY3:
arg = arg.decode(getpreferredencoding() or 'utf-8')
if isfile(arg):
if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
recipe_dir = tempfile.mkdtemp()
t = tarfile.open(arg, 'r:*')
t.extractall(path=recipe_dir)
t.close()
need_cleanup = True
elif arg.endswith('.yaml'):
recipe_dir = os.path.dirname(arg)
need_cleanup = False
else:
print("Ignoring non-recipe: %s" % arg)
return None, None
else:
recipe_dir = abspath(arg)
need_cleanup = False
if not isdir(recipe_dir):
sys.exit("Error: no such directory: %s" % recipe_dir)
try:
m = MetaData(recipe_dir, config=config)
except exceptions.YamlParsingError as e:
sys.stderr.write(e.error_msg())
sys.exit(1)
if config.set_build_id:
m.config.compute_build_id(m.name(), reset=reset_build_id)
if m.needs_source_for_render and (not os.path.isdir(m.config.work_dir) or
len(os.listdir(m.config.work_dir)) == 0):
try_download(m, no_download_source=no_download_source)
rendered_metadata = {}
if m.final:
rendered_metadata = [
(m, False, False),
]
else:
index, index_ts = get_build_index(m.config, m.config.build_subdir)
# when building, we don't want to fully expand all outputs into metadata, only expand
# whatever variants we have.
variants = (dict_of_lists_to_list_of_dicts(variants)
if variants else get_package_variants(m))
rendered_metadata = distribute_variants(
m,
variants,
permit_unsatisfiable_variants=permit_unsatisfiable_variants,
stub_subpackages=True)
if need_cleanup:
utils.rm_rf(recipe_dir)
return rendered_metadata
def create_env(prefix, specs_or_actions, env, config, subdir, clear_cache=True, retry=0,
locks=None, is_cross=False):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
external_logger_context = utils.LoggingContext(logging.DEBUG)
else:
external_logger_context = utils.LoggingContext(logging.ERROR)
with external_logger_context:
log = utils.get_logger(__name__)
# if os.path.isdir(prefix):
# utils.rm_rf(prefix)
if specs_or_actions: # Don't waste time if there is nothing to do
log.debug("Creating environment in %s", prefix)
log.debug(str(specs_or_actions))
with utils.path_prepended(prefix):
if not locks:
locks = utils.get_conda_operation_locks(config)
try:
with utils.try_acquire_locks(locks, timeout=config.timeout):
# input is a list - it's specs in MatchSpec format
if not hasattr(specs_or_actions, 'keys'):
specs = list(set(specs_or_actions))
actions = get_install_actions(prefix, tuple(specs), env,
subdir=subdir,
verbose=config.verbose,
debug=config.debug,
locking=config.locking,
bldpkgs_dirs=tuple(config.bldpkgs_dirs),
timeout=config.timeout,
disable_pip=config.disable_pip,
max_env_retry=config.max_env_retry,
output_folder=config.output_folder,
channel_urls=tuple(config.channel_urls))
else:
actions = specs_or_actions
index, index_ts = get_build_index(subdir=subdir,
bldpkgs_dir=config.bldpkgs_dir,
output_folder=config.output_folder,
channel_urls=config.channel_urls,
debug=config.debug,
verbose=config.verbose,
locking=config.locking,
timeout=config.timeout)
utils.trim_empty_keys(actions)
display_actions(actions, index)
if utils.on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
execute_actions(actions, index, verbose=config.debug)
except (SystemExit, PaddingError, LinkError, DependencyNeedsBuildingError,
CondaError) as exc:
if (("too short in" in str(exc) or
re.search('post-link failed for: (?:[a-zA-Z]*::)?openssl', str(exc)) or
isinstance(exc, PaddingError)) and
config.prefix_length > 80):
if config.prefix_length_fallback:
log.warn("Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn("One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn("Falling back to legacy prefix length of 80 characters.")
log.warn("Your package will not install into prefixes > 80 characters.")
config.prefix_length = 80
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.build_prefix
actions['PREFIX'] = prefix
create_env(prefix, actions, config=config, subdir=subdir, env=env,
clear_cache=clear_cache, is_cross=is_cross)
else:
raise
elif 'lock' in str(exc):
if retry < config.max_env_retry:
log.warn("failed to create env, retrying. exception was: %s", str(exc))
create_env(prefix, actions, config=config, subdir=subdir, env=env,
clear_cache=clear_cache, retry=retry + 1, is_cross=is_cross)
elif ('requires a minimum conda version' in str(exc) or
'link a source that does not' in str(exc)):
with utils.try_acquire_locks(locks, timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn("I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn("failed to create env, retrying. exception was: %s", str(exc))
create_env(prefix, actions, config=config, subdir=subdir, env=env,
clear_cache=clear_cache, retry=retry + 1, is_cross=is_cross)
else:
log.error("Failed to create env, max retries exceeded.")
raise
else:
raise
# HACK: some of the time, conda screws up somehow and incomplete packages result.
# Just retry.
except (AssertionError, IOError, ValueError, RuntimeError, LockError) as exc:
if isinstance(exc, AssertionError):
with utils.try_acquire_locks(locks, timeout=config.timeout):
pkg_dir = os.path.dirname(os.path.dirname(str(exc)))
log.warn("I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn("failed to create env, retrying. exception was: %s", str(exc))
create_env(prefix, actions, config=config, subdir=subdir, env=env,
clear_cache=clear_cache, retry=retry + 1, is_cross=is_cross)
else:
log.error("Failed to create env, max retries exceeded.")
raise
# We must not symlink conda across different platforms when cross-compiling.
# On second thought, I think we must, because activating the host env does
# the symlink for us anyway, and when activate does it, we end up with
# conda symlinks in every package. =()
# if os.path.basename(prefix) == '_build_env' or not is_cross:
if utils.on_win:
shell = "cmd.exe"
else:
shell = "bash"
symlink_conda(prefix, sys.prefix, shell)
def finalize_metadata(m, index=None):
"""Fully render a recipe. Fill in versions for build dependencies."""
if not index:
index = get_build_index(m.config, m.config.build_subdir)
exclude_pattern = None
excludes = m.config.variant.get('exclude_from_build_hash', [])
if excludes:
exclude_pattern = re.compile('|'.join('(?:^{}(?:\s|$|\Z))'.format(exc) for exc in excludes))
# these are obtained from a sort of dry-run of conda. These are the actual packages that would
# be installed in the environment.
build_deps = get_env_dependencies(m, 'build', m.config.variant, index, exclude_pattern)
# optimization: we don't need the index after here, and copying them takes a lot of time.
rendered_metadata = m.copy()
build_dep_versions = {dep.split()[0]: " ".join(dep.split()[1:]) for dep in build_deps}
extra_run_specs = get_upstream_pins(m, build_deps, index)
reset_index = False
if m.config.build_subdir != m.config.host_subdir:
index = get_build_index(m.config, m.config.host_subdir)
reset_index = True
# IMPORTANT: due to the statefulness of conda's index, this index invalidates the earlier one!
# To avoid confusion, any index passed around is always the native build platform.
if reset_index:
index = None
# here's where we pin run dependencies to their build time versions. This happens based
# on the keys in the 'pin_run_as_build' key in the variant, which is a list of package
# names to have this behavior.
requirements = rendered_metadata.meta.get('requirements', {})
run_deps = requirements.get('run', [])
versioned_run_deps = [get_pin_from_build(m, dep, build_dep_versions) for dep in run_deps]
versioned_run_deps.extend(extra_run_specs)
rendered_metadata.meta['requirements'] = rendered_metadata.meta.get('requirements', {})
for env, values in (('build', build_deps), ('run', versioned_run_deps)):
if values:
requirements[env] = list({strip_channel(dep) for dep in values})
rendered_metadata.meta['requirements'] = requirements
test_deps = rendered_metadata.get_value('test/requires')
if test_deps:
versioned_test_deps = list({get_pin_from_build(m, dep, build_dep_versions)
for dep in test_deps})
rendered_metadata.meta['test']['requires'] = versioned_test_deps
# if source/path is relative, then the output package makes no sense at all. The next
# best thing is to hard-code the absolute path. This probably won't exist on any
# system other than the original build machine, but at least it will work there.
if m.meta.get('source'):
if 'path' in m.meta['source'] and not os.path.isabs(m.meta['source']['path']):
rendered_metadata.meta['source']['path'] = os.path.normpath(
os.path.join(m.path, m.meta['source']['path']))
elif ('git_url' in m.meta['source'] and not (
# absolute paths are not relative paths
os.path.isabs(m.meta['source']['git_url']) or
# real urls are not relative paths
":" in m.meta['source']['git_url'])):
rendered_metadata.meta['source']['git_url'] = os.path.normpath(
os.path.join(m.path, m.meta['source']['git_url']))
if not rendered_metadata.meta.get('build'):
rendered_metadata.meta['build'] = {}
# hard-code build string so that any future "renderings" can't go wrong based on user env
rendered_metadata.meta['build']['string'] = rendered_metadata.build_id()
rendered_metadata.final = True
rendered_metadata.config.index = index
return rendered_metadata
def collect_tasks(path,
folders,
matrix_base_dir,
channels=None,
steps=0,
test=False,
max_downstream=5,
variant_config_files=None,
platform_filters=None,
clobber_sections_file=None,
append_sections_file=None,
pass_throughs=None,
skip_existing=True,
build_config_vars={}):
""" Return a graph of build tasks """
task_graph = nx.DiGraph()
parsed_cli_args = _parse_python_numpy_from_pass_throughs(pass_throughs)
config = conda_build.api.Config(
clobber_sections_file=clobber_sections_file,
append_sections_file=append_sections_file,
skip_existing=skip_existing,
**parsed_cli_args,
)
platform_filters = ensure_list(platform_filters) if platform_filters else [
'*'
]
platforms = parse_platforms(matrix_base_dir, platform_filters,
build_config_vars)
# loop over platforms here because each platform may have different dependencies
# each platform will be submitted with a different label
for platform in platforms:
subdir = f"{platform['platform']}-{platform['arch']}"
config.variants = get_package_variants(path, config,
platform.get('variants'))
config.channel_urls = channels or []
config.variant_config_files = variant_config_files or []
conda_resolve = Resolve(
get_build_index(subdir=subdir,
bldpkgs_dir=config.bldpkgs_dir,
channel_urls=channels)[0])
# this graph is potentially different for platform and for build or test mode ("run")
graph = construct_graph(
path,
worker=platform,
folders=folders,
run="build",
matrix_base_dir=matrix_base_dir,
conda_resolve=conda_resolve,
config=config,
)
# Apply the build label to any nodes that need (re)building or testing
expand_run(
graph,
config=config.copy(),
conda_resolve=conda_resolve,
worker=platform,
run="build",
steps=steps,
max_downstream=max_downstream,
recipes_dir=path,
matrix_base_dir=matrix_base_dir,
)
# merge this graph with the main one
task_graph = nx.compose(task_graph, graph)
collapse_noarch_python_nodes(task_graph)
return task_graph
def execute_download_actions(m,
actions,
env,
package_subset=None,
require_files=False):
index, _, _ = get_build_index(getattr(m.config, '{}_subdir'.format(env)),
bldpkgs_dir=m.config.bldpkgs_dir,
output_folder=m.config.output_folder,
channel_urls=m.config.channel_urls,
debug=m.config.debug,
verbose=m.config.verbose,
locking=m.config.locking,
timeout=m.config.timeout)
# this should be just downloading packages. We don't need to extract them -
download_actions = {
k: v
for k, v in actions.items() if k in ('FETCH', 'EXTRACT', 'PREFIX')
}
if 'FETCH' in actions or 'EXTRACT' in actions:
# this is to force the download
execute_actions(download_actions, index, verbose=m.config.debug)
pkg_files = {}
packages = actions.get('LINK', [])
package_subset = utils.ensure_list(package_subset)
selected_packages = set()
if package_subset:
for pkg in package_subset:
if hasattr(pkg, 'name'):
if pkg in packages:
selected_packages.add(pkg)
else:
pkg_name = pkg.split()[0]
for link_pkg in packages:
if pkg_name == link_pkg.name:
selected_packages.add(link_pkg)
break
packages = selected_packages
for pkg in packages:
if hasattr(pkg, 'dist_name'):
pkg_dist = pkg.dist_name
else:
pkg = strip_channel(pkg)
pkg_dist = pkg.split(' ')[0]
pkg_loc = find_pkg_dir_or_file_in_pkgs_dirs(pkg_dist,
m,
files_only=require_files)
# ran through all pkgs_dirs, and did not find package or folder. Download it.
# TODO: this is a vile hack reaching into conda's internals. Replace with
# proper conda API when available.
if not pkg_loc and conda_43:
try:
pkg_record = [_ for _ in index if _.dist_name == pkg_dist][0]
# the conda 4.4 API uses a single `link_prefs` kwarg
# whereas conda 4.3 used `index` and `link_dists` kwargs
pfe = ProgressiveFetchExtract(link_prefs=(index[pkg_record], ))
except TypeError:
# TypeError: __init__() got an unexpected keyword argument 'link_prefs'
pfe = ProgressiveFetchExtract(link_dists=[pkg], index=index)
with utils.LoggingContext():
pfe.execute()
for pkg_dir in pkgs_dirs:
_loc = os.path.join(pkg_dir, index[pkg].fn)
if os.path.isfile(_loc):
pkg_loc = _loc
break
pkg_files[pkg] = pkg_loc, pkg_dist
return pkg_files
def create_env(prefix,
specs_or_actions,
env,
config,
subdir,
clear_cache=True,
retry=0,
locks=None,
is_cross=False,
always_include_files=[]):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
external_logger_context = utils.LoggingContext(logging.DEBUG)
else:
external_logger_context = utils.LoggingContext(logging.ERROR)
with external_logger_context:
log = utils.get_logger(__name__)
# if os.path.isdir(prefix):
# utils.rm_rf(prefix)
if specs_or_actions: # Don't waste time if there is nothing to do
log.debug("Creating environment in %s", prefix)
log.debug(str(specs_or_actions))
with utils.path_prepended(prefix):
if not locks:
locks = utils.get_conda_operation_locks(config)
try:
with utils.try_acquire_locks(locks,
timeout=config.timeout):
# input is a list - it's specs in MatchSpec format
if not hasattr(specs_or_actions, 'keys'):
specs = list(set(specs_or_actions))
actions = get_install_actions(
prefix,
tuple(specs),
env,
subdir=subdir,
verbose=config.verbose,
debug=config.debug,
locking=config.locking,
bldpkgs_dirs=tuple(config.bldpkgs_dirs),
timeout=config.timeout,
disable_pip=config.disable_pip,
max_env_retry=config.max_env_retry,
output_folder=config.output_folder,
channel_urls=tuple(config.channel_urls))
else:
actions = specs_or_actions
index, index_ts = get_build_index(
subdir=subdir,
bldpkgs_dir=config.bldpkgs_dir,
output_folder=config.output_folder,
channel_urls=config.channel_urls,
debug=config.debug,
verbose=config.verbose,
locking=config.locking,
timeout=config.timeout)
utils.trim_empty_keys(actions)
display_actions(actions, index)
if utils.on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
execute_actions(actions, index, verbose=config.debug)
except (SystemExit, PaddingError, LinkError,
DependencyNeedsBuildingError, CondaError) as exc:
if (("too short in" in str(exc) or re.search(
'post-link failed for: (?:[a-zA-Z]*::)?openssl',
str(exc)) or isinstance(exc, PaddingError))
and config.prefix_length > 80):
if config.prefix_length_fallback:
log.warn(
"Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.build_prefix
actions['PREFIX'] = prefix
create_env(prefix,
actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
is_cross=is_cross)
else:
raise
elif 'lock' in str(exc):
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
elif ('requires a minimum conda version' in str(exc)
or 'link a source that does not' in str(exc)):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = str(exc)
folder = 0
while os.path.dirname(
pkg_dir) not in pkgs_dirs and folder < 20:
pkg_dir = os.path.dirname(pkg_dir)
folder += 1
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
else:
raise
# HACK: some of the time, conda screws up somehow and incomplete packages result.
# Just retry.
except (AssertionError, IOError, ValueError, RuntimeError,
LockError) as exc:
if isinstance(exc, AssertionError):
with utils.try_acquire_locks(locks,
timeout=config.timeout):
pkg_dir = os.path.dirname(os.path.dirname(
str(exc)))
log.warn(
"I think conda ended up with a partial extraction for %s. "
"Removing the folder and retrying", pkg_dir)
if os.path.isdir(pkg_dir):
utils.rm_rf(pkg_dir)
if retry < config.max_env_retry:
log.warn(
"failed to create env, retrying. exception was: %s",
str(exc))
create_env(prefix,
actions,
config=config,
subdir=subdir,
env=env,
clear_cache=clear_cache,
retry=retry + 1,
is_cross=is_cross)
else:
log.error(
"Failed to create env, max retries exceeded.")
raise
# We must not symlink conda across different platforms when cross-compiling.
# On second thought, I think we must, because activating the host env does
# the symlink for us anyway, and when activate does it, we end up with
# conda symlinks in every package. =()
# if os.path.basename(prefix) == '_build_env' or not is_cross:
# Hack, do not SYMLINK_CONDA when we're building conda.
if not any(include in ('bin/deactivate', 'Scripts/deactivate.bat')
for include in always_include_files):
if utils.on_win:
shell = "cmd.exe"
else:
shell = "bash"
symlink_conda(prefix, sys.prefix, shell)