def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package, and figure out
# which python version we prefer. `python_age` can use used to tweak which
# python gets used here.
if metadata.noarch or metadata.noarch_python:
from .conda_interface import VersionOrder
age = int(
metadata.get_value('build/noarch_python_build_age',
metadata.config.noarch_python_build_age))
versions = []
for variant in variants:
if 'python' in variant:
vo = variant['python']
if vo not in versions:
versions.append(vo)
version_indices = sorted(
range(len(versions)),
key=lambda k: VersionOrder(versions[k].split(' ')[0]))
if age < 0:
age = 0
elif age > len(versions) - 1:
age = len(versions) - 1
build_ver = versions[version_indices[len(versions) - 1 - age]]
variants = filter_by_key_value(variants, 'python', build_ver,
'noarch_python_reduction')
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
metadata.config.variant = variants[0]
used_variables = metadata.get_used_loop_vars(force_global=False)
top_loop = metadata.get_reduced_variant_set(used_variables)
for variant in top_loop:
from conda_build.build import get_all_replacements
get_all_replacements(variant)
mv = metadata.copy()
mv.config.variant = variant
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
conform_dict = {}
for key in used_variables:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
conform_dict[key] = variant[key]
for key, values in conform_dict.items():
mv.config.variants = (filter_by_key_value(
mv.config.variants, key, values,
'distribute_variants_reduction') or mv.config.variants)
get_all_replacements(mv.config.variants)
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
mv.config.squished_variants = list_of_dicts_to_dict_of_lists(
mv.config.variants)
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
try:
mv.parse_until_resolved(
allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
except SystemExit:
pass
need_source_download = (not mv.needs_source_for_render
or not mv.source_provided)
rendered_metadata[(mv.dist(),
mv.config.variant.get('target_platform', mv.config.subdir),
tuple((var, mv.config.variant.get(var))
for var in mv.get_used_vars()))] = \
(mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package
if metadata.noarch or metadata.noarch_python:
conform_dict = {'python': variants[0]['python']}
variants = conform_variants_to_value(variants, conform_dict)
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
squished_variants = list_of_dicts_to_dict_of_lists(variants)
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
for variant in variants:
mv = metadata.copy()
# this determines which variants were used, and thus which ones should be locked for
# future rendering
mv.final = False
mv.config.variant = {}
mv.parse_again(permit_undefined_jinja=True,
allow_no_other_outputs=True,
bypass_env_check=True)
vars_in_recipe = set(mv.undefined_jinja_vars)
mv.config.variant = variant
conform_dict = {}
for key in vars_in_recipe:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
if re.search(r"\s*\{\{\s*%s\s*(?:.*?)?\}\}" % key, recipe_text):
if key in variant:
variant_index = squished_variants[key].index(variant[key])
zipped_keys = [key]
if 'zip_keys' in variant:
zip_key_groups = variant['zip_keys']
if zip_key_groups and not isinstance(
zip_key_groups[0], list):
zip_key_groups = [zip_key_groups]
for group in zip_key_groups:
if key in group:
zipped_keys = group
break
for zipped_key in zipped_keys:
conform_dict[zipped_key] = squished_variants[
zipped_key][variant_index]
conform_dict.update({
key: val
for key, val in variant.items()
if key in mv.meta.get('requirements').get('build', []) +
mv.meta.get('requirements').get('host', [])
})
compiler_matches = re.findall(r"compiler\([\'\"](.*)[\'\"].*\)",
recipe_requirements)
if compiler_matches:
from conda_build.jinja_context import native_compiler
for match in compiler_matches:
compiler_key = '{}_compiler'.format(match)
conform_dict[compiler_key] = variant.get(
compiler_key, native_compiler(match, mv.config))
conform_dict['target_platform'] = variant['target_platform']
build_reqs = mv.meta.get('requirements', {}).get('build', [])
host_reqs = mv.meta.get('requirements', {}).get('host', [])
if 'python' in build_reqs or 'python' in host_reqs:
conform_dict['python'] = variant['python']
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
mv.config.variants = conform_variants_to_value(mv.config.variants,
conform_dict)
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
mv.parse_until_resolved(allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
need_source_download = (bool(mv.meta.get('source'))
and not mv.needs_source_for_render
and not os.listdir(mv.config.work_dir))
# if python is in the build specs, but doesn't have a specific associated
# version, make sure to add one to newly parsed 'requirements/build'.
for env in ('build', 'host', 'run'):
utils.insert_variant_versions(mv, env)
fm = mv.copy()
# HACK: trick conda-build into thinking this is final, and computing a hash based
# on the current meta.yaml. The accuracy doesn't matter, all that matters is
# our ability to differentiate configurations
fm.final = True
rendered_metadata[fm.dist()] = (mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package
if metadata.noarch or metadata.noarch_python:
variants = filter_by_key_value(variants, 'python',
variants[0]['python'],
'noarch_reduction')
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
for variant in variants:
mv = metadata.copy()
mv.config.variant = variant
used_variables = mv.get_used_loop_vars()
conform_dict = {}
for key in used_variables:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
conform_dict[key] = variant[key]
build_reqs = mv.meta.get('requirements', {}).get('build', [])
host_reqs = mv.meta.get('requirements', {}).get('host', [])
if 'python' in build_reqs or 'python' in host_reqs:
conform_dict['python'] = variant['python']
if 'r-base' in build_reqs or 'r-base' in host_reqs:
conform_dict['r_base'] = variant['r_base']
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
for key, values in conform_dict.items():
mv.config.variants = (filter_by_key_value(
mv.config.variants, key, values,
'distribute_variants_reduction') or mv.config.variants)
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
mv.config.squished_variants = list_of_dicts_to_dict_of_lists(
mv.config.variants)
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
mv.parse_until_resolved(allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
need_source_download = (not mv.needs_source_for_render
or not mv.source_provided)
rendered_metadata[(mv.dist(),
mv.config.variant.get('target_platform', mv.config.subdir),
tuple((var, mv.config.variant[var])
for var in mv.get_used_vars()))] = \
(mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def compute_builds(path,
base_name,
git_rev=None,
stop_rev=None,
folders=None,
matrix_base_dir=None,
steps=0,
max_downstream=5,
test=False,
public=True,
output_dir='../output',
output_folder_label='git',
config_overrides=None,
**kw):
if not git_rev and not folders:
raise ValueError(
"Either git_rev or folders list are required to know what to compute"
)
checkout_rev = stop_rev or git_rev
folders = folders
path = path.replace('"', '')
if not folders:
folders = git_changed_recipes(git_rev, stop_rev, git_root=path)
if not folders:
print(
"No folders specified to build, and nothing changed in git. Exiting."
)
return
matrix_base_dir = matrix_base_dir or path
# clean up quoting from concourse template evaluation
matrix_base_dir = matrix_base_dir.replace('"', '')
repo_commit = ''
git_identifier = ''
if checkout_rev:
with checkout_git_rev(checkout_rev, path):
git_identifier = _get_current_git_rev(path)
task_graph = collect_tasks(path,
folders=folders,
steps=steps,
max_downstream=max_downstream,
test=test,
matrix_base_dir=matrix_base_dir)
try:
repo_commit = _get_current_git_rev(path)
except subprocess.CalledProcessError:
repo_commit = 'master'
else:
task_graph = collect_tasks(path,
folders=folders,
steps=steps,
max_downstream=max_downstream,
test=test,
matrix_base_dir=matrix_base_dir)
with open(os.path.join(matrix_base_dir, 'config.yml')) as src:
data = yaml.load(src)
data['recipe-repo-commit'] = repo_commit
if config_overrides:
data.update(config_overrides)
plan = graph_to_plan_with_jobs(os.path.abspath(path),
task_graph,
commit_id=repo_commit,
matrix_base_dir=matrix_base_dir,
config_vars=data,
public=public)
output_dir = output_dir.format(base_name=base_name,
git_identifier=git_identifier)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with open(os.path.join(output_dir, 'plan.yml'), 'w') as f:
yaml.dump(plan, f, default_flow_style=False)
# expand folders to include any dependency builds or tests
if not os.path.isabs(path):
path = os.path.normpath(os.path.join(os.getcwd(), path))
for fn in glob.glob(os.path.join(output_dir, 'output_order*')):
os.remove(fn)
for node in nx.topological_sort(task_graph, reverse=True):
meta = task_graph.node[node]['meta']
if meta.meta_path:
recipe = os.path.dirname(meta.meta_path)
else:
recipe = meta.get('extra', {}).get('parent_recipe', {})
assert recipe, ("no parent recipe set, and no path associated "
"with this metadata")
# make recipe path relative
recipe = recipe.replace(path + '/', '')
# copy base recipe into a folder named for this node
out_folder = os.path.join(output_dir, node)
if os.path.isdir(out_folder):
shutil.rmtree(out_folder)
shutil.copytree(os.path.join(path, recipe), out_folder)
# write the conda_build_config.yml for this particular metadata into that recipe
# This should sit alongside meta.yaml, where conda-build will be able to find it
squished_variants = list_of_dicts_to_dict_of_lists(
meta.config.variants)
with open(os.path.join(out_folder, 'conda_build_config.yaml'),
'w') as f:
yaml.dump(squished_variants, f, default_flow_style=False)
order_fn = 'output_order_' + task_graph.node[node]['worker']['label']
with open(os.path.join(output_dir, order_fn), 'a') as f:
f.write(node + '\n')
def distribute_variants(metadata, variants, permit_unsatisfiable_variants=False,
allow_no_other_outputs=False, bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package
if metadata.noarch or metadata.noarch_python:
variants = filter_by_key_value(variants, 'python', variants[0]['python'],
'noarch_reduction')
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
metadata.config.variant = variants[0]
used_variables = metadata.get_used_loop_vars(force_global=False)
top_loop = metadata.get_reduced_variant_set(used_variables)
for variant in top_loop:
mv = metadata.copy()
mv.config.variant = variant
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
conform_dict = {}
for key in used_variables:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
conform_dict[key] = variant[key]
for key, values in conform_dict.items():
mv.config.variants = (filter_by_key_value(mv.config.variants, key, values,
'distribute_variants_reduction') or
mv.config.variants)
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
mv.config.squished_variants = list_of_dicts_to_dict_of_lists(mv.config.variants)
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
try:
mv.parse_until_resolved(allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
except SystemExit:
pass
need_source_download = (not mv.needs_source_for_render or not mv.source_provided)
rendered_metadata[(mv.dist(),
mv.config.variant.get('target_platform', mv.config.subdir),
tuple((var, mv.config.variant.get(var))
for var in mv.get_used_vars()))] = \
(mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
def distribute_variants(metadata,
variants,
permit_unsatisfiable_variants=False,
allow_no_other_outputs=False,
bypass_env_check=False):
rendered_metadata = {}
need_source_download = True
# don't bother distributing python if it's a noarch package
if metadata.noarch or metadata.noarch_python:
variants = filter_by_key_value(variants, 'python',
variants[0]['python'],
'noarch_reduction')
# store these for reference later
metadata.config.variants = variants
# These are always the full set. just 'variants' is the one that gets
# used mostly, and can be reduced
metadata.config.input_variants = variants
squished_variants = list_of_dicts_to_dict_of_lists(variants)
recipe_requirements = metadata.extract_requirements_text()
recipe_package_and_build_text = metadata.extract_package_and_build_text()
recipe_text = recipe_package_and_build_text + recipe_requirements
if PY3 and hasattr(recipe_text, 'decode'):
recipe_text = recipe_text.decode()
elif not PY3 and hasattr(recipe_text, 'encode'):
recipe_text = recipe_text.encode()
for variant in variants:
mv = metadata.copy()
mv.config.variant = variant
used_variables = mv.get_used_loop_vars()
conform_dict = {}
for key in used_variables:
# We use this variant in the top-level recipe.
# constrain the stored variants to only this version in the output
# variant mapping
conform_dict[key] = variant[key]
# handle grouping from zip_keys for everything in conform_dict
if 'zip_keys' in variant:
zip_key_groups = variant['zip_keys']
if zip_key_groups and not isinstance(zip_key_groups[0], list):
zip_key_groups = [zip_key_groups]
for key in list(conform_dict.keys()):
zipped_keys = None
for group in zip_key_groups:
if key in group:
zipped_keys = group
if zipped_keys:
# here we zip the values of the keys, so that we can match the combination
zipped_values = list(
zip(*[
squished_variants[key] for key in zipped_keys
]))
variant_index = zipped_values.index(
tuple(variant[key] for key in zipped_keys))
for zipped_key in zipped_keys:
conform_dict[zipped_key] = squished_variants[
zipped_key][variant_index]
build_reqs = mv.meta.get('requirements', {}).get('build', [])
host_reqs = mv.meta.get('requirements', {}).get('host', [])
if 'python' in build_reqs or 'python' in host_reqs:
conform_dict['python'] = variant['python']
if 'r-base' in build_reqs or 'r-base' in host_reqs:
conform_dict['r_base'] = variant['r_base']
pin_run_as_build = variant.get('pin_run_as_build', {})
if mv.numpy_xx and 'numpy' not in pin_run_as_build:
pin_run_as_build['numpy'] = {'min_pin': 'x.x', 'max_pin': 'x.x'}
for key, values in conform_dict.items():
mv.config.variants = (filter_by_key_value(
mv.config.variants, key, values,
'distribute_variants_reduction') or mv.config.variants)
numpy_pinned_variants = []
for _variant in mv.config.variants:
_variant['pin_run_as_build'] = pin_run_as_build
numpy_pinned_variants.append(_variant)
mv.config.variants = numpy_pinned_variants
mv.config.squished_variants = list_of_dicts_to_dict_of_lists(
mv.config.variants)
if mv.needs_source_for_render and mv.variant_in_source:
mv.parse_again()
utils.rm_rf(mv.config.work_dir)
source.provide(mv)
mv.parse_again()
mv.parse_until_resolved(allow_no_other_outputs=allow_no_other_outputs,
bypass_env_check=bypass_env_check)
need_source_download = (not mv.needs_source_for_render
or not mv.source_provided)
# if python is in the build specs, but doesn't have a specific associated
# version, make sure to add one to newly parsed 'requirements/build'.
for env in ('build', 'host', 'run'):
utils.insert_variant_versions(mv.meta.get('requirements', {}),
mv.config.variant, env)
fm = mv.copy()
# HACK: trick conda-build into thinking this is final, and computing a hash based
# on the current meta.yaml. The accuracy doesn't matter, all that matters is
# our ability to differentiate configurations
fm.final = True
rendered_metadata[(fm.dist(),
fm.config.variant.get('target_platform', fm.config.subdir),
tuple((var, fm.config.variant[var])
for var in fm.get_used_loop_vars()))] = \
(mv, need_source_download, None)
# list of tuples.
# each tuple item is a tuple of 3 items:
# metadata, need_download, need_reparse_in_env
return list(rendered_metadata.values())
