def get_config(folder, variant=None, additional_files=None):
if not additional_files:
additional_files = []
if not variant:
variant = {}
config = get_or_merge_config(None, variant)
if cb_split_version >= (3, 20, 5):
config_files = find_config_files(folder, config)
else:
config_files = find_config_files(folder)
all_files = [os.path.abspath(p) for p in config_files + additional_files]
# reverse files an uniquify
def make_unique_list(lx):
seen = set()
return [x for x in lx if not (x in seen or seen.add(x))]
# we reverse the order so that command line can overwrite the hierarchy
all_files = make_unique_list(all_files[::-1])[::-1]
console.print(f"\nLoading config files: [green]{', '.join(all_files)}\n")
parsed_cfg = collections.OrderedDict()
for f in all_files:
parsed_cfg[f] = parse_config_file(f, config)
# this merges each of the specs, providing a debug message when a given setting is overridden
# by a later spec
combined_spec = combine_specs(parsed_cfg, log_output=config.verbose)
# console.print(combined_spec)
return combined_spec, config
def get_config(folder):
config = get_or_merge_config(None, {})
config_files = find_config_files(folder)
parsed_cfg = collections.OrderedDict()
for f in config_files:
parsed_cfg[f] = parse_config_file(f, config)
normalized = {}
for k in parsed_cfg[f].keys():
if "_" in k:
n = k.replace("_", "-")
normalized[n] = parsed_cfg[f][k]
parsed_cfg[f].update(normalized)
# TODO just using latest config here, should merge!
if len(config_files):
cbc = parsed_cfg[config_files[-1]]
else:
cbc = {}
return cbc, config
def get_config(folder, variant=None):
if not variant:
variant = {}
config = get_or_merge_config(None, variant)
if cb_split_version >= (3, 20, 5):
config_files = find_config_files(folder, config)
else:
config_files = find_config_files(folder)
console.print(f"\nLoading config files: [green]{', '.join(config_files)}\n")
parsed_cfg = collections.OrderedDict()
for f in config_files:
parsed_cfg[f] = parse_config_file(f, config)
# TODO just using latest config here, should merge!
if len(config_files):
cbc = parsed_cfg[config_files[-1]]
else:
cbc = {}
return cbc, config
def create_migration_yaml_creator(migrators: MutableSequence[Migrator], gx: nx.DiGraph):
with indir(os.environ["CONDA_PREFIX"]):
pinnings = parse_config_file(
"conda_build_config.yaml", config=Config(**CB_CONFIG),
)
feedstocks_to_be_repinned = []
for k, package_pin_list in pinnings.items():
# we need the package names for the migrator itself but need the
# feedstock for everything else
package_name = k
# exclude non-package keys
if k not in gx.nodes and k not in gx.graph["outputs_lut"]:
# conda_build_config.yaml can't have `-` unlike our package names
k = k.replace("_", "-")
# replace sub-packages with their feedstock names
k = gx.graph["outputs_lut"].get(k, k)
if (
(k in gx.nodes)
and not gx.nodes[k]["payload"].get("archived", False)
and gx.nodes[k]["payload"].get("version")
and k not in feedstocks_to_be_repinned
):
current_pins = list(map(str, package_pin_list))
current_version = str(gx.nodes[k]["payload"]["version"])
# we need a special parsing for pinning stuff
meta_yaml = parse_meta_yaml(
gx.nodes[k]["payload"]["raw_meta_yaml"], for_pinning=True,
)
# find the most stringent max pin for this feedstock if any
pin_spec = ""
for block in [meta_yaml] + meta_yaml.get("outputs", []) or []:
build = block.get("build", {}) or {}
# and check the exported package is within the feedstock
exports = [
p.get("max_pin", "")
for p in build.get("run_exports", [{}])
# make certain not direct hard pin
if isinstance(p, MutableMapping)
# if the pinned package is in an output of the parent feedstock
and (
gx.graph["outputs_lut"].get(p.get("package_name", ""), "") == k
# if the pinned package is the feedstock itself
or p.get("package_name", "") == k
)
]
if not exports:
continue
# get the most stringent pin spec from the recipe block
max_pin = max(exports, key=len)
if len(max_pin) > len(pin_spec):
pin_spec = max_pin
# fall back to the pinning file or "x"
if not pin_spec:
pin_spec = (
pinnings["pin_run_as_build"].get(k, {}).get("max_pin", "x") or "x"
)
current_pins = list(
map(lambda x: re.sub("[^0-9.]", "", x).rstrip("."), current_pins),
)
current_version = re.sub("[^0-9.]", "", current_version).rstrip(".")
if current_pins == [""]:
continue
current_pin = str(max(map(VersionOrder, current_pins)))
# If the current pin and the current version is the same nothing
# to do even if the pin isn't accurate to the spec
if current_pin != current_version and _outside_pin_range(
pin_spec, current_pin, current_version,
):
feedstocks_to_be_repinned.append(k)
print(package_name, current_version, current_pin, pin_spec)
migrators.append(
MigrationYamlCreator(
package_name, current_version, current_pin, pin_spec, k, gx,
),
)
def create_migration_yaml_creator(migrators: MutableSequence[Migrator],
gx: nx.DiGraph):
cfp_gx = copy.deepcopy(gx)
for node in list(cfp_gx.nodes):
if node != "conda-forge-pinning":
pluck(cfp_gx, node)
print("pinning migrations", flush=True)
with indir(os.environ["CONDA_PREFIX"]):
pinnings = parse_config_file(
"conda_build_config.yaml",
config=Config(**CB_CONFIG),
)
feedstocks_to_be_repinned = []
for pinning_name, package_pin_list in pinnings.items():
# there are three things:
# pinning_name - entry in pinning file
# package_name - the actual package, could differ via `-` -> `_`
# from pinning_name
# feedstock_name - the feedstock that outputs the package
# we need the package names for the migrator itself but need the
# feedstock for everything else
# exclude non-package keys
if pinning_name not in gx.graph["outputs_lut"]:
# conda_build_config.yaml can't have `-` unlike our package names
package_name = pinning_name.replace("_", "-")
else:
package_name = pinning_name
# replace sub-packages with their feedstock names
# TODO - we are grabbing one element almost at random here
# the sorted call makes it stable at least?
fs_name = next(
iter(
sorted(gx.graph["outputs_lut"].get(package_name,
{package_name})), ), )
if ((fs_name in gx.nodes)
and not gx.nodes[fs_name]["payload"].get("archived", False)
and gx.nodes[fs_name]["payload"].get("version")
and fs_name not in feedstocks_to_be_repinned):
current_pins = list(map(str, package_pin_list))
current_version = str(gx.nodes[fs_name]["payload"]["version"])
# we need a special parsing for pinning stuff
meta_yaml = parse_meta_yaml(
gx.nodes[fs_name]["payload"]["raw_meta_yaml"],
for_pinning=True,
)
# find the most stringent max pin for this feedstock if any
pin_spec = ""
for block in [meta_yaml] + meta_yaml.get("outputs", []) or []:
build = block.get("build", {}) or {}
# and check the exported package is within the feedstock
exports = [
p.get("max_pin", "")
for p in build.get("run_exports", [{}])
# make certain not direct hard pin
if isinstance(p, MutableMapping)
# ensure the export is for this package
and p.get("package_name", "") == package_name
# ensure the pinned package is in an output of the parent feedstock
and (fs_name in gx.graph["outputs_lut"].get(
p.get("package_name", ""), set()))
]
if not exports:
continue
# get the most stringent pin spec from the recipe block
max_pin = max(exports, key=len)
if len(max_pin) > len(pin_spec):
pin_spec = max_pin
# fall back to the pinning file or "x"
if not pin_spec:
pin_spec = (pinnings["pin_run_as_build"].get(
pinning_name, {}).get("max_pin", "x")) or "x"
current_pins = list(
map(lambda x: re.sub("[^0-9.]", "", x).rstrip("."),
current_pins), )
current_pins = [
cp.strip() for cp in current_pins if cp.strip() != ""
]
current_version = re.sub("[^0-9.]", "",
current_version).rstrip(".")
if not current_pins or current_version == "":
continue
current_pin = str(max(map(VersionOrder, current_pins)))
# If the current pin and the current version is the same nothing
# to do even if the pin isn't accurate to the spec
if current_pin != current_version and _outside_pin_range(
pin_spec,
current_pin,
current_version,
):
feedstocks_to_be_repinned.append(fs_name)
print(
" %s:\n"
" curr version: %s\n"
" curr pin: %s\n"
" pin_spec: %s" %
(pinning_name, current_version, current_pin, pin_spec),
flush=True,
)
migrators.append(
MigrationYamlCreator(
pinning_name,
current_version,
current_pin,
pin_spec,
fs_name,
cfp_gx,
full_graph=gx,
), )
print(" ", flush=True)
def main(config=None):
folder = sys.argv[1]
config = get_or_merge_config(None, {})
config_files = find_config_files(folder)
parsed_cfg = collections.OrderedDict()
for f in config_files:
parsed_cfg[f] = parse_config_file(f, config)
print(parsed_cfg[f])
normalized = {}
for k in parsed_cfg[f].keys():
if '_' in k:
n = k.replace('_', '-')
normalized[n] = parsed_cfg[f][k]
parsed_cfg[f].update(normalized)
print(parsed_cfg[f].keys())
# TODO just using latest config here, should merge!
cbc = parsed_cfg[config_files[-1]]
recipe_path = os.path.join(folder, "recipe.yaml")
# step 1: parse YAML
with open(recipe_path) as fi:
loader = YAML(typ='safe')
ydoc = loader.load(fi)
print(ydoc)
# step 2: fill out context dict
context_dict = ydoc.get("context") or {}
jenv = jinja2.Environment()
for key, value in context_dict.items():
if isinstance(value, str):
tmpl = jenv.from_string(value)
context_dict[key] = tmpl.render(context_dict)
if ydoc.get("context"):
del ydoc["context"]
# step 3: recursively loop over the entire recipe and render jinja with context
jenv.globals.update(jinja_functions(config, context_dict))
for key in ydoc:
render_recursive(ydoc[key], context_dict, jenv)
flatten_selectors(ydoc, ns_cfg(config))
# We need to assemble the variants for each output
variants = {}
# if we have a outputs section, use that order the outputs
if ydoc.get("outputs"):
# if ydoc.get("build"):
# raise InvalidRecipeError("You can either declare outputs, or build?")
for o in ydoc["outputs"]:
variants[o["package"]["name"]] = get_dependency_variants(
o["requirements"], cbc, config)
else:
# we only have one output
variants[ydoc["package"]["name"]] = get_dependency_variants(
ydoc["requirements"], cbc, config)
# this takes in all variants and outputs, builds a dependency tree and returns
# the final metadata
sorted_outputs = to_build_tree(ydoc, variants, config)
def main(config=None):
print(banner)
parser = argparse.ArgumentParser(
description='Boa, the fast build tool for conda packages.')
subparsers = parser.add_subparsers(help='sub-command help', dest='command')
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('recipe_dir', type=str)
render_parser = subparsers.add_parser('render',
parents=[parent_parser],
help='render a recipe')
build_parser = subparsers.add_parser('build',
parents=[parent_parser],
help='build a recipe')
args = parser.parse_args()
command = args.command
folder = args.recipe_dir
config = get_or_merge_config(None, {})
config_files = find_config_files(folder)
parsed_cfg = collections.OrderedDict()
for f in config_files:
parsed_cfg[f] = parse_config_file(f, config)
normalized = {}
for k in parsed_cfg[f].keys():
if "_" in k:
n = k.replace("_", "-")
normalized[n] = parsed_cfg[f][k]
parsed_cfg[f].update(normalized)
# TODO just using latest config here, should merge!
if len(config_files):
cbc = parsed_cfg[config_files[-1]]
else:
cbc = {}
update_index(os.path.dirname(config.output_folder),
verbose=config.debug,
threads=1)
recipe_path = os.path.join(folder, "recipe.yaml")
# step 1: parse YAML
with open(recipe_path) as fi:
loader = YAML(typ="safe")
ydoc = loader.load(fi)
# step 2: fill out context dict
context_dict = ydoc.get("context") or {}
jenv = jinja2.Environment()
for key, value in context_dict.items():
if isinstance(value, str):
tmpl = jenv.from_string(value)
context_dict[key] = tmpl.render(context_dict)
if ydoc.get("context"):
del ydoc["context"]
# step 3: recursively loop over the entire recipe and render jinja with context
jenv.globals.update(jinja_functions(config, context_dict))
for key in ydoc:
render_recursive(ydoc[key], context_dict, jenv)
flatten_selectors(ydoc, ns_cfg(config))
# We need to assemble the variants for each output
variants = {}
# if we have a outputs section, use that order the outputs
if ydoc.get("outputs"):
# if ydoc.get("build"):
# raise InvalidRecipeError("You can either declare outputs, or build?")
for o in ydoc["outputs"]:
# inherit from global package
pkg_meta = {}
pkg_meta.update(ydoc["package"])
pkg_meta.update(o["package"])
o["package"] = pkg_meta
build_meta = {}
build_meta.update(ydoc.get("build"))
build_meta.update(o.get("build") or {})
o["build"] = build_meta
variants[o["package"]["name"]] = get_dependency_variants(
o["requirements"], cbc, config)
else:
# we only have one output
variants[ydoc["package"]["name"]] = get_dependency_variants(
ydoc["requirements"], cbc, config)
# this takes in all variants and outputs, builds a dependency tree and returns
# the final metadata
sorted_outputs = to_build_tree(ydoc, variants, config)
# then we need to solve and build from the bottom up
# we can't first solve all packages without finalizing everything
# - solve the package
# - solv build, add weak run exports to
# - add run exports from deps!
if command == 'render':
for o in sorted_outputs:
print(o)
exit()
solver = MambaSolver(["conda-forge"], "linux-64")
for o in sorted_outputs:
solver.replace_channels()
o.finalize_solve(sorted_outputs, solver)
print(o)
o.config.compute_build_id(o.name)
print(o.config.host_prefix)
if 'build' in o.transactions:
mkdir_p(o.config.build_prefix)
print(o.transactions)
o.transactions['build'].execute(
PrefixData(o.config.build_prefix),
PackageCacheData.first_writable().pkgs_dir)
if 'host' in o.transactions:
mkdir_p(o.config.host_prefix)
print(o.transactions)
o.transactions['host'].execute(
PrefixData(o.config.host_prefix),
PackageCacheData.first_writable().pkgs_dir)
print(o.sections)
stats = {}
print("Final variant config")
print(config.variant)
print(o.variant)
build(MetaData(recipe_path, o), None)
# sorted_outputs
# print(sorted_outputs[0].config.host_prefix)
exit()
for o in sorted_outputs:
print("\n")
print(o)
