def get_metadata(yml, config):
with open(yml, "r") as fi:
d = ruamel.yaml.safe_load(fi)
o = Output(d, config)
return MetaData(os.path.dirname(yml), o)
def to_build_tree(ydoc, variants, config, selected_features):
for k in variants:
table = Table(show_header=True, header_style="bold")
table.title = f"Output: [bold white]{k}[/bold white]"
table.add_column("Package")
table.add_column("Variant versions")
for pkg, var in variants[k].items():
table.add_row(pkg, "\n".join(var))
console.print(table)
# first we need to perform a topological sort taking into account all the outputs
if ydoc.get("outputs"):
outputs = [
Output(o, config, parent=ydoc, selected_features=selected_features)
for o in ydoc["outputs"]
]
outputs = {o.name: o for o in outputs}
else:
outputs = [Output(ydoc, config, selected_features=selected_features)]
outputs = {o.name: o for o in outputs}
if len(outputs) > 1:
sort_dict = {
k: [x.name for x in o.all_requirements()] for k, o in outputs.items()
}
tsorted = toposort.toposort(sort_dict)
tsorted = [o for o in tsorted if o in sort_dict.keys()]
else:
tsorted = [o for o in outputs.keys()]
final_outputs = []
for name in tsorted:
output = outputs[name]
if variants.get(output.name):
v = variants[output.name]
combos = []
differentiating_keys = []
for k in v:
if len(v[k]) > 1:
differentiating_keys.append(k)
combos.append([(k, x) for x in v[k]])
all_combinations = tuple(itertools.product(*combos))
all_combinations = [dict(x) for x in all_combinations]
for c in all_combinations:
x = output.apply_variant(c, differentiating_keys)
final_outputs.append(x)
else:
x = output.apply_variant({})
final_outputs.append(x)
temp = final_outputs
final_outputs = []
has_intermediate = False
for o in temp:
if o.sections["build"].get("intermediate"):
if has_intermediate:
raise RuntimeError(
"Already found an intermediate build. There can be only one!"
)
final_outputs.insert(0, o)
has_intermediate = True
else:
final_outputs.append(o)
# Note: maybe this should happen _before_ apply variant?!
if has_intermediate:
# inherit dependencies
def merge_requirements(a, b):
b_names = [x.name for x in b]
for r in a:
if r.name in b_names:
continue
else:
b.append(r)
intermediate = final_outputs[0]
for o in final_outputs[1:]:
merge_requirements(
intermediate.requirements["host"], o.requirements["host"]
)
merge_requirements(
intermediate.requirements["build"], o.requirements["build"]
)
merged_variant = {}
merged_variant.update(intermediate.config.variant)
merged_variant.update(o.config.variant)
o.config.variant = merged_variant
return final_outputs
def to_build_tree(ydoc, variants, config, cbc, selected_features):
for k in variants:
table = Table(show_header=True, header_style="bold")
table.title = f"Output: [bold white]{k}[/bold white]"
table.add_column("Package")
table.add_column("Variant versions")
for pkg, var in variants[k].items():
table.add_row(pkg, "\n".join(var))
console.print(table)
# first we need to perform a topological sort taking into account all the outputs
if ydoc.get("outputs"):
outputs = [
Output(
o,
config,
parent=ydoc,
conda_build_config=cbc,
selected_features=selected_features,
)
for o in ydoc["outputs"]
]
outputs = {o.name: o for o in outputs}
else:
outputs = [
Output(
ydoc,
config,
conda_build_config=cbc,
selected_features=selected_features,
)
]
outputs = {o.name: o for o in outputs}
if len(outputs) > 1:
sort_dict = {
k: [x.name for x in o.all_requirements()] for k, o in outputs.items()
}
tsorted = toposort.toposort(sort_dict)
tsorted = [o for o in tsorted if o in sort_dict.keys()]
else:
tsorted = [o for o in outputs.keys()]
final_outputs = []
# need to strip static away from output name... :/
static_feature = selected_features.get("static", False)
for name in tsorted:
output = outputs[name]
# this is all a bit hacky ... will have to clean that up eventually
variant_name = name
if static_feature and name.endswith("-static"):
variant_name = name[: -len("-static")]
# zip keys need to be contracted
zipped_keys = cbc.get("zip_keys", [])
if variants.get(variant_name):
v = variants[variant_name]
import copy
vzipped = copy.copy(v)
zippers = {}
for zkeys in zipped_keys:
# we check if our variant contains keys that need to be zipped
if sum(k in v for k in zkeys) > 1:
filtered_zip_keys = [k for k in v if k in zkeys]
print("Filtered zip keys: ", filtered_zip_keys)
zkname = "__zip_" + "_".join(filtered_zip_keys)
zklen = None
for zk in filtered_zip_keys:
if zk not in cbc:
raise RuntimeError(
f"Trying to zip keys, but not all zip keys found on conda-build-config {zk}"
)
zkl = len(cbc[zk])
if not zklen:
zklen = zkl
if zklen and zkl != zklen:
raise RuntimeError(
f"Trying to zip keys, but not all zip keys have the same length {zkeys}"
)
vzipped[zkname] = [str(i) for i in range(zklen)]
zippers[zkname] = {zk: cbc[zk] for zk in filtered_zip_keys}
for zk in filtered_zip_keys:
del vzipped[zk]
combos = []
differentiating_keys = []
for k, vz in vzipped.items():
if len(vz) > 1:
differentiating_keys.append(k)
combos.append([(k, x) for x in vz])
all_combinations = tuple(itertools.product(*combos))
all_combinations = [dict(x) for x in all_combinations]
# unzip the zipped keys
unzipped_combinations = []
for c in all_combinations:
unz_combo = {}
for vc in c:
if vc.startswith("__zip_"):
ziptask = zippers[vc]
zipindex = int(c[vc])
for zippkg in ziptask:
unz_combo[zippkg] = ziptask[zippkg][zipindex]
if vc in differentiating_keys:
differentiating_keys.remove(vc)
differentiating_keys.extend(zippers[vc].keys())
else:
unz_combo[vc] = c[vc]
unzipped_combinations.append(unz_combo)
for c in unzipped_combinations:
x = output.apply_variant(c, differentiating_keys)
final_outputs.append(x)
else:
x = output.apply_variant({})
final_outputs.append(x)
temp = final_outputs
final_outputs = []
has_intermediate = False
for o in temp:
if o.sections["build"].get("intermediate"):
if has_intermediate:
raise RuntimeError(
"Already found an intermediate build. There can be only one!"
)
final_outputs.insert(0, o)
has_intermediate = True
else:
final_outputs.append(o)
# Note: maybe this should happen _before_ apply variant?!
if has_intermediate:
# inherit dependencies
def merge_requirements(a, b):
b_names = [x.name for x in b]
for r in a:
if r.name in b_names:
continue
else:
b.append(r)
intermediate = final_outputs[0]
for o in final_outputs[1:]:
merge_requirements(
intermediate.requirements["host"], o.requirements["host"]
)
merge_requirements(
intermediate.requirements["build"], o.requirements["build"]
)
merged_variant = {}
merged_variant.update(intermediate.config.variant)
merged_variant.update(o.config.variant)
o.config.variant = merged_variant
return final_outputs