def __init__(
self, graph: nx.DiGraph = None, name: Optional[str] = None, pr_limit: int = 0,
):
super().__init__(graph=graph, pr_limit=pr_limit)
# We are constraining the scope of this migrator
with indir("../conda-forge-pinning-feedstock/recipe/migrations"), open(
"arch_rebuild.txt", "r"
) as f:
self.target_packages = set(f.read().split())
self.name = name
# filter the graph down to the target packages
if self.target_packages:
packages = self.target_packages.copy()
for target in self.target_packages:
if target in self.graph.nodes:
packages.update(nx.ancestors(self.graph, target))
self.graph.remove_nodes_from([n for n in self.graph if n not in packages])
# filter out stub packages and ignored packages
for node in list(self.graph.nodes):
if (
node.endswith("_stub")
or (node.startswith("m2-"))
or (node.startswith("m2w64-"))
or (node in self.ignored_packages)
or (
self.graph.nodes[node]
.get("payload", {})
.get("meta_yaml", {})
.get("build", {})
.get("noarch")
)
):
pluck(self.graph, node)
self.graph.remove_edges_from(nx.selfloop_edges(self.graph))
def __init__(
self,
graph: nx.DiGraph = None,
name: Optional[str] = None,
pr_limit: int = 0,
):
# rebuild the graph to only use edges from the arm osx requirements
graph2 = nx.create_empty_copy(graph)
for node, attrs in graph.nodes(data="payload"):
for plat_arch in self.arches:
reqs = attrs.get(
f"{plat_arch}_requirements",
attrs.get("osx_64_requirements",
attrs.get("requirements", {})),
)
host_deps = set(as_iterable(reqs.get("host", set())))
run_deps = set(as_iterable(reqs.get("run", set())))
deps = host_deps.union(run_deps)
for dep in deps:
dep = graph.graph["outputs_lut"].get(dep, dep)
graph2.add_edge(dep, node)
super().__init__(graph=graph2, pr_limit=pr_limit, check_solvable=False)
assert (not self.check_solvable
), "We don't want to check solvability for arm osx!"
# We are constraining the scope of this migrator
with indir("../conda-forge-pinning-feedstock/recipe/migrations"), open(
"osx_arm64.txt",
"r",
) as f:
self.target_packages = set(f.read().split())
self.name = name
# filter the graph down to the target packages
if self.target_packages:
self.target_packages.add("python") # hack that is ~harmless?
packages = self.target_packages.copy()
for target in self.target_packages:
if target in self.graph.nodes:
packages.update(nx.ancestors(self.graph, target))
self.graph.remove_nodes_from(
[n for n in self.graph if n not in packages])
for name in self.excluded_dependencies:
self.graph.remove_nodes_from(nx.descendants(graph, name))
# filter out stub packages and ignored packages
for node, attrs in list(self.graph.nodes("payload")):
if not attrs:
print(node)
if (not attrs or node.endswith("_stub") or (node.startswith("m2-"))
or (node.startswith("m2w64-"))
or (node in self.ignored_packages) or
(attrs.get("meta_yaml", {}).get("build", {}).get("noarch"))):
pluck(self.graph, node)
self.graph.remove_edges_from(nx.selfloop_edges(self.graph))
def create_rebuild_graph(
gx: nx.DiGraph,
package_names: Sequence[str],
excluded_feedstocks: MutableSet[str] = None,
include_noarch: bool = False,
) -> nx.DiGraph:
total_graph = copy.deepcopy(gx)
excluded_feedstocks = set() if excluded_feedstocks is None else excluded_feedstocks
# Always exclude the packages themselves from the migration
for node in package_names:
excluded_feedstocks.update(gx.graph["outputs_lut"].get(node, {node}))
included_nodes = set()
for node, node_attrs in gx.nodes.items():
# always keep pinning
if node == "conda-forge-pinning":
continue
attrs: "AttrsTypedDict" = node_attrs["payload"]
requirements = attrs.get("requirements", {})
host = requirements.get("host", set())
build = requirements.get("build", set())
bh = host or build
only_python = "python" in package_names
inclusion_criteria = bh & set(package_names) and (
include_noarch or not all_noarch(attrs, only_python=only_python)
)
# get host/build, run and test and launder them through outputs
# this should fix outputs related issues (eg gdal)
all_reqs = requirements.get("run", set())
if inclusion_criteria:
all_reqs = all_reqs | requirements.get("test", set())
all_reqs = all_reqs | (host or build)
rq = get_deps_from_outputs_lut(
all_reqs,
gx.graph["outputs_lut"],
)
for e in list(total_graph.in_edges(node)):
if e[0] not in rq:
total_graph.remove_edge(*e)
if inclusion_criteria:
included_nodes.add(node)
# all nodes have the conda-forge-pinning as child package
total_graph.add_edges_from([(n, "conda-forge-pinning") for n in total_graph.nodes])
included_nodes.add("conda-forge-pinning") # it does not get added above
# finally remove all nodes that should not be built from the graph
for node in list(total_graph.nodes):
# if there isn't a strict dependency or if the feedstock is excluded,
# remove it while retaining the edges to its parents and children
if (node not in included_nodes) or (node in excluded_feedstocks):
pluck(total_graph, node)
# post plucking we can have several strange cases, lets remove all selfloops
total_graph.remove_edges_from(nx.selfloop_edges(total_graph))
return total_graph
def __init__(
self,
graph: nx.DiGraph = None,
name: Optional[str] = None,
pr_limit: int = 0,
piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None,
):
# rebuild the graph to only use edges from the arm and power requirements
graph2 = nx.create_empty_copy(graph)
for node, attrs in graph.nodes(data="payload"):
for plat_arch in self.arches:
deps = set().union(
*attrs.get(
f"{plat_arch}_requirements",
attrs.get("requirements", {}),
).values()
)
for dep in get_deps_from_outputs_lut(deps, graph.graph["outputs_lut"]):
graph2.add_edge(dep, node)
super().__init__(
graph=graph2,
pr_limit=pr_limit,
check_solvable=False,
piggy_back_migrations=piggy_back_migrations,
)
assert not self.check_solvable, "We don't want to check solvability for aarch!"
# We are constraining the scope of this migrator
with indir("../conda-forge-pinning-feedstock/recipe/migrations"), open(
"arch_rebuild.txt",
) as f:
self.target_packages = set(f.read().split())
self.name = name
# filter the graph down to the target packages
if self.target_packages:
self.target_packages.add("python") # hack that is ~harmless?
packages = self.target_packages.copy()
for target in self.target_packages:
if target in self.graph.nodes:
packages.update(nx.ancestors(self.graph, target))
self.graph.remove_nodes_from([n for n in self.graph if n not in packages])
# filter out stub packages and ignored packages
for node, attrs in list(self.graph.nodes("payload")):
if (
node.endswith("_stub")
or (node.startswith("m2-"))
or (node.startswith("m2w64-"))
or (node in self.ignored_packages)
or all_noarch(attrs)
):
pluck(self.graph, node)
self.graph.remove_edges_from(nx.selfloop_edges(self.graph))
def __init__(
self,
graph: nx.DiGraph = None,
name: Optional[str] = None,
pr_limit: int = 0,
piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None,
):
# rebuild the graph to only use edges from the arm osx requirements
graph2 = nx.create_empty_copy(graph)
for node, attrs in graph.nodes(data="payload"):
for plat_arch in self.arches:
reqs = attrs.get(
f"{plat_arch}_requirements",
attrs.get("osx_64_requirements",
attrs.get("requirements", {})),
)
host_deps = set(as_iterable(reqs.get("host", set())))
run_deps = set(as_iterable(reqs.get("run", set())))
deps = host_deps.union(run_deps)
# We are including the compiler stubs here so that
# excluded_dependencies work correctly.
# Edges to these compiler stubs are removed afterwards
build_deps = set(as_iterable(reqs.get("build", set())))
for build_dep in build_deps:
if build_dep.endswith("_stub"):
deps.add(build_dep)
for dep in get_deps_from_outputs_lut(
deps, graph.graph["outputs_lut"]):
graph2.add_edge(dep, node)
super().__init__(
graph=graph2,
pr_limit=pr_limit,
check_solvable=False,
piggy_back_migrations=piggy_back_migrations,
)
assert (not self.check_solvable
), "We don't want to check solvability for arm osx!"
self.name = name
# Excluded dependencies need to be removed before no target_packages are
# filtered out so that if a target_package is excluded, its dependencies
# are not added to the graph
for excluded_dep in self.excluded_dependencies:
self.graph.remove_nodes_from(
nx.descendants(self.graph, excluded_dep))
# We are constraining the scope of this migrator
with indir("../conda-forge-pinning-feedstock/recipe/migrations"), open(
"osx_arm64.txt", ) as f:
self.target_packages = set(f.read().split())
# filter the graph down to the target packages
if self.target_packages:
self.target_packages.add("python") # hack that is ~harmless?
packages = self.target_packages.copy()
for target in self.target_packages:
if target in self.graph.nodes:
packages.update(nx.ancestors(self.graph, target))
self.graph.remove_nodes_from(
[n for n in self.graph if n not in packages])
# filter out stub packages and ignored packages
for node, attrs in list(self.graph.nodes("payload")):
if (not attrs or node.endswith("_stub") or (node.startswith("m2-"))
or (node.startswith("m2w64-"))
or (node in self.ignored_packages) or all_noarch(attrs)):
pluck(self.graph, node)
self.graph.remove_edges_from(nx.selfloop_edges(self.graph))
def create_migration_yaml_creator(migrators: MutableSequence[Migrator],
gx: nx.DiGraph):
cfp_gx = copy.deepcopy(gx)
for node in list(cfp_gx.nodes):
if node != "conda-forge-pinning":
pluck(cfp_gx, node)
print("pinning migrations", flush=True)
with indir(os.environ["CONDA_PREFIX"]):
pinnings = parse_config_file(
"conda_build_config.yaml",
config=Config(**CB_CONFIG),
)
feedstocks_to_be_repinned = []
for pinning_name, package_pin_list in pinnings.items():
# there are three things:
# pinning_name - entry in pinning file
# package_name - the actual package, could differ via `-` -> `_`
# from pinning_name
# feedstock_name - the feedstock that outputs the package
# we need the package names for the migrator itself but need the
# feedstock for everything else
# exclude non-package keys
if pinning_name not in gx.graph["outputs_lut"]:
# conda_build_config.yaml can't have `-` unlike our package names
package_name = pinning_name.replace("_", "-")
else:
package_name = pinning_name
# replace sub-packages with their feedstock names
# TODO - we are grabbing one element almost at random here
# the sorted call makes it stable at least?
fs_name = next(
iter(
sorted(gx.graph["outputs_lut"].get(package_name,
{package_name})), ), )
if ((fs_name in gx.nodes)
and not gx.nodes[fs_name]["payload"].get("archived", False)
and gx.nodes[fs_name]["payload"].get("version")
and fs_name not in feedstocks_to_be_repinned):
current_pins = list(map(str, package_pin_list))
current_version = str(gx.nodes[fs_name]["payload"]["version"])
# we need a special parsing for pinning stuff
meta_yaml = parse_meta_yaml(
gx.nodes[fs_name]["payload"]["raw_meta_yaml"],
for_pinning=True,
)
# find the most stringent max pin for this feedstock if any
pin_spec = ""
for block in [meta_yaml] + meta_yaml.get("outputs", []) or []:
build = block.get("build", {}) or {}
# and check the exported package is within the feedstock
exports = [
p.get("max_pin", "")
for p in build.get("run_exports", [{}])
# make certain not direct hard pin
if isinstance(p, MutableMapping)
# ensure the export is for this package
and p.get("package_name", "") == package_name
# ensure the pinned package is in an output of the parent feedstock
and (fs_name in gx.graph["outputs_lut"].get(
p.get("package_name", ""), set()))
]
if not exports:
continue
# get the most stringent pin spec from the recipe block
max_pin = max(exports, key=len)
if len(max_pin) > len(pin_spec):
pin_spec = max_pin
# fall back to the pinning file or "x"
if not pin_spec:
pin_spec = (pinnings["pin_run_as_build"].get(
pinning_name, {}).get("max_pin", "x")) or "x"
current_pins = list(
map(lambda x: re.sub("[^0-9.]", "", x).rstrip("."),
current_pins), )
current_pins = [
cp.strip() for cp in current_pins if cp.strip() != ""
]
current_version = re.sub("[^0-9.]", "",
current_version).rstrip(".")
if not current_pins or current_version == "":
continue
current_pin = str(max(map(VersionOrder, current_pins)))
# If the current pin and the current version is the same nothing
# to do even if the pin isn't accurate to the spec
if current_pin != current_version and _outside_pin_range(
pin_spec,
current_pin,
current_version,
):
feedstocks_to_be_repinned.append(fs_name)
print(
" %s:\n"
" curr version: %s\n"
" curr pin: %s\n"
" pin_spec: %s" %
(pinning_name, current_version, current_pin, pin_spec),
flush=True,
)
migrators.append(
MigrationYamlCreator(
pinning_name,
current_version,
current_pin,
pin_spec,
fs_name,
cfp_gx,
full_graph=gx,
), )
print(" ", flush=True)
def add_replacement_migrator(
migrators: MutableSequence[Migrator],
gx: nx.DiGraph,
old_pkg: "PackageName",
new_pkg: "PackageName",
rationale: str,
alt_migrator: Union[Migrator, None] = None,
) -> None:
"""Adds a migrator to replace one package with another.
Parameters
----------
migrators : list of Migrator
The list of migrators to run.
gx : graph
The conda-forge dependency graph.
old_pkg : str
The package to be replaced.
new_pkg : str
The package to replace the `old_pkg`.
rationale : str
The reason the for the migration. Should be a full statement.
alt_migrator : Replacement migrator or a sublcass thereof
An alternate Replacement migrator to use for special tasks.
"""
print(
"========================================"
"========================================",
flush=True,
)
print(f"making replacement migrator for {old_pkg} -> {new_pkg}",
flush=True)
total_graph = copy.deepcopy(gx)
for node, node_attrs in gx.nodes.items():
requirements = node_attrs["payload"].get("requirements", {})
rq = (requirements.get("build", set())
| requirements.get("host", set())
| requirements.get("run", set())
| requirements.get("test", set()))
pkgs = {old_pkg}
old_pkg_c = pkgs.intersection(rq)
if not old_pkg_c:
pluck(total_graph, node)
# post plucking we can have several strange cases, lets remove all selfloops
total_graph.remove_edges_from(nx.selfloop_edges(total_graph))
if alt_migrator is not None:
migrators.append(
alt_migrator(
old_pkg=old_pkg,
new_pkg=new_pkg,
rationale=rationale,
pr_limit=PR_LIMIT,
graph=total_graph,
), )
else:
migrators.append(
Replacement(
old_pkg=old_pkg,
new_pkg=new_pkg,
rationale=rationale,
pr_limit=PR_LIMIT,
graph=total_graph,
), )