def __init__(
self,
graph: nx.DiGraph = None,
name: Optional[str] = None,
pr_limit: int = 0,
):
# rebuild the graph to only use edges from the arm osx requirements
graph2 = nx.create_empty_copy(graph)
for node, attrs in graph.nodes(data="payload"):
for plat_arch in self.arches:
reqs = attrs.get(
f"{plat_arch}_requirements",
attrs.get("osx_64_requirements",
attrs.get("requirements", {})),
)
host_deps = set(as_iterable(reqs.get("host", set())))
run_deps = set(as_iterable(reqs.get("run", set())))
deps = host_deps.union(run_deps)
for dep in deps:
dep = graph.graph["outputs_lut"].get(dep, dep)
graph2.add_edge(dep, node)
super().__init__(graph=graph2, pr_limit=pr_limit, check_solvable=False)
assert (not self.check_solvable
), "We don't want to check solvability for arm osx!"
# We are constraining the scope of this migrator
with indir("../conda-forge-pinning-feedstock/recipe/migrations"), open(
"osx_arm64.txt",
"r",
) as f:
self.target_packages = set(f.read().split())
self.name = name
# filter the graph down to the target packages
if self.target_packages:
self.target_packages.add("python") # hack that is ~harmless?
packages = self.target_packages.copy()
for target in self.target_packages:
if target in self.graph.nodes:
packages.update(nx.ancestors(self.graph, target))
self.graph.remove_nodes_from(
[n for n in self.graph if n not in packages])
for name in self.excluded_dependencies:
self.graph.remove_nodes_from(nx.descendants(graph, name))
# filter out stub packages and ignored packages
for node, attrs in list(self.graph.nodes("payload")):
if not attrs:
print(node)
if (not attrs or node.endswith("_stub") or (node.startswith("m2-"))
or (node.startswith("m2w64-"))
or (node in self.ignored_packages) or
(attrs.get("meta_yaml", {}).get("build", {}).get("noarch"))):
pluck(self.graph, node)
self.graph.remove_edges_from(nx.selfloop_edges(self.graph))
def extract_missing_packages(
required_imports,
questionable_imports,
run_packages,
package_by_import,
import_by_package,
node,
nodes,
):
exclude_packages = STATIC_EXCLUDES.union(
{node, node.replace("-", "_"),
node.replace("_", "-")}, )
questionable_packages = set().union(*list(
as_iterable(package_by_import.get(k, k))
for k in questionable_imports))
required_packages = set().union(*list(
as_iterable(package_by_import.get(k, k)) for k in required_imports))
run_imports = set().union(*list(
as_iterable(import_by_package.get(k, k)) for k in run_packages))
exclude_imports = set().union(*list(
as_iterable(import_by_package.get(k, k)) for k in exclude_packages))
d = {}
# These are all normalized to packages
# packages who's libraries are not imported
cf_minus_df = (run_packages - required_packages - exclude_packages -
questionable_packages) & nodes
if cf_minus_df:
d.update(cf_minus_df=cf_minus_df)
# These are all normalized to imports
# imports which have no associated package in the meta.yaml
df_minus_cf_imports = required_imports - run_imports - exclude_imports
# Normalize to packages, the native interface for conda-forge
# Note that the set overlap is a bit of a hack, sources could have imports we don't ship at all
df_minus_cf = (set().union(*list(
as_iterable(package_by_import.get(k, k)) for k in df_minus_cf_imports))
& nodes)
if df_minus_cf:
d.update(df_minus_cf=df_minus_cf)
return d
def populate_feedstock_attributes(
name: str,
sub_graph: LazyJson,
meta_yaml: typing.Union[str, Response] = "",
conda_forge_yaml: typing.Union[str, Response] = "",
mark_not_archived=False,
# build_sh: typing.Union[str, Response] = "",
# pre_unlink: typing.Union[str, Response] = "",
# post_link: typing.Union[str, Response] = "",
# pre_link: typing.Union[str, Response] = "",
# activate: typing.Union[str, Response] = "",
) -> LazyJson:
"""Parse the various configuration information into something usable
Notes
-----
If the return is bad hand the response itself in so that it can be parsed
for meaning.
"""
sub_graph.update({"feedstock_name": name, "bad": False})
if mark_not_archived:
sub_graph.update({"archived": False})
# handle all the raw strings
if isinstance(meta_yaml, Response):
sub_graph["bad"] = f"make_graph: {meta_yaml.status_code}"
return sub_graph
sub_graph["raw_meta_yaml"] = meta_yaml
# Get the conda-forge.yml
if isinstance(conda_forge_yaml, str):
sub_graph["conda-forge.yml"] = {
k: v
for k, v in yaml.safe_load(conda_forge_yaml).items() if k in {
"provider",
"min_r_ver",
"min_py_ver",
"max_py_ver",
"max_r_ver",
"compiler_stack",
"bot",
}
}
yaml_dict = ChainDB(*[
parse_meta_yaml(meta_yaml, platform=plat)
for plat in ["win", "osx", "linux"]
])
if not yaml_dict:
logger.error(f"Something odd happened when parsing recipe {name}")
sub_graph["bad"] = "make_graph: Could not parse"
return sub_graph
sub_graph["meta_yaml"] = _convert_to_dict(yaml_dict)
meta_yaml = sub_graph["meta_yaml"]
sub_graph["strong_exports"] = False
# TODO: make certain to remove None
requirements_dict = defaultdict(set)
for block in [meta_yaml] + meta_yaml.get("outputs", []) or []:
req: "RequirementsTypedDict" = block.get("requirements", {}) or {}
if isinstance(req, list):
requirements_dict["run"].update(set(req))
continue
for section in ["build", "host", "run"]:
requirements_dict[section].update(
list(as_iterable(req.get(section, []) or [])), )
test: "TestTypedDict" = block.get("test", {})
requirements_dict["test"].update(test.get("requirements", []) or [])
requirements_dict["test"].update(test.get("requires", []) or [])
run_exports = (block.get("build", {}) or {}).get("run_exports", {})
if isinstance(run_exports, dict) and run_exports.get("strong"):
sub_graph["strong_exports"] = True
for k in list(requirements_dict.keys()):
requirements_dict[k] = {v for v in requirements_dict[k] if v}
sub_graph["total_requirements"] = dict(requirements_dict)
sub_graph["requirements"] = {
k: {pin_sep_pat.split(x)[0].lower()
for x in v}
for k, v in sub_graph["total_requirements"].items()
}
# handle multi outputs
if "outputs" in yaml_dict:
sub_graph["outputs_names"] = sorted(
list({d.get("name", "")
for d in yaml_dict["outputs"]}), )
# TODO: Write schema for dict
# TODO: remove this
req = get_requirements(yaml_dict)
sub_graph["req"] = req
keys = [("package", "name"), ("package", "version")]
missing_keys = [k[1] for k in keys if k[1] not in yaml_dict.get(k[0], {})]
source = yaml_dict.get("source", [])
if isinstance(source, collections.abc.Mapping):
source = [source]
source_keys: Set[str] = set()
for s in source:
if not sub_graph.get("url"):
sub_graph["url"] = s.get("url")
source_keys |= s.keys()
if "url" not in source_keys:
missing_keys.append("url")
if missing_keys:
logger.error(f"Recipe {name} doesn't have a {', '.join(missing_keys)}")
for k in keys:
if k[1] not in missing_keys:
sub_graph[k[1]] = yaml_dict[k[0]][k[1]]
kl = list(sorted(source_keys & hashlib.algorithms_available, reverse=True))
if kl:
sub_graph["hash_type"] = kl[0]
return sub_graph
def filter(self,
attrs: "AttrsTypedDict",
not_bad_str_start: str = "") -> bool:
scripts = as_iterable(
attrs.get("meta_yaml", {}).get("build", {}).get("script", []), )
return not bool(set(self.bad_install) & set(scripts))
def __init__(
self,
graph: nx.DiGraph = None,
name: Optional[str] = None,
pr_limit: int = 0,
piggy_back_migrations: Optional[Sequence[MiniMigrator]] = None,
):
# rebuild the graph to only use edges from the arm osx requirements
graph2 = nx.create_empty_copy(graph)
for node, attrs in graph.nodes(data="payload"):
for plat_arch in self.arches:
reqs = attrs.get(
f"{plat_arch}_requirements",
attrs.get("osx_64_requirements",
attrs.get("requirements", {})),
)
host_deps = set(as_iterable(reqs.get("host", set())))
run_deps = set(as_iterable(reqs.get("run", set())))
deps = host_deps.union(run_deps)
# We are including the compiler stubs here so that
# excluded_dependencies work correctly.
# Edges to these compiler stubs are removed afterwards
build_deps = set(as_iterable(reqs.get("build", set())))
for build_dep in build_deps:
if build_dep.endswith("_stub"):
deps.add(build_dep)
for dep in get_deps_from_outputs_lut(
deps, graph.graph["outputs_lut"]):
graph2.add_edge(dep, node)
super().__init__(
graph=graph2,
pr_limit=pr_limit,
check_solvable=False,
piggy_back_migrations=piggy_back_migrations,
)
assert (not self.check_solvable
), "We don't want to check solvability for arm osx!"
self.name = name
# Excluded dependencies need to be removed before no target_packages are
# filtered out so that if a target_package is excluded, its dependencies
# are not added to the graph
for excluded_dep in self.excluded_dependencies:
self.graph.remove_nodes_from(
nx.descendants(self.graph, excluded_dep))
# We are constraining the scope of this migrator
with indir("../conda-forge-pinning-feedstock/recipe/migrations"), open(
"osx_arm64.txt", ) as f:
self.target_packages = set(f.read().split())
# filter the graph down to the target packages
if self.target_packages:
self.target_packages.add("python") # hack that is ~harmless?
packages = self.target_packages.copy()
for target in self.target_packages:
if target in self.graph.nodes:
packages.update(nx.ancestors(self.graph, target))
self.graph.remove_nodes_from(
[n for n in self.graph if n not in packages])
# filter out stub packages and ignored packages
for node, attrs in list(self.graph.nodes("payload")):
if (not attrs or node.endswith("_stub") or (node.startswith("m2-"))
or (node.startswith("m2w64-"))
or (node in self.ignored_packages) or all_noarch(attrs)):
pluck(self.graph, node)
self.graph.remove_edges_from(nx.selfloop_edges(self.graph))
