def run(
feedstock_ctx: FeedstockContext,
migrator: Migrator,
protocol: str = "ssh",
pull_request: bool = True,
rerender: bool = True,
fork: bool = True,
**kwargs: typing.Any,
) -> Tuple["MigrationUidTypedDict", dict]:
"""For a given feedstock and migration run the migration
Parameters
----------
feedstock_ctx: FeedstockContext
The node attributes
migrator: Migrator instance
The migrator to run on the feedstock
protocol : str, optional
The git protocol to use, defaults to ``ssh``
pull_request : bool, optional
If true issue pull request, defaults to true
rerender : bool
Whether to rerender
fork : bool
If true create a fork, defaults to true
gh : github3.GitHub instance, optional
Object for communicating with GitHub, if None build from $USERNAME
and $PASSWORD, defaults to None
kwargs: dict
The key word arguments to pass to the migrator
Returns
-------
migrate_return: MigrationUidTypedDict
The migration return dict used for tracking finished migrations
pr_json: dict
The PR json object for recreating the PR as needed
"""
# get the repo
# TODO: stop doing this.
migrator.attrs = feedstock_ctx.attrs # type: ignore
branch_name = migrator.remote_branch(feedstock_ctx) + "_h" + uuid4().hex[0:6]
# TODO: run this in parallel
feedstock_dir, repo = get_repo(
ctx=migrator.ctx.session,
fctx=feedstock_ctx,
branch=branch_name,
feedstock=feedstock_ctx.feedstock_name,
protocol=protocol,
pull_request=pull_request,
fork=fork,
)
recipe_dir = os.path.join(feedstock_dir, "recipe")
# migrate the feedstock
migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs)
# TODO - make a commit here if the repo changed
migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs, **kwargs)
if not migrate_return:
logger.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO - commit main migration here
migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs, **kwargs)
# TODO commit post migration here
# rerender, maybe
diffed_files: typing.List[str] = []
with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
msg = migrator.commit_message(feedstock_ctx) # noqa
eval_cmd("git add --all .")
eval_cmd(f"git commit -am '{msg}'")
if rerender:
head_ref = eval_cmd("git rev-parse HEAD") # noqa
logger.info("Rerendering the feedstock")
# In the event we can't rerender, try to update the pinnings,
# then bail if it does not work again
try:
eval_cmd(
"conda smithy rerender -c auto --no-check-uptodate", timeout=300,
)
except SubprocessError:
return False, False
# If we tried to run the MigrationYaml and rerender did nothing (we only
# bumped the build number and dropped a yaml file in migrations) bail
# for instance platform specific migrations
gdiff = eval_cmd(f"git diff --name-only {head_ref}...HEAD")
diffed_files = [
_
for _ in gdiff.split()
if not (
_.startswith("recipe")
or _.startswith("migrators")
or _.startswith("README")
)
]
if (
migrator.check_solvable
or feedstock_ctx.attrs["conda-forge.yml"]
.get("bot", {})
.get("check_solvable", False)
) and not is_recipe_solvable(feedstock_dir):
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO: Better annotation here
pr_json: typing.Union[MutableMapping, None, bool]
if (
isinstance(migrator, MigrationYaml)
and not diffed_files
and feedstock_ctx.attrs["name"] != "conda-forge-pinning"
):
# spoof this so it looks like the package is done
pr_json = {
"state": "closed",
"merged_at": "never issued",
"id": str(uuid4()),
}
else:
# push up
try:
pr_json = push_repo(
session_ctx=migrator.ctx.session,
fctx=feedstock_ctx,
feedstock_dir=feedstock_dir,
body=migrator.pr_body(feedstock_ctx),
repo=repo,
title=migrator.pr_title(feedstock_ctx),
head=f"{migrator.ctx.github_username}:{branch_name}",
branch=branch_name,
)
# This shouldn't happen too often any more since we won't double PR
except github3.GitHubError as e:
if e.msg != "Validation Failed":
raise
else:
print(f"Error during push {e}")
# If we just push to the existing PR then do nothing to the json
pr_json = False
ljpr = False
if pr_json:
ljpr = LazyJson(
os.path.join(migrator.ctx.session.prjson_dir, str(pr_json["id"]) + ".json"),
)
ljpr.update(**pr_json)
# from .dynamo_models import PRJson
# PRJson.dump(pr_json)
# If we've gotten this far then the node is good
feedstock_ctx.attrs["bad"] = False
logger.info("Removing feedstock dir")
eval_cmd(f"rm -rf {feedstock_dir}")
return migrate_return, ljpr
def graph_migrator_status(
migrator: Migrator,
gx: nx.DiGraph,
) -> Tuple[dict, list, nx.DiGraph]:
"""Gets the migrator progress for a given migrator"""
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
num_viz = 0
out: Dict[str, Set[str]] = {
"done": set(),
"in-pr": set(),
"awaiting-pr": set(),
"not-solvable": set(),
"awaiting-parents": set(),
"bot-error": set(),
}
gx2 = copy.deepcopy(getattr(migrator, "graph", gx))
top_level = {node for node in gx2 if not list(gx2.predecessors(node))}
build_sequence = list(cyclic_topological_sort(gx2, top_level))
feedstock_metadata = dict()
import graphviz
from streamz.graph import _clean_text
gv = graphviz.Digraph(graph_attr={"packmode": "array_3"})
# pinning isn't actually in the migration
if "conda-forge-pinning" in gx2.nodes():
gx2.remove_node("conda-forge-pinning")
for node, node_attrs in gx2.nodes.items():
attrs = node_attrs["payload"]
# remove archived from status
if attrs.get("archived", False):
continue
node_metadata: Dict = {}
feedstock_metadata[node] = node_metadata
nuid = migrator.migrator_uid(attrs)
all_pr_jsons = []
for pr_json in attrs.get("PRed", []):
all_pr_jsons.append(copy.deepcopy(pr_json))
feedstock_ctx = FeedstockContext(
package_name=node,
feedstock_name=attrs.get("feedstock_name", node),
attrs=attrs,
)
# hack around bug in migrator vs graph data for this one
if isinstance(migrator, MatplotlibBase):
if "name" in nuid:
del nuid["name"]
for i in range(len(all_pr_jsons)):
if (all_pr_jsons[i] and "name" in all_pr_jsons[i]["data"]
and all_pr_jsons[i]["data"]["migrator_name"]
== "MatplotlibBase"):
del all_pr_jsons[i]["data"]["name"]
for pr_json in all_pr_jsons:
if pr_json and pr_json["data"] == frozen_to_json_friendly(
nuid)["data"]:
break
else:
pr_json = None
# No PR was ever issued but the migration was performed.
# This is only the case when the migration was done manually
# before the bot could issue any PR.
manually_done = pr_json is None and frozen_to_json_friendly(
nuid)["data"] in (z["data"] for z in all_pr_jsons)
buildable = not migrator.filter(attrs)
fntc = "black"
status_icon = ""
if manually_done:
out["done"].add(node)
fc = "#440154"
fntc = "white"
elif pr_json is None:
if buildable:
if "not solvable" in (attrs.get("pre_pr_migrator_status",
{}).get(migrator_name, "")):
out["not-solvable"].add(node)
fc = "#ff8c00"
elif "bot error" in (attrs.get("pre_pr_migrator_status",
{}).get(migrator_name, "")):
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
else:
out["awaiting-pr"].add(node)
fc = "#35b779"
elif not isinstance(migrator, Replacement):
if "bot error" in (attrs.get("pre_pr_migrator_status",
{}).get(migrator_name, "")):
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
else:
out["awaiting-parents"].add(node)
fc = "#fde725"
elif "PR" not in pr_json:
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
elif pr_json["PR"]["state"] == "closed":
out["done"].add(node)
fc = "#440154"
fntc = "white"
else:
out["in-pr"].add(node)
fc = "#31688e"
fntc = "white"
pr_status = pr_json["PR"]["mergeable_state"]
if pr_status == "clean":
status_icon = " ✓"
else:
status_icon = " ❎"
if node not in out["done"]:
num_viz += 1
gv.node(
node,
label=_clean_text(node) + status_icon,
fillcolor=fc,
style="filled",
fontcolor=fntc,
URL=(pr_json or {}).get("PR", {}).get(
"html_url",
feedstock_url(fctx=feedstock_ctx,
protocol="https").strip(".git"),
),
)
# additional metadata for reporting
node_metadata["num_descendants"] = len(nx.descendants(gx2, node))
node_metadata["immediate_children"] = [
k for k in sorted(gx2.successors(node))
if not gx2[k].get("payload", {}).get("archived", False)
]
if node in out["not-solvable"] or node in out["bot-error"]:
node_metadata["pre_pr_migrator_status"] = attrs.get(
"pre_pr_migrator_status",
{},
).get(migrator_name, "")
else:
node_metadata["pre_pr_migrator_status"] = ""
if pr_json and "PR" in pr_json:
# I needed to fake some PRs they don't have html_urls though
node_metadata["pr_url"] = pr_json["PR"].get(
"html_url",
feedstock_url(fctx=feedstock_ctx,
protocol="https").strip(".git"),
)
node_metadata["pr_status"] = pr_json["PR"].get("mergeable_state")
out2: Dict = {}
for k in out.keys():
out2[k] = list(
sorted(
out[k],
key=lambda x: build_sequence.index(x)
if x in build_sequence else -1,
), )
out2["_feedstock_status"] = feedstock_metadata
for (e0, e1), edge_attrs in gx2.edges.items():
if (e0 not in out["done"] and e1 not in out["done"]
and not gx2.nodes[e0]["payload"].get("archived", False)
and not gx2.nodes[e1]["payload"].get("archived", False)):
gv.edge(e0, e1)
print(" len(gv):", num_viz, flush=True)
out2["_num_viz"] = num_viz
return out2, build_sequence, gv
def migrator_status(
migrator: Migrator, gx: nx.DiGraph,
) -> Tuple[dict, list, nx.DiGraph]:
"""Gets the migrator progress for a given migrator
Returns
-------
out : dict
Dictionary of statuses with the feedstocks in them
order :
Build order for this migrator
"""
out: Dict[str, Set[str]] = {
"done": set(),
"in-pr": set(),
"awaiting-pr": set(),
"awaiting-parents": set(),
"bot-error": set(),
}
gx2 = copy.deepcopy(getattr(migrator, "graph", gx))
top_level = {node for node in gx2 if not list(gx2.predecessors(node))}
build_sequence = list(cyclic_topological_sort(gx2, top_level))
feedstock_metadata = dict()
import graphviz
from streamz.graph import _clean_text
gv = graphviz.Digraph(graph_attr={"packmode": "array_3"})
for node, node_attrs in gx2.nodes.items():
attrs = node_attrs["payload"]
# remove archived from status
if attrs.get("archived", False):
continue
node_metadata: Dict = {}
feedstock_metadata[node] = node_metadata
nuid = migrator.migrator_uid(attrs)
all_pr_jsons = []
for pr_json in attrs.get("PRed", []):
all_pr_jsons.append(copy.deepcopy(pr_json))
# hack around bug in migrator vs graph data for this one
if isinstance(migrator, MatplotlibBase):
if "name" in nuid:
del nuid["name"]
for i in range(len(all_pr_jsons)):
if (
all_pr_jsons[i]
and "name" in all_pr_jsons[i]["data"]
and all_pr_jsons[i]["data"]["migrator_name"] == "MatplotlibBase"
):
del all_pr_jsons[i]["data"]["name"]
for pr_json in all_pr_jsons:
if pr_json and pr_json["data"] == frozen_to_json_friendly(nuid)["data"]:
break
else:
pr_json = None
# No PR was ever issued but the migration was performed.
# This is only the case when the migration was done manually
# before the bot could issue any PR.
manually_done = pr_json is None and frozen_to_json_friendly(nuid)["data"] in (
z["data"] for z in all_pr_jsons
)
buildable = not migrator.filter(attrs)
fntc = "black"
if manually_done:
out["done"].add(node)
fc = "#440154"
fntc = "white"
elif pr_json is None:
if buildable:
out["awaiting-pr"].add(node)
fc = "#35b779"
elif not isinstance(migrator, Replacement):
out["awaiting-parents"].add(node)
fc = "#fde725"
elif "PR" not in pr_json:
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
elif pr_json["PR"]["state"] == "closed":
out["done"].add(node)
fc = "#440154"
fntc = "white"
else:
out["in-pr"].add(node)
fc = "#31688e"
fntc = "white"
if node not in out["done"]:
gv.node(
node,
label=_clean_text(node),
fillcolor=fc,
style="filled",
fontcolor=fntc,
URL=(pr_json or {}).get("PR", {}).get("html_url", ""),
)
# additional metadata for reporting
node_metadata["num_descendants"] = len(nx.descendants(gx2, node))
node_metadata["immediate_children"] = [
k
for k in sorted(gx2.successors(node))
if not gx2[k].get("payload", {}).get("archived", False)
]
if pr_json and "PR" in pr_json:
# I needed to fake some PRs they don't have html_urls though
node_metadata["pr_url"] = pr_json["PR"].get("html_url", "")
out2: Dict = {}
for k in out.keys():
out2[k] = list(
sorted(
out[k],
key=lambda x: build_sequence.index(x) if x in build_sequence else -1,
),
)
out2["_feedstock_status"] = feedstock_metadata
for (e0, e1), edge_attrs in gx2.edges.items():
if (
e0 not in out["done"]
and e1 not in out["done"]
and not gx2.nodes[e0]["payload"].get("archived", False)
and not gx2.nodes[e1]["payload"].get("archived", False)
):
gv.edge(e0, e1)
return out2, build_sequence, gv
def run(
feedstock_ctx: FeedstockContext,
migrator: Migrator,
protocol: str = "ssh",
pull_request: bool = True,
rerender: bool = True,
fork: bool = True,
base_branch: str = "master",
**kwargs: typing.Any,
) -> Tuple["MigrationUidTypedDict", dict]:
"""For a given feedstock and migration run the migration
Parameters
----------
feedstock_ctx: FeedstockContext
The node attributes
migrator: Migrator instance
The migrator to run on the feedstock
protocol : str, optional
The git protocol to use, defaults to ``ssh``
pull_request : bool, optional
If true issue pull request, defaults to true
rerender : bool
Whether to rerender
fork : bool
If true create a fork, defaults to true
base_branch : str, optional
The base branch to which the PR will be targeted. Defaults to "master".
kwargs: dict
The key word arguments to pass to the migrator
Returns
-------
migrate_return: MigrationUidTypedDict
The migration return dict used for tracking finished migrations
pr_json: dict
The PR json object for recreating the PR as needed
"""
# get the repo
# TODO: stop doing this.
migrator.attrs = feedstock_ctx.attrs # type: ignore
branch_name = migrator.remote_branch(
feedstock_ctx) + "_h" + uuid4().hex[0:6]
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
# TODO: run this in parallel
feedstock_dir, repo = get_repo(
ctx=migrator.ctx.session,
fctx=feedstock_ctx,
branch=branch_name,
feedstock=feedstock_ctx.feedstock_name,
protocol=protocol,
pull_request=pull_request,
fork=fork,
base_branch=base_branch,
)
if not feedstock_dir or not repo:
LOGGER.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
return False, False
recipe_dir = os.path.join(feedstock_dir, "recipe")
# migrate the feedstock
migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO - make a commit here if the repo changed
migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs,
**kwargs)
if not migrate_return:
LOGGER.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO - commit main migration here
migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO commit post migration here
# rerender, maybe
diffed_files: typing.List[str] = []
with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
msg = migrator.commit_message(feedstock_ctx) # noqa
try:
eval_cmd("git add --all .")
eval_cmd(f"git commit -am '{msg}'")
except CalledProcessError as e:
LOGGER.info(
"could not commit to feedstock - "
"likely no changes - error is '%s'" % (repr(e)), )
if rerender:
head_ref = eval_cmd("git rev-parse HEAD").strip()
LOGGER.info("Rerendering the feedstock")
try:
eval_cmd(
"conda smithy rerender -c auto --no-check-uptodate",
timeout=300,
)
make_rerender_comment = False
except Exception as e:
# I am trying this bit of code to force these errors
# to be surfaced in the logs at the right time.
print(f"RERENDER ERROR: {e}", flush=True)
if not isinstance(migrator, Version):
raise
else:
# for check solvable or automerge, we always raise rerender errors
if feedstock_ctx.attrs["conda-forge.yml"].get(
"bot", {}).get(
"check_solvable",
False,
) or (feedstock_ctx.attrs["conda-forge.yml"].get(
"bot", {}).get(
"automerge",
False,
)):
raise
else:
make_rerender_comment = True
# If we tried to run the MigrationYaml and rerender did nothing (we only
# bumped the build number and dropped a yaml file in migrations) bail
# for instance platform specific migrations
gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD")
diffed_files = [
_ for _ in gdiff.split()
if not (_.startswith("recipe") or _.startswith("migrators")
or _.startswith("README"))
]
else:
make_rerender_comment = False
if (feedstock_ctx.feedstock_name != "conda-forge-pinning"
and base_branch == "master" and
((
migrator.check_solvable
# we always let stuff in cycles go
and feedstock_ctx.attrs["name"] not in getattr(
migrator, "cycles", set())
# we always let stuff at the top go
and feedstock_ctx.attrs["name"] not in getattr(
migrator, "top_level", set())
# for solveability always assume automerge is on.
and
(feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"automerge", True)))
or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"check_solvable",
False,
))):
solvable, errors, _ = is_recipe_solvable(
feedstock_dir,
build_platform=feedstock_ctx.attrs["conda-forge.yml"].get(
"build_platform",
None,
),
)
if not solvable:
_solver_err_str = "not solvable ({}): {}: {}".format(
('<a href="' + os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>'),
base_branch,
sorted(set(errors)),
)
if isinstance(migrator, Version):
_new_ver = feedstock_ctx.attrs["new_version"]
if _new_ver in feedstock_ctx.attrs["new_version_errors"]:
feedstock_ctx.attrs["new_version_errors"][
_new_ver] += "\n\nsolver error - {}".format(
_solver_err_str, )
else:
feedstock_ctx.attrs["new_version_errors"][
_new_ver] = _solver_err_str
feedstock_ctx.attrs["new_version_errors"][
_new_ver] = sanitize_string(
feedstock_ctx.attrs["new_version_errors"][_new_ver], )
# remove part of a try for solver errors to make those slightly
# higher priority
feedstock_ctx.attrs["new_version_attempts"][_new_ver] -= 0.8
pre_key = "pre_pr_migrator_status"
if pre_key not in feedstock_ctx.attrs:
feedstock_ctx.attrs[pre_key] = {}
feedstock_ctx.attrs[pre_key][migrator_name] = sanitize_string(
_solver_err_str, )
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO: Better annotation here
pr_json: typing.Union[MutableMapping, None, bool]
if (isinstance(migrator, MigrationYaml) and not diffed_files
and feedstock_ctx.attrs["name"] != "conda-forge-pinning"):
# spoof this so it looks like the package is done
pr_json = {
"state": "closed",
"merged_at": "never issued",
"id": str(uuid4()),
}
else:
# push up
try:
# TODO: remove this hack, but for now this is the only way to get
# the feedstock dir into pr_body
feedstock_ctx.feedstock_dir = feedstock_dir
pr_json = push_repo(
session_ctx=migrator.ctx.session,
fctx=feedstock_ctx,
feedstock_dir=feedstock_dir,
body=migrator.pr_body(feedstock_ctx),
repo=repo,
title=migrator.pr_title(feedstock_ctx),
head=f"{migrator.ctx.github_username}:{branch_name}",
branch=branch_name,
base_branch=base_branch,
)
# This shouldn't happen too often any more since we won't double PR
except github3.GitHubError as e:
if e.msg != "Validation Failed":
raise
else:
print(f"Error during push {e}")
# If we just push to the existing PR then do nothing to the json
pr_json = False
ljpr = False
if pr_json and pr_json["state"] != "closed" and make_rerender_comment:
comment_on_pr(
pr_json,
"""\
Hi! This feedstock was not able to be rerendered after the version update changes. I
have pushed the version update changes anyways and am trying to rerender again with this
comment. Hopefully you all can fix this!
@conda-forge-admin rerender""",
repo,
)
if pr_json:
ljpr = LazyJson(
os.path.join(migrator.ctx.session.prjson_dir,
str(pr_json["id"]) + ".json"), )
ljpr.update(**pr_json)
# from .dynamo_models import PRJson
# PRJson.dump(pr_json)
else:
ljpr = False
# If we've gotten this far then the node is good
feedstock_ctx.attrs["bad"] = False
LOGGER.info("Removing feedstock dir")
eval_cmd(f"rm -rf {feedstock_dir}")
return migrate_return, ljpr
