def initialize_migrators(github_username="",
github_password="",
github_token=None,
dry_run=False):
'''
Setup graph, required contexts, and migrators
Parameters
----------
github_username: str, optional
Username for bot on GitHub
github_password: str, optional
Password for bot on GitHub
github_token: str, optional
Token for bot on GitHub
dry_run: bool, optional
If true, does not submit pull requests on GitHub
Returns
-------
tuple
Migrator session to interact with GitHub and list of migrators.
Currently only returns pre-defined migrators.
'''
gx = load_graph()
smithy_version = eval_cmd("conda smithy --version").strip()
pinning_version = json.loads(
eval_cmd("conda list conda-forge-pinning --json"))[0]["version"]
for m in MIGRATORS:
print(
f'{getattr(m, "name", m)} graph size: {len(getattr(m, "graph", []))}'
)
ctx = MigratorSessionContext(
circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""),
graph=gx,
smithy_version=smithy_version,
pinning_version=pinning_version,
github_username=github_username,
github_password=github_password,
github_token=github_token,
dry_run=dry_run,
)
return ctx, MIGRATORS
def migrate(
self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any
) -> "MigrationUidTypedDict":
migration_yaml_dict = {
"__migrator": {"build_number": 1, "kind": "version", "migration_number": 1},
self.package_name: [self.new_pin_version],
"migrator_ts": float(time.time()),
}
with indir(os.path.join(recipe_dir, "migrations")):
mig_fname = "{}{}.yaml".format(
self.package_name,
self.new_pin_version.replace(".", ""),
)
with open(mig_fname, "w") as f:
yaml.dump(migration_yaml_dict, f, default_flow_style=False, indent=2)
eval_cmd("git add .")
return super().migrate(recipe_dir, attrs)
def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict",
**kwargs: Any) -> "MigrationUidTypedDict":
# if conda-forge-pinning update the pins and close the migration
if attrs.get("name", "") == "conda-forge-pinning":
# read up the conda build config
with indir(recipe_dir), open("conda_build_config.yaml") as f:
cbc_contents = f.read()
merged_cbc = merge_migrator_cbc(self.yaml_contents, cbc_contents)
with indir(os.path.join(recipe_dir, "migrations")):
os.remove(f"{self.name}.yaml")
# replace the conda build config with the merged one
with indir(recipe_dir), open("conda_build_config.yaml", "w") as f:
f.write(merged_cbc)
# don't need to bump build number once we move to datetime
# version numbers for pinning
return super().migrate(recipe_dir, attrs)
else:
# in case the render is old
os.makedirs(os.path.join(recipe_dir, "../.ci_support"),
exist_ok=True)
with indir(os.path.join(recipe_dir, "../.ci_support")):
os.makedirs("migrations", exist_ok=True)
with indir("migrations"):
with open(f"{self.name}.yaml", "w") as f:
f.write(self.yaml_contents)
eval_cmd("git add .")
if self.conda_forge_yml_patches is not None:
with indir(os.path.join(recipe_dir, "..")):
with open("conda-forge.yml", "r") as fp:
cfg = yaml.safe_load(fp.read())
_patch_dict(cfg, self.conda_forge_yml_patches)
with open("conda-forge.yml", "w") as fp:
yaml.dump(cfg, fp, default_flow_style=False, indent=2)
eval_cmd("git add conda-forge.yml")
with indir(recipe_dir):
self.set_build_number("meta.yaml")
return super().migrate(recipe_dir, attrs)
def main(args: "CLIArgs") -> None:
# logging
if args.debug:
setup_logger(logging.getLogger("conda_forge_tick"), level="debug")
else:
setup_logger(logging.getLogger("conda_forge_tick"))
github_username = env.get("USERNAME", "")
github_password = env.get("PASSWORD", "")
github_token = env.get("GITHUB_TOKEN")
mctx, temp, migrators = initialize_migrators(
github_username=github_username,
github_password=github_password,
dry_run=args.dry_run,
github_token=github_token,
)
# compute the time per migrator
print("computing time per migration", flush=True)
(num_nodes, time_per_migrator,
tot_time_per_migrator) = _compute_time_per_migrator(
mctx,
migrators,
)
for i, migrator in enumerate(migrators):
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
print(
" %s%s: %d - gets %f seconds (%f percent)" % (
migrator.__class__.__name__,
extra_name,
num_nodes[i],
time_per_migrator[i],
time_per_migrator[i] / max(tot_time_per_migrator, 1) * 100,
),
flush=True,
)
for mg_ind, migrator in enumerate(migrators):
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
time_per = time_per_migrator[mg_ind]
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
print(
"\n========================================"
"========================================"
"\n"
"========================================"
"========================================",
flush=True,
)
print(
"Running migrations for %s%s: %d\n" % (
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
),
flush=True,
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
LOGGER.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
LOGGER.info(
" node|curr|new|attempts: %s|%s|%s|%d",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""),
0,
)),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
base_branches = migrator.get_possible_feedstock_branches(attrs)
orig_branch = attrs.get("branch", "master")
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if ((_now - int(env.get("START_TIME", time.time())) > int(
env.get("TIMEOUT", 600)))
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per):
break
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
try:
for base_branch in base_branches:
attrs["branch"] = base_branch
if migrator.filter(attrs):
continue
print("\n", flush=True, end="")
LOGGER.info(
"%s%s IS MIGRATING %s:%s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
base_branch,
)
try:
# Don't bother running if we are at zero
if mctx.gh_api_requests_left == 0:
break
migrator_uid, pr_json = run(
feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get("hash_type", "sha256"),
base_branch=base_branch,
)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"]
for existing_pr in attrs.get(
"PRed", [])
]:
pass
else:
if not pr_json:
pr_json = {
"state": "closed",
"head": {
"ref": "<this_is_not_a_branch>"
},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version":
mctx.pinning_version,
}, )
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
LOGGER.critical(
"GITHUB ERROR ON FEEDSTOCK: %s",
fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
LOGGER.exception("URLError ERROR")
attrs["bad"] = {
"exception":
str(e),
"traceback":
str(traceback.format_exc()).split("\n"),
"code":
getattr(e, "code"),
"url":
getattr(e, "url"),
}
pre_key = "pre_pr_migrator_status"
if pre_key not in attrs:
attrs[pre_key] = {}
attrs[pre_key][migrator_name] = sanitize_string(
"bot error (%s): %s: %s" % (
'<a href="' +
os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
), )
except Exception as e:
LOGGER.exception("NON GITHUB ERROR")
# we don't set bad for rerendering errors
if ("conda smithy rerender -c auto --no-check-uptodate"
not in str(e)):
attrs["bad"] = {
"exception":
str(e),
"traceback":
str(traceback.format_exc()).split("\n", ),
}
pre_key = "pre_pr_migrator_status"
if pre_key not in attrs:
attrs[pre_key] = {}
attrs[pre_key][migrator_name] = sanitize_string(
"bot error (%s): %s: %s" % (
'<a href="' +
os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
), )
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# reset branch
attrs["branch"] = orig_branch
# Write graph partially through
if not args.dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
LOGGER.info(os.getcwd())
for f in glob.glob("/tmp/*"):
if f not in temp:
try:
eval_cmd(f"rm -rf {f}")
except Exception:
pass
if mctx.gh_api_requests_left == 0:
break
print("\n", flush=True)
LOGGER.info("API Calls Remaining: %d", mctx.gh_api_requests_left)
LOGGER.info("Done")
def initialize_migrators(
github_username: str = "",
github_password: str = "",
github_token: Optional[str] = None,
dry_run: bool = False,
) -> Tuple[MigratorSessionContext, list, MutableSequence[Migrator]]:
temp = glob.glob("/tmp/*")
gx = load_graph()
smithy_version = eval_cmd("conda smithy --version").strip()
pinning_version = json.loads(
eval_cmd("conda list conda-forge-pinning --json"))[0]["version"]
migrators = []
add_arch_migrate(migrators, gx)
migration_factory(migrators, gx)
add_replacement_migrator(
migrators,
gx,
"matplotlib",
"matplotlib-base",
("Unless you need `pyqt`, recipes should depend only on "
"`matplotlib-base`."),
alt_migrator=MatplotlibBase,
)
create_migration_yaml_creator(migrators=migrators, gx=gx)
print("rebuild migration graph sizes:", flush=True)
for m in migrators:
print(
f' {getattr(m, "name", m)} graph size: '
f'{len(getattr(m, "graph", []))}',
flush=True,
)
print(" ", flush=True)
mctx = MigratorSessionContext(
circle_build_url=os.getenv("CIRCLE_BUILD_URL", ""),
graph=gx,
smithy_version=smithy_version,
pinning_version=pinning_version,
github_username=github_username,
github_password=github_password,
github_token=github_token,
dry_run=dry_run,
)
print("building package import maps and version migrator", flush=True)
python_nodes = {
n
for n, v in mctx.graph.nodes("payload")
if "python" in v.get("req", "")
}
python_nodes.update([
k for node_name, node in mctx.graph.nodes("payload")
for k in node.get("outputs_names", []) if node_name in python_nodes
], )
version_migrator = Version(
python_nodes=python_nodes,
pr_limit=PR_LIMIT * 4,
piggy_back_migrations=[
Jinja2VarsCleanup(),
DuplicateLinesCleanup(),
PipMigrator(),
LicenseMigrator(),
CondaForgeYAMLCleanup(),
ExtraJinja2KeysCleanup(),
Build2HostMigrator(),
NoCondaInspectMigrator(),
Cos7Config(),
],
)
migrators = [version_migrator] + migrators
print(" ", flush=True)
return mctx, temp, migrators
def run(
feedstock_ctx: FeedstockContext,
migrator: Migrator,
protocol: str = "ssh",
pull_request: bool = True,
rerender: bool = True,
fork: bool = True,
base_branch: str = "master",
**kwargs: typing.Any,
) -> Tuple["MigrationUidTypedDict", dict]:
"""For a given feedstock and migration run the migration
Parameters
----------
feedstock_ctx: FeedstockContext
The node attributes
migrator: Migrator instance
The migrator to run on the feedstock
protocol : str, optional
The git protocol to use, defaults to ``ssh``
pull_request : bool, optional
If true issue pull request, defaults to true
rerender : bool
Whether to rerender
fork : bool
If true create a fork, defaults to true
base_branch : str, optional
The base branch to which the PR will be targeted. Defaults to "master".
kwargs: dict
The key word arguments to pass to the migrator
Returns
-------
migrate_return: MigrationUidTypedDict
The migration return dict used for tracking finished migrations
pr_json: dict
The PR json object for recreating the PR as needed
"""
# get the repo
# TODO: stop doing this.
migrator.attrs = feedstock_ctx.attrs # type: ignore
branch_name = migrator.remote_branch(
feedstock_ctx) + "_h" + uuid4().hex[0:6]
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
# TODO: run this in parallel
feedstock_dir, repo = get_repo(
ctx=migrator.ctx.session,
fctx=feedstock_ctx,
branch=branch_name,
feedstock=feedstock_ctx.feedstock_name,
protocol=protocol,
pull_request=pull_request,
fork=fork,
base_branch=base_branch,
)
if not feedstock_dir or not repo:
LOGGER.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
return False, False
recipe_dir = os.path.join(feedstock_dir, "recipe")
# migrate the feedstock
migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO - make a commit here if the repo changed
migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs,
**kwargs)
if not migrate_return:
LOGGER.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO - commit main migration here
migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO commit post migration here
# rerender, maybe
diffed_files: typing.List[str] = []
with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
msg = migrator.commit_message(feedstock_ctx) # noqa
try:
eval_cmd("git add --all .")
eval_cmd(f"git commit -am '{msg}'")
except CalledProcessError as e:
LOGGER.info(
"could not commit to feedstock - "
"likely no changes - error is '%s'" % (repr(e)), )
if rerender:
head_ref = eval_cmd("git rev-parse HEAD").strip()
LOGGER.info("Rerendering the feedstock")
try:
eval_cmd(
"conda smithy rerender -c auto --no-check-uptodate",
timeout=300,
)
make_rerender_comment = False
except Exception as e:
# I am trying this bit of code to force these errors
# to be surfaced in the logs at the right time.
print(f"RERENDER ERROR: {e}", flush=True)
if not isinstance(migrator, Version):
raise
else:
# for check solvable or automerge, we always raise rerender errors
if feedstock_ctx.attrs["conda-forge.yml"].get(
"bot", {}).get(
"check_solvable",
False,
) or (feedstock_ctx.attrs["conda-forge.yml"].get(
"bot", {}).get(
"automerge",
False,
)):
raise
else:
make_rerender_comment = True
# If we tried to run the MigrationYaml and rerender did nothing (we only
# bumped the build number and dropped a yaml file in migrations) bail
# for instance platform specific migrations
gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD")
diffed_files = [
_ for _ in gdiff.split()
if not (_.startswith("recipe") or _.startswith("migrators")
or _.startswith("README"))
]
else:
make_rerender_comment = False
if (feedstock_ctx.feedstock_name != "conda-forge-pinning"
and base_branch == "master" and
((
migrator.check_solvable
# we always let stuff in cycles go
and feedstock_ctx.attrs["name"] not in getattr(
migrator, "cycles", set())
# we always let stuff at the top go
and feedstock_ctx.attrs["name"] not in getattr(
migrator, "top_level", set())
# for solveability always assume automerge is on.
and
(feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"automerge", True)))
or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"check_solvable",
False,
))):
solvable, errors, _ = is_recipe_solvable(
feedstock_dir,
build_platform=feedstock_ctx.attrs["conda-forge.yml"].get(
"build_platform",
None,
),
)
if not solvable:
_solver_err_str = "not solvable ({}): {}: {}".format(
('<a href="' + os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>'),
base_branch,
sorted(set(errors)),
)
if isinstance(migrator, Version):
_new_ver = feedstock_ctx.attrs["new_version"]
if _new_ver in feedstock_ctx.attrs["new_version_errors"]:
feedstock_ctx.attrs["new_version_errors"][
_new_ver] += "\n\nsolver error - {}".format(
_solver_err_str, )
else:
feedstock_ctx.attrs["new_version_errors"][
_new_ver] = _solver_err_str
feedstock_ctx.attrs["new_version_errors"][
_new_ver] = sanitize_string(
feedstock_ctx.attrs["new_version_errors"][_new_ver], )
# remove part of a try for solver errors to make those slightly
# higher priority
feedstock_ctx.attrs["new_version_attempts"][_new_ver] -= 0.8
pre_key = "pre_pr_migrator_status"
if pre_key not in feedstock_ctx.attrs:
feedstock_ctx.attrs[pre_key] = {}
feedstock_ctx.attrs[pre_key][migrator_name] = sanitize_string(
_solver_err_str, )
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO: Better annotation here
pr_json: typing.Union[MutableMapping, None, bool]
if (isinstance(migrator, MigrationYaml) and not diffed_files
and feedstock_ctx.attrs["name"] != "conda-forge-pinning"):
# spoof this so it looks like the package is done
pr_json = {
"state": "closed",
"merged_at": "never issued",
"id": str(uuid4()),
}
else:
# push up
try:
# TODO: remove this hack, but for now this is the only way to get
# the feedstock dir into pr_body
feedstock_ctx.feedstock_dir = feedstock_dir
pr_json = push_repo(
session_ctx=migrator.ctx.session,
fctx=feedstock_ctx,
feedstock_dir=feedstock_dir,
body=migrator.pr_body(feedstock_ctx),
repo=repo,
title=migrator.pr_title(feedstock_ctx),
head=f"{migrator.ctx.github_username}:{branch_name}",
branch=branch_name,
base_branch=base_branch,
)
# This shouldn't happen too often any more since we won't double PR
except github3.GitHubError as e:
if e.msg != "Validation Failed":
raise
else:
print(f"Error during push {e}")
# If we just push to the existing PR then do nothing to the json
pr_json = False
ljpr = False
if pr_json and pr_json["state"] != "closed" and make_rerender_comment:
comment_on_pr(
pr_json,
"""\
Hi! This feedstock was not able to be rerendered after the version update changes. I
have pushed the version update changes anyways and am trying to rerender again with this
comment. Hopefully you all can fix this!
@conda-forge-admin rerender""",
repo,
)
if pr_json:
ljpr = LazyJson(
os.path.join(migrator.ctx.session.prjson_dir,
str(pr_json["id"]) + ".json"), )
ljpr.update(**pr_json)
# from .dynamo_models import PRJson
# PRJson.dump(pr_json)
else:
ljpr = False
# If we've gotten this far then the node is good
feedstock_ctx.attrs["bad"] = False
LOGGER.info("Removing feedstock dir")
eval_cmd(f"rm -rf {feedstock_dir}")
return migrate_return, ljpr
def auto_tick(dry_run=False,
debug=False,
fork=False,
organization='nsls-ii-forge'):
'''
Automatically update package versions and submit pull requests to
associated feedstocks
Parameters
----------
dry_run: bool, optional
Generate version migration yamls but do not run them
debug: bool, optional
Setup logging to be in debug mode
fork: bool, optional
Create a fork of the repo from the organization to $GITHUB_USERNAME
organization: str, optional
GitHub organization that manages feedstock repositories
'''
from conda_forge_tick.xonsh_utils import env
if debug:
setup_logger(logger, level="debug")
else:
setup_logger(logger)
# set Version.pr_body to custom pr_body function
Version.pr_body = bot_pr_body
# TODO: use ~/.netrc instead
github_username = env.get("GITHUB_USERNAME", "")
github_password = env.get("GITHUB_TOKEN", "")
github_token = env.get("GITHUB_TOKEN")
global MIGRATORS
print('Initializing migrators...')
mctx, MIGRATORS = initialize_migrators(
github_username=github_username,
github_password=github_password,
dry_run=dry_run,
github_token=github_token,
)
# compute the time per migrator
print('Computing time per migrator')
(num_nodes, time_per_migrator,
tot_time_per_migrator) = _compute_time_per_migrator(mctx, )
for i, migrator in enumerate(MIGRATORS):
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
logger.info(
"Total migrations for %s%s: %d - gets %f seconds (%f percent)",
migrator.__class__.__name__,
extra_name,
num_nodes[i],
time_per_migrator[i],
time_per_migrator[i] / tot_time_per_migrator * 100,
)
print('Performing migrations...')
for mg_ind, migrator in enumerate(MIGRATORS):
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
time_per = time_per_migrator[mg_ind]
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
logger.info(
"Running migrations for %s%s: %d",
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
logger.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
logger.info(
" node|curr|new|attempts: %s|%s|%s|%d",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""),
0,
)),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if ((_now - int(env.get("START_TIME", time.time())) > int(
env.get("TIMEOUT", 600)))
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per):
break
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
print("\n", flush=True, end="")
logger.info(
"%s%s IS MIGRATING %s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
)
try:
# Don't bother running if we are at zero
if (dry_run or mctx.gh.rate_limit()["resources"]["core"]
["remaining"] == 0):
break
migrator_uid, pr_json = run(feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get(
"hash_type", "sha256"),
fork=fork,
organization=organization)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"]
for existing_pr in attrs.get("PRed", [])
]:
pass
else:
if pr_json is None:
pr_json = {
"state": "closed",
"head": {
"ref": "<this_is_not_a_branch>"
},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version": mctx.pinning_version,
}, )
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
logger.critical(
"GITHUB ERROR ON FEEDSTOCK: %s",
fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
logger.exception("URLError ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
"code": getattr(e, "code"),
"url": getattr(e, "url"),
}
except Exception as e:
logger.exception("NON GITHUB ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
}
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# Write graph partially through
if not dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
logger.info(os.getcwd())
if not dry_run:
logger.info(
"API Calls Remaining: %d",
mctx.gh.rate_limit()["resources"]["core"]["remaining"],
)
logger.info("Done")
def run(feedstock_ctx,
migrator,
protocol='ssh',
pull_request=True,
rerender=True,
fork=False,
organization='nsls-ii-forge',
**kwargs):
"""
For a given feedstock and migration run the migration and possibly submit
pull request
Parameters
----------
feedstock_ctx: FeedstockContext
The node attributes of the feedstock
migrator: Migrator
The migrator to run on the feedstock
protocol: str, optional
The git protocol to use, defaults to ``ssh``
pull_request: bool, optional
If true issue pull request, defaults to true
rerender: bool
Whether to rerender, defaults to true
fork: bool
If true create a fork, defaults to false
organization: str, optional
GitHub organization to get repo from
gh: github3.GitHub, optional
Object for communicating with GitHub, if None, build from $GITHUB_USERNAME
and $GITHUB_PASSWORD, defaults to None
kwargs: dict
The key word arguments to pass to the migrator
Returns
-------
migrate_return: MigrationUidTypedDict
The migration return dict used for tracking finished migrations
pr_json: dict
The PR json object for recreating the PR as needed
"""
# get the repo
migrator.attrs = feedstock_ctx.attrs
branch_name = migrator.remote_branch(
feedstock_ctx) + "_h" + uuid4().hex[:6]
# TODO: run this in parallel
feedstock_dir, repo = get_repo(
ctx=migrator.ctx.session,
fctx=feedstock_ctx,
branch=branch_name,
organization=organization,
feedstock=feedstock_ctx.feedstock_name,
protocol=protocol,
pull_request=pull_request,
fork=fork,
)
recipe_dir = os.path.join(feedstock_dir, "recipe")
# migrate the feedstock
migrator.run_pre_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO - make a commit here if the repo changed
migrate_return = migrator.migrate(recipe_dir, feedstock_ctx.attrs,
**kwargs)
if not migrate_return:
logger.critical(
"Failed to migrate %s, %s",
feedstock_ctx.package_name,
feedstock_ctx.attrs.get("bad"),
)
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
# TODO - commit main migration here
migrator.run_post_piggyback_migrations(recipe_dir, feedstock_ctx.attrs,
**kwargs)
# TODO commit post migration here
# rerender, maybe
diffed_files = []
with indir(feedstock_dir), env.swap(RAISE_SUBPROC_ERROR=False):
msg = migrator.commit_message(feedstock_ctx) # noqa
try:
eval_cmd("git add --all .")
eval_cmd(f"git commit -am '{msg}'")
except CalledProcessError as e:
logger.info(
"could not commit to feedstock - "
"likely no changes - error is '%s'" % (repr(e)), )
if rerender:
head_ref = eval_cmd("git rev-parse HEAD").strip()
logger.info("Rerendering the feedstock")
# In the event we can't rerender, try to update the pinnings,
# then bail if it does not work again
try:
eval_cmd(
"conda smithy rerender -c auto --no-check-uptodate",
timeout=300,
)
except SubprocessError:
return False, False
# If we tried to run the MigrationYaml and rerender did nothing (we only
# bumped the build number and dropped a yaml file in migrations) bail
# for instance platform specific migrations
gdiff = eval_cmd(f"git diff --name-only {head_ref.strip()}...HEAD")
diffed_files = [
_ for _ in gdiff.split()
if not (_.startswith("recipe") or _.startswith("migrators")
or _.startswith("README"))
]
if ((migrator.check_solvable
and feedstock_ctx.attrs["conda-forge.yml"].get("bot",
{}).get("automerge"))
or feedstock_ctx.attrs["conda-forge.yml"].get("bot", {}).get(
"check_solvable",
False)) and not is_recipe_solvable(feedstock_dir):
eval_cmd(f"rm -rf {feedstock_dir}")
return False, False
if (isinstance(migrator, MigrationYaml) and not diffed_files
and feedstock_ctx.attrs["name"] != "conda-forge-pinning"):
# spoof this so it looks like the package is done
pr_json = {
"state": "closed",
"merged_at": "never issued",
"id": str(uuid4()),
}
else:
# push up
try:
if fork:
head = f"{migrator.ctx.github_username}:{branch_name}"
else:
head = f"{organization}:{branch_name}"
pr_json = push_repo(session_ctx=migrator.ctx.session,
fctx=feedstock_ctx,
feedstock_dir=feedstock_dir,
body=migrator.pr_body(feedstock_ctx),
repo=repo,
title=migrator.pr_title(feedstock_ctx),
head=head,
branch=branch_name,
fork=fork,
organization=organization)
# This shouldn't happen too often any more since we won't double PR
except github3.GitHubError as e:
if e.msg != "Validation Failed":
raise
else:
print(f"Error during push {e}")
print(f'Errors: {e.errors}')
# If we just push to the existing PR then do nothing to the json
pr_json = None
ljpr = None
if pr_json is not None:
ljpr = LazyJson(
os.path.join(migrator.ctx.session.prjson_dir,
str(pr_json["id"]) + ".json"), )
ljpr.update(**pr_json)
else:
ljpr = None
# If we've gotten this far then the node is good
feedstock_ctx.attrs["bad"] = False
logger.info("Removing feedstock dir")
eval_cmd(f"rm -rf {feedstock_dir}")
return migrate_return, ljpr
def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict",
**kwargs: Any) -> None:
# r- recipes have a special syntax here
if (attrs.get("feedstock_name", "").startswith("r-") or attrs.get(
"name",
"").startswith("r-")) and "r-base" in attrs["raw_meta_yaml"]:
if attrs.get("feedstock_name", None) is not None:
if attrs.get("feedstock_name", None).endswith("-feedstock"):
name = attrs.get("feedstock_name")[:-len("-feedstock")]
else:
name = attrs.get("feedstock_name")
else:
name = attrs.get("name", None)
_do_r_license_munging(name, recipe_dir)
return
try:
cb_work_dir = _get_source_code(recipe_dir)
except Exception:
return
if cb_work_dir is None:
return
with indir(cb_work_dir):
# look for a license file
license_files = [
s for s in os.listdir(".")
if any(s.lower().startswith(k)
for k in ["license", "copying", "copyright"])
]
eval_cmd(f"rm -r {cb_work_dir}")
# if there is a license file in tarball update things
if license_files:
with indir(recipe_dir):
"""BSD 3-Clause License
Copyright (c) 2017, Anthony Scopatz
Copyright (c) 2018, The Regro Developers
All rights reserved."""
with open("meta.yaml") as f:
raw = f.read()
lines = raw.splitlines()
ptn = re.compile(r"(\s*?)" + "license:")
for i, line in enumerate(lines):
m = ptn.match(line)
if m is not None:
break
# TODO: Sketchy type assertion
assert m is not None
ws = m.group(1)
if len(license_files) == 1:
replace_in_file(
line,
line + "\n" + ws +
f"license_file: {list(license_files)[0]}",
"meta.yaml",
)
else:
# note that this white space is not perfect but works for
# most of the situations
replace_in_file(
line,
line + "\n" + ws + "license_file: \n" +
"".join(f"{ws*2}- {z} \n" for z in license_files),
"meta.yaml",
)
def _run_migrator(migrator, mctx, temp, time_per, dry_run):
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
print(
"Running migrations for %s%s: %d\n"
% (
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
),
flush=True,
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
LOGGER.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
LOGGER.info(
" node|curr|new|attempts: %s|%s|%s|%f",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(
attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""),
0,
)
),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if (
(
_now - int(env.get("START_TIME", time.time()))
> int(env.get("TIMEOUT", 600))
)
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per
):
break
base_branches = migrator.get_possible_feedstock_branches(attrs)
if "branch" in attrs:
has_attrs_branch = True
orig_branch = attrs.get("branch")
else:
has_attrs_branch = False
orig_branch = None
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
# map main to current default branch
base_branches = [
br if br != "main" else fctx.default_branch for br in base_branches
]
try:
for base_branch in base_branches:
attrs["branch"] = base_branch
if migrator.filter(attrs):
continue
print("\n", flush=True, end="")
sys.stderr.flush()
sys.stdout.flush()
LOGGER.info(
"%s%s IS MIGRATING %s:%s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
base_branch,
)
try:
# Don't bother running if we are at zero
if mctx.gh_api_requests_left == 0:
break
migrator_uid, pr_json = run(
feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get("hash_type", "sha256"),
base_branch=base_branch,
)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"]
for existing_pr in attrs.get("PRed", [])
]:
pass
else:
if not pr_json:
pr_json = {
"state": "closed",
"head": {"ref": "<this_is_not_a_branch>"},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version": mctx.pinning_version,
},
)
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
LOGGER.critical(
"GITHUB ERROR ON FEEDSTOCK: %s",
fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
LOGGER.exception("URLError ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
"code": getattr(e, "code"),
"url": getattr(e, "url"),
}
_set_pre_pr_migrator_fields(
attrs,
migrator_name,
sanitize_string(
"bot error (%s): %s: %s"
% (
'<a href="'
+ os.getenv("CIRCLE_BUILD_URL", "")
+ '">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
),
),
)
except Exception as e:
LOGGER.exception("NON GITHUB ERROR")
# we don't set bad for rerendering errors
if (
"conda smithy rerender -c auto --no-check-uptodate"
not in str(e)
):
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split(
"\n",
),
}
_set_pre_pr_migrator_fields(
attrs,
migrator_name,
sanitize_string(
"bot error (%s): %s: %s"
% (
'<a href="'
+ os.getenv("CIRCLE_BUILD_URL", "")
+ '">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
),
),
)
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# reset branch
if has_attrs_branch:
attrs["branch"] = orig_branch
# do this but it is crazy
gc.collect()
# Write graph partially through
if not dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
LOGGER.info(os.getcwd())
for f in glob.glob("/tmp/*"):
if f not in temp:
try:
eval_cmd(f"rm -rf {f}")
except Exception:
pass
if mctx.gh_api_requests_left == 0:
break
return good_prs