def migrate(d: dict):
pred = d.get('PRed', None)
if pred:
l = []
for element in d['PRed']:
l.append(frozen_to_json_friendly(element))
d['PRed'] = l
pred_json = d.get('PRed_json', None)
if pred_json:
l = []
for element, pr in d['PRed_json'].items():
l.append(frozen_to_json_friendly(element, pr))
d['PRed_json'] = l
def predecessors_not_yet_built(self, attrs: "AttrsTypedDict") -> bool:
# Check if all upstreams have been built
for node, payload in _gen_active_feedstocks_payloads(
self.graph.predecessors(attrs["feedstock_name"]),
self.graph,
):
muid = frozen_to_json_friendly(self.migrator_uid(payload))
if muid not in _sanitized_muids(payload.get("PRed", []), ):
LOGGER.debug("not yet built: %s" % node)
return True
# This is due to some PRed_json loss due to bad graph deploy outage
for m_pred_json in payload.get("PRed", []):
if m_pred_json["data"] == muid["data"]:
break
else:
m_pred_json = None
# note that if the bot is missing the PR we assume it is open
# so that errors halt the migration and can be fixed
if (m_pred_json and m_pred_json.get("PR", {
"state": "open"
}).get("state", "") == "open"):
LOGGER.debug("not yet built dataloss: %s" % node)
return True
return False
def predecessors_not_yet_built(self, attrs: "AttrsTypedDict") -> bool:
# Check if all upstreams have been built
for node in self.graph.predecessors(attrs["feedstock_name"]):
try:
payload = self.graph.nodes[node]["payload"]
except KeyError as e:
print(node)
raise e
muid = frozen_to_json_friendly(self.migrator_uid(payload))
if muid not in _sanitized_muids(
payload.get("PRed", []),
) and not payload.get("archived", False):
return True
# This is due to some PRed_json loss due to bad graph deploy outage
for m_pred_json in payload.get("PRed", []):
if m_pred_json["data"] == muid["data"]:
break
else:
m_pred_json = None
# note that if the bot is missing the PR we assume it is open
# so that errors halt the migration and can be fixed
if (
m_pred_json
and m_pred_json.get("PR", {"state": "open"}).get("state", "") == "open"
):
return True
return False
def parse_already_pred() -> bool:
pr_data = frozen_to_json_friendly(self.migrator_uid(attrs))
migrator_uid: "MigrationUidTypedDict" = typing.cast(
"MigrationUidTypedDict",
pr_data["data"],
)
already_migrated_uids: typing.Iterable[
"MigrationUidTypedDict"] = list(z["data"]
for z in attrs.get("PRed", []))
already_pred = migrator_uid in already_migrated_uids
if already_pred:
ind = already_migrated_uids.index(migrator_uid)
LOGGER.debug(f"{__name}: already PRed: uid: {migrator_uid}")
if "PR" in attrs.get("PRed", [])[ind]:
if isinstance(attrs.get("PRed", [])[ind]["PR"], LazyJson):
with attrs.get("PRed", [])[ind]["PR"] as mg_attrs:
LOGGER.debug(
"%s: already PRed: PR file: %s" %
(__name, mg_attrs.file_name), )
html_url = mg_attrs.get("html_url", "no url")
LOGGER.debug(
f"{__name}: already PRed: url: {html_url}")
return already_pred
def parse_already_pred() -> bool:
migrator_uid: "MigrationUidTypedDict" = typing.cast(
"MigrationUidTypedDict",
frozen_to_json_friendly(self.migrator_uid(attrs))["data"],
)
already_migrated_uids: typing.Iterable["MigrationUidTypedDict"] = (
z["data"] for z in attrs.get("PRed", []))
return migrator_uid in already_migrated_uids
def all_predecessors_issued_and_stale(self,
attrs: "AttrsTypedDict") -> bool:
# Check if all upstreams have been issue and are stale
for node, payload in _gen_active_feedstocks_payloads(
self.graph.predecessors(attrs["feedstock_name"]),
self.graph,
):
if node in self.ignored_deps_per_node.get(
attrs.get("feedstock_name", None),
[],
):
continue
muid = frozen_to_json_friendly(self.migrator_uid(payload))
pr_muids = _sanitized_muids(payload.get("PRed", []))
if muid not in pr_muids:
LOGGER.debug(
"node %s PR %s not yet issued!",
node,
muid.get("data", {}).get("name", None),
)
# not yet issued
return False
else:
# issued so check timestamp
pr_index = pr_muids.index(muid)
ts = (payload.get("PRed", [])[pr_index].get("PR", {}).get(
"created_at", None))
state = (payload.get("PRed",
[])[pr_index].get("PR", {
"state": "open"
}).get("state", ""))
if state == "open":
if ts is not None:
now = datetime.datetime.now(datetime.timezone.utc)
ts = dateutil.parser.parse(ts)
if now - ts < datetime.timedelta(days=14):
LOGGER.debug(
"node %s has PR %s open for %s",
node,
muid.get("data", {}).get("name", None),
now - ts,
)
return False
else:
# no timestamp so keep things open
LOGGER.debug(
"node %s has PR %s:%s with no timestamp",
node,
muid.get("data", {}).get("name", None),
payload.get("PRed", [])[pr_index]["PR"].file_name,
)
return False
return True
def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
if super().filter(attrs):
return True
muid = frozen_to_json_friendly(self.migrator_uid(attrs))
for arch in self.arches:
configured_arch = (
attrs.get("conda-forge.yml", {}).get("provider", {}).get(arch)
)
if configured_arch:
return muid in _sanitized_muids(attrs.get("PRed", []))
else:
return False
def run_test_yaml_migration(m,
*,
inp,
output,
kwargs,
prb,
mr_out,
tmpdir,
should_filter=False):
os.makedirs(os.path.join(tmpdir, "recipe"), exist_ok=True)
with open(os.path.join(tmpdir, "recipe", "meta.yaml"), "w") as f:
f.write(inp)
with indir(tmpdir):
subprocess.run(["git", "init"])
# Load the meta.yaml (this is done in the graph)
try:
pmy = parse_meta_yaml(inp)
except Exception:
pmy = {}
if pmy:
pmy["version"] = pmy["package"]["version"]
pmy["req"] = set()
for k in ["build", "host", "run"]:
pmy["req"] |= set(pmy.get("requirements", {}).get(k, set()))
try:
pmy["meta_yaml"] = parse_meta_yaml(inp)
except Exception:
pmy["meta_yaml"] = {}
pmy["raw_meta_yaml"] = inp
pmy.update(kwargs)
assert m.filter(pmy) is should_filter
if should_filter:
return
mr = m.migrate(os.path.join(tmpdir, "recipe"), pmy)
assert mr_out == mr
pmy.update(PRed=[frozen_to_json_friendly(mr)])
with open(os.path.join(tmpdir, "recipe/meta.yaml")) as f:
actual_output = f.read()
assert actual_output == output
assert os.path.exists(
os.path.join(tmpdir, ".ci_support/migrations/hi.yaml"))
with open(os.path.join(tmpdir, ".ci_support/migrations/hi.yaml")) as f:
saved_migration = f.read()
assert saved_migration == m.yaml_contents
def parse_already_pred() -> bool:
migrator_uid: "MigrationUidTypedDict" = typing.cast(
"MigrationUidTypedDict",
frozen_to_json_friendly(self.migrator_uid(attrs))["data"],
)
already_migrated_uids: typing.Iterable[
"MigrationUidTypedDict"] = list(
copy.deepcopy(z["data"]) for z in attrs.get("PRed", []))
# we shipped a bug, so fixing this here -
# need to ignore name in uuid
for uid in already_migrated_uids:
if uid["migrator_name"] == "MatplotlibBase" and "name" in uid:
del uid["name"]
del migrator_uid["name"]
return migrator_uid in already_migrated_uids
def test_migration(m, inp, output, kwargs, prb, mr_out, should_filter, tmpdir):
mr_out.update(bot_rerun=False)
with open(os.path.join(tmpdir, "meta.yaml"), "w") as f:
f.write(inp)
# Load the meta.yaml (this is done in the graph)
try:
pmy = parse_meta_yaml(inp)
except Exception:
pmy = {}
if pmy:
pmy["version"] = pmy["package"]["version"]
pmy["req"] = set()
for k in ["build", "host", "run"]:
pmy["req"] |= set(pmy.get("requirements", {}).get(k, set()))
try:
pmy["meta_yaml"] = parse_meta_yaml(inp)
except Exception:
pmy["meta_yaml"] = {}
pmy["raw_meta_yaml"] = inp
pmy.update(kwargs)
assert m.filter(pmy) is should_filter
if should_filter:
return
mr = m.migrate(tmpdir, pmy)
assert mr_out == mr
pmy.update(PRed=[frozen_to_json_friendly(mr)])
with open(os.path.join(tmpdir, "meta.yaml"), "r") as f:
actual_output = f.read()
assert actual_output == output
if isinstance(m, Compiler):
assert m.messages in m.pr_body()
# TODO: fix subgraph here (need this to be xsh file)
elif isinstance(m, Version):
pass
elif isinstance(m, Rebuild):
return
else:
assert prb in m.pr_body()
assert m.filter(pmy) is True
def _test_node_built(node):
# check to see if node has been built
payload = self.graph.nodes[node]["payload"]
muid = frozen_to_json_friendly(self.migrator_uid(payload))
if muid not in _sanitized_muids(payload.get(
"PRed", []), ) and not payload.get("archived", False):
return False
# This is due to some PRed_json loss due to bad graph deploy outage
for m_pred_json in payload.get("PRed", []):
if m_pred_json["data"] == muid["data"]:
break
else:
m_pred_json = None
# note that if the bot is missing the PR we assume it is open
# so that errors halt the migration and can be fixed
if (m_pred_json and (m_pred_json.get("PR", {
"state": "open"
}).get("state", "")) == "open"):
return False
return True
def main(args: "CLIArgs") -> None:
# logging
from .xonsh_utils import env
debug = env.get("CONDA_FORGE_TICK_DEBUG", False)
if debug:
setup_logger(logging.getLogger("conda_forge_tick"), level="debug")
else:
setup_logger(logging.getLogger("conda_forge_tick"))
github_username = env.get("USERNAME", "")
github_password = env.get("PASSWORD", "")
github_token = env.get("GITHUB_TOKEN")
global MIGRATORS
mctx, temp, MIGRATORS = initialize_migrators(
github_username=github_username,
github_password=github_password,
dry_run=args.dry_run,
github_token=github_token,
)
# compute the time per migrator
(num_nodes, time_per_migrator, tot_time_per_migrator) = _compute_time_per_migrator(
mctx,
)
for i, migrator in enumerate(MIGRATORS):
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
logger.info(
"Total migrations for %s%s: %d - gets %f seconds (%f percent)",
migrator.__class__.__name__,
extra_name,
num_nodes[i],
time_per_migrator[i],
time_per_migrator[i] / tot_time_per_migrator * 100,
)
for mg_ind, migrator in enumerate(MIGRATORS):
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
time_per = time_per_migrator[mg_ind]
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
logger.info(
"Running migrations for %s%s: %d",
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
logger.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
logger.info(
" node|curr|new|attempts: %s|%s|%s|%d",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(
attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""), 0,
)
),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if (
(
_now - int(env.get("START_TIME", time.time()))
> int(env.get("TIMEOUT", 600))
)
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per
):
break
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
print("\n", flush=True, end="")
logger.info(
"%s%s IS MIGRATING %s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
)
try:
# Don't bother running if we are at zero
if (
args.dry_run
or mctx.gh.rate_limit()["resources"]["core"]["remaining"] == 0
):
break
migrator_uid, pr_json = run(
feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get("hash_type", "sha256"),
)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"] for existing_pr in attrs.get("PRed", [])
]:
pass
else:
if not pr_json:
pr_json = {
"state": "closed",
"head": {"ref": "<this_is_not_a_branch>"},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version": mctx.pinning_version,
},
)
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
logger.critical(
"GITHUB ERROR ON FEEDSTOCK: %s", fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
logger.exception("URLError ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
"code": getattr(e, "code"),
"url": getattr(e, "url"),
}
except Exception as e:
logger.exception("NON GITHUB ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
}
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# Write graph partially through
if not args.dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
logger.info(os.getcwd())
for f in glob.glob("/tmp/*"):
if f not in temp:
eval_cmd(f"rm -rf {f}")
if not args.dry_run:
logger.info(
"API Calls Remaining: %d",
mctx.gh.rate_limit()["resources"]["core"]["remaining"],
)
logger.info("Done")
def _run_migrator(migrator, mctx, temp, time_per, dry_run):
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
print(
"Running migrations for %s%s: %d\n"
% (
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
),
flush=True,
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
LOGGER.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
LOGGER.info(
" node|curr|new|attempts: %s|%s|%s|%f",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(
attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""),
0,
)
),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if (
(
_now - int(env.get("START_TIME", time.time()))
> int(env.get("TIMEOUT", 600))
)
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per
):
break
base_branches = migrator.get_possible_feedstock_branches(attrs)
if "branch" in attrs:
has_attrs_branch = True
orig_branch = attrs.get("branch")
else:
has_attrs_branch = False
orig_branch = None
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
# map main to current default branch
base_branches = [
br if br != "main" else fctx.default_branch for br in base_branches
]
try:
for base_branch in base_branches:
attrs["branch"] = base_branch
if migrator.filter(attrs):
continue
print("\n", flush=True, end="")
sys.stderr.flush()
sys.stdout.flush()
LOGGER.info(
"%s%s IS MIGRATING %s:%s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
base_branch,
)
try:
# Don't bother running if we are at zero
if mctx.gh_api_requests_left == 0:
break
migrator_uid, pr_json = run(
feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get("hash_type", "sha256"),
base_branch=base_branch,
)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"]
for existing_pr in attrs.get("PRed", [])
]:
pass
else:
if not pr_json:
pr_json = {
"state": "closed",
"head": {"ref": "<this_is_not_a_branch>"},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version": mctx.pinning_version,
},
)
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
LOGGER.critical(
"GITHUB ERROR ON FEEDSTOCK: %s",
fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
LOGGER.exception("URLError ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
"code": getattr(e, "code"),
"url": getattr(e, "url"),
}
_set_pre_pr_migrator_fields(
attrs,
migrator_name,
sanitize_string(
"bot error (%s): %s: %s"
% (
'<a href="'
+ os.getenv("CIRCLE_BUILD_URL", "")
+ '">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
),
),
)
except Exception as e:
LOGGER.exception("NON GITHUB ERROR")
# we don't set bad for rerendering errors
if (
"conda smithy rerender -c auto --no-check-uptodate"
not in str(e)
):
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split(
"\n",
),
}
_set_pre_pr_migrator_fields(
attrs,
migrator_name,
sanitize_string(
"bot error (%s): %s: %s"
% (
'<a href="'
+ os.getenv("CIRCLE_BUILD_URL", "")
+ '">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
),
),
)
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# reset branch
if has_attrs_branch:
attrs["branch"] = orig_branch
# do this but it is crazy
gc.collect()
# Write graph partially through
if not dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
LOGGER.info(os.getcwd())
for f in glob.glob("/tmp/*"):
if f not in temp:
try:
eval_cmd(f"rm -rf {f}")
except Exception:
pass
if mctx.gh_api_requests_left == 0:
break
return good_prs
def run_test_migration(
m,
inp,
output,
kwargs,
prb,
mr_out,
should_filter=False,
tmpdir=None,
):
mm_ctx = MigratorSessionContext(
graph=G,
smithy_version="",
pinning_version="",
github_username="",
github_password="",
circle_build_url=env["CIRCLE_BUILD_URL"],
)
m_ctx = MigratorContext(mm_ctx, m)
m.bind_to_ctx(m_ctx)
if mr_out:
mr_out.update(bot_rerun=False)
with open(os.path.join(tmpdir, "meta.yaml"), "w") as f:
f.write(inp)
# read the conda-forge.yml
if os.path.exists(os.path.join(tmpdir, "..", "conda-forge.yml")):
with open(os.path.join(tmpdir, "..", "conda-forge.yml")) as fp:
cf_yml = fp.read()
else:
cf_yml = "{}"
# Load the meta.yaml (this is done in the graph)
try:
name = parse_meta_yaml(inp)["package"]["name"]
except Exception:
name = "blah"
pmy = populate_feedstock_attributes(name, {}, inp, cf_yml)
# these are here for legacy migrators
pmy["version"] = pmy["meta_yaml"]["package"]["version"]
pmy["req"] = set()
for k in ["build", "host", "run"]:
req = pmy["meta_yaml"].get("requirements", {}) or {}
_set = req.get(k) or set()
pmy["req"] |= set(_set)
pmy["raw_meta_yaml"] = inp
pmy.update(kwargs)
assert m.filter(pmy) is should_filter
if should_filter:
return
m.run_pre_piggyback_migrations(
tmpdir,
pmy,
hash_type=pmy.get("hash_type", "sha256"),
)
mr = m.migrate(tmpdir, pmy, hash_type=pmy.get("hash_type", "sha256"))
m.run_post_piggyback_migrations(
tmpdir,
pmy,
hash_type=pmy.get("hash_type", "sha256"),
)
assert mr_out == mr
if not mr:
return
pmy.update(PRed=[frozen_to_json_friendly(mr)])
with open(os.path.join(tmpdir, "meta.yaml")) as f:
actual_output = f.read()
# strip jinja comments
pat = re.compile(r"{#.*#}")
actual_output = pat.sub("", actual_output)
output = pat.sub("", output)
assert actual_output == output
def auto_tick(dry_run=False,
debug=False,
fork=False,
organization='nsls-ii-forge'):
'''
Automatically update package versions and submit pull requests to
associated feedstocks
Parameters
----------
dry_run: bool, optional
Generate version migration yamls but do not run them
debug: bool, optional
Setup logging to be in debug mode
fork: bool, optional
Create a fork of the repo from the organization to $GITHUB_USERNAME
organization: str, optional
GitHub organization that manages feedstock repositories
'''
from conda_forge_tick.xonsh_utils import env
if debug:
setup_logger(logger, level="debug")
else:
setup_logger(logger)
# set Version.pr_body to custom pr_body function
Version.pr_body = bot_pr_body
# TODO: use ~/.netrc instead
github_username = env.get("GITHUB_USERNAME", "")
github_password = env.get("GITHUB_TOKEN", "")
github_token = env.get("GITHUB_TOKEN")
global MIGRATORS
print('Initializing migrators...')
mctx, MIGRATORS = initialize_migrators(
github_username=github_username,
github_password=github_password,
dry_run=dry_run,
github_token=github_token,
)
# compute the time per migrator
print('Computing time per migrator')
(num_nodes, time_per_migrator,
tot_time_per_migrator) = _compute_time_per_migrator(mctx, )
for i, migrator in enumerate(MIGRATORS):
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
logger.info(
"Total migrations for %s%s: %d - gets %f seconds (%f percent)",
migrator.__class__.__name__,
extra_name,
num_nodes[i],
time_per_migrator[i],
time_per_migrator[i] / tot_time_per_migrator * 100,
)
print('Performing migrations...')
for mg_ind, migrator in enumerate(MIGRATORS):
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
time_per = time_per_migrator[mg_ind]
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
logger.info(
"Running migrations for %s%s: %d",
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
logger.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
logger.info(
" node|curr|new|attempts: %s|%s|%s|%d",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""),
0,
)),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if ((_now - int(env.get("START_TIME", time.time())) > int(
env.get("TIMEOUT", 600)))
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per):
break
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
print("\n", flush=True, end="")
logger.info(
"%s%s IS MIGRATING %s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
)
try:
# Don't bother running if we are at zero
if (dry_run or mctx.gh.rate_limit()["resources"]["core"]
["remaining"] == 0):
break
migrator_uid, pr_json = run(feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get(
"hash_type", "sha256"),
fork=fork,
organization=organization)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"]
for existing_pr in attrs.get("PRed", [])
]:
pass
else:
if pr_json is None:
pr_json = {
"state": "closed",
"head": {
"ref": "<this_is_not_a_branch>"
},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version": mctx.pinning_version,
}, )
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
logger.critical(
"GITHUB ERROR ON FEEDSTOCK: %s",
fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
logger.exception("URLError ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
"code": getattr(e, "code"),
"url": getattr(e, "url"),
}
except Exception as e:
logger.exception("NON GITHUB ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
}
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# Write graph partially through
if not dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
logger.info(os.getcwd())
if not dry_run:
logger.info(
"API Calls Remaining: %d",
mctx.gh.rate_limit()["resources"]["core"]["remaining"],
)
logger.info("Done")
def main(args: "CLIArgs") -> None:
# logging
if args.debug:
setup_logger(logging.getLogger("conda_forge_tick"), level="debug")
else:
setup_logger(logging.getLogger("conda_forge_tick"))
github_username = env.get("USERNAME", "")
github_password = env.get("PASSWORD", "")
github_token = env.get("GITHUB_TOKEN")
mctx, temp, migrators = initialize_migrators(
github_username=github_username,
github_password=github_password,
dry_run=args.dry_run,
github_token=github_token,
)
# compute the time per migrator
print("computing time per migration", flush=True)
(num_nodes, time_per_migrator,
tot_time_per_migrator) = _compute_time_per_migrator(
mctx,
migrators,
)
for i, migrator in enumerate(migrators):
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
print(
" %s%s: %d - gets %f seconds (%f percent)" % (
migrator.__class__.__name__,
extra_name,
num_nodes[i],
time_per_migrator[i],
time_per_migrator[i] / max(tot_time_per_migrator, 1) * 100,
),
flush=True,
)
for mg_ind, migrator in enumerate(migrators):
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
_mg_start = time.time()
effective_graph = mmctx.effective_graph
time_per = time_per_migrator[mg_ind]
if hasattr(migrator, "name"):
extra_name = "-%s" % migrator.name
else:
extra_name = ""
print(
"\n========================================"
"========================================"
"\n"
"========================================"
"========================================",
flush=True,
)
print(
"Running migrations for %s%s: %d\n" % (
migrator.__class__.__name__,
extra_name,
len(effective_graph.nodes),
),
flush=True,
)
possible_nodes = list(migrator.order(effective_graph, mctx.graph))
# version debugging info
if isinstance(migrator, Version):
LOGGER.info("possible version migrations:")
for node_name in possible_nodes:
with effective_graph.nodes[node_name]["payload"] as attrs:
LOGGER.info(
" node|curr|new|attempts: %s|%s|%s|%d",
node_name,
attrs.get("version"),
attrs.get("new_version"),
(attrs.get("new_version_attempts", {}).get(
attrs.get("new_version", ""),
0,
)),
)
for node_name in possible_nodes:
with mctx.graph.nodes[node_name]["payload"] as attrs:
base_branches = migrator.get_possible_feedstock_branches(attrs)
orig_branch = attrs.get("branch", "master")
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
_now = time.time()
if ((_now - int(env.get("START_TIME", time.time())) > int(
env.get("TIMEOUT", 600)))
or good_prs >= migrator.pr_limit
or (_now - _mg_start) > time_per):
break
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
try:
for base_branch in base_branches:
attrs["branch"] = base_branch
if migrator.filter(attrs):
continue
print("\n", flush=True, end="")
LOGGER.info(
"%s%s IS MIGRATING %s:%s",
migrator.__class__.__name__.upper(),
extra_name,
fctx.package_name,
base_branch,
)
try:
# Don't bother running if we are at zero
if mctx.gh_api_requests_left == 0:
break
migrator_uid, pr_json = run(
feedstock_ctx=fctx,
migrator=migrator,
rerender=migrator.rerender,
protocol="https",
hash_type=attrs.get("hash_type", "sha256"),
base_branch=base_branch,
)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"]
for existing_pr in attrs.get(
"PRed", [])
]:
pass
else:
if not pr_json:
pr_json = {
"state": "closed",
"head": {
"ref": "<this_is_not_a_branch>"
},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version":
mctx.pinning_version,
}, )
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
LOGGER.critical(
"GITHUB ERROR ON FEEDSTOCK: %s",
fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
LOGGER.exception("URLError ERROR")
attrs["bad"] = {
"exception":
str(e),
"traceback":
str(traceback.format_exc()).split("\n"),
"code":
getattr(e, "code"),
"url":
getattr(e, "url"),
}
pre_key = "pre_pr_migrator_status"
if pre_key not in attrs:
attrs[pre_key] = {}
attrs[pre_key][migrator_name] = sanitize_string(
"bot error (%s): %s: %s" % (
'<a href="' +
os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
), )
except Exception as e:
LOGGER.exception("NON GITHUB ERROR")
# we don't set bad for rerendering errors
if ("conda smithy rerender -c auto --no-check-uptodate"
not in str(e)):
attrs["bad"] = {
"exception":
str(e),
"traceback":
str(traceback.format_exc()).split("\n", ),
}
pre_key = "pre_pr_migrator_status"
if pre_key not in attrs:
attrs[pre_key] = {}
attrs[pre_key][migrator_name] = sanitize_string(
"bot error (%s): %s: %s" % (
'<a href="' +
os.getenv("CIRCLE_BUILD_URL", "") +
'">bot CI job</a>',
base_branch,
str(traceback.format_exc()),
), )
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# reset branch
attrs["branch"] = orig_branch
# Write graph partially through
if not args.dry_run:
dump_graph(mctx.graph)
eval_cmd(f"rm -rf {mctx.rever_dir}/*")
LOGGER.info(os.getcwd())
for f in glob.glob("/tmp/*"):
if f not in temp:
try:
eval_cmd(f"rm -rf {f}")
except Exception:
pass
if mctx.gh_api_requests_left == 0:
break
print("\n", flush=True)
LOGGER.info("API Calls Remaining: %d", mctx.gh_api_requests_left)
LOGGER.info("Done")
def main(args: "CLIArgs") -> None:
github_username = env.get("USERNAME", "")
github_password = env.get("PASSWORD", "")
github_token = env.get("GITHUB_TOKEN")
global MIGRATORS
mctx, temp, MIGRATORS = initialize_migrators(
github_username=github_username,
github_password=github_password,
dry_run=args.dry_run,
github_token=github_token,
)
for migrator in MIGRATORS:
mmctx = MigratorContext(session=mctx, migrator=migrator)
migrator.bind_to_ctx(mmctx)
good_prs = 0
effective_graph = mmctx.effective_graph
logger.info(
"Total migrations for %s: %d",
migrator.__class__.__name__,
len(effective_graph.nodes),
)
for node_name in migrator.order(effective_graph, mctx.graph):
with mctx.graph.nodes[node_name]["payload"] as attrs:
# Don't let CI timeout, break ahead of the timeout so we make certain
# to write to the repo
# TODO: convert these env vars
if (
time.time() - int(env.get("START_TIME", time.time()))
> int(env.get("TIMEOUT", 600))
or good_prs >= migrator.pr_limit
):
break
fctx = FeedstockContext(
package_name=node_name,
feedstock_name=attrs["feedstock_name"],
attrs=attrs,
)
logger.info(
"%s IS MIGRATING %s",
migrator.__class__.__name__.upper(),
fctx.package_name,
)
try:
# Don't bother running if we are at zero
if (
args.dry_run
or mctx.gh.rate_limit()["resources"]["core"]["remaining"] == 0
):
break
# FIXME: this causes the bot to not-rerender things when it
# should. For instance, if the bot rerenders but the PR is
# left open then we don't rerender again even though we should.
# This need logic to check if the rerender has been merged.
rerender = (
attrs.get("smithy_version") != mctx.smithy_version
or attrs.get("pinning_version") != mctx.pinning_version
or migrator.rerender
)
migrator_uid, pr_json = run(
feedstock_ctx=fctx,
migrator=migrator,
rerender=rerender,
protocol="https",
hash_type=attrs.get("hash_type", "sha256"),
)
# if migration successful
if migrator_uid:
d = frozen_to_json_friendly(migrator_uid)
# if we have the PR already do nothing
if d["data"] in [
existing_pr["data"] for existing_pr in attrs.get("PRed", [])
]:
pass
else:
if not pr_json:
pr_json = {
"state": "closed",
"head": {"ref": "<this_is_not_a_branch>"},
}
d["PR"] = pr_json
attrs.setdefault("PRed", []).append(d)
attrs.update(
{
"smithy_version": mctx.smithy_version,
"pinning_version": mctx.pinning_version,
},
)
except github3.GitHubError as e:
if e.msg == "Repository was archived so is read-only.":
attrs["archived"] = True
else:
logger.critical(
"GITHUB ERROR ON FEEDSTOCK: %s", fctx.feedstock_name,
)
if is_github_api_limit_reached(e, mctx.gh):
break
except URLError as e:
logger.exception("URLError ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
"code": getattr(e, "code"),
"url": getattr(e, "url"),
}
except Exception as e:
logger.exception("NON GITHUB ERROR")
attrs["bad"] = {
"exception": str(e),
"traceback": str(traceback.format_exc()).split("\n"),
}
else:
if migrator_uid:
# On successful PR add to our counter
good_prs += 1
finally:
# Write graph partially through
dump_graph(mctx.graph)
eval_xonsh(f"rm -rf {mctx.rever_dir}/*")
logger.info(os.getcwd())
for f in glob.glob("/tmp/*"):
if f not in temp:
eval_xonsh(f"rm -rf {f}")
if not args.dry_run:
logger.info(
"API Calls Remaining: %d",
mctx.gh.rate_limit()["resources"]["core"]["remaining"],
)
logger.info("Done")
def graph_migrator_status(
migrator: Migrator,
gx: nx.DiGraph,
) -> Tuple[dict, list, nx.DiGraph]:
"""Gets the migrator progress for a given migrator"""
if hasattr(migrator, "name"):
assert isinstance(migrator.name, str)
migrator_name = migrator.name.lower().replace(" ", "")
else:
migrator_name = migrator.__class__.__name__.lower()
num_viz = 0
out: Dict[str, Set[str]] = {
"done": set(),
"in-pr": set(),
"awaiting-pr": set(),
"not-solvable": set(),
"awaiting-parents": set(),
"bot-error": set(),
}
gx2 = copy.deepcopy(getattr(migrator, "graph", gx))
top_level = {node for node in gx2 if not list(gx2.predecessors(node))}
build_sequence = list(cyclic_topological_sort(gx2, top_level))
feedstock_metadata = dict()
import graphviz
from streamz.graph import _clean_text
gv = graphviz.Digraph(graph_attr={"packmode": "array_3"})
# pinning isn't actually in the migration
if "conda-forge-pinning" in gx2.nodes():
gx2.remove_node("conda-forge-pinning")
for node, node_attrs in gx2.nodes.items():
attrs = node_attrs["payload"]
# remove archived from status
if attrs.get("archived", False):
continue
node_metadata: Dict = {}
feedstock_metadata[node] = node_metadata
nuid = migrator.migrator_uid(attrs)
all_pr_jsons = []
for pr_json in attrs.get("PRed", []):
all_pr_jsons.append(copy.deepcopy(pr_json))
feedstock_ctx = FeedstockContext(
package_name=node,
feedstock_name=attrs.get("feedstock_name", node),
attrs=attrs,
)
# hack around bug in migrator vs graph data for this one
if isinstance(migrator, MatplotlibBase):
if "name" in nuid:
del nuid["name"]
for i in range(len(all_pr_jsons)):
if (all_pr_jsons[i] and "name" in all_pr_jsons[i]["data"]
and all_pr_jsons[i]["data"]["migrator_name"]
== "MatplotlibBase"):
del all_pr_jsons[i]["data"]["name"]
for pr_json in all_pr_jsons:
if pr_json and pr_json["data"] == frozen_to_json_friendly(
nuid)["data"]:
break
else:
pr_json = None
# No PR was ever issued but the migration was performed.
# This is only the case when the migration was done manually
# before the bot could issue any PR.
manually_done = pr_json is None and frozen_to_json_friendly(
nuid)["data"] in (z["data"] for z in all_pr_jsons)
buildable = not migrator.filter(attrs)
fntc = "black"
status_icon = ""
if manually_done:
out["done"].add(node)
fc = "#440154"
fntc = "white"
elif pr_json is None:
if buildable:
if "not solvable" in (attrs.get("pre_pr_migrator_status",
{}).get(migrator_name, "")):
out["not-solvable"].add(node)
fc = "#ff8c00"
elif "bot error" in (attrs.get("pre_pr_migrator_status",
{}).get(migrator_name, "")):
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
else:
out["awaiting-pr"].add(node)
fc = "#35b779"
elif not isinstance(migrator, Replacement):
if "bot error" in (attrs.get("pre_pr_migrator_status",
{}).get(migrator_name, "")):
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
else:
out["awaiting-parents"].add(node)
fc = "#fde725"
elif "PR" not in pr_json:
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
elif pr_json["PR"]["state"] == "closed":
out["done"].add(node)
fc = "#440154"
fntc = "white"
else:
out["in-pr"].add(node)
fc = "#31688e"
fntc = "white"
pr_status = pr_json["PR"]["mergeable_state"]
if pr_status == "clean":
status_icon = " ✓"
else:
status_icon = " ❎"
if node not in out["done"]:
num_viz += 1
gv.node(
node,
label=_clean_text(node) + status_icon,
fillcolor=fc,
style="filled",
fontcolor=fntc,
URL=(pr_json or {}).get("PR", {}).get(
"html_url",
feedstock_url(fctx=feedstock_ctx,
protocol="https").strip(".git"),
),
)
# additional metadata for reporting
node_metadata["num_descendants"] = len(nx.descendants(gx2, node))
node_metadata["immediate_children"] = [
k for k in sorted(gx2.successors(node))
if not gx2[k].get("payload", {}).get("archived", False)
]
if node in out["not-solvable"] or node in out["bot-error"]:
node_metadata["pre_pr_migrator_status"] = attrs.get(
"pre_pr_migrator_status",
{},
).get(migrator_name, "")
else:
node_metadata["pre_pr_migrator_status"] = ""
if pr_json and "PR" in pr_json:
# I needed to fake some PRs they don't have html_urls though
node_metadata["pr_url"] = pr_json["PR"].get(
"html_url",
feedstock_url(fctx=feedstock_ctx,
protocol="https").strip(".git"),
)
node_metadata["pr_status"] = pr_json["PR"].get("mergeable_state")
out2: Dict = {}
for k in out.keys():
out2[k] = list(
sorted(
out[k],
key=lambda x: build_sequence.index(x)
if x in build_sequence else -1,
), )
out2["_feedstock_status"] = feedstock_metadata
for (e0, e1), edge_attrs in gx2.edges.items():
if (e0 not in out["done"] and e1 not in out["done"]
and not gx2.nodes[e0]["payload"].get("archived", False)
and not gx2.nodes[e1]["payload"].get("archived", False)):
gv.edge(e0, e1)
print(" len(gv):", num_viz, flush=True)
out2["_num_viz"] = num_viz
return out2, build_sequence, gv
from conda_forge_tick.utils import load_graph, dump_graph, frozen_to_json_friendly
gx = load_graph()
for k, node_attrs in gx.nodes.items():
prs = node_attrs.get('PRed', [])
for i, pr in enumerate(prs):
pr['data']['bot_rerun'] = False
pr.update(frozen_to_json_friendly(pr['data']))
dump_graph(gx)
def migrator_status(
migrator: Migrator, gx: nx.DiGraph,
) -> Tuple[dict, list, nx.DiGraph]:
"""Gets the migrator progress for a given migrator
Returns
-------
out : dict
Dictionary of statuses with the feedstocks in them
order :
Build order for this migrator
"""
out: Dict[str, Set[str]] = {
"done": set(),
"in-pr": set(),
"awaiting-pr": set(),
"awaiting-parents": set(),
"bot-error": set(),
}
gx2 = copy.deepcopy(getattr(migrator, "graph", gx))
top_level = {node for node in gx2 if not list(gx2.predecessors(node))}
build_sequence = list(cyclic_topological_sort(gx2, top_level))
feedstock_metadata = dict()
import graphviz
from streamz.graph import _clean_text
gv = graphviz.Digraph(graph_attr={"packmode": "array_3"})
for node, node_attrs in gx2.nodes.items():
attrs = node_attrs["payload"]
# remove archived from status
if attrs.get("archived", False):
continue
node_metadata: Dict = {}
feedstock_metadata[node] = node_metadata
nuid = migrator.migrator_uid(attrs)
all_pr_jsons = []
for pr_json in attrs.get("PRed", []):
all_pr_jsons.append(copy.deepcopy(pr_json))
# hack around bug in migrator vs graph data for this one
if isinstance(migrator, MatplotlibBase):
if "name" in nuid:
del nuid["name"]
for i in range(len(all_pr_jsons)):
if (
all_pr_jsons[i]
and "name" in all_pr_jsons[i]["data"]
and all_pr_jsons[i]["data"]["migrator_name"] == "MatplotlibBase"
):
del all_pr_jsons[i]["data"]["name"]
for pr_json in all_pr_jsons:
if pr_json and pr_json["data"] == frozen_to_json_friendly(nuid)["data"]:
break
else:
pr_json = None
# No PR was ever issued but the migration was performed.
# This is only the case when the migration was done manually
# before the bot could issue any PR.
manually_done = pr_json is None and frozen_to_json_friendly(nuid)["data"] in (
z["data"] for z in all_pr_jsons
)
buildable = not migrator.filter(attrs)
fntc = "black"
if manually_done:
out["done"].add(node)
fc = "#440154"
fntc = "white"
elif pr_json is None:
if buildable:
out["awaiting-pr"].add(node)
fc = "#35b779"
elif not isinstance(migrator, Replacement):
out["awaiting-parents"].add(node)
fc = "#fde725"
elif "PR" not in pr_json:
out["bot-error"].add(node)
fc = "#000000"
fntc = "white"
elif pr_json["PR"]["state"] == "closed":
out["done"].add(node)
fc = "#440154"
fntc = "white"
else:
out["in-pr"].add(node)
fc = "#31688e"
fntc = "white"
if node not in out["done"]:
gv.node(
node,
label=_clean_text(node),
fillcolor=fc,
style="filled",
fontcolor=fntc,
URL=(pr_json or {}).get("PR", {}).get("html_url", ""),
)
# additional metadata for reporting
node_metadata["num_descendants"] = len(nx.descendants(gx2, node))
node_metadata["immediate_children"] = [
k
for k in sorted(gx2.successors(node))
if not gx2[k].get("payload", {}).get("archived", False)
]
if pr_json and "PR" in pr_json:
# I needed to fake some PRs they don't have html_urls though
node_metadata["pr_url"] = pr_json["PR"].get("html_url", "")
out2: Dict = {}
for k in out.keys():
out2[k] = list(
sorted(
out[k],
key=lambda x: build_sequence.index(x) if x in build_sequence else -1,
),
)
out2["_feedstock_status"] = feedstock_metadata
for (e0, e1), edge_attrs in gx2.edges.items():
if (
e0 not in out["done"]
and e1 not in out["done"]
and not gx2.nodes[e0]["payload"].get("archived", False)
and not gx2.nodes[e1]["payload"].get("archived", False)
):
gv.edge(e0, e1)
return out2, build_sequence, gv
from conda_forge_tick.utils import load_graph, dump_graph, frozen_to_json_friendly
gx = load_graph()
for k, node_attrs in gx.nodes.items():
prs = node_attrs.get("PRed", [])
for i, pr in enumerate(prs):
pr["data"]["bot_rerun"] = False
pr.update(frozen_to_json_friendly(pr["data"]))
dump_graph(gx)
