def _update_pr(update_function, dry_run, gx):
failed_refresh = 0
succeeded_refresh = 0
gh = "" if dry_run else github_client()
futures = {}
node_ids = list(gx.nodes)
# this makes sure that github rate limits are dispersed
random.shuffle(node_ids)
with executor("thread", NUM_GITHUB_THREADS) as pool:
for node_id in node_ids:
node = gx.nodes[node_id]["payload"]
prs = node.get("PRed", [])
for i, migration in enumerate(prs):
pr_json = migration.get("PR", None)
# allow for false
if pr_json:
future = pool.submit(update_function, ghctx, pr_json, gh,
dry_run)
futures[future] = (node_id, i)
for f in as_completed(futures):
name, i = futures[f]
try:
res = f.result()
if res:
succeeded_refresh += 1
with gx.nodes[name]["payload"] as node:
node["PRed"][i]["PR"].update(**res)
# XXX: This is a bit of a hack
if update_function == close_out_labels:
node["PRed"][i]["bot_rerun"] = time.time()
logger.info(f"Updated json for {name}: {res['id']}")
except github3.GitHubError as e:
logger.error(f"GITHUB ERROR ON FEEDSTOCK: {name}")
failed_refresh += 1
if is_github_api_limit_reached(e, gh):
break
except github3.exceptions.ConnectionError:
logger.error(f"GITHUB ERROR ON FEEDSTOCK: {name}")
failed_refresh += 1
except Exception:
logger.critical(
"ERROR ON FEEDSTOCK: {}: {}".format(
name,
gx.nodes[name]["payload"]["PRed"][i],
), )
raise
return succeeded_refresh, failed_refresh
def close_labels(gx: nx.DiGraph) -> nx.DiGraph:
failed_refresh = 0
succeeded_refresh = 0
gh = github_client()
futures = {}
node_ids = list(gx.nodes)
# this makes sure that github rate limits are dispersed
random.shuffle(node_ids)
with executor('thread', NUM_GITHUB_THREADS) as (pool, as_completed):
for node_id in node_ids:
node = gx.nodes[node_id]['payload']
prs = node.get("PRed", [])
for i, migration in enumerate(prs):
pr_json = migration.get('PR', None)
# allow for false
if pr_json:
future = pool.submit(close_out_labels, pr_json, gh)
futures[future] = (node_id, i)
for f in as_completed(futures):
name, i = futures[f]
try:
res = f.result()
if res:
succeeded_refresh += 1
# add a piece of metadata which makes the muid matchup
# fail
with gx.node[name]['payload'] as node:
node['PRed'][i]['data']['bot_rerun'] = time.time()
if 'bot_rerun' not in gx.node[name]['payload']["PRed"][i]['keys']:
node['PRed'][i]['keys'].append('bot_rerun')
logger.info(
"Closed and removed PR and branch for "
"{}: {}".format(name, res["id"])
)
except github3.GitHubError as e:
logger.critical("GITHUB ERROR ON FEEDSTOCK: {}".format(name))
failed_refresh += 1
if is_github_api_limit_reached(e, gh):
break
except Exception as e:
logger.critical("ERROR ON FEEDSTOCK: {}: {}".format(
name, gx.nodes[name]['payload']["PRed"][i]['data']))
raise
logger.info("bot re-run failed for {} PRs".format(failed_refresh))
logger.info("bot re-run succeed for {} PRs".format(succeeded_refresh))
return gx
def update_graph_pr_status(gx: nx.DiGraph) -> nx.DiGraph:
failed_refresh = 0
succeeded_refresh = 0
gh = github_client()
futures = {}
node_ids = list(gx.nodes)
# this makes sure that github rate limits are dispersed
random.shuffle(node_ids)
with executor('thread', NUM_GITHUB_THREADS) as (pool, as_completed):
for node_id in node_ids:
node = gx.nodes[node_id]['payload']
prs = node.get("PRed", [])
for i, migration in enumerate(prs):
pr_json = migration.get('PR', None)
# allow for false
if pr_json:
future = pool.submit(refresh_pr, pr_json, gh)
futures[future] = (node_id, i)
for f in as_completed(futures):
name, i = futures[f]
try:
res = f.result()
if res:
succeeded_refresh += 1
with gx.node[name]['payload'] as node:
node["PRed"][i]['PR'].update(**res)
logger.info("Updated json for {}: {}".format(name, res["id"]))
except github3.GitHubError as e:
logger.critical("GITHUB ERROR ON FEEDSTOCK: {}".format(name))
failed_refresh += 1
if is_github_api_limit_reached(e, gh):
break
except github3.exceptions.ConnectionError as e:
logger.critical("GITHUB ERROR ON FEEDSTOCK: {}".format(name))
failed_refresh += 1
except Exception as e:
logger.critical("ERROR ON FEEDSTOCK: {}: {}".format(
name,
gx.nodes[name]['payload']["PRed"][i]['data']))
raise
logger.info("JSON Refresh failed for {} PRs".format(failed_refresh))
logger.info("JSON Refresh succeed for {} PRs".format(succeeded_refresh))
return gx
def _update_pr(update_function, dry_run, gx):
failed_refresh = 0
succeeded_refresh = 0
gh = "" if dry_run else github_client()
futures = {}
node_ids = list(gx.nodes)
# this makes sure that github rate limits are dispersed
random.shuffle(node_ids)
pr_info_ordered = OrderedDict()
if not dry_run:
last_prs = _get_last_updated_prs()
else:
last_prs = []
# Setting them here first gives them the highest priority in the OrderedDict
for pr_id in last_prs:
pr_info_ordered[pr_id] = None
pr_json_regex = re.compile(r"^pr_json/([0-9]*).json$")
with executor("thread", NUM_GITHUB_THREADS) as pool:
for node_id in tqdm.tqdm(node_ids, desc='ordering PRs', leave=False):
node = gx.nodes[node_id]["payload"]
prs = node.get("PRed", [])
for i, migration in enumerate(prs):
pr_json = migration.get("PR", None)
# allow for false
if pr_json:
if '__lazy_json__' in pr_json:
m = pr_json_regex.match(pr_json['__lazy_json__'])
if m:
pr_id = int(m.group(1))
else:
pr_id = object()
else:
pr_id = object()
pr_info_ordered[pr_id] = (pr_json, node_id, i)
for pr_id, v in pr_info_ordered.items():
if v:
(pr_json, node_id, i) = v
future = pool.submit(update_function, ghctx, pr_json, gh,
dry_run)
futures[future] = (node_id, i)
for f in as_completed(futures):
name, i = futures[f]
try:
res = f.result()
if res:
succeeded_refresh += 1
with gx.nodes[name]["payload"] as node:
node["PRed"][i]["PR"].update(**res)
# XXX: This is a bit of a hack
if update_function == close_out_labels:
node["PRed"][i]["data"]["bot_rerun"] = time.time()
logger.info(f"Updated json for {name}: {res['id']}")
except github3.GitHubError as e:
logger.error(f"GITHUB ERROR ON FEEDSTOCK: {name}")
failed_refresh += 1
if is_github_api_limit_reached(e, gh):
break
except github3.exceptions.ConnectionError:
logger.error(f"GITHUB ERROR ON FEEDSTOCK: {name}")
failed_refresh += 1
except Exception:
logger.critical(
"ERROR ON FEEDSTOCK: {}: {}".format(
name,
gx.nodes[name]["payload"]["PRed"][i],
), )
raise
return succeeded_refresh, failed_refresh