"old_issue_number": issue["issue_number"],
"pipeline_id": pipe.get("id"),
"pipeline_name": pipe.get("name"),
"position": issue.get("position"),
"repo_id": DEST_REPO_ID,
}
# assign positions to issues
issues = _utils.load_issues(DIR)
for issue in issues:
repo_id = issue.get("repo_id")
if repo["id"] == 140626918:
"""
we skip in atd-data-tech
Those issue will not have pipelines updated,
but do need to reconnect the dependencies, etc.
"""
continue
issue_number = issue.get("number")
key = f"{repo_id}${issue_number}"
issue["migration"]["pipeline"] = issues_with_positions.get(key)
write_issue(issue, DIR)
if __name__ == "__main__":
logger = get_logger("apply_pipeline_positions")
main()
new_issue = {
"repo_name": r["name"],
"repo_id": r["id"],
"assignees": [person.login for person in issue.assignees],
"labels": [label.name for label in issue.labels],
"state": issue.state,
"url": issue.url,
"number": issue.number,
"title": issue.title,
"body": issue.body,
}
if issue.milestone:
new_issue["milestone"] = issue.milestone.title
fname = f"{DIR}/{new_issue['repo_name']}${new_issue['number']}.json"
new_issue["migration"] = {}
new_issue["path"] = fname
with open(fname, "w") as fout:
logger.info(f"{new_issue['repo_name']} {new_issue['number']}")
fout.write(json.dumps(new_issue))
issue_count += 1
logger.info(f"Issues Processed: {issue_count}")
if __name__ == "__main__":
logger = get_logger("download_github")
main()
e["new_issue_number"]
})
else:
logger.error(
f"Child issue for issue #{issue_number} does not exist: {e}"
)
if payload["add_issues"]:
res = zenhub_request(DEST_REPO_ID, issue_number, payload)
if not res:
logger.error(f"ERROR: {issue['path']}")
issue["migration"]["epic_created"] = False
else:
logger.info(issue["path"])
issue["migration"]["epic_created"] = True
write_issue(issue, DIR)
epic_count += 1
logger.info(f"Issues Processed: {issue_count}")
logger.info(f"Epics Processed: {epic_count}")
logger.info(f"Child Issues Processed: {child_issue_count}")
if __name__ == "__main__":
logger = get_logger("convert_epics")
main()
else:
print(e)
return None
return res.json()
def main():
with open("releases_with_new_issue_numbers.json", "r") as fin:
data = json.loads(fin.read())
for rel_id in data.keys():
rel = data[rel_id]
if rel.get("state") == "closed":
continue
rel.pop("issues")
rel.pop("state")
rel["repositories"] = [DEST_REPO_ID]
rel.pop("release_id")
rel.pop("created_at")
rel.pop("closed_at")
res = zenhub_request(DEST_REPO_ID, rel)
pdb.set_trace()
if __name__ == "__main__":
logger = get_logger("delete")
main()
child_issue_count += 1
if not new_issue_number:
# child issue has not been processed, it's probably a closed issue
# which we're not migrating
missing_epic_issues.append({
"repo_id":
child_issue["repo_id"],
"issue_number":
child_issue["issue_number"],
})
# write update issue to file
issue["migration"]["epics_staged"] = True
write_issue(issue, DIR)
epic_count += 1
logger.info(f"Issues Processed: {issue_count}")
logger.info(f"Epics Processed: {epic_count}")
logger.info(f"Child Issues Processed: {child_issue_count}")
with open(MISSING_CHILDREN_FILE, "w") as fout:
fout.write(json.dumps(missing_epic_issues))
# logger.info(f"Issues Processed: {issue_count}")
if __name__ == "__main__":
logger = get_logger("stage_epics")
main()
for issue in issues:
if issue.get("is_epic") and issue.get("repo_id") != 140626918:
"""
we skip existing atd-data-tech epics.
These issues already exist, but we need to connect the dependencies, etc.
"""
# new issue number of issue that will be converted to epic
issue_number = issue["migration"].get("new_issue_number")
payload = {"issues": []}
res = zenhub_request(DEST_REPO_ID, issue_number, payload)
if not res:
logger.error(f"ERROR: {issue['path']}")
issue["migration"]["epic_created"] = False
else:
logger.info(issue["path"])
issue["migration"]["epic_created"] = True
write_issue(issue, DIR)
issue_count += 1
logger.info(f"Issues Processed: {issue_count}")
if __name__ == "__main__":
logger = get_logger("creat_epics")
main()
# for relid in rels_temp.keys():
# name = rels_temp[relid]["title"]
# rel_map[name] = relid
for rel_id in rels:
if rels[rel_id]["state"] == "closed":
continue
# title = rels[rel_id]["title"] # this mapping is only needed in test
# try:
# mapped_id = rel_map[title]
# except:
# pdb.set_trace() # emoji are killing this // will not be an issue in prod
# print("emoji fail")
payload = {"add_issues": [], "remove_issues": []}
if rels[rel_id]["issues"]:
payload["add_issues"] = rels[rel_id]["issues"]
res = zenhub_request(DEST_REPO_ID, payload)
if not res:
logger.error(f"ERROR: {payload}")
else:
logger.info(payload)
if __name__ == "__main__":
logger = get_logger("set_releases")
main()
print(res.text)
return None
if res.status_code == 403:
print(res.text)
return None
else:
print(e)
return None
return res.json()
def main():
dp = load(DEPEND_FILE)
for d in dp:
res = zenhub_request(d)
if not res:
logger.error(f"ERROR: {d}")
else:
logger.info(d)
if __name__ == "__main__":
logger = get_logger("update_dependencies")
main()
But we do need to reference these isssue
To connect the dependencies and epics to new issues.
"""
issue["migration"]["created_github"] = True
issue["migration"]["new_issue_number"] = issue["number"]
write_issue(issue, DIR)
issue_count += 1
continue
else:
res = create_issue(issue, DEST_REPO)
issue["migration"]["created_github"] = True
issue["migration"]["new_issue_number"] = res["number"]
issue_count += 1
created_count += 1
write_issue(issue, DIR)
logger.info(f"Issues Processed: {issue_count}")
logger.info(f"Issues Created: {created_count}")
# issue["migration"]["source_body_update"] = update_body(
# issue["body"], res["number"], DEST_REPO
# )
# res = update_issue(issue)
# issue["migration"]["updated_source_github"] = True
# write_issue(issue, DIR)
if __name__ == "__main__":
logger = get_logger("github_create")
main()
# disgusting hack because i'm tired of writing this migation
# we have to be able to distinguish between map and map/append
# label actions. i don't want to refactor all the code so
# we're squeezing the action into the destination label_name
label_dest = label_dest + "_&_" + label_action
lookup[label_src] = label_dest
return lookup
def main():
with open(LABEL_FILE, "r") as fin:
reader = csv.DictReader(fin)
label_map = [row for row in reader if "map" in row["action"]]
label_lookup = build_lookup(label_map)
issues = _utils.load_issues(DIR)
for issue in issues:
labels = issue.get("labels")
labels = map_labels(labels, label_lookup)
labels = map_repos(labels, issue["repo_name"], REPO_MAP)
issue["migration"]["labels"] = labels
write_issue(issue, DIR)
if __name__ == "__main__":
logger = get_logger("apply_labels")
main()
print(res.text)
return None
else:
print(e)
return none
return res.json()
def main():
all_dependencies = []
for repo in SOURCE_REPOS:
repo_id = repo["id"]
res = zenhub_request(repo_id)
if res == None:
logger.error(repo_id)
all_dependencies.extend(res["dependencies"])
with open(outfile, "w") as fout:
fout.write(json.dumps(all_dependencies))
if __name__ == "__main__":
logger = get_logger("get_dependencies")
main()
issues = _utils.load_issues(DIR)
# get the new issue number of every issue in a release
for issue in issues:
repo_id = issue.get("repo_id")
issue_number = issue.get("number")
key = f"{repo_id}${issue_number}"
if key in lookup:
new_issue_number = issue["migration"].get("new_issue_number")
if not new_issue_number:
# issue has apparently not been created in github
logger.error(key)
raise Exception(f"New {issue_number} does not exist in github!")
lookup[key]["new_issue_number"] = new_issue_number
# build new releases file
releases = update_releases(releases, lookup)
releases = drop_null_issues(releases)
with open(DEST_FILE, "w") as fout:
fout.write(json.dumps(releases))
if __name__ == "__main__":
logger = get_logger("process_releases")
main()
links = requests.utils.parse_header_links(res.headers["Link"])
except KeyError:
# if there's only one page there will be no link headers
break
if links:
for link in links:
if link.get("rel") == "next":
url = link.get("url")
print(url)
elif link.get("rel") == "last":
last_url = link.get("url")
return milestones
def main():
issues = _utils.load_issues(DIR)
dest_milestones = get_milestones_from_repo(DEST_REPO)
issues, update_count = update_milestones(issues, dest_milestones)
issue_count = write_issues(issues, DIR)
logger.info(f"Issues Processed: {issue_count}")
logger.info(f"Milestones Updated: {update_count}")
if __name__ == "__main__":
logger = get_logger("set_milestones")
main()
if issue["migration"].get("zenhub_downloaded"):
continue
time.sleep(.6) # zenhub rate limit is 100 requests/minute
try:
issue = get_zenhub_issue(issue)
except:
issue["migration"]["zenhub_downloaded"] = False
logger.error(f"Error: {issue['path']}")
error_count += 1
continue
if issue["is_epic"]:
get_epic_issues(issue)
fname = issue["path"]
with open(fname, "w") as fout:
logger.info(f"{issue['repo_name']} {issue['number']}")
fout.write(json.dumps(issue))
issue_count += 1
logger.info(f"Issues Processed: {issue_count}")
logger.info(f"Errors: {error_count}")
if __name__ == "__main__":
logger = get_logger("download_zenhub")
main()
def main():
issues = _utils.load_issues(DIR)
issue_count = 0
for issue in issues:
if (
not issue.get("migration").get("comments_retreived")
and issue.get("repo_id") != 140626918
):
"""
we skip comments atd-data-tech
The issues already exist, but we need to connect the dependencies, etc.
"""
issue["comments"] = get_comments(issue["repo_name"], issue["number"])
issue["comments"] = parse_comments(issue["comments"])
issue["migration"]["comments_retreived"] = True
logger.info(issue["number"])
write_issue(issue, DEST_REPO)
issue_count += 1
logger.info(f"Issues Processed: {issue_count}")
if __name__ == "__main__":
logger = get_logger("download_comments")
main()
dest_labels = get_labels_from_repo(DEST_REPO)
dest_labels = [label["name"] for label in dest_labels]
# get all labels from source repos
source_labels = get_labels_from_source_repos(SOURCE_REPOS)
# get all labels from label file
file_labels = get_labels_from_csv(LABEL_FILE)
# merge label definitions from file and source repos
source_labels.update(file_labels)
# get all labels from labeled issues
issue_labels = get_labels_from_issues(DIR)
create_labels = get_missing_labels(dest_labels, source_labels,
issue_labels)
for label in create_labels:
try:
create_label(label, DEST_REPO)
logger.info(f"Created {label['name']}")
except:
logger.error(f"ERROR: {label['name']}")
if __name__ == "__main__":
logger = get_logger("create_labels")
main()
if key in d_lookup:
new_issue_number = issue["migration"].get("new_issue_number")
if not new_issue_number:
# issue has apparently not been created in github
logger.error(key)
raise Exception(
f"New {issue_number} does not exist in github!")
d_lookup[key]["new_issue_number"] = new_issue_number
else:
# issue must be closed, because we haven't downloaded it
missing_dependency_issues.append(key)
# build new dependencies file
depends = build_new_dependencies(depends, d_lookup)
depends = drop_null_dependencies(depends)
with open(DEST_FILE, "w") as fout:
fout.write(json.dumps(depends))
with open(MISSING_DEPEND_FILE, "w") as fout:
fout.write(json.dumps(missing_dependency_issues))
if __name__ == "__main__":
logger = get_logger("process_dependencies")
main()
issues = _utils.load_issues(DIR)
sorted_issues = sort_issues(issues)
for issue_element in sorted_issues:
issue = issue_element.get("data")
issue_number = issue.get("migration").get("new_issue_number")
pos = issue.get("migration").get("pipeline").get("position")
# TODO: TEST ONLY. Prod just use existing pipeline ID
# pipe_id = issue.get("migration").get("pipeline").get("pipeline_id")
pipe_name = issue.get("migration").get("pipeline").get("pipeline_name")
pipe_id = replace_pipe(test_pipes, pipe_name)
payload = {"pipeline_id": pipe_id, "position": pos}
res = zenhub_request(DEST_REPO_ID, WORKSPACE_ID, issue_number, payload)
if not res:
logger.error(f"ERROR: {issue['path']}")
issue["migration"]["pipeline_processed"] = False
else:
logger.info(issue["path"])
issue["migration"]["pipeline_processed"] = True
write_issue(issue, DIR)
if __name__ == "__main__":
logger = get_logger("update_pipelines")
main()