def commit(self, repo,
file_descriptions,
commit_message,
branch='master',
force=False):
"""Make a commit on GitHub.
If a path exists, it will be replaced; if not, it will be created.
See http://developer.github.com/v3/git/"""
gh_repo = self._gh.get_user().get_repo(repo)
head_ref = gh_repo.get_git_ref("heads/%s" % branch)
latest_commit = gh_repo.get_git_commit(head_ref.object.sha)
base_tree = latest_commit.tree
tree_els = [github.InputGitTreeElement(
path=desc.path,
mode='100755' if desc.executable else '100644',
type='blob',
content=desc.contents
) for desc in file_descriptions]
new_tree = gh_repo.create_git_tree(tree_els, base_tree)
new_commit = gh_repo.create_git_commit(
message=commit_message,
parents=[latest_commit],
tree=new_tree)
head_ref.edit(sha=new_commit.sha, force=force)
def push(repo_name, branch, files, commit_message='test'):
'''
Pushes changes to Github repo using PyGithub
Arguments:
repo_name: Repository name
branch: Pre-existing remote GitHub branch
files: Dictionary keys containing file paths relative to the repository's root directory and values containing the content within the file path
commit_message: Commit message used for commit
'''
repo = github.Github(
os.environ['TF_VAR_testing_github_token']).get_user().get_repo(
repo_name)
elements = []
head_ref = repo.get_git_ref('heads/' + branch)
for filepath, content in files.items():
log.debug(f'Creating file: {filepath}')
blob = repo.create_git_blob(content, "utf-8")
elements.append(
github.InputGitTreeElement(path=filepath,
mode='100644',
type='blob',
sha=blob.sha))
head_sha = repo.get_branch(branch).commit.sha
base_tree = repo.get_git_tree(sha=head_sha)
tree = repo.create_git_tree(elements, base_tree)
parent = repo.get_git_commit(sha=head_sha)
commit_id = repo.create_git_commit(commit_message, tree, [parent]).sha
head_ref.edit(sha=commit_id)
log.debug(f'Commit ID: {commit_id}')
return commit_id
def create_pr(branch: str, message: str, files: typing.Dict[str, str]):
repo = _get_repo()
main = repo.get_branch("main")
# check for branch already existing
try:
ref = repo.get_git_ref(f"heads/{branch}")
except github.GithubException:
ref = None
updates = [
github.InputGitTreeElement(path,
mode="100644",
type="blob",
content=content)
for path, content in files.items()
]
new_tree = repo.create_git_tree(updates,
repo.get_git_tree(main.commit.sha))
new_commit = repo.create_git_commit(message, new_tree,
[main.commit.commit])
if not ref:
repo.create_git_ref(f"refs/heads/{branch}", sha=new_commit.sha)
else:
ref.edit(sha=new_commit.sha, force=True)
try:
pr = repo.create_pull(title=message,
body=message,
base="main",
head=branch)
return pr.url
except github.GithubException:
pass
def commit(self,
commit_files,
commit_msg,
branch="heads/master",
type="text",
mode="100644"):
"""
Commit a list of files on specified branch.
Notes:
At this moment it supports only text file.
Parameters:
commit_files (list):Tuples like (file name, content as str).
commit_msg (str): Commit message.
branch (str): Branch name (default: heads/master).
type (str): Default is 'text'), may be 'blob', or others also.
mode (str): File mode (default: '100644')."""
index_files = []
master_ref = self.repo.get_git_ref(branch)
base_tree = self.repo.get_git_tree(master_ref.object.sha)
for file_entry in commit_files:
file_name, file_content = file_entry[0], file_entry[1]
tree_el = github.InputGitTreeElement(
file_name, mode, type, file_content)
index_files.append(tree_el)
tree = self.repo.create_git_tree(index_files, base_tree)
parent = self.repo.get_git_commit(master_ref.object.sha)
commit = self.repo.create_git_commit(commit_msg, tree, [parent])
master_ref.edit(commit.sha)
def create_file_element(file_path, file_content):
repo = get_repo()
file_blob = repo.create_git_blob(file_content, "utf-8")
file_element = github.InputGitTreeElement(path=file_path,
mode="100644",
type="blob",
sha=file_blob.sha)
return file_element
def testCreateGitTree(self):
tree = self.repo.create_git_tree([
github.InputGitTreeElement("Foobar.txt",
"100644",
"blob",
content="File created by PyGithub")
])
self.assertEqual(tree.sha, "41cf8c178c636a018d537cb20daae09391efd70b")
def testCreateGitTreeWithSha(self):
tree = self.repo.create_git_tree([
github.InputGitTreeElement(
"Barbaz.txt",
"100644",
"blob",
sha="5dd930f591cd5188e9ea7200e308ad355182a1d8")
])
self.assertEqual(tree.sha, "fae707821159639589bf94f3fb0a7154ec5d441b")
def testCreateGitTreeWithBaseTree(self):
base_tree = self.repo.get_git_tree(
"41cf8c178c636a018d537cb20daae09391efd70b")
tree = self.repo.create_git_tree([
github.InputGitTreeElement("Barbaz.txt",
"100644",
"blob",
content="File also created by PyGithub")
], base_tree)
self.assertEqual(tree.sha, "107139a922f33bab6fbeb9f9eb8787e7f19e0528")
def create_pull_request(installation_token, repo):
pulls = list(repo.get_pulls(state="all"))[:config.INITIAL_PULLS_LOOKUP]
contexts = set()
for pull in pulls:
contexts.update(
sanitize_context(s.context)
for s in repo.get_commit(pull.head.sha).get_statuses()
if not s.context.startswith("mergify/")
)
mergify_config = {
"rules": {
"default": {
"protection": {
"required_status_checks": {
"contexts": list(contexts),
},
"required_pull_request_reviews": {
"required_approving_review_count": 1
},
},
},
}
}
content = yaml.dump(mergify_config, default_flow_style=False)
default_branch = repo.get_branch(repo.default_branch)
message = "Mergify initial configuration"
parents = [repo.get_git_commit(default_branch.commit.sha)]
tree = repo.create_git_tree([
github.InputGitTreeElement(".mergify.yml", "100644", "blob", content)
], base_tree=default_branch.commit.commit.tree)
commit = repo.create_git_commit(message, tree, parents)
repo.create_git_ref("refs/heads/%s" % INITIAL_CONFIG_BRANCH, commit.sha)
repo.create_pull(
title=message,
body="""This is an initial configuration for Mergify.
This pull request will require one reviewer approval.
Those required [status checks](https://doc.mergify.io/configuration.html#required-status-checks) have been discovered from the %d more recent pull requests of your project:
%s
More information about Mergify configuration can be found at https://doc.mergify.io
To modify this pull request, you can check it out locally.
See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/
""" % (config.INITIAL_PULLS_LOOKUP, "\n *".join(contexts)), # noqa
base=repo.default_branch,
head=INITIAL_CONFIG_BRANCH,
)
LOG.info('Initial configuration created for repo %s', repo.full_name)
def create_or_update_file(self,
filename,
content,
commit_message,
branch="master"):
"""
Given a filename, content, commit message and branch, create
or update the file with the given content on the given
branch.
If no branch is given, assume master. If a branch is given,
update/create the file in a commit on the given branch.
"""
branch_exists = self.branch_exists(branch)
if branch_exists:
sha = self.get_latest_sha(branch)
else:
sha = self.get_latest_sha()
base_tree = self.get_tree(sha)
blob = self.create_blob(content, "utf-8")
new_tree = self.create_tree(
tree=[
github.InputGitTreeElement(
path=filename,
mode='100644', # plain files
type='blob',
sha=blob.sha,
)
],
base_tree=base_tree,
)
new_commit = self.create_commit(
message=commit_message,
tree=new_tree,
parents=[self.get_commit(sha)],
)
if branch_exists:
ref_to_update = self.get_ref(branch)
else:
ref_to_update = self.create_branch(branch, new_commit.sha)
# Update the given branch HEAD reference to point to the newest commit
ref_to_update.edit(sha=new_commit.sha, force=False)
return new_commit.sha
def git():
# create a github instance with the current token
g = Github(token)
# get user will get me, because I authenticated my account
user = g.get_user()
# I want to get the OSF-test repo
repo = user.get_repo('OSF-test')
# we're going to create a blob. our first step is to get the content
# of that blob
file = open('/Users/samportnow/Documents/git-test/test.txt', 'r')
contents = file.read()
# now we can create a blob of the contents
my_blob = repo.create_git_blob(contents, 'utf-8')
# now we need to get the master branch
master_branch = repo.get_git_ref('heads/master')
# and the base commit of that master branch
base_commit = repo.get_commit(sha=master_branch._object._sha)
# and the tree were are going to committing to
tree = github.InputGitTreeElement(path='test.txt',
mode='100755',
type='blob',
sha=my_blob.sha)
# now we create a NEW Tree!
new_tree = repo.create_git_tree(base_tree=base_commit._commit.tree,
tree=[tree])
# now we can finally commit!
# lets try to use a DIFFERENT author, whose on my collaborator team
# (this works)
contributor = g.get_user('sullytesting1987')
email = contributor._email
name = contributor._name
author = github.InputGitAuthor(name=name,
email=email,
date=str(datetime.datetime.now()))
commit = repo.create_git_commit("This is a commit",
tree=new_tree,
parents=[master_branch._object._sha],
author=author)
# note: i changed the pygithub code for the parents list.
# they are looking for github commit objects, but all that
# is really needed is the sha of the master branch. so i got that instead
# and finally we update the ref
# note: pygithub's equivalent of update ref is edit!
master_branch.edit(commit.sha)
return 'SUCCESS'
def test_basic():
gw = GithubWriter(user="******",repo="testing")
assert_is_instance(gw, GithubWriter)
sha = gw.get_latest_sha()
assert_equals( len(sha) , 40, 'Got a reasonable looking sha back:%s ' % sha)
print("sha of master = %s" % sha)
assert_false( gw.branch_exists("really_please_dont_exist") )
assert_true( gw.branch_exists("master") )
tree = gw.get_tree(sha)
assert_equals( len(tree.sha) , 40, 'Got a reasonable looking sha back:%s ' % tree.sha)
print("tree sha of master = %s" % tree.sha)
blob = gw.create_blob("some other content", "utf-8")
# Create a new Tree, which will be part of our new commit
new_tree = gw.create_tree(
tree = [github.InputGitTreeElement(
path = "foo.json",
mode = '100644',
type = 'blob',
sha = blob.sha,
)],
base_tree = tree,
)
# Actually create the Git commit from our tree and parent commit
new_commit = gw.create_commit(
message = "test commit message",
tree = new_tree,
parents = [ gw.get_commit(sha) ],
)
branch = gw.create_branch("testing_%d" % os.getpid(), new_commit.sha )
# Update the branch to point to our new commit ref
branch.edit(sha=new_commit.sha, force=False)
def push_to_github(request):
body = json.loads(request.body)
user_name = body["user"]
repo_name = body["repository"]
branch = body["branch"]
commit_message = body["message"]
contents = body["contents"]
logged_in_user = logged_in(request)
github_user = json.loads(logged_in_user.content)
if (github_user["github_handle"] != user_name
or not is_github_token_valid(github_user["access_token"])):
return HttpResponseForbidden()
g = github.Github(github_user["access_token"])
element_list = list()
for key, value in contents.items():
file_code = "100644"
element = github.InputGitTreeElement(key, file_code, "blob", value)
element_list.append(element)
repo = g.get_repo(f"{user_name}/{repo_name}")
try:
master_ref = repo.get_git_ref(f"heads/{branch}")
except: # Ignore PycodestyleBear (E722)
# with a bit of luck, the reason is that there is no first commit yet
# TODO: code this up more cleanly. The create_file solution is a hack
repo.create_file("README.md", "Initial commit", "")
master_ref = repo.get_git_ref(f"heads/{branch}")
master_sha = master_ref.object.sha
base_tree = repo.get_git_tree(master_sha)
tree = repo.create_git_tree(element_list, base_tree)
parent = repo.get_git_commit(master_sha)
commit = repo.create_git_commit(commit_message, tree, [parent])
master_ref.edit(commit.sha)
if master_ref.object.sha == commit.sha:
return HttpResponse(status=200, content_type="application/json")
else:
return HttpResponse(status=400, content_type="application/json")
def build(
# fmt: off
repo: str,
commit: str,
package_name: str = Option(None,
help="Package name (if different from repo)"),
py35: bool = Option(False, "--py35", help="Build wheels for Python 3.5"),
llvm: bool = Option(False, "--llvm", help="Requires LLVM to be installed"),
rust: bool = Option(False, "--rust", help="Requires Rust to be installed"),
universal: bool = Option(
False,
"--universal",
help="Build universal (pure Python) wheel and sdist"),
skip_tests: bool = Option(
False,
"--skip-tests",
help="Don't run tests (e.g. if package doesn't have any)"),
build_constraints: bool = Option(
False,
"--build-constraints",
help="Use build constraints for build requirements"),
# fmt: on
):
"""Build wheels for a given repo and commit / tag."""
print(LOGO)
repo_id = get_repo_id()
user, package = repo.lower().split("/", 1)
if package_name is None:
package_name = package.replace("-", "_")
msg.info(f"Building in repo {repo_id}")
msg.info(f"Building wheels for {user}/{package}\n")
if universal:
msg.warn(
"Building only universal sdist and wheel, no cross-platform wheels"
)
if skip_tests:
msg.warn("Not running any tests")
clone_url = DEFAULT_CLONE_TEMPLATE.format(f"{user}/{package}")
repo = get_gh().get_repo(repo_id)
with msg.loading("Finding a unique name for this release..."):
# Pick the release_name by finding an unused one
i = 1
while True:
release_name = f"{package_name}-{commit}"
if i > 1:
release_name += f"-{i}"
try:
repo.get_release(release_name)
except github.UnknownObjectException:
break
i += 1
branch_name = f"branch-for-{release_name}"
bs = {
"clone-url": clone_url,
"package-name": package_name,
"commit": commit,
"options": {
"llvm": llvm,
"rust": rust,
"py35": py35,
"universal": universal,
"skip_tests": skip_tests,
"build_constraints": build_constraints,
},
"upload-to": {
"type": "github-release",
"repo-id": repo_id,
"release-id": release_name,
},
}
bs_json = json.dumps(bs)
bs_json_formatted = json.dumps(bs, indent=4)
msg.text(f"Creating release {release_name} to collect assets")
release_text = f"https://github.com/{user}/{package}\n\n### Build spec\n\n```json\n{bs_json_formatted}\n```"
release = repo.create_git_release(release_name, release_name, release_text)
with msg.loading("Creating build branch..."):
# 'master' is a 'Commit'. 'master.commit' is a 'GitCommit'. These are
# different types that are mostly *not* interchangeable:
# https://pygithub.readthedocs.io/en/latest/github_objects/Commit.html
# https://pygithub.readthedocs.io/en/latest/github_objects/GitCommit.html
master = repo.get_commit("master")
master_gitcommit = master.commit
patch = github.InputGitTreeElement(
"build-spec.json",
"100644",
"blob",
content=bs_json,
)
tree = repo.create_git_tree([patch], master_gitcommit.tree)
our_gitcommit = repo.create_git_commit(f"Building: {release_name}",
tree, [master_gitcommit])
repo.create_git_ref(f"refs/heads/{branch_name}", our_gitcommit.sha)
msg.good(f"Commit is {our_gitcommit.sha[:8]} in branch {branch_name}")
msg.text(f"Release: {release.html_url}")
msg.text(
f"Checks: https://github.com/{repo_id}/commit/{our_gitcommit.sha}/checks"
)
def magic_build(magic_build_repo_id, clone_url, package_name, commit):
if clone_url is None:
clone_url = DEFAULT_CLONE_TEMPLATE.format(package_name)
repo = get_gh().get_repo(magic_build_repo_id)
print("Finding a unique name for this release...")
# Pick the release_name by finding an unused one
i = 1
while True:
release_name = "{}-{}-wheels".format(package_name, commit)
if i > 1:
release_name += "-{}".format(i)
try:
repo.get_release(release_name)
except github.UnknownObjectException:
break
i += 1
branch_name = "branch-for-" + release_name
bs = {
"clone-url": clone_url,
"package-name": package_name,
"commit": commit,
"upload-to": {
"type": "github-release",
"repo-id": MAGIC_BUILD_REPO,
"release-id": release_name,
},
}
bs_json = json.dumps(bs)
print("Creating release {!r} to collect assets...".format(release_name))
release = repo.create_git_release(
release_name,
release_name,
"Build spec:\n\n```json\n{}\n```".format(bs_json),
)
print(" {}".format(release.html_url))
print("Creating build branch...".format(MAGIC_BUILD_REPO))
# 'master' is a 'Commit'. 'master.commit' is a 'GitCommit'. These are
# different types that are mostly *not* interchangeable:
# https://pygithub.readthedocs.io/en/latest/github_objects/Commit.html
# https://pygithub.readthedocs.io/en/latest/github_objects/GitCommit.html
master = repo.get_commit("master")
master_gitcommit = master.commit
patch = github.InputGitTreeElement(
"build-spec.json",
"100644",
"blob",
content=bs_json,
)
tree = repo.create_git_tree([patch], master_gitcommit.tree)
our_gitcommit = repo.create_git_commit("Building: {}".format(release_name),
tree, [master_gitcommit])
repo.create_git_ref("refs/heads/" + branch_name, our_gitcommit.sha)
print(" Commit is {} in branch {!r}.".format(our_gitcommit.sha[:8],
branch_name))
print("Waiting for build to complete...")
# get_combined_status needs a Commit, not a GitCommit
our_commit = repo.get_commit(our_gitcommit.sha)
showed_urls = {}
while True:
time.sleep(10)
combined_status = our_commit.get_combined_status()
display_name_to_state = {}
for display_name in STATUSES.values():
display_name_to_state[display_name] = "not available"
for status in combined_status.statuses:
if status.context in STATUSES:
display_name = STATUSES[status.context]
display_name_to_state[display_name] = status.state
if display_name not in showed_urls:
print(" {} logs: {}".format(display_name,
status.target_url))
showed_urls[display_name] = status.target_url
displays = [
"[{} - {}]".format(display_name, state)
for (display_name, state) in display_name_to_state.items()
]
print(" ".join(displays))
pending = False
failed = False
# The Github states are: "error", "failure", "success", "pending"
for state in display_name_to_state.values():
if state not in FINAL_STATES:
pending = True
if state in BAD_STATES:
failed = True
if failed or not pending:
break
if failed:
print("*** Failed! ***")
for display_name, url in showed_urls.items():
print(" {} logs: {}".format(display_name, url))
sys.exit(1)
else:
_download_release_assets(magic_build_repo_id, release_name)
def send_pull_request(form_data):
# read metadata of new quickstart repo
qs_u = form_data['github-username']
qs_r = form_data['github-repository']
qs_n = form_data['alternate-name'] or qs_r
qs_c = form_data['cartridges'] or []
qs_t = form_data['type'] or []
try:
qs = _read_quickstart_repo(qs_u, qs_r)
qs['alternate_name'] = qs_n
qs['cartridges'] = qs_c
qs['type'] = qs_t
except PyGitHub.UnknownObjectException:
raise OOIndexError("Username or repository not found: %s/%s" %
(qs_u, qs_r))
try:
owner = PyGitHub.Github().get_user(qs_u)
qs['owner_name'] = owner.name
qs['owner_avatar_url'] = owner.avatar_url
except:
qs['owner_name'] = qs['owner']
qs['owner_avatar_url'] = ''
# read content of original quickstar.json
# fork repo if needed
u = app.config['OO_INDEX_GITHUB_USERNAME']
r = app.config['OO_INDEX_GITHUB_REPONAME']
q = app.config['OO_INDEX_QUICKSTART_JSON']
repo, head, tree, quickstart = _read_github_file(u, r, q)
# add quickstart to quickstart.json
quickstart.append(qs)
# create new blob with updated quickstart.json
print "Creating blob...",
sys.stdout.flush()
new_blob = repo.create_git_blob(
json.dumps(quickstart, indent=3, encoding='utf-8'), 'utf-8')
# create tree with new blob
element = _get_tree_element(repo, tree, q)
element = PyGitHub.InputGitTreeElement(path=element.path,
mode=element.mode,
type=element.type,
sha=new_blob.sha)
if not element:
flash("File not found: %s/%s/%s" % (u, r, q), "error")
return
print "Updating tree...",
sys.stdout.flush()
new_tree = repo.create_git_tree([element], tree)
# create commit for new tree
print "Creating commit...",
sys.stdout.flush()
message = 'Quickstart add request: %s/%s' % (qs_u, qs_r)
new_commit = repo.create_git_commit(message, new_tree,
[repo.get_git_commit(head.sha)])
# create new branch for new commit
print "Creating branch...",
sys.stdout.flush()
try:
new_branch = repo.create_git_ref('refs/heads/%s-%s' % (qs_u, qs_r),
new_commit.sha)
except PyGitHub.UnknownObjectException:
raise OOIndexError("Username or repository not found: %s/%s" %
(qs_u, qs_r))
# and finally, we send our pull request
print "Creating pull request...",
sys.stdout.flush()
upstream = _get_repo_for(u, r, session['token'])
pr_params = {
'title': message,
'body': 'Automatically generated PR for oo-index',
'base': 'master',
'head': '%s:%s-%s' % (g.user, qs_u, qs_r),
}
pr = upstream.create_pull(**pr_params)
return pr
def create_pull_request(installation_token, repo):
pulls = list(repo.get_pulls(state="all"))[:config.INITIAL_PULLS_LOOKUP]
contexts = set()
for pull in pulls:
contexts.update(
sanitize_context(s.context)
for s in repo.get_commit(pull.head.sha).get_statuses()
if not s.context.startswith("mergify/"))
mergify_config = {
"rules": {
"default": {
"protection": {
"required_status_checks": {
"contexts": list(contexts),
},
"required_pull_request_reviews": {
"required_approving_review_count": 1
},
},
},
}
}
content = yaml.dump(mergify_config, default_flow_style=False)
try:
default_branch = repo.get_branch(repo.default_branch)
except github.GithubException as e:
if e.status != 404:
raise
# TODO(sileht): When an empty repo is created we can't get the default
# branch this one doesn't yet exists. We may want to pospone the first
# PR in this case. For now just return to not raise backtrace
return
try:
parents = [repo.get_git_commit(default_branch.commit.sha)]
except github.GithubException as e:
if e.status == 409 and e.data['message'] == 'Git Repository is empty.':
return
raise
message = "Mergify initial configuration"
tree = repo.create_git_tree([
github.InputGitTreeElement(".mergify.yml", "100644", "blob", content)
],
base_tree=default_branch.commit.commit.tree)
commit = repo.create_git_commit(message, tree, parents)
repo.create_git_ref("refs/heads/%s" % INITIAL_CONFIG_BRANCH, commit.sha)
repo.create_pull(
title=message,
body="""This is an initial configuration for Mergify.
This pull request will require one reviewer approval.
Those required [status checks](https://doc.mergify.io/configuration.html#required-status-checks) have been discovered from the %d more recent pull requests of your project:
%s
More information about Mergify configuration can be found at https://doc.mergify.io
To modify this pull request, you can check it out locally.
See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/
""" % (config.INITIAL_PULLS_LOOKUP, "\n *".join(contexts)), # noqa
base=repo.default_branch,
head=INITIAL_CONFIG_BRANCH,
)
LOG.info('Initial configuration created', repository=repo.full_name)
def cmd_coalesce(config, gh, org, args):
repos = get_challenge_repos(config, org)
if len(repos) == 0:
log.error("No repos to coalesce")
return
master_repo_name = config.prefix + "challenges"
description = "%s master challenge repository." % config.ctfname
# Case 1: Brand new master repo
# 1. Create repo
# 2. Build up .gitmodules blob
# 3. Build git trees, one per category
# 4. Insert submodule blobs into their respective category trees
# 5. Create new commit!
# Case 2: Updating master repo
# TODO
# Get the admin team (if any)
ctf_admin_team = get_admin_team(config, org)
# Create the new repo
new_repo = create_repo(org, master_repo_name, description, admin_user=ctf_admin_team)
if new_repo is None:
return
if ctf_admin_team:
try:
ctf_admin_team.set_repo_permission(new_repo, 'admin')
except github.GithubException as e:
log.warning("Failed to set the repository permissions to admin for team '%s'",
str(t))
try:
head = new_repo.get_git_ref('heads/master')
base_tree = new_repo.get_git_tree(head.object.sha)
base_commit = new_repo.get_git_commit(head.object.sha)
log.info('Retrieved base commit for %s', master_repo_name)
except github.GithubException as e:
log.error('Unable to retrieve base commit, tree, or reference')
log.error('Reason: %s', str(e))
log.error("Rolling back created repo...")
delete_repo(new_repo)
return None
master_repo = None
by_category = {}
for repo in repos:
cat = repo.category
if cat not in by_category:
by_category[cat] = []
by_category[cat] += [repo]
tl_elements = []
submodules = []
for cat, repo_list in sorted(by_category.items()):
elements = []
for r in repo_list:
repo_name = '%s%d' % (cat, r.chal_num)
path = '%s/%s' % (cat, repo_name)
branch = 'master'
url = r.ssh_url
try:
r_head = r.get_git_ref('heads/%s' % branch)
r_base_tree = r.get_git_tree(r_head.object.sha)
r_base_commit = r.get_git_commit(r_head.object.sha)
log.info('Retrieved base commit for %s on the %s branch', r.name, branch)
except github.GithubException as e:
log.error('Unable to retrieve base commit, tree, or reference')
log.error('Reason: %s', str(e))
log.error("Rolling back created repo...")
delete_repo(new_repo)
return None
commit = r_head.object.sha
submodule = GitSubmodule(path, url, branch, commit)
submodules += [submodule]
elements += [github.InputGitTreeElement(repo_name, object_types["submodule"],
'commit', sha=commit)]
# Create the directory for this category
# Store sha for later reference
try:
new_tree = new_repo.create_git_tree(elements)
except github.GithubException as e:
log.error("Failed to create a git tree object for category '%s'", cat)
log.error("Reason: %s", e)
log.error("Rolling back created repo...")
delete_repo(new_repo)
return None
tl_elements += [github.InputGitTreeElement(cat, object_types["directory"], 'tree', sha=new_tree.sha)]
# Emit .gitmodules blob
gitmodules_content = "".join([str(x) for x in submodules])
tl_elements += [github.InputGitTreeElement(".gitmodules", object_types["file"],
'blob', content=gitmodules_content)]
# Create top level tree
try:
tl_tree = new_repo.create_git_tree(tl_elements)
except github.GithubException as e:
log.error("Failed to create the top-level git tree object")
log.error("Reason: %s", e)
log.error("Rolling back created repo...")
delete_repo(new_repo)
return None
# Create commit linking to top of tree
try:
new_commit = new_repo.create_git_commit('Added initial challenge submodules', tl_tree, [base_commit])
head.edit(new_commit.sha)
except github.GithubException as e:
log.error("Failed to create commit object")
log.error("Reason: %s", e)
delete_repo(new_repo)
return
log.info("Master repository ready: %s", new_repo.html_url)
def save_git_hub_file():
"""
FLASK POST routing method for '/saveFileToRemoteGithub'
Save a file to a GitHub repository. The POST request content is a JSON string containing the file name, repository name, branch, access token, the graph data in JSON format and a commit message.
"""
# Extract parameters and file content from json.
content = request.get_json(silent=True)
filename = content["filename"]
repo_name = content["repositoryName"]
repo_branch = content["repositoryBranch"]
repo_token = content["token"]
graph = content["jsonData"]
commit_message = content["commitMessage"]
# Extracting the true repo name and repo folder.
folder_name, repo_name = extract_folder_and_repo_names(repo_name)
if folder_name != "":
filename = folder_name + "/" + filename
g = github.Github(repo_token)
# get repo
try:
repo = g.get_repo(repo_name)
except github.GithubException as e:
print("Error in get_repo({0})! Repo: {1} Status: {2} Data: {3}".format(
"heads/" + repo_branch, str(repo_name), e.status, e.data))
return jsonify({"error": e.data["message"]}), 400
# Set branch
try:
branch_ref = repo.get_git_ref("heads/" + repo_branch)
except github.GithubException as e:
# repository might be empty
print("Error in get_git_ref({0})! Repo: {1} Status: {2} Data: {3}".
format("heads/" + repo_branch, str(repo_name), e.status, e.data))
return jsonify({"error": e.data["message"]}), 400
# get SHA from branch
branch_sha = branch_ref.object.sha
# Add repo and file name in the graph.
graph["modelData"]["repo"] = repo_name
graph["modelData"]["repoBranch"] = repo_branch
graph["modelData"]["repoService"] = "GitHub"
graph["modelData"]["filePath"] = filename
# Clean the GitHub file reference.
graph["modelData"]["sha"] = ""
graph["modelData"]["git_url"] = ""
# The 'indent=4' option is used for nice formatting. Without it the file is stored as a single line.
json_data = json.dumps(graph, indent=4)
# Commit to GitHub repo.
latest_commit = repo.get_git_commit(branch_sha)
base_tree = latest_commit.tree
try:
new_tree = repo.create_git_tree(
[
github.InputGitTreeElement(path=filename,
mode="100644",
type="blob",
content=json_data)
],
base_tree,
)
except github.GithubException as e:
# repository might not have permission
print("Error in create_git_tree({0})! Repo: {1} Status: {2} Data: {3}".
format("heads/" + repo_branch, str(repo_name), e.status, e.data))
return jsonify({"error": e.data["message"]}), 400
new_commit = repo.create_git_commit(message=commit_message,
parents=[latest_commit],
tree=new_tree)
branch_ref.edit(sha=new_commit.sha, force=False)
return "ok"
def build(repo, commit, package_name=None):
"""Build wheels for a given repo and commit / tag."""
click.secho(LOGO, fg='cyan')
repo_id = _get_repo_id()
user, package = repo.split('/', 1)
if package_name is None:
package_name = package
click.secho("Building in repo {}".format(repo_id))
click.secho("Building wheels for {}/{}\n".format(user, package))
clone_url = DEFAULT_CLONE_TEMPLATE.format("{}/{}".format(user, package))
repo = get_gh().get_repo(repo_id)
click.secho("Finding a unique name for this release...", fg='yellow')
# Pick the release_name by finding an unused one
i = 1
while True:
release_name = "{}-{}".format(package_name, commit)
if i > 1:
release_name += '-{}'.format(i)
try:
repo.get_release(release_name)
except github.UnknownObjectException:
break
i += 1
branch_name = 'branch-for-' + release_name
bs = {
'clone-url': clone_url,
'package-name': package_name,
'repo': '{}/{}'.format(user, package),
'commit': commit,
'upload-to': {
'type': 'github-release',
'repo-id': repo_id,
'release-id': release_name,
}
}
bs_json = json.dumps(bs)
click.secho(
"Creating release {} to collect assets...".format(release_name),
fg='yellow')
release_template = "https://github.com/{}/{}\n\n### Build spec\n\n```json\n{}\n```"
release = repo.create_git_release(
release_name,
release_name,
release_template.format(user, package, json.dumps(bs, indent=4)),
)
print(release.html_url)
click.secho("Creating build branch...", fg='yellow')
# 'master' is a 'Commit'. 'master.commit' is a 'GitCommit'. These are
# different types that are mostly *not* interchangeable:
# https://pygithub.readthedocs.io/en/latest/github_objects/Commit.html
# https://pygithub.readthedocs.io/en/latest/github_objects/GitCommit.html
master = repo.get_commit('master')
master_gitcommit = master.commit
patch = github.InputGitTreeElement(
'build-spec.json',
'100644',
'blob',
content=bs_json,
)
tree = repo.create_git_tree([patch], master_gitcommit.tree)
our_gitcommit = repo.create_git_commit("Building: {}".format(release_name),
tree, [master_gitcommit])
repo.create_git_ref('refs/heads/' + branch_name, our_gitcommit.sha)
print("Commit is {} in branch {}.".format(our_gitcommit.sha[:8],
branch_name))
click.secho("Waiting for build to complete...", fg='yellow')
# get_combined_status needs a Commit, not a GitCommit
our_commit = repo.get_commit(our_gitcommit.sha)
showed_urls = {}
while True:
time.sleep(10)
combined_status = our_commit.get_combined_status()
display_name_to_state = {}
for display_name in STATUSES.values():
display_name_to_state[display_name] = NA_STATE
for status in combined_status.statuses:
if status.context in STATUSES:
display_name = STATUSES[status.context]
display_name_to_state[display_name] = status.state
if display_name not in showed_urls:
print("{} logs: {}".format(display_name,
status.target_url))
showed_urls[display_name] = status.target_url
displays = [
click.style("[{} - {}]".format(name, state),
fg=STATUS_COLORS.get(state, 'white'))
for name, state in display_name_to_state.items()
]
click.echo(" ".join(displays))
pending = False
failed = False
# The Github states are: "error", "failure", "success", "pending"
for state in display_name_to_state.values():
if state not in FINAL_STATES:
pending = True
if state in BAD_STATES:
failed = True
if failed or not pending:
break
release = get_release(repo_id, release_name)
if failed:
click.secho("*** Failed! ***", bg='red', fg='black')
for display_name, url in showed_urls.items():
print("{} logs: {}".format(display_name, url))
release.update_release('\u274c ' + release.title, release.body)
sys.exit(1)
else:
_download_release_assets(repo_id, release_name)
release.update_release('\u2705 ' + release.title, release.body)
def create_git_tree(repo, directory, file_read_hook=None):
object_tree = { "blobs" : [], "trees" : {} }
# counters
file_bytes = 0
file_count = 0
directory_count = 0
log.info("Building file system tree for '%s'", directory)
# Walk the filesystem and build a tree structure
for root, dirs, files in os.walk(directory):
file_objects = []
components = root.split(os.sep)
# skip leading and any blank components
components = components[1:len(components)]
components = filter(lambda x: x != "", components)
# each call is for one directory
directory_count += 1
# top-level directory
if len(components) == 0:
level = object_tree
else:
base = object_tree
# drill down in the tree to the level we're at
for component in components:
if component not in base["trees"]:
base["trees"][component] = { "blobs" : [], "trees" : {} }
base = base["trees"][component]
level = base
# for each file at the current level
for f in files:
path = os.path.join(root, f)
ty = object_types['executable'] if os.access(path, os.X_OK) else object_types['file']
# read in all of the file contents (this could be deferred for streaming)
contents = ""
if file_read_hook:
contents = file_read_hook(path)
if contents is None:
return None
else:
try:
with open(path, 'rb') as fp:
contents = fp.read()
except IOError:
log.error("Unable to open file '%s' for reading", path)
return None
file_bytes += len(contents)
file_count += 1
# create a tree element for this
level["blobs"] += [GitBlob(f, ty, content=contents)]
log.info('Walked %d directories and %d files (total bytes %d)',
directory_count, file_count, file_bytes)
log.info('Creating git trees...')
# rewalk the data structure and build the tree nodes
# do this using DFS to build the bottom most trees first
stack = [TreeObj("", object_tree)]
while len(stack):
node = stack.pop()
# if we haven't visited the node and it has child directories
if not node.visited and len(node.tree["trees"]):
node.visited = True
# we're at a leaf directory node
stack.append(node)
for name, tree in node.tree["trees"].iteritems():
stack.append(TreeObj(name, tree))
# otherwise, process the directory node
else:
# map all directories to git tree elements
elements = map(lambda x: github.InputGitTreeElement(x[0],
object_types["directory"], 'tree', sha=x[1]["obj"].sha), node.tree["trees"].items())
# upload our blobs :)
for blob in node.tree["blobs"]:
import base64
try:
git_blob = repo.create_git_blob(base64.encodestring(blob.content).strip(), 'base64')
elements += [github.InputGitTreeElement(blob.path, blob.ty, 'blob', sha=git_blob.sha)]
except github.GithubExeception as e:
log.error("Failed to upload git blob for file '%s'", blob.path)
log.error("Reason: %s", e)
# XXX: can we roll back trees/blobs created before the failure?
return None
# git trees CANNOT be empty
if len(elements) == 0:
elements += [github.InputGitTreeElement(".keep", object_types["file"], 'blob', content="")]
# talk to the API and create the damn tree
try:
new_tree = repo.create_git_tree(elements)
except github.GithubException as e:
log.error("Failed to create a git tree object for node '%s'", node.path)
log.error("Reason: %s", e)
# XXX: can we roll back trees created before the failure?
return None
node.tree["obj"] = new_tree
return object_tree["obj"]
