def commit(self, repo, file_descriptions, commit_message, branch='master', force=False): """Make a commit on GitHub. If a path exists, it will be replaced; if not, it will be created. See http://developer.github.com/v3/git/""" gh_repo = self._gh.get_user().get_repo(repo) head_ref = gh_repo.get_git_ref("heads/%s" % branch) latest_commit = gh_repo.get_git_commit(head_ref.object.sha) base_tree = latest_commit.tree tree_els = [github.InputGitTreeElement( path=desc.path, mode='100755' if desc.executable else '100644', type='blob', content=desc.contents ) for desc in file_descriptions] new_tree = gh_repo.create_git_tree(tree_els, base_tree) new_commit = gh_repo.create_git_commit( message=commit_message, parents=[latest_commit], tree=new_tree) head_ref.edit(sha=new_commit.sha, force=force)
def push(repo_name, branch, files, commit_message='test'): ''' Pushes changes to Github repo using PyGithub Arguments: repo_name: Repository name branch: Pre-existing remote GitHub branch files: Dictionary keys containing file paths relative to the repository's root directory and values containing the content within the file path commit_message: Commit message used for commit ''' repo = github.Github( os.environ['TF_VAR_testing_github_token']).get_user().get_repo( repo_name) elements = [] head_ref = repo.get_git_ref('heads/' + branch) for filepath, content in files.items(): log.debug(f'Creating file: {filepath}') blob = repo.create_git_blob(content, "utf-8") elements.append( github.InputGitTreeElement(path=filepath, mode='100644', type='blob', sha=blob.sha)) head_sha = repo.get_branch(branch).commit.sha base_tree = repo.get_git_tree(sha=head_sha) tree = repo.create_git_tree(elements, base_tree) parent = repo.get_git_commit(sha=head_sha) commit_id = repo.create_git_commit(commit_message, tree, [parent]).sha head_ref.edit(sha=commit_id) log.debug(f'Commit ID: {commit_id}') return commit_id
def create_pr(branch: str, message: str, files: typing.Dict[str, str]): repo = _get_repo() main = repo.get_branch("main") # check for branch already existing try: ref = repo.get_git_ref(f"heads/{branch}") except github.GithubException: ref = None updates = [ github.InputGitTreeElement(path, mode="100644", type="blob", content=content) for path, content in files.items() ] new_tree = repo.create_git_tree(updates, repo.get_git_tree(main.commit.sha)) new_commit = repo.create_git_commit(message, new_tree, [main.commit.commit]) if not ref: repo.create_git_ref(f"refs/heads/{branch}", sha=new_commit.sha) else: ref.edit(sha=new_commit.sha, force=True) try: pr = repo.create_pull(title=message, body=message, base="main", head=branch) return pr.url except github.GithubException: pass
def commit(self, commit_files, commit_msg, branch="heads/master", type="text", mode="100644"): """ Commit a list of files on specified branch. Notes: At this moment it supports only text file. Parameters: commit_files (list):Tuples like (file name, content as str). commit_msg (str): Commit message. branch (str): Branch name (default: heads/master). type (str): Default is 'text'), may be 'blob', or others also. mode (str): File mode (default: '100644').""" index_files = [] master_ref = self.repo.get_git_ref(branch) base_tree = self.repo.get_git_tree(master_ref.object.sha) for file_entry in commit_files: file_name, file_content = file_entry[0], file_entry[1] tree_el = github.InputGitTreeElement( file_name, mode, type, file_content) index_files.append(tree_el) tree = self.repo.create_git_tree(index_files, base_tree) parent = self.repo.get_git_commit(master_ref.object.sha) commit = self.repo.create_git_commit(commit_msg, tree, [parent]) master_ref.edit(commit.sha)
def create_file_element(file_path, file_content): repo = get_repo() file_blob = repo.create_git_blob(file_content, "utf-8") file_element = github.InputGitTreeElement(path=file_path, mode="100644", type="blob", sha=file_blob.sha) return file_element
def testCreateGitTree(self): tree = self.repo.create_git_tree([ github.InputGitTreeElement("Foobar.txt", "100644", "blob", content="File created by PyGithub") ]) self.assertEqual(tree.sha, "41cf8c178c636a018d537cb20daae09391efd70b")
def testCreateGitTreeWithSha(self): tree = self.repo.create_git_tree([ github.InputGitTreeElement( "Barbaz.txt", "100644", "blob", sha="5dd930f591cd5188e9ea7200e308ad355182a1d8") ]) self.assertEqual(tree.sha, "fae707821159639589bf94f3fb0a7154ec5d441b")
def testCreateGitTreeWithBaseTree(self): base_tree = self.repo.get_git_tree( "41cf8c178c636a018d537cb20daae09391efd70b") tree = self.repo.create_git_tree([ github.InputGitTreeElement("Barbaz.txt", "100644", "blob", content="File also created by PyGithub") ], base_tree) self.assertEqual(tree.sha, "107139a922f33bab6fbeb9f9eb8787e7f19e0528")
def create_pull_request(installation_token, repo): pulls = list(repo.get_pulls(state="all"))[:config.INITIAL_PULLS_LOOKUP] contexts = set() for pull in pulls: contexts.update( sanitize_context(s.context) for s in repo.get_commit(pull.head.sha).get_statuses() if not s.context.startswith("mergify/") ) mergify_config = { "rules": { "default": { "protection": { "required_status_checks": { "contexts": list(contexts), }, "required_pull_request_reviews": { "required_approving_review_count": 1 }, }, }, } } content = yaml.dump(mergify_config, default_flow_style=False) default_branch = repo.get_branch(repo.default_branch) message = "Mergify initial configuration" parents = [repo.get_git_commit(default_branch.commit.sha)] tree = repo.create_git_tree([ github.InputGitTreeElement(".mergify.yml", "100644", "blob", content) ], base_tree=default_branch.commit.commit.tree) commit = repo.create_git_commit(message, tree, parents) repo.create_git_ref("refs/heads/%s" % INITIAL_CONFIG_BRANCH, commit.sha) repo.create_pull( title=message, body="""This is an initial configuration for Mergify. This pull request will require one reviewer approval. Those required [status checks](https://doc.mergify.io/configuration.html#required-status-checks) have been discovered from the %d more recent pull requests of your project: %s More information about Mergify configuration can be found at https://doc.mergify.io To modify this pull request, you can check it out locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/ """ % (config.INITIAL_PULLS_LOOKUP, "\n *".join(contexts)), # noqa base=repo.default_branch, head=INITIAL_CONFIG_BRANCH, ) LOG.info('Initial configuration created for repo %s', repo.full_name)
def create_or_update_file(self, filename, content, commit_message, branch="master"): """ Given a filename, content, commit message and branch, create or update the file with the given content on the given branch. If no branch is given, assume master. If a branch is given, update/create the file in a commit on the given branch. """ branch_exists = self.branch_exists(branch) if branch_exists: sha = self.get_latest_sha(branch) else: sha = self.get_latest_sha() base_tree = self.get_tree(sha) blob = self.create_blob(content, "utf-8") new_tree = self.create_tree( tree=[ github.InputGitTreeElement( path=filename, mode='100644', # plain files type='blob', sha=blob.sha, ) ], base_tree=base_tree, ) new_commit = self.create_commit( message=commit_message, tree=new_tree, parents=[self.get_commit(sha)], ) if branch_exists: ref_to_update = self.get_ref(branch) else: ref_to_update = self.create_branch(branch, new_commit.sha) # Update the given branch HEAD reference to point to the newest commit ref_to_update.edit(sha=new_commit.sha, force=False) return new_commit.sha
def git(): # create a github instance with the current token g = Github(token) # get user will get me, because I authenticated my account user = g.get_user() # I want to get the OSF-test repo repo = user.get_repo('OSF-test') # we're going to create a blob. our first step is to get the content # of that blob file = open('/Users/samportnow/Documents/git-test/test.txt', 'r') contents = file.read() # now we can create a blob of the contents my_blob = repo.create_git_blob(contents, 'utf-8') # now we need to get the master branch master_branch = repo.get_git_ref('heads/master') # and the base commit of that master branch base_commit = repo.get_commit(sha=master_branch._object._sha) # and the tree were are going to committing to tree = github.InputGitTreeElement(path='test.txt', mode='100755', type='blob', sha=my_blob.sha) # now we create a NEW Tree! new_tree = repo.create_git_tree(base_tree=base_commit._commit.tree, tree=[tree]) # now we can finally commit! # lets try to use a DIFFERENT author, whose on my collaborator team # (this works) contributor = g.get_user('sullytesting1987') email = contributor._email name = contributor._name author = github.InputGitAuthor(name=name, email=email, date=str(datetime.datetime.now())) commit = repo.create_git_commit("This is a commit", tree=new_tree, parents=[master_branch._object._sha], author=author) # note: i changed the pygithub code for the parents list. # they are looking for github commit objects, but all that # is really needed is the sha of the master branch. so i got that instead # and finally we update the ref # note: pygithub's equivalent of update ref is edit! master_branch.edit(commit.sha) return 'SUCCESS'
def test_basic(): gw = GithubWriter(user="******",repo="testing") assert_is_instance(gw, GithubWriter) sha = gw.get_latest_sha() assert_equals( len(sha) , 40, 'Got a reasonable looking sha back:%s ' % sha) print("sha of master = %s" % sha) assert_false( gw.branch_exists("really_please_dont_exist") ) assert_true( gw.branch_exists("master") ) tree = gw.get_tree(sha) assert_equals( len(tree.sha) , 40, 'Got a reasonable looking sha back:%s ' % tree.sha) print("tree sha of master = %s" % tree.sha) blob = gw.create_blob("some other content", "utf-8") # Create a new Tree, which will be part of our new commit new_tree = gw.create_tree( tree = [github.InputGitTreeElement( path = "foo.json", mode = '100644', type = 'blob', sha = blob.sha, )], base_tree = tree, ) # Actually create the Git commit from our tree and parent commit new_commit = gw.create_commit( message = "test commit message", tree = new_tree, parents = [ gw.get_commit(sha) ], ) branch = gw.create_branch("testing_%d" % os.getpid(), new_commit.sha ) # Update the branch to point to our new commit ref branch.edit(sha=new_commit.sha, force=False)
def push_to_github(request): body = json.loads(request.body) user_name = body["user"] repo_name = body["repository"] branch = body["branch"] commit_message = body["message"] contents = body["contents"] logged_in_user = logged_in(request) github_user = json.loads(logged_in_user.content) if (github_user["github_handle"] != user_name or not is_github_token_valid(github_user["access_token"])): return HttpResponseForbidden() g = github.Github(github_user["access_token"]) element_list = list() for key, value in contents.items(): file_code = "100644" element = github.InputGitTreeElement(key, file_code, "blob", value) element_list.append(element) repo = g.get_repo(f"{user_name}/{repo_name}") try: master_ref = repo.get_git_ref(f"heads/{branch}") except: # Ignore PycodestyleBear (E722) # with a bit of luck, the reason is that there is no first commit yet # TODO: code this up more cleanly. The create_file solution is a hack repo.create_file("README.md", "Initial commit", "") master_ref = repo.get_git_ref(f"heads/{branch}") master_sha = master_ref.object.sha base_tree = repo.get_git_tree(master_sha) tree = repo.create_git_tree(element_list, base_tree) parent = repo.get_git_commit(master_sha) commit = repo.create_git_commit(commit_message, tree, [parent]) master_ref.edit(commit.sha) if master_ref.object.sha == commit.sha: return HttpResponse(status=200, content_type="application/json") else: return HttpResponse(status=400, content_type="application/json")
def build( # fmt: off repo: str, commit: str, package_name: str = Option(None, help="Package name (if different from repo)"), py35: bool = Option(False, "--py35", help="Build wheels for Python 3.5"), llvm: bool = Option(False, "--llvm", help="Requires LLVM to be installed"), rust: bool = Option(False, "--rust", help="Requires Rust to be installed"), universal: bool = Option( False, "--universal", help="Build universal (pure Python) wheel and sdist"), skip_tests: bool = Option( False, "--skip-tests", help="Don't run tests (e.g. if package doesn't have any)"), build_constraints: bool = Option( False, "--build-constraints", help="Use build constraints for build requirements"), # fmt: on ): """Build wheels for a given repo and commit / tag.""" print(LOGO) repo_id = get_repo_id() user, package = repo.lower().split("/", 1) if package_name is None: package_name = package.replace("-", "_") msg.info(f"Building in repo {repo_id}") msg.info(f"Building wheels for {user}/{package}\n") if universal: msg.warn( "Building only universal sdist and wheel, no cross-platform wheels" ) if skip_tests: msg.warn("Not running any tests") clone_url = DEFAULT_CLONE_TEMPLATE.format(f"{user}/{package}") repo = get_gh().get_repo(repo_id) with msg.loading("Finding a unique name for this release..."): # Pick the release_name by finding an unused one i = 1 while True: release_name = f"{package_name}-{commit}" if i > 1: release_name += f"-{i}" try: repo.get_release(release_name) except github.UnknownObjectException: break i += 1 branch_name = f"branch-for-{release_name}" bs = { "clone-url": clone_url, "package-name": package_name, "commit": commit, "options": { "llvm": llvm, "rust": rust, "py35": py35, "universal": universal, "skip_tests": skip_tests, "build_constraints": build_constraints, }, "upload-to": { "type": "github-release", "repo-id": repo_id, "release-id": release_name, }, } bs_json = json.dumps(bs) bs_json_formatted = json.dumps(bs, indent=4) msg.text(f"Creating release {release_name} to collect assets") release_text = f"https://github.com/{user}/{package}\n\n### Build spec\n\n```json\n{bs_json_formatted}\n```" release = repo.create_git_release(release_name, release_name, release_text) with msg.loading("Creating build branch..."): # 'master' is a 'Commit'. 'master.commit' is a 'GitCommit'. These are # different types that are mostly *not* interchangeable: # https://pygithub.readthedocs.io/en/latest/github_objects/Commit.html # https://pygithub.readthedocs.io/en/latest/github_objects/GitCommit.html master = repo.get_commit("master") master_gitcommit = master.commit patch = github.InputGitTreeElement( "build-spec.json", "100644", "blob", content=bs_json, ) tree = repo.create_git_tree([patch], master_gitcommit.tree) our_gitcommit = repo.create_git_commit(f"Building: {release_name}", tree, [master_gitcommit]) repo.create_git_ref(f"refs/heads/{branch_name}", our_gitcommit.sha) msg.good(f"Commit is {our_gitcommit.sha[:8]} in branch {branch_name}") msg.text(f"Release: {release.html_url}") msg.text( f"Checks: https://github.com/{repo_id}/commit/{our_gitcommit.sha}/checks" )
def magic_build(magic_build_repo_id, clone_url, package_name, commit): if clone_url is None: clone_url = DEFAULT_CLONE_TEMPLATE.format(package_name) repo = get_gh().get_repo(magic_build_repo_id) print("Finding a unique name for this release...") # Pick the release_name by finding an unused one i = 1 while True: release_name = "{}-{}-wheels".format(package_name, commit) if i > 1: release_name += "-{}".format(i) try: repo.get_release(release_name) except github.UnknownObjectException: break i += 1 branch_name = "branch-for-" + release_name bs = { "clone-url": clone_url, "package-name": package_name, "commit": commit, "upload-to": { "type": "github-release", "repo-id": MAGIC_BUILD_REPO, "release-id": release_name, }, } bs_json = json.dumps(bs) print("Creating release {!r} to collect assets...".format(release_name)) release = repo.create_git_release( release_name, release_name, "Build spec:\n\n```json\n{}\n```".format(bs_json), ) print(" {}".format(release.html_url)) print("Creating build branch...".format(MAGIC_BUILD_REPO)) # 'master' is a 'Commit'. 'master.commit' is a 'GitCommit'. These are # different types that are mostly *not* interchangeable: # https://pygithub.readthedocs.io/en/latest/github_objects/Commit.html # https://pygithub.readthedocs.io/en/latest/github_objects/GitCommit.html master = repo.get_commit("master") master_gitcommit = master.commit patch = github.InputGitTreeElement( "build-spec.json", "100644", "blob", content=bs_json, ) tree = repo.create_git_tree([patch], master_gitcommit.tree) our_gitcommit = repo.create_git_commit("Building: {}".format(release_name), tree, [master_gitcommit]) repo.create_git_ref("refs/heads/" + branch_name, our_gitcommit.sha) print(" Commit is {} in branch {!r}.".format(our_gitcommit.sha[:8], branch_name)) print("Waiting for build to complete...") # get_combined_status needs a Commit, not a GitCommit our_commit = repo.get_commit(our_gitcommit.sha) showed_urls = {} while True: time.sleep(10) combined_status = our_commit.get_combined_status() display_name_to_state = {} for display_name in STATUSES.values(): display_name_to_state[display_name] = "not available" for status in combined_status.statuses: if status.context in STATUSES: display_name = STATUSES[status.context] display_name_to_state[display_name] = status.state if display_name not in showed_urls: print(" {} logs: {}".format(display_name, status.target_url)) showed_urls[display_name] = status.target_url displays = [ "[{} - {}]".format(display_name, state) for (display_name, state) in display_name_to_state.items() ] print(" ".join(displays)) pending = False failed = False # The Github states are: "error", "failure", "success", "pending" for state in display_name_to_state.values(): if state not in FINAL_STATES: pending = True if state in BAD_STATES: failed = True if failed or not pending: break if failed: print("*** Failed! ***") for display_name, url in showed_urls.items(): print(" {} logs: {}".format(display_name, url)) sys.exit(1) else: _download_release_assets(magic_build_repo_id, release_name)
def send_pull_request(form_data): # read metadata of new quickstart repo qs_u = form_data['github-username'] qs_r = form_data['github-repository'] qs_n = form_data['alternate-name'] or qs_r qs_c = form_data['cartridges'] or [] qs_t = form_data['type'] or [] try: qs = _read_quickstart_repo(qs_u, qs_r) qs['alternate_name'] = qs_n qs['cartridges'] = qs_c qs['type'] = qs_t except PyGitHub.UnknownObjectException: raise OOIndexError("Username or repository not found: %s/%s" % (qs_u, qs_r)) try: owner = PyGitHub.Github().get_user(qs_u) qs['owner_name'] = owner.name qs['owner_avatar_url'] = owner.avatar_url except: qs['owner_name'] = qs['owner'] qs['owner_avatar_url'] = '' # read content of original quickstar.json # fork repo if needed u = app.config['OO_INDEX_GITHUB_USERNAME'] r = app.config['OO_INDEX_GITHUB_REPONAME'] q = app.config['OO_INDEX_QUICKSTART_JSON'] repo, head, tree, quickstart = _read_github_file(u, r, q) # add quickstart to quickstart.json quickstart.append(qs) # create new blob with updated quickstart.json print "Creating blob...", sys.stdout.flush() new_blob = repo.create_git_blob( json.dumps(quickstart, indent=3, encoding='utf-8'), 'utf-8') # create tree with new blob element = _get_tree_element(repo, tree, q) element = PyGitHub.InputGitTreeElement(path=element.path, mode=element.mode, type=element.type, sha=new_blob.sha) if not element: flash("File not found: %s/%s/%s" % (u, r, q), "error") return print "Updating tree...", sys.stdout.flush() new_tree = repo.create_git_tree([element], tree) # create commit for new tree print "Creating commit...", sys.stdout.flush() message = 'Quickstart add request: %s/%s' % (qs_u, qs_r) new_commit = repo.create_git_commit(message, new_tree, [repo.get_git_commit(head.sha)]) # create new branch for new commit print "Creating branch...", sys.stdout.flush() try: new_branch = repo.create_git_ref('refs/heads/%s-%s' % (qs_u, qs_r), new_commit.sha) except PyGitHub.UnknownObjectException: raise OOIndexError("Username or repository not found: %s/%s" % (qs_u, qs_r)) # and finally, we send our pull request print "Creating pull request...", sys.stdout.flush() upstream = _get_repo_for(u, r, session['token']) pr_params = { 'title': message, 'body': 'Automatically generated PR for oo-index', 'base': 'master', 'head': '%s:%s-%s' % (g.user, qs_u, qs_r), } pr = upstream.create_pull(**pr_params) return pr
def create_pull_request(installation_token, repo): pulls = list(repo.get_pulls(state="all"))[:config.INITIAL_PULLS_LOOKUP] contexts = set() for pull in pulls: contexts.update( sanitize_context(s.context) for s in repo.get_commit(pull.head.sha).get_statuses() if not s.context.startswith("mergify/")) mergify_config = { "rules": { "default": { "protection": { "required_status_checks": { "contexts": list(contexts), }, "required_pull_request_reviews": { "required_approving_review_count": 1 }, }, }, } } content = yaml.dump(mergify_config, default_flow_style=False) try: default_branch = repo.get_branch(repo.default_branch) except github.GithubException as e: if e.status != 404: raise # TODO(sileht): When an empty repo is created we can't get the default # branch this one doesn't yet exists. We may want to pospone the first # PR in this case. For now just return to not raise backtrace return try: parents = [repo.get_git_commit(default_branch.commit.sha)] except github.GithubException as e: if e.status == 409 and e.data['message'] == 'Git Repository is empty.': return raise message = "Mergify initial configuration" tree = repo.create_git_tree([ github.InputGitTreeElement(".mergify.yml", "100644", "blob", content) ], base_tree=default_branch.commit.commit.tree) commit = repo.create_git_commit(message, tree, parents) repo.create_git_ref("refs/heads/%s" % INITIAL_CONFIG_BRANCH, commit.sha) repo.create_pull( title=message, body="""This is an initial configuration for Mergify. This pull request will require one reviewer approval. Those required [status checks](https://doc.mergify.io/configuration.html#required-status-checks) have been discovered from the %d more recent pull requests of your project: %s More information about Mergify configuration can be found at https://doc.mergify.io To modify this pull request, you can check it out locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/ """ % (config.INITIAL_PULLS_LOOKUP, "\n *".join(contexts)), # noqa base=repo.default_branch, head=INITIAL_CONFIG_BRANCH, ) LOG.info('Initial configuration created', repository=repo.full_name)
def cmd_coalesce(config, gh, org, args): repos = get_challenge_repos(config, org) if len(repos) == 0: log.error("No repos to coalesce") return master_repo_name = config.prefix + "challenges" description = "%s master challenge repository." % config.ctfname # Case 1: Brand new master repo # 1. Create repo # 2. Build up .gitmodules blob # 3. Build git trees, one per category # 4. Insert submodule blobs into their respective category trees # 5. Create new commit! # Case 2: Updating master repo # TODO # Get the admin team (if any) ctf_admin_team = get_admin_team(config, org) # Create the new repo new_repo = create_repo(org, master_repo_name, description, admin_user=ctf_admin_team) if new_repo is None: return if ctf_admin_team: try: ctf_admin_team.set_repo_permission(new_repo, 'admin') except github.GithubException as e: log.warning("Failed to set the repository permissions to admin for team '%s'", str(t)) try: head = new_repo.get_git_ref('heads/master') base_tree = new_repo.get_git_tree(head.object.sha) base_commit = new_repo.get_git_commit(head.object.sha) log.info('Retrieved base commit for %s', master_repo_name) except github.GithubException as e: log.error('Unable to retrieve base commit, tree, or reference') log.error('Reason: %s', str(e)) log.error("Rolling back created repo...") delete_repo(new_repo) return None master_repo = None by_category = {} for repo in repos: cat = repo.category if cat not in by_category: by_category[cat] = [] by_category[cat] += [repo] tl_elements = [] submodules = [] for cat, repo_list in sorted(by_category.items()): elements = [] for r in repo_list: repo_name = '%s%d' % (cat, r.chal_num) path = '%s/%s' % (cat, repo_name) branch = 'master' url = r.ssh_url try: r_head = r.get_git_ref('heads/%s' % branch) r_base_tree = r.get_git_tree(r_head.object.sha) r_base_commit = r.get_git_commit(r_head.object.sha) log.info('Retrieved base commit for %s on the %s branch', r.name, branch) except github.GithubException as e: log.error('Unable to retrieve base commit, tree, or reference') log.error('Reason: %s', str(e)) log.error("Rolling back created repo...") delete_repo(new_repo) return None commit = r_head.object.sha submodule = GitSubmodule(path, url, branch, commit) submodules += [submodule] elements += [github.InputGitTreeElement(repo_name, object_types["submodule"], 'commit', sha=commit)] # Create the directory for this category # Store sha for later reference try: new_tree = new_repo.create_git_tree(elements) except github.GithubException as e: log.error("Failed to create a git tree object for category '%s'", cat) log.error("Reason: %s", e) log.error("Rolling back created repo...") delete_repo(new_repo) return None tl_elements += [github.InputGitTreeElement(cat, object_types["directory"], 'tree', sha=new_tree.sha)] # Emit .gitmodules blob gitmodules_content = "".join([str(x) for x in submodules]) tl_elements += [github.InputGitTreeElement(".gitmodules", object_types["file"], 'blob', content=gitmodules_content)] # Create top level tree try: tl_tree = new_repo.create_git_tree(tl_elements) except github.GithubException as e: log.error("Failed to create the top-level git tree object") log.error("Reason: %s", e) log.error("Rolling back created repo...") delete_repo(new_repo) return None # Create commit linking to top of tree try: new_commit = new_repo.create_git_commit('Added initial challenge submodules', tl_tree, [base_commit]) head.edit(new_commit.sha) except github.GithubException as e: log.error("Failed to create commit object") log.error("Reason: %s", e) delete_repo(new_repo) return log.info("Master repository ready: %s", new_repo.html_url)
def save_git_hub_file(): """ FLASK POST routing method for '/saveFileToRemoteGithub' Save a file to a GitHub repository. The POST request content is a JSON string containing the file name, repository name, branch, access token, the graph data in JSON format and a commit message. """ # Extract parameters and file content from json. content = request.get_json(silent=True) filename = content["filename"] repo_name = content["repositoryName"] repo_branch = content["repositoryBranch"] repo_token = content["token"] graph = content["jsonData"] commit_message = content["commitMessage"] # Extracting the true repo name and repo folder. folder_name, repo_name = extract_folder_and_repo_names(repo_name) if folder_name != "": filename = folder_name + "/" + filename g = github.Github(repo_token) # get repo try: repo = g.get_repo(repo_name) except github.GithubException as e: print("Error in get_repo({0})! Repo: {1} Status: {2} Data: {3}".format( "heads/" + repo_branch, str(repo_name), e.status, e.data)) return jsonify({"error": e.data["message"]}), 400 # Set branch try: branch_ref = repo.get_git_ref("heads/" + repo_branch) except github.GithubException as e: # repository might be empty print("Error in get_git_ref({0})! Repo: {1} Status: {2} Data: {3}". format("heads/" + repo_branch, str(repo_name), e.status, e.data)) return jsonify({"error": e.data["message"]}), 400 # get SHA from branch branch_sha = branch_ref.object.sha # Add repo and file name in the graph. graph["modelData"]["repo"] = repo_name graph["modelData"]["repoBranch"] = repo_branch graph["modelData"]["repoService"] = "GitHub" graph["modelData"]["filePath"] = filename # Clean the GitHub file reference. graph["modelData"]["sha"] = "" graph["modelData"]["git_url"] = "" # The 'indent=4' option is used for nice formatting. Without it the file is stored as a single line. json_data = json.dumps(graph, indent=4) # Commit to GitHub repo. latest_commit = repo.get_git_commit(branch_sha) base_tree = latest_commit.tree try: new_tree = repo.create_git_tree( [ github.InputGitTreeElement(path=filename, mode="100644", type="blob", content=json_data) ], base_tree, ) except github.GithubException as e: # repository might not have permission print("Error in create_git_tree({0})! Repo: {1} Status: {2} Data: {3}". format("heads/" + repo_branch, str(repo_name), e.status, e.data)) return jsonify({"error": e.data["message"]}), 400 new_commit = repo.create_git_commit(message=commit_message, parents=[latest_commit], tree=new_tree) branch_ref.edit(sha=new_commit.sha, force=False) return "ok"
def build(repo, commit, package_name=None): """Build wheels for a given repo and commit / tag.""" click.secho(LOGO, fg='cyan') repo_id = _get_repo_id() user, package = repo.split('/', 1) if package_name is None: package_name = package click.secho("Building in repo {}".format(repo_id)) click.secho("Building wheels for {}/{}\n".format(user, package)) clone_url = DEFAULT_CLONE_TEMPLATE.format("{}/{}".format(user, package)) repo = get_gh().get_repo(repo_id) click.secho("Finding a unique name for this release...", fg='yellow') # Pick the release_name by finding an unused one i = 1 while True: release_name = "{}-{}".format(package_name, commit) if i > 1: release_name += '-{}'.format(i) try: repo.get_release(release_name) except github.UnknownObjectException: break i += 1 branch_name = 'branch-for-' + release_name bs = { 'clone-url': clone_url, 'package-name': package_name, 'repo': '{}/{}'.format(user, package), 'commit': commit, 'upload-to': { 'type': 'github-release', 'repo-id': repo_id, 'release-id': release_name, } } bs_json = json.dumps(bs) click.secho( "Creating release {} to collect assets...".format(release_name), fg='yellow') release_template = "https://github.com/{}/{}\n\n### Build spec\n\n```json\n{}\n```" release = repo.create_git_release( release_name, release_name, release_template.format(user, package, json.dumps(bs, indent=4)), ) print(release.html_url) click.secho("Creating build branch...", fg='yellow') # 'master' is a 'Commit'. 'master.commit' is a 'GitCommit'. These are # different types that are mostly *not* interchangeable: # https://pygithub.readthedocs.io/en/latest/github_objects/Commit.html # https://pygithub.readthedocs.io/en/latest/github_objects/GitCommit.html master = repo.get_commit('master') master_gitcommit = master.commit patch = github.InputGitTreeElement( 'build-spec.json', '100644', 'blob', content=bs_json, ) tree = repo.create_git_tree([patch], master_gitcommit.tree) our_gitcommit = repo.create_git_commit("Building: {}".format(release_name), tree, [master_gitcommit]) repo.create_git_ref('refs/heads/' + branch_name, our_gitcommit.sha) print("Commit is {} in branch {}.".format(our_gitcommit.sha[:8], branch_name)) click.secho("Waiting for build to complete...", fg='yellow') # get_combined_status needs a Commit, not a GitCommit our_commit = repo.get_commit(our_gitcommit.sha) showed_urls = {} while True: time.sleep(10) combined_status = our_commit.get_combined_status() display_name_to_state = {} for display_name in STATUSES.values(): display_name_to_state[display_name] = NA_STATE for status in combined_status.statuses: if status.context in STATUSES: display_name = STATUSES[status.context] display_name_to_state[display_name] = status.state if display_name not in showed_urls: print("{} logs: {}".format(display_name, status.target_url)) showed_urls[display_name] = status.target_url displays = [ click.style("[{} - {}]".format(name, state), fg=STATUS_COLORS.get(state, 'white')) for name, state in display_name_to_state.items() ] click.echo(" ".join(displays)) pending = False failed = False # The Github states are: "error", "failure", "success", "pending" for state in display_name_to_state.values(): if state not in FINAL_STATES: pending = True if state in BAD_STATES: failed = True if failed or not pending: break release = get_release(repo_id, release_name) if failed: click.secho("*** Failed! ***", bg='red', fg='black') for display_name, url in showed_urls.items(): print("{} logs: {}".format(display_name, url)) release.update_release('\u274c ' + release.title, release.body) sys.exit(1) else: _download_release_assets(repo_id, release_name) release.update_release('\u2705 ' + release.title, release.body)
def create_git_tree(repo, directory, file_read_hook=None): object_tree = { "blobs" : [], "trees" : {} } # counters file_bytes = 0 file_count = 0 directory_count = 0 log.info("Building file system tree for '%s'", directory) # Walk the filesystem and build a tree structure for root, dirs, files in os.walk(directory): file_objects = [] components = root.split(os.sep) # skip leading and any blank components components = components[1:len(components)] components = filter(lambda x: x != "", components) # each call is for one directory directory_count += 1 # top-level directory if len(components) == 0: level = object_tree else: base = object_tree # drill down in the tree to the level we're at for component in components: if component not in base["trees"]: base["trees"][component] = { "blobs" : [], "trees" : {} } base = base["trees"][component] level = base # for each file at the current level for f in files: path = os.path.join(root, f) ty = object_types['executable'] if os.access(path, os.X_OK) else object_types['file'] # read in all of the file contents (this could be deferred for streaming) contents = "" if file_read_hook: contents = file_read_hook(path) if contents is None: return None else: try: with open(path, 'rb') as fp: contents = fp.read() except IOError: log.error("Unable to open file '%s' for reading", path) return None file_bytes += len(contents) file_count += 1 # create a tree element for this level["blobs"] += [GitBlob(f, ty, content=contents)] log.info('Walked %d directories and %d files (total bytes %d)', directory_count, file_count, file_bytes) log.info('Creating git trees...') # rewalk the data structure and build the tree nodes # do this using DFS to build the bottom most trees first stack = [TreeObj("", object_tree)] while len(stack): node = stack.pop() # if we haven't visited the node and it has child directories if not node.visited and len(node.tree["trees"]): node.visited = True # we're at a leaf directory node stack.append(node) for name, tree in node.tree["trees"].iteritems(): stack.append(TreeObj(name, tree)) # otherwise, process the directory node else: # map all directories to git tree elements elements = map(lambda x: github.InputGitTreeElement(x[0], object_types["directory"], 'tree', sha=x[1]["obj"].sha), node.tree["trees"].items()) # upload our blobs :) for blob in node.tree["blobs"]: import base64 try: git_blob = repo.create_git_blob(base64.encodestring(blob.content).strip(), 'base64') elements += [github.InputGitTreeElement(blob.path, blob.ty, 'blob', sha=git_blob.sha)] except github.GithubExeception as e: log.error("Failed to upload git blob for file '%s'", blob.path) log.error("Reason: %s", e) # XXX: can we roll back trees/blobs created before the failure? return None # git trees CANNOT be empty if len(elements) == 0: elements += [github.InputGitTreeElement(".keep", object_types["file"], 'blob', content="")] # talk to the API and create the damn tree try: new_tree = repo.create_git_tree(elements) except github.GithubException as e: log.error("Failed to create a git tree object for node '%s'", node.path) log.error("Reason: %s", e) # XXX: can we roll back trees created before the failure? return None node.tree["obj"] = new_tree return object_tree["obj"]