def clone(repo_name, internal):
uniq_path = hashlib.sha224(repo_name).hexdigest()
uniq_path = hashlib.sha224(repo_name).hexdigest()
if os.path.isdir(os.getcwd() + '/clones/' + uniq_path):
shutil.rmtree(os.getcwd() + '/clones/' + uniq_path)
repo_url = 'https://github.com/%s.git' % (repo_name)
try:
clone_dir = os.getcwd() + '/clones/'
if not os.path.isdir(clone_dir):
os.makedirs(clone_dir)
repo_path = clone_dir + uniq_path
login_info = ''
if internal==True:
username = '******' #your-github-username-here
password = str(keyring.get_password('github', username))
login_info = git.UserPass(username, password)
git_obj = git.clone_repository(repo_url, repo_path, credentials=login_info)
return repo_path
except Exception, e:
print e
if str(e).find('Unexpected HTTP status code: 404'):
print "Repo doesn't exists"
return "Repo doesn't exists"
def loc_main(args):
if args.input == '-':
input = sys.stdin
else:
input = codecs.open(args.input, 'r', 'utf-8')
if args.output == '-':
output = sys.stdout
else:
output = codecs.open(args.output, 'w', 'utf-8')
credentials = pygit2.UserPass(
args.username or os.environ.get('GIT_USERNAME', None), args.password
or os.environ.get('GIT_PASSWORD', None))
if not credentials.credential_tuple[0] or not credentials.credential_tuple[
1]:
# NB the user must supply both or none.
credentials = None
with input, output:
for line in input:
url = line.strip()
if url == '' or url.startswith('#'):
continue
cloc(url, credentials, output)
def push(repo, branch="master"):
remote = repo.remotes["origin"]
remote.credentials = pygit2.UserPass(settings.github_user,
settings.github_token)
remote.push(
[f"refs/heads/{branch}"],
callbacks=pygit2.RemoteCallbacks(credentials=remote.credentials))
def _get_credentials(_auth):
if not _auth:
return None
_agent = _auth.get("agent", False)
_public_key = _auth.get("public_key", None)
_private_key = _auth.get("private_key", None)
_username = _auth.get("user", None)
_passphrase = _auth.get("pass", None)
if _agent:
credentials = pygit2.KeypairFromAgent(_username or "git")
elif _public_key and _private_key:
if os.path.exists(_public_key) and os.path.exists(_private_key):
credentials = pygit2.Keypair(_username or "git", _public_key,
_private_key, _passphrase or "")
else:
credentials = pygit2.KeypairFromMemory(_username or "git",
_public_key, _private_key,
_passphrase or "")
elif _username and _passphrase:
credentials = pygit2.UserPass(_username, _passphrase)
elif _username:
credentials = pygit2.Username(_username)
else:
credentials = None
return credentials
def git_push(repo, profile, tag):
git_token, git_username = github_config(profile)
callbacks = pygit2.RemoteCallbacks(
pygit2.UserPass(git_token, 'x-oauth-basic'))
remote = repo.remotes["origin"]
remote.push(['refs/heads/master'], callbacks=callbacks)
remote.push(['refs/tags/{}'.format(tag)], callbacks=callbacks)
def clone(repo_name, internal):
clone_directory = os.environ['git_clone_dir']
uniq_path = hashlib.sha224(repo_name).hexdigest()
if os.path.isdir(os.path.join(clone_directory, uniq_path)):
shutil.rmtree(os.path.join(clone_directory, uniq_path))
if internal:
repo_url = '%s/%s.git' % (os.environ['int_git_url'], repo_name)
else:
repo_url = '%s/%s.git' % (os.environ['ext_git_url'], repo_name)
try:
clone_dir = clone_directory
if not os.path.isdir(clone_dir):
os.makedirs(clone_dir)
repo_path = os.path.join(clone_dir, uniq_path)
if internal == True:
username = os.environ['int_git_user']
password = os.environ['int_git_token']
else:
username = os.environ['ext_git_user']
password = os.environ['ext_git_token']
login_info = git.UserPass(username, password)
git_obj = git.clone_repository(repo_url,
repo_path,
credentials=login_info)
return repo_path
except Exception, e:
if str(e).find('Unexpected HTTP status code: 404'):
log.logger.error("repo doesn't exists")
return "Repo doesn't exists"
log.logger.error(e)
def CloneRepo(username, password, repo_name, repo_clone_url, signature_name,
signature_email, first_commit_msg):
g = Github(username, password)
user = g.get_user()
repo = user.create_repo(repo_name)
if os.path.exists(os.getcwd() + '/tmp/'):
shutil.rmtree(os.getcwd() + '/tmp/')
file_path = os.path.abspath(os.getcwd() + '/tmp')
#Clone it
repoClone = pygit2.clone_repository(repo_clone_url, file_path)
#Commit it
repoClone.remotes.set_url("origin", repo.clone_url)
index = repoClone.index
index.add_all()
index.write()
author = pygit2.Signature(signature_name, signature_email)
commiter = pygit2.Signature(signature_name, signature_email)
tree = index.write_tree()
oid = repoClone.create_commit('refs/heads/master', author, commiter,
first_commit_msg, tree,
[repoClone.head.target])
remote = repoClone.remotes["origin"]
credentials = pygit2.UserPass(username, password)
remote.credentials = credentials
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
remote.push(['refs/heads/master'], callbacks=callbacks)
if os.path.exists(os.getcwd() + '/tmp/'):
shutil.rmtree(os.getcwd() + '/tmp/')
def __init__(self):
self.repo = pygit2.Repository(
os.path.join(os.path.relpath(GIT_REPO), '.git'))
creds = pygit2.UserPass(GIT_USERNAME, GIT_PASSWORD)
self.remote = pygit2.RemoteCallbacks(credentials=creds)
self.author = pygit2.Signature(GIT_USERNAME, GIT_EMAIL)
def test_clone_with_credentials(self):
credentials = pygit2.UserPass("libgit2", "libgit2")
repo = clone_repository(
"https://bitbucket.org/libgit2/testgitrepository.git",
self._temp_dir, credentials=credentials)
self.assertFalse(repo.is_empty)
def main():
"""Get the CSV file of students' names and the destination
repository from the command line, then transfer the files.
Afterwards, commit and push using provided credentials."""
parser = argparse.ArgumentParser(
description="Read a file of "
"student GitHub usernames, then transfer files in their "
"PE repositories to a given destination repository.")
parser.add_argument("students", help="CSV or plaintext list of students")
parser.add_argument("destination",
help="destination repository, "
"in the form owner/name (e.g. nus-cs2103-AY1920S1/pe)")
parser.add_argument("credentials",
help="credentials file - "
"four lines of author, email, username, access token")
parser.add_argument(
"-p",
help="number of download processes to "
"run in parallel (default is number of cores on machine)",
type=int,
default=os.cpu_count())
args = parser.parse_args()
students = read_student_names(args.students)
AUTHOR, EMAIL, USERNAME, TOKEN = read_credentials(args.credentials)
user_pass = pygit2.UserPass(USERNAME, TOKEN)
key = pygit2.RemoteCallbacks(credentials=user_pass)
processes = 1 if args.p is None or args.p < 1 else args.p
endpoint = clone_destination(args.destination, key)
files_path = make_files_folder(endpoint)
print_clashes(collate_files(students, files_path, processes))
commit_and_push(endpoint, AUTHOR, EMAIL, "Collect PE files", key)
def vgit_push(self, path, message, user_name_commiter, user_name_author,
email_commiter, email_author, branch, user_name_pusher,
user_passoword_pusher, log_cb):
try:
repo = Repository(path)
index = repo.index
reference = 'refs/heads/' + branch
tree = index.write_tree()
author = pygit2.Signature(user_name_author, email_author)
commiter = pygit2.Signature(user_name_commiter, email_commiter)
oid = repo.create_commit(reference, author, commiter, message,
tree, [repo.head.target])
credentials = pygit2.UserPass(user_name_pusher,
user_passoword_pusher)
remo = repo.remotes["origin"]
remo.credentials = credentials
aux = remo.url
repo.remotes.set_push_url(user_name_pusher, aux)
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
remo.push([reference], callbacks=callbacks)
except ValueError as err:
log_cb(str(err))
return False
except Exception as err:
log_cb(str(err))
return False
return True
def test_clone_with_credentials(self):
url = 'https://github.com/libgit2/TestGitRepository'
credentials = pygit2.UserPass("libgit2", "libgit2")
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
repo = clone_repository(url, self._temp_dir, callbacks=callbacks)
assert not repo.is_empty
def main():
"""Main entry point to the program"""
target_dir = input("Please enter target dir > ")
u_name = input("Please enter github username > ")
pss_wrd = getpass.getpass(prompt="Please enter github password > ")
check_interval = input(
"Please enter desired check interval (seconds)[10] > ")
if check_interval == "":
check_interval = 10
if target_dir == "" or u_name == "" or pss_wrd == "":
print_help()
return
# getting set up for the pull request
cred = pygit2.UserPass(u_name, pss_wrd)
callback = pygit2.RemoteCallbacks(credentials=cred)
repo = pygit2.Repository(target_dir)
print(f"Watching {target_dir} for updates every {check_interval} seconds.")
start_time = time.time()
while True:
cur_time = time.time()
if cur_time - start_time >= check_interval:
print(f"Checking for updates...")
# initiate the pull
pull(repo, credentials_callback=callback)
# reset start time to check new interval
start_time = time.time()
def credentials(self, url, username_from_url, allowed_types):
"""
The callback to return a suitable authentication method.
it supports GIT_CREDTYPE_SSH_KEY and GIT_CREDTYPE_USERPASS_PLAINTEXT
GIT_CREDTYPE_SSH_KEY with an ssh agent configured in the env variable SSH_AUTH_SOCK
or with id_rsa and id_rsa.pub in ~/.ssh (password must be the empty string)
GIT_CREDTYPE_USERPASS_PLAINTEXT from the env variables GIT_USERNAME and GIT_PASSWORD
"""
if pygit2.credentials.GIT_CREDTYPE_SSH_KEY & allowed_types:
if "SSH_AUTH_SOCK" in os.environ:
# Use ssh agent for authentication
return pygit2.KeypairFromAgent(username_from_url)
else:
ssh = join(expanduser('~'), '.ssh')
if "QUIT_SSH_KEY_HOME" in os.environ:
ssh = os.environ["QUIT_SSH_KEY_HOME"]
# public key is still needed because:
# pygit2._pygit2.GitError: Failed to authenticate SSH session:
# Unable to extract public key from private key file:
# Method unimplemented in libgcrypt backend
pubkey = join(ssh, 'id_rsa.pub')
privkey = join(ssh, 'id_rsa')
# check if ssh key is available in the directory
if os.path.isfile(pubkey) and os.path.isfile(privkey):
return pygit2.Keypair(username_from_url, pubkey, privkey,
"")
else:
raise Exception(
"No SSH keys could be found, please specify SSH_AUTH_SOCK or add keys to "
+ "your ~/.ssh/")
elif pygit2.credentials.GIT_CREDTYPE_USERPASS_PLAINTEXT & allowed_types:
if self.session and "OAUTH_TOKEN" in self.session:
return pygit2.UserPass(self.session["OAUTH_TOKEN"],
'x-oauth-basic')
elif "GIT_USERNAME" in os.environ and "GIT_PASSWORD" in os.environ:
return pygit2.UserPass(os.environ["GIT_USERNAME"],
os.environ["GIT_PASSWORD"])
else:
raise Exception(
"Remote requested plaintext username and password authentication but "
+ "GIT_USERNAME or GIT_PASSWORD are not set.")
else:
raise Exception(
"Only unsupported credential types allowed by remote end")
def credentials(self, url, username_from_url, allowed_types):
username = username_from_url if username_from_url else self.username
if allowed_types & pygit2.credentials.GIT_CREDENTIAL_SSH_KEY:
return pygit2.Keypair(username, self.public_key_file,
self.private_key_file, '')
elif allowed_types & pygit2.credentials.GIT_CREDENTIAL_USERPASS_PLAINTEXT:
return pygit2.UserPass(username, self.password)
else:
return None
def credentials(self, url, username_from_url, allowed_types):
print(self, url, username_from_url, allowed_types)
# allowed_types tells us what type of Credential object the server wants
if allowed_types == pygit2.GIT_CREDTYPE_USERPASS_PLAINTEXT:
# Our problem: do what you need to do in order to find out what to
# put into the UserPass object.
password = getpass.getpass('Enter password for url={}'.format(url))
return pygit2.UserPass('philippecarphin', password)
elif allowed_types == pygit2.GITCREDTYPE_USERNAME:
raise NotImplemented
def credentials(self, url, username_from_url, allowed_types):
# print('[credentials]', url, username_from_url, allowed_types)
if allowed_types == pygit2.GIT_CREDTYPE_SSH_KEY:
return pygit2.Keypair('git', self.__id_rsa_pub, self.__id_rsa,
'')
elif allowed_types == pygit2.GIT_CREDTYPE_USERPASS_PLAINTEXT:
return pygit2.UserPass(self.__username, self.__password)
else:
raise RuntimeError(
'unsupported authentication type: {}'.format(
allowed_types))
def get_cred(repo_url, repo_region):
credentials = session.get_credentials()
credentials = credentials.get_frozen_credentials()
username = credentials.access_key + '%' + credentials.token
password = sign_request(repo_region, repo_url)
cred = pygit2.UserPass(username, password)
print credentials
return RemoteCallbacks(credentials=cred)
def credentials(self, url, username_from_url, allowed_types):
# its a libgit2 bug, that it infinitly retries the authentication
self.attempts += 1
if self.attempts >= 5:
# pygit2 doesn't propagate the exception properly
msg = 'Wrong oauth personal access token'
print(msg)
raise ValueError(msg)
if allowed_types & pygit2.credentials.GIT_CREDTYPE_USERPASS_PLAINTEXT:
return pygit2.UserPass(self.token, 'x-oauth-basic')
else:
return None
def backup_gh_repos(user,
backupdir,
token=None,
backup_list=None,
exclude_list=None):
callbacks = None
if token != None:
log.info("Authenticating with Github, and pulling all repositories")
response = requests.get(
"https://api.github.com/user/repos?visibility=all",
auth=(user, token))
repos = json.loads(response.text)
cred = pygit2.UserPass(user, token)
callbacks = pygit2.RemoteCallbacks(credentials=cred)
else:
response = requests.get("https://api.github.com/users/" + user +
"/repos")
repos = json.loads(response.text)
to_backup = []
if backup_list != None:
for repo in repos:
if repo["name"] in backup_list:
to_backup.append(repo)
else:
if exclude_list != None:
for repo in repos:
if repo["name"] not in exclude_list:
to_backup.append(repo)
else:
to_backup = repos
for repo in to_backup:
log.info("Backing up: " + repo["name"])
dest = backupdir / "repo" / repo["full_name"]
localrepopath = pygit2.discover_repository(str(dest))
if localrepopath == None:
log.info("Cloning repo...")
pygit2.clone_repository(repo["clone_url"],
str(dest),
bare=True,
callbacks=callbacks)
else:
log.info("Fetching updates...")
localrepo = (
pygit2.Repository(localrepopath).remotes["origin"].fetch(
callbacks=callbacks))
def initialize(request):
if path.exists(settings.MODREPO_DIR) and path.exists(path.join(settings.MODREPO_DIR, ".git")):
repo = pygit2.Repository(settings.MODREPO_DIR)
for remote in repo.remotes:
if remote.name == 'origin':
result = remote.fetch()
repo.checkout('refs/remotes/origin/master')
else:
shutil.rmtree(settings.MODREPO_DIR)
cred = pygit2.UserPass(settings.MODREPO_USER, settings.MODREPO_PASS)
pygit2.clone_repository(settings.MODREPO_REMOTE, settings.MODREPO_DIR, credentials=cred)
return redirect(modrepo)
def commit_and_push(repo_clone, commit_message):
index = repo_clone.index
index.add_all()
index.write()
author = pygit2.Signature(userName, email)
commiter = pygit2.Signature(userName, email)
tree = index.write_tree()
repo_clone.create_commit('refs/heads/master', author, commiter,
commit_message, tree,
[repo_clone.head.get_object().hex])
remote = repo_clone.remotes["origin"]
credentials = pygit2.UserPass(repoToken, 'x-oauth-basic')
remote.credentials = credentials
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
remote.push(['refs/heads/master'], callbacks=callbacks)
async def git_verify(self, ctx, filepath):
try:
repo = pygit2.Repository(await self.config.pipeline_base())
except pygit2.GitError:
return
if not repo.diff():
return
if not repo.lookup_branch("master").is_checked_out():
await send_cancellation_message(
ctx,
f"Hey {ctx.author.mention}, the pipeline branch is currently **not**"
f" set to master! Please inform a sysadmin that this crud change"
f" was only **temporarily** made!")
return
if any(diff for diff in repo.diff().deltas
if diff.old_file.path != filepath):
return await send_cancellation_message(
ctx,
f"Hey {ctx.author.mention}, there are currently staged changes."
f" Please inform a sysadmin so that your changes can be"
f" manually committed.")
keys = await self.bot.get_shared_api_tokens("github")
if "username" not in keys or "token" not in keys:
return await send_cancellation_message(
ctx, f"Github credentials unset. Add via `{ctx.prefix}set api"
f" github username <username> token <access token>`")
try:
index = repo.index
index.add(filepath)
index.write()
tree = index.write_tree()
email = await self.config.user(ctx.author).email()
author = pygit2.Signature(str(ctx.author), email
or "*****@*****.**")
commiter = pygit2.Signature("Famiel", "*****@*****.**")
parent, ref = repo.resolve_refish(refish=repo.head.name)
repo.create_commit(ref.name, author, commiter, "Updating JSON",
tree, [parent.oid])
upcred = pygit2.UserPass(keys['username'], keys['token'])
remote = discord.utils.get(repo.remotes, name="origin")
remote.push(['refs/heads/master'],
callbacks=pygit2.RemoteCallbacks(credentials=upcred))
except Exception as e:
logger.exception("Failed to push.")
await send_cancellation_message(
ctx, f"Failed to push to github. Please alert a sysadmin.")
def clone_from_github(self) -> bool:
try:
callbacks = RemoteCallbacks(
pygit2.UserPass(self.api_key, 'x-oauth-basic'))
clone_repository(url='https://github.com/{}'.format(
self.repo_name),
path=self.old_repo,
callbacks=callbacks,
checkout_branch=self.branch)
if os.path.exists(path='{}/.git'.format(self.old_repo)):
return True
else:
return False
except (GitError, ValueError) as error:
print(error)
return False
def org_checkout(organization, github_url, github_token, clone_dir, verbose,
filter, exclude):
"""Checkout repositories from a GitHub organization."""
logging.basicConfig(
format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
level=(verbose and logging.DEBUG or logging.INFO))
callbacks = pygit2.RemoteCallbacks(
pygit2.UserPass(github_token, 'x-oauth-basic'))
repos = []
for r in github_repos(organization, github_url, github_token):
if filter:
found = False
for f in filter:
if fnmatch(r['name'], f):
found = True
break
if not found:
continue
if exclude:
found = False
for e in exclude:
if fnmatch(r['name'], e):
found = True
break
if found:
continue
repo_path = os.path.join(clone_dir, r['name'])
repos.append(repo_path)
if not os.path.exists(repo_path):
log.debug("Cloning repo: %s/%s" % (organization, r['name']))
repo = pygit2.clone_repository(r['url'],
repo_path,
callbacks=callbacks)
else:
repo = pygit2.Repository(repo_path)
if repo.status():
log.warning('repo %s not clean skipping update')
continue
log.debug("Syncing repo: %s/%s" % (organization, r['name']))
pull(repo, callbacks)
return repos
def clone_repo(self, repo_path, rev=''):
try:
repo = pygit2.clone_repository(self.repo_url, repo_path)
except pygit2.GitError:
username = input('Enter git username: '******'Enter git password: ')
cred = pygit2.UserPass(username, password)
try:
repo = pygit2.clone_repository(self.repo_url, repo_path,
callbacks=pygit2.RemoteCallbacks(credentials=cred))
except ValueError:
print("Invalid URL!")
sys.exit(1)
except Exception as e:
print(e)
sys.exit(1)
return repo
def commit (
project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff github commit -v local_dir=<local_dir> commit_msg=<msg> branch=<branch>`
Commit all the outstanding changes to the remote branch. If the branch does not exist,
it will be created.
"""
state = _get_state()
access_token = vars.get("access_token", state.get(GITHUB_ACCESS_TOKEN))
github = _get_github(access_token)
local_dir = vars["local_dir"] # typically the "./{repository_name}"
repo = pygit2.Repository(local_dir + "/.git")
user = github.get_user()
branch = vars.get("branch", "master")
commit_msg = vars["commit_msg"]
# repo.remotes.set_url("origin", repo.clone_url)
index = repo.index
index.add_all()
index.remove_all("*.secrets*")
index.write()
author = pygit2.Signature(user.name, user.email)
commiter = pygit2.Signature(user.name, user.email)
tree = index.write_tree()
oid = repo.create_commit(
"refs/heads/" + branch,
author,
commiter,
commit_msg,
tree,
[repo.head.target],
# [repo.head.get_object().hex],
)
credentials = pygit2.UserPass(access_token, "x-oauth-basic")
# callbacks = pygit2.RemoteCallbacks(credentials=credentials)
remote = repo.remotes["origin"]
remote.credentials = credentials
remote.push(["refs/heads/" + branch], callbacks=_get_callbacks(access_token))
def _push_to_repo(user_name, cloned_repo_directory):
"""
:param user_name: github username
:param cloned_repo_directory: path/to/repo_directory
:return:
"""
repoclone = pygit2.Repository(cloned_repo_directory + "/.git/")
remote = repoclone.remotes["origin"]
username = user_name
password = get_github_password(username, refresh=False)
credentials = pygit2.UserPass(username, password)
remote.credentials = credentials
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
try:
remote.push(['refs/heads/master'], callbacks=callbacks)
except GitCommandError as e:
_LOG.error(str(e))
raise PushFailedException('Uh oh, it seems like something went'
' wrong with the push. "{0}"'.format(str(e)))
def credentials(
self, url: str, username_from_url: typing.Optional[str], allowed_types: int
) -> typing.Optional[
typing.Union[pygit2.Keypair, pygit2.UserPass, pygit2.Username]
]:
"""
If the remote server requires authentication, this function will be called and
its return value used for authentication.
"""
if allowed_types & pygit2.credentials.GIT_CREDTYPE_SSH_KEY:
return pygit2.KeypairFromAgent(username_from_url)
elif allowed_types & pygit2.credentials.GIT_CREDTYPE_USERPASS_PLAINTEXT:
username = username_from_url or click.prompt("Username")
password = click.prompt("Password", hide_input=True)
return pygit2.UserPass(username=username, password=password)
elif allowed_types & pygit2.credentials.GIT_CREDTYPE_USERNAME:
return pygit2.Username(username_from_url)
else:
return None
def __init__(self, password):
self.host_keys = paramiko.util.load_host_keys(
os.path.expanduser("~/.ssh/known_hosts"))
self.userpass = pygit2.UserPass("robobot", password)
self.callbacks = pygit2.RemoteCallbacks(credentials=self.userpass)
self.callbacks.push_update_reference = self.push_update_ref
try:
self.key = paramiko.RSAKey.from_private_key_file(
os.path.join(os.environ["HOME"], ".ssh", "id_rsa"), password)
except paramiko.ssh_exception.SSHException as e:
print("Wrong password!")
sys.exit(-1)
self.sock = {
"91": None,
"92": None,
"93": None,
"94": None,
"95": None
}
self.transport = {}