def get_remote_repo_path(remote_git, branch_from, loop=False):
""" Return the path of the remote git repository corresponding to the
provided information.
"""
repopath = os.path.join(
APP.config['REMOTE_GIT_FOLDER'],
werkzeug.secure_filename('%s_%s' % (remote_git, branch_from))
)
if not os.path.exists(repopath):
try:
pygit2.clone_repository(
remote_git, repopath, checkout_branch=branch_from)
except Exception as err:
LOG.debug(err)
LOG.exception(err)
flask.abort(500, 'Could not clone the remote git repository')
else:
repo = pagure.lib.repo.PagureRepo(repopath)
try:
repo.pull(branch=branch_from, force=True)
except pagure.exceptions.PagureException as err:
LOG.debug(err)
LOG.exception(err)
flask.abort(500, err.message)
return repopath
def clone(repo_name, internal):
clone_directory = os.environ['git_clone_dir']
uniq_path = hashlib.sha224(repo_name).hexdigest()
if os.path.isdir(os.path.join(clone_directory, uniq_path)):
shutil.rmtree(os.path.join(clone_directory, uniq_path))
if internal:
repo_url = '%s/%s.git' % (os.environ['int_git_url'], repo_name)
else:
repo_url = '%s/%s.git' % (os.environ['ext_git_url'], repo_name)
try:
clone_dir = clone_directory
if not os.path.isdir(clone_dir):
os.makedirs(clone_dir)
repo_path = os.path.join(clone_dir, uniq_path)
if internal==True:
username = os.environ['int_git_user']
password = os.environ['int_git_token']
login_info = git.UserPass(username, password)
git_obj = git.clone_repository(repo_url, repo_path, credentials=login_info)
else:
#username = os.environ['ext_git_user']
#password = os.environ['ext_git_token']
git_obj = git.clone_repository(repo_url, repo_path)
return repo_path
except Exception, e:
if str(e).find('Unexpected HTTP status code: 404'):
log.logger.error("repo doesn't exists")
return "Repo doesn't exists"
log.logger.error(e)
def open_repository(url,
target_directory):
"""
Make a connection from a remote git repository into a local
directory.
Args:
url(string): The remote github url of the repository
target_directory(string): The local target directory
Returns:
pygit2.repo: The repository object created
"""
git_url = urlparse.urlparse(url)
username = git_url.netloc.split('@')[0]\
if '@' in git_url.netloc else 'git'
try:
credentials = pygit2.credentials.KeypairFromAgent(username)
except AttributeError as e:
pygit2_parse_error(e)
# If local directory exists, then make a connection to it
# Otherwise, clone the remote repo into the new directory
if os.path.isdir(target_directory):
shaker.libs.logger.Logger().debug("open_repository: "
"Opening url '%s' "
"with existing local repository '%s'"
% (url, target_directory))
repo = pygit2.Repository(target_directory)
else:
# Try to use pygit2 0.22 cloning
try:
shaker.libs.logger.Logger().debug("open_repository: "
"Trying to open repository "
"using pygit2 0.22 format")
repo = pygit2.clone_repository(url,
target_directory,
credentials=credentials)
except TypeError as e:
shaker.libs.logger.Logger().debug("open_repository: "
"Failed to detect pygit2 0.22")
shaker.libs.logger.Logger().debug("open_repository: "
"Trying to open repository "
"using pygit2 0.23 format")
# Try to use pygit2 0.23 cloning
callbacks = pygit2.RemoteCallbacks(credentials)
repo = pygit2.clone_repository(url,
target_directory,
callbacks=callbacks)
shaker.libs.logger.Logger().debug(":open_repository: "
"Cloning url '%s' into local repository '%s'"
% (url, target_directory))
origin = filter(lambda x: x.name == 'origin', repo.remotes)
if not origin:
repo.create_remote('origin', url)
origin = filter(lambda x: x.name == 'origin', repo.remotes)
origin[0].credentials = credentials
return repo
def init_repo(repo_path, clone_from=None, clone_refs=False,
alternate_repo_paths=None, is_bare=True):
"""Initialise a new git repository or clone from existing."""
assert is_valid_new_path(repo_path)
init_repository(repo_path, is_bare)
if clone_from:
# The clone_from's objects and refs are in fact cloned into a
# subordinate tree that's then set as an alternate for the real
# repo. This lets git-receive-pack expose available commits as
# extra haves without polluting refs in the real repo.
sub_path = os.path.join(repo_path, 'turnip-subordinate')
clone_repository(clone_from, sub_path, True)
assert is_bare
alt_path = os.path.join(repo_path, 'objects/info/alternates')
with open(alt_path, 'w') as f:
f.write('../turnip-subordinate/objects\n')
if clone_refs:
# With the objects all accessible via the subordinate, we
# can just copy all refs from the origin. Unlike
# pygit2.clone_repository, this won't set up a remote.
# TODO: Filter out internal (eg. MP) refs.
from_repo = Repository(clone_from)
to_repo = Repository(repo_path)
for ref in from_repo.listall_references():
to_repo.create_reference(
ref, from_repo.lookup_reference(ref).target)
if alternate_repo_paths:
write_alternates(repo_path, alternate_repo_paths)
ensure_config(repo_path) # set repository configuration defaults
return repo_path
def clone_repo(cls, url, dest_path, bare=False):
""" Clone a git repo from the provided url at the specified dest_path.
:arg url: the url of the git.
:type url: str
:arg dest_path: the path where to clone the git repo (including the
name of the git repo itself, ie: if you clone foo in /home/bb/bar
the dest_path will be /home/bb/bar/foo).
:type dest_path: str
:kwarg bare: a boolean specifying whether the cloned repo should be
a bare repo or not
:type bare: bool
:return: a `GitRepo` object instanciated at the provided path
:rtype: GitRepo
:raises OSError: raised when the directory where is cloned the repo
exists and is not empty
:raises pygit2.GitError: raised then the url provided is invalid
"""
if os.path.exists(dest_path):
raise OSError(
errno.EEXIST, '%s exists and is not empty' % dest_path)
pygit2.clone_repository(url, dest_path, bare=bare)
return cls(path=dest_path)
def initRepositories(self, repositoryIDs):
"""
Creates a Review folder in the current directory
and then creates a folder for each repository that
is specified in the repos dictionary.
"""
repositories = {}
isDir = path.isdir(self.absPath + "/Review")
if not isDir:
os.mkdir(self.absPath + "/Review")
for currentID in repositoryIDs.keys():
currentPath = self.absPath + "/Review/" + currentID
isDir = path.isdir(currentPath)
if not isDir:
os.mkdir(currentPath)
repositories[currentID] = pygit2.clone_repository(
"git://" + repositoryIDs[currentID].getUrl(), currentPath
)
repositories[currentID].checkout(
repositories[currentID].lookup_reference(repositoryIDs[currentID].getTag())
)
else:
shutil.rmtree(currentPath)
os.mkdir(currentPath)
repositories[currentID] = pygit2.clone_repository(
"git://" + repositoryIDs[currentID].getUrl(), currentPath
)
repositories[currentID].checkout(
repositories[currentID].lookup_reference(repositoryIDs[currentID].getTag())
)
def get_remote_repo_path(remote_git, branch_from, loop=False):
""" Return the path of the remote git repository corresponding to the
provided information.
"""
repopath = os.path.join(
APP.config["REMOTE_GIT_FOLDER"], werkzeug.secure_filename("%s_%s" % (remote_git, branch_from))
)
if not os.path.exists(repopath):
try:
pygit2.clone_repository(remote_git, repopath, checkout_branch=branch_from)
except Exception as err:
LOG.exception(err)
flask.abort(500, "The following error was raised when trying to clone the " "remote repo: %s" % str(err))
else:
repo = pagure.lib.repo.PagureRepo(repopath)
try:
repo.pull(branch=branch_from, force=True)
except pygit2.GitError as err:
LOG.exception(err)
flask.abort(
500, "The following error was raised when trying to pull the " "changes from the remote: %s" % str(err)
)
except pagure.exceptions.PagureException as err:
LOG.exception(err)
flask.abort(500, str(err))
return repopath
def prepare_git(url, path):
''' Prepare git pkgbuild directory '''
try:
pygit2.clone_repository(url=url, path=path)
except pygit2.GitError as error:
print('Failed to clone repository')
error_print_exit(error)
print('Successfully cloned repository: ' + url)
def test_clone_with_checkout_branch(self):
# create a test case which isolates the remote
test_repo = clone_repository('./test/data/testrepo.git',
os.path.join(self._temp_dir, 'testrepo-orig.git'),
bare=True)
test_repo.create_branch('test', test_repo[test_repo.head.target])
repo = clone_repository(test_repo.path,
os.path.join(self._temp_dir, 'testrepo.git'),
checkout_branch='test', bare=True)
self.assertEqual(repo.lookup_reference('HEAD').target, 'refs/heads/test')
def before_all(context):
"""Do the preparation that can be done at the very beginning.
The "tito" executable must be available.
:param context: the context as described in the environment file
:type context: behave.runner.Context
:raises exceptions.IOError: if the tito-enabled project cannot be
created
:raises exceptions.ValueError: if the tito-enabled project cannot be
created
:raises exceptions.OSError: if the executable cannot be executed
:raises subprocess.CalledProcessError: if the tito-enabled project
cannot be created
"""
signature = pygit2.Signature(
stackci.NAME, '{}@example.com'.format(stackci.NAME))
context.titodn = tempfile.mkdtemp()
dst_spec = os.path.join(context.titodn, b'foo.spec')
shutil.copy(
os.path.join(os.path.dirname(__file__), b'resources', b'foo.spec'),
dst_spec)
try:
titorepo = pygit2.init_repository(context.titodn)
titorepo.index.add(os.path.relpath(dst_spec, titorepo.workdir))
titorepo.index.write()
titorepo.create_commit(
'refs/heads/master', signature, signature, 'Add a spec file.',
titorepo.index.write_tree(), [])
# FIXME: https://github.com/libgit2/pygit2/issues/531
except Exception as err:
raise ValueError('Git repository creation failed: {}'.format(err))
# FIXME: https://github.com/dgoodwin/tito/issues/171
subprocess.check_call(['tito', 'init'], cwd=context.titodn)
context.librepodn = tempfile.mkdtemp()
try:
libreporepo = pygit2.clone_repository(
'https://github.com/rpm-software-management/librepo.git',
context.librepodn)
libreporepo.reset(
'd9bed0d9f96b505fb86a1adc50b3d6f8275fab93', pygit2.GIT_RESET_HARD)
# FIXME: https://github.com/libgit2/pygit2/issues/531
except Exception as err:
raise ValueError('Git repository creation failed: {}'.format(err))
context.libcompsdn = tempfile.mkdtemp()
try:
libcompsrepo = pygit2.clone_repository(
'https://github.com/rpm-software-management/libcomps.git',
context.libcompsdn)
libcompsrepo.reset(
'eb966bc43097c0d00e154abe7f40f4d1d75fbcd1', pygit2.GIT_RESET_HARD)
# FIXME: https://github.com/libgit2/pygit2/issues/531
except Exception as err:
raise ValueError('Git repository creation failed: {}'.format(err))
def clone_worker(git_remote_callback, protocol, dry_run, part):
project, path = part[0], part[1]
try:
if not dry_run:
clone_repository(project.url[protocol], path, callbacks=git_remote_callback)
else:
msg("@{cf}Fetching@|: %s\n" % escape(textify(project.url[protocol])))
except (GitError, AuthorizationFailed) as e:
shutil.rmtree(path, ignore_errors=True)
return project, str(e)
return project, ""
def collectRepo(repo):
repo_url = repo
urlChunks = repo_url.split('/')
repo_path = 'repos/' + urlChunks[len(urlChunks)-1].replace('.git', '').lower()
if not os.path.exists(repo_path):
print 'Checking out ' + repo_path
clone_repository(repo_url, repo_path)
else:
print 'Repository ' + repo_path + ' already exists!'
return repo_path + '/.git'
def git_clone(repo_url, branch='master'):
'''
clones repo to a /tmp/ dir
'''
if repo_url == '' or not check_available_repo(repo_url):
return False
_tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
return _tmp_dir
def create_repository(user, git_repo):
from .models import Repository as LocalRepository
try:
git_user, git_name = GitOperations.git_uri_parse(git_repo)
creds = GitOperations.get_credentials(git_user, user)
where = GitOperations.get_repository_location(user, git_name)
clone_repository(git_repo, where, bare=True, credentials=creds)
repo = LocalRepository(user, git_user, git_name, git_repo).save()
return repo
except GitError as e:
raise GitException(e.args)
except ValueError as e: # pragma: no cover
raise GitException(e.args)
def clean_clone(url, path):
try:
shutil.rmtree(path)
except shutil.Error:
raise TaskException(errno.EACCES, 'Cannot remove templates directory: {0}'.format(path))
except FileNotFoundError:
pass
try:
pygit2.clone_repository(url, path)
except pygit2.GitError:
self.add_warning(TaskWarning(
errno.EACCES,
'Cannot update template cache. Result is outdated. Check networking.'))
return
def temp_clone(self, url):
"""Clone a url into a temporary place, yield that place, then delete it"""
tmpdir = None
try:
tmpdir = tempfile.mkdtemp()
repo = os.path.join(tmpdir, "repo")
try:
pygit2.clone_repository(url, repo)
except pygit2.GitError as error:
raise GitError("Couldn't clone the new remote", url=url, repo=self.location, error_type=error.__class__.__name__, error=error)
yield repo
finally:
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
def initialize(request):
if path.exists(settings.MODREPO_DIR) and path.exists(path.join(settings.MODREPO_DIR, ".git")):
repo = pygit2.Repository(settings.MODREPO_DIR)
for remote in repo.remotes:
if remote.name == 'origin':
result = remote.fetch()
repo.checkout('refs/remotes/origin/master')
else:
shutil.rmtree(settings.MODREPO_DIR)
cred = pygit2.UserPass(settings.MODREPO_USER, settings.MODREPO_PASS)
pygit2.clone_repository(settings.MODREPO_REMOTE, settings.MODREPO_DIR, credentials=cred)
return redirect(modrepo)
def setup_git_repo(self):
""" Create a basic git repo withing the tests folder that can be used
then for the tests.
"""
# Create a bare git repo
bare_repo_path = os.path.join(self.gitroot, 'test_repo.git')
os.makedirs(bare_repo_path)
pygit2.init_repository(bare_repo_path, bare=True)
# Clone the bare git repo and add the elements we need in it
git_repo_path = os.path.join(self.gitroot, 'test_repo')
pygit2.clone_repository(bare_repo_path, git_repo_path, bare=False)
repo = pygit2.Repository(git_repo_path)
# Add basic files needed
open(os.path.join(git_repo_path, '.gitignore'), 'w').close()
repo.index.add('.gitignore')
open(os.path.join(git_repo_path, 'sources'), 'w').close()
repo.index.add('sources')
repo.index.write()
with open(os.path.join(
git_repo_path, '.git', 'config'), 'a') as stream:
stream.write(GITCONFIG)
# Commits the files added
tree = repo.index.write_tree()
author = pygit2.Signature(
'Alice Author', '*****@*****.**')
committer = pygit2.Signature(
'Cecil Committer', '*****@*****.**')
repo.create_commit(
'refs/heads/master', # the name of the reference to update
author,
committer,
'Add basic file required',
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
[]
)
# Push to the remote repo
master_ref = repo.lookup_reference('HEAD').resolve()
refname = '%s:%s' % (master_ref.name, master_ref.name)
ori_remote = repo.remotes[0]
ori_remote.push(refname)
def clone_modpack(gitrepo, targetdir):
"""
Clones git repo in a new directory
"""
log = logging.getLogger("clone_modpack")
log.addHandler(DatabaseLogger())
log.info("clone_modpack task started")
cleandir = sanitize_path(targetdir)
if path.isdir(path.join(MODPACKPATH, cleandir)):
log.error('NOPE. There\'s a dir named like this.')
return None
pygit2.clone_repository(gitrepo, path.join(MODPACKPATH, cleandir))
log.info("Repo created. Building")
build_all_caches()
def clone_modpack(gitrepo, targetdir):
"""
Clones git repo in a new directory
"""
log = logging.getLogger("clone_modpack")
log.addHandler(DatabaseLogger())
log.info("clone_modpack task started")
cleandir = sanitize_path(targetdir)
if path.isdir(path.join(MODPACKPATH, cleandir)):
log.error('NOPE. There\'s a dir named like this.')
return None
pygit2.clone_repository(gitrepo, path.join(MODPACKPATH, cleandir))
log.info("Repo created. Building")
build_all_caches()
def test_keypair_from_memory(tmp_path, pygit2_empty_key):
url = 'ssh://[email protected]/pygit2/empty'
with pytest.raises(pygit2.GitError):
pygit2.clone_repository(url, tmp_path)
prv, pub, secret = pygit2_empty_key
with open(prv) as f:
prv_mem = f.read()
with open(pub) as f:
pub_mem = f.read()
keypair = pygit2.KeypairFromMemory("git", pub_mem, prv_mem, secret)
callbacks = pygit2.RemoteCallbacks(credentials=keypair)
pygit2.clone_repository(url, tmp_path, callbacks=callbacks)
def test_create_branch(self):
non_bare_repo_path = os.path.join(
self.environment_path, 'remote-non-bare')
pygit2.clone_repository(
os.path.join(self.environment_path, 'remote'),
non_bare_repo_path,
)
clone = pygit2.Repository(non_bare_repo_path)
gitrepo = Repository(non_bare_repo_path)
gitcs = gitrepo[clone.head.get_object().hex]
branch = gitcs.create_branch('fakebranch')
self.assertEquals(branch.get_changeset(), gitrepo.tip())
self.assertIsNotNone(clone.lookup_branch('fakebranch'))
self.assertEquals(
'fakebranch', clone.lookup_branch('fakebranch').branch_name)
def set_up_git_repo(self, name="test"):
""" Set up the git repo to play with. """
# Create a git repo to play with
gitrepo = os.path.join(self.path, "repos", "%s.git" % name)
repo = pygit2.init_repository(gitrepo, bare=True)
newpath = tempfile.mkdtemp(prefix="pagure-other-test")
repopath = os.path.join(newpath, "test")
clone_repo = pygit2.clone_repository(gitrepo, repopath)
# Create a file in that git repo
with open(os.path.join(repopath, "sources"), "w") as stream:
stream.write("foo\n bar")
clone_repo.index.add("sources")
clone_repo.index.write()
# Commits the files added
tree = clone_repo.index.write_tree()
author = pygit2.Signature("Alice Author", "*****@*****.**")
committer = pygit2.Signature("Cecil Committer", "*****@*****.**")
clone_repo.create_commit(
"refs/heads/master", # the name of the reference to update
author,
committer,
"Add sources file for testing",
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
[],
)
refname = "refs/heads/master"
ori_remote = clone_repo.remotes[0]
PagureRepo.push(ori_remote, refname)
def create_local_repo_for_project(vcsMongo, path, project_name):
url = vcsMongo["url"]
# removes the https and replaces it with git
repo_url = "git" + url[5:]
# if os.path.isdir(path):
# shutil.rmtree(path)
# check if we have the project file locally (HPC SYSTEM ONLY!)
hpc_path = '/mnt/jgrabow1/bin/projects/{}'.format(project_name)
if os.path.isdir(hpc_path):
# shutil.copytree(hpc_path, path)
# workaround because our tmpdir path exists already, adopted from: https://stackoverflow.com/a/12514470
for item in os.listdir(hpc_path):
s = os.path.join(hpc_path, item)
d = os.path.join(path, item)
if os.path.isdir(s):
shutil.copytree(s, d)
else:
shutil.copy2(s, d)
# complete path
if not path.endswith('.git'):
if not path.endswith('/'):
path += '/'
path += '.git'
repo = pygit2.Repository(path)
else:
repo = pygit2.clone_repository(repo_url, path)
# this is a workaround for: https://github.com/libgit2/pygit2/issues/818
repo = pygit2.Repository(path)
return repo
def test_clone_with_credentials(self):
credentials = pygit2.UserPass("libgit2", "libgit2")
repo = clone_repository(
"https://bitbucket.org/libgit2/testgitrepository.git",
self._temp_dir, credentials=credentials)
self.assertFalse(repo.is_empty)
def stream(repo_uri, stream_uri, verbose, assume):
"""Stream policy changes to destination"""
logging.basicConfig(
format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
with contextlib.closing(TempDir().open()) as temp_dir:
if repo_uri is None:
repo_uri = pygit2.discover_repository(os.getcwd())
log.debug("Using repository %s", repo_uri)
if repo_uri.startswith('http') or repo_uri.startswith('git@'):
log.info("Cloning repository: %s", repo_uri)
repo = pygit2.clone_repository(repo_uri, temp_dir.path)
else:
repo = pygit2.Repository(repo_uri)
load_resources()
policy_repo = PolicyRepo(repo_uri, repo)
change_count = 0
with contextlib.closing(transport(stream_uri, assume)) as t:
for change in policy_repo.delta_stream():
change_count += 1
t.send(change)
log.info("Streamed %d policy changes", change_count)
def clone(repourl, repopath):
try:
return pygit2.clone_repository(repourl,
repopath,
callbacks=GitRemoteCallbacks())
except pygit2.GitError as e:
raise errors.GitError(e)
def tamagotchi_repo(tmpdir_factory):
repo_root = str(tmpdir_factory.mktemp('repo'))
repo = git.clone_repository(tamagotchi_github_url, repo_root, checkout_branch='for-rendering')
branch = repo.lookup_branch('origin/start', git.GIT_BRANCH_REMOTE)
commit = repo[branch.target]
repo.create_branch('start', commit)
return repo
def __init__(self,
path,
origin=None,
create=False,
garbageCollection=False):
"""Initialize a quit repo at a given location of the filesystem.
Keyword arguments:
path -- the location of an existing git repository or where the repository should be created
origin -- if the repository does not exist and origin is given the origin repository is
cloned to the given path (default: None)
create -- boolean whether to create a new repository if the specified path is empty
(default: False)
garbageCollection -- boolean whether to activate the garbage collection on the git
repository (default: False)
"""
self.callback = QuitRemoteCallbacks()
try:
self._repository = pygit2.Repository(path)
except (KeyError, GitError):
if not create:
raise RepositoryNotFound('Repository "%s" does not exist' %
path)
if origin:
self._repository = pygit2.clone_repository(
url=origin, path=path, bare=False, callbacks=self.callback)
else:
self._repository = pygit2.init_repository(path)
if garbageCollection:
self.init_garbageCollection(path)
self.path = path
def __enter__(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = self.tmp.name
self.path = os.path.join(self.root, "repo-" + fresh.salt(5))
if self.master is None:
self.repo = pygit2.init_repository(self.path)
else:
self.repo = pygit2.clone_repository(self.master.repo.path,
self.path)
self.log_file_path = os.path.join(self.root, "log")
self.log_thread = threading.Thread(target=self._log_reader)
self.log_thread.start()
if self.flags.get("global_config") is False:
self.global_config = None
else:
self.global_config = os.path.join(self.root, "global.json")
self.config = {
"ethereum": {
"private_key": self.owner_key.hex(),
"rpc_target": ethereum_rpc_target,
"chain_id": w3.eth.chainId,
},
"logging": {
"file": self.log_file_path,
},
}
return self
def test_clone_remote_name(self):
repo_path = "./test/data/testrepo.git/"
repo = clone_repository(
repo_path, self._temp_dir, remote_name="custom_remote"
)
self.assertFalse(repo.is_empty)
self.assertEqual(repo.remotes[0].name, "custom_remote")
def init_remote(remote_path, local_path):
remote_repo = pygit2.init_repository(str(remote_path), bare=True)
coid = create_commit(remote_repo, 'refs/heads/master', [], 'Hello')
coid = create_commit(remote_repo, 'refs/heads/master', [coid], 'Goodbyte')
local_repo = pygit2.clone_repository(str(remote_path), str(local_path))
commit = local_repo.revparse_single('HEAD')
local_repo.branches.create('pushed', commit)
pushed_commit_oid = create_commit(local_repo,
'refs/heads/pushed',
[commit.id],
'Pushed')
# master
# \
# `---pushed
pushed_commit = local_repo[pushed_commit_oid]
local_repo.branches.create('notpushed', pushed_commit)
# master
# \
# `---pushed, notpushed
local_repo.remotes.add_push('origin', '+refs/heads/*:refs/remotes/origin/*')
origin = local_repo.remotes['origin']
commit = local_repo.revparse_single('refs/heads/master')
merged_remote_coid = create_commit(local_repo, 'refs/heads/master', [commit.id], 'MergedRemote')
# -------master
# \
# `---pushed, notpushed
commit = local_repo.get(merged_remote_coid)
local_repo.branches.create('merged_remote', commit)
local_repo.branches.create('old_master', commit)
# -------master, merged_remote, old_master
# \
# `---pushed, notpushed
commit = local_repo.revparse_single('refs/heads/master')
merged_remote_coid = create_commit(local_repo, 'refs/heads/master', [commit.id], 'MergedRemote')
# -------merged_remote, old_master--master
# \
# `---pushed, notpushed
local_repo.remotes['origin'].push(['refs/heads/master',
'refs/heads/old_master',
'refs/heads/pushed',
'refs/heads/notpushed',
'refs/heads/merged_remote'])
# -------merged_remote, old_master, origin/merged_remote, origin/old_master--master, origin/master
# \
# `---pushed, notpushed, origin/notpushed, origin/pushed
yield remote_repo
def archive_repository(archive_path, url, github_slug, time_series, token):
''' Download previous versions of the chosen repository at the specified
intervals and commit these to slave account for future analysis '''
# archive_path = create_archive(url)
time_intervals = get_time_intervals(time_series)
for timestamp in time_intervals:
sha = get_sha(github_slug, timestamp, token)
if sha:
oid = pygit2.Oid(hex=sha)
download_path = os.path.join(archive_path, str(timestamp))
try:
repo = pygit2.clone_repository(url, download_path, bare=False)
init_repo(oid, download_path)
u_name = github_slug.split('/')[1] + str(timestamp).replace(
'-', '')
bare_url = create_bare_repo(u_name)
if bare_url:
commit_files(bare_url, download_path)
except Exception as ex:
return str(type(ex).__name__)
# return '{}: {}'.format(type(ex).__name__, ex.args[0])
# os.system('rmdir /S /Q "{}"'.format(archive_path))
else:
return 'Error: Could not access commit history for given time period'.format(
url)
# os.system('rmdir /S /Q "{}"'.format(archive_path))
return 'Successfully archived {}'.format(url)
def _clone(
self,
workspace,
ssh_pub_key,
ssh_priv_key,
ssh_user='******',
ssh_pass='',
):
ws = util.create_workspace(workspace, self.alias)
keypair = Keypair(
username=ssh_user,
pubkey=ssh_pub_key,
privkey=ssh_priv_key,
passphrase=ssh_pass,
)
try:
cb = RemoteCallbacks(credentials=keypair)
repo = clone_repository(
url=self.url,
path=ws,
callbacks=cb,
)
try:
repo.checkout(self.refs)
print(f'Cloned {repo} to {ws}')
except Exception as err:
raise errors.CannotFetchRef(
f'Cannot fetch ref: {self.refs}'
)
except Exception as err:
raise errors.CannotCloneRepository(
f'Cannot clone repository: {err}'
)
def clone_repository(self, repo: str, url: str, lang: str = None):
"""
"""
path = Path("%s%s" % (self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
try:
pygit2.clone_repository(
"ssh://%s" % (url.replace('org:', 'org/')),
"%s%s" % (self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
callbacks=MyRemoteCallbacks())
logging.info('Successfully cloned repo %s to %s%s' %
(repo, self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
except Exception as e:
print(e)
logging.debug('Error while cloning repo %s to %s%s' %
(repo, self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
return
def test_clone_with_credentials(self):
url = 'https://github.com/libgit2/TestGitRepository'
credentials = pygit2.UserPass("libgit2", "libgit2")
callbacks = pygit2.RemoteCallbacks(credentials=credentials)
repo = clone_repository(url, self._temp_dir, callbacks=callbacks)
assert not repo.is_empty
def analyze(url, api_token):
github_url = url.strip().replace("http://www.github.com/", "")
github_url = github_url.strip().replace("https://www.github.com/", "")
github_url = github_url.strip().replace("http://github.com/", "")
github_url = github_url.strip().replace("https://github.com/", "")
print("Start analyzing: %s" % github_url)
start_time = time.perf_counter()
g = Github(api_token)
repo = g.get_repo(github_url)
stats = open(os.path.join("github", repo.owner.login + "-" + repo.name), "w")
stats.write("forks;branches;watchers;stars\n")
stats.write("%d;%d;%d;%d\n"
% (repo.forks_count, len(list(repo.get_branches())),
repo.watchers_count, repo.stargazers_count))
stats.flush()
stats.close()
repo_path = os.path.join(os.getcwd(), "git", repo.owner.login, repo.name)
# initialize repo
if os.path.exists(repo_path):
cloned_repo = Repository(discover_repository(repo_path))
else:
cloned_repo = clone_repository(repo.clone_url, repo_path)
# run churny
run("churny " + repo_path, os.path.join("overall", repo.owner.login + "-" + repo.name))
run("churny -m " + repo_path, os.path.join("monthly", repo.owner.login + "-" + repo.name))
elapsed_time = time.perf_counter() - start_time
print("Stop analyzing: %s (%d s)" % (github_url, elapsed_time))
def git_clone_update(git_url, git_branch, git_folder):
# git clone if needed
if '~' in git_folder:
git_folder = os.path.expanduser(git_folder)
if not os.path.exists(git_folder):
LOG.info('Cloning %s', git_url)
repo = pygit2.clone_repository(git_url, git_folder, False, False, 'origin', git_branch)
else:
repo = pygit2.Repository(git_folder)
# git pull remote branch
cwd = os.getcwd()
os.chdir(git_folder)
pull = subprocess.Popen(
["git", "pull", git_url, git_branch],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = pull.communicate()
os.chdir(cwd)
if pull.returncode:
raise DgrocException('Strange result of the git pull from \'%s\':\n%s' % (git_url, out[1] if not out[0] else out[0]))
return
return repo;
def test_clone_remote_name(self):
repo_path = "./test/data/testrepo.git/"
repo = clone_repository(repo_path,
self._temp_dir,
remote_name="custom_remote")
self.assertFalse(repo.is_empty)
self.assertEqual(repo.remotes[0].name, "custom_remote")
def clone(location):
"""
Clone a Git repository if it doesn't exist into a temporary directory.
:param str location: the Git URL to clone
:rtype: pygit2.Repository
"""
path = join(config.CLONE_ROOT, _simple_location_name(location))
log.info('Cloning to path {}'.format(path))
try:
log.info('Looking for existing repository')
repository_path = discover_repository(path)
except KeyError:
log.info('No existing repository found, cloning fresh')
return clone_repository(location, path)
else:
log.info('Repository already exists at {}'.format(repository_path))
repository = Repository(repository_path)
remote = repository.remotes[0]
remote.sideband_progress = log.info
log.info('Fetching to get the latest objects from remote')
remote.fetch()
return repository
def install_package(self, pkg_name, expression):
package_cache_path = path.join(APP_CACHE_DIR, pkg_name)
# Check if the package is cached
try:
repo = Repository(package_cache_path)
# TODO: check lastest version
# If lastest is cached, advance
# If no, prompt user
except KeyError as e:
repo = clone_repository(APP_REMOTE['url'], package_cache_path)
# Get suitable version for the expression
versions = utils.get_versions_cached(repo)
version = max_satisfy(versions=versions.keys(), expression='')
package_archive_name = '{0}-{1}.tar'.format(pkg_name, version)
package_archive_path = path.join(APP_ARCHIVE_DIR, package_archive_name)
# Create archive file
# If the file already exists, move forward
if path.isfile(package_archive_path):
with tarfile.open(package_archive_path, 'w') as archive:
repo.write_archive(archive=archive, treeish=versions[version].tree_id)
# TODO: use strategy
# Extract archive to package dir
path_to_package = path.join(USER_PACKAGE_FOLDER, 'user', pkg_name)
tar = tarfile.open(package_archive_path)
tar.extractall(path=path_to_package)
tar.close()
def _clone_and_top_commits(folder, branch, branch_ref=False):
""" Clone the repository, checkout the specified branch and return
the top commit of that branch if there is one.
Returns the repo, the path to the clone and the top commit(s) in a tuple
or the repo, the path to the clone and the reference to the branch
object if branch_ref is True.
"""
if not os.path.exists(folder):
os.makedirs(folder)
brepo = pygit2.init_repository(folder, bare=True)
newfolder = tempfile.mkdtemp(prefix='pagure-tests')
repo = pygit2.clone_repository(folder, newfolder)
branch_ref_obj = None
if "origin/%s" % branch in repo.listall_branches(pygit2.GIT_BRANCH_ALL):
branch_ref_obj = pagure.lib.git.get_branch_ref(repo, branch)
repo.checkout(branch_ref_obj)
if branch_ref:
return (repo, newfolder, branch_ref_obj)
parents = []
commit = None
try:
if branch_ref_obj:
commit = repo[branch_ref_obj.get_object().hex]
else:
commit = repo.revparse_single('HEAD')
except KeyError:
pass
if commit:
parents = [commit.oid.hex]
return (repo, newfolder, parents)
def commits_pygit2(repo_url, remove=True):
""" Iterate commits using Python libgit2 binding.
Unlike GitPython, it can clone repository for you and works in the same
memory space so it is much faster. It is kind of heavy, but can be handy if
you need to work with repository/commits content (e.g. code analysis)
:param repo_url Git repository URL (not GitHub URL!).
Example: git://github.com/user/repo.git
"""
import os
import tempfile
import shutil
import pygit2
org, repo_name = get_repo_name(repo_url)
folder = tempfile.mkdtemp(prefix='_'.join(('ghd', org, repo_name, '')))
repo = pygit2.clone_repository(repo_url, folder, bare=True)
try:
for commit in repo.walk(repo.head.target):
# http://www.pygit2.org/objects.html#commits
yield {
'sha': commit.oid,
'author_name': commit.author.name,
'author_email': commit.author.email,
'committer_name': commit.committer.name,
'committer_email': commit.committer.email,
'message': commit.message.strip(),
'parent_ids': "\n".join(str(pid) for pid in commit.parent_ids),
'time': commit.commit_time,
}
finally:
if remove:
os.chdir('/tmp')
shutil.rmtree(folder)
def clone_git_repository():
try:
repo = Repository(GIT_REPO_PATH + '/.git')
except GitError as e:
print("Cloning Repository")
repo = clone_repository(GIT_REPO_URL, GIT_REPO_PATH)
return repo
def test_clone_with_credentials(self):
credentials = pygit2.UserPass("libgit2", "libgit2")
repo = clone_repository(
"https://bitbucket.org/libgit2/testgitrepository.git",
self._temp_dir, credentials=credentials)
self.assertFalse(repo.is_empty)
def __init__(self, repo_url, dir_manager, credentials_manager):
"""
:repo_url: URL of the repository
:repo_dir: directory where the repository is expected to reside
"""
self.up2date = False
self.repo_url = repo_url
dir_path = dir_manager.get_repo_dir(repo_url)
my_credentials = credentials_manager.get_credentials(repo_url)
if not os.path.isfile(dir_path + '/HEAD'):
try:
self.repo = pygit2.clone_repository(repo_url, dir_path, bare=True, credentials=my_credentials)
except pygit2.GitError as e:
raise GitException("Cloning failed. {}".format(e.message))
else:
self.repo = pygit2.Repository(pygit2.discover_repository(dir_path))
self.up2date = True
def _remote_credentials(url, username_from_url, allowed_types):
return credentials_manager.get_credentials(url)
for remote in self.repo.remotes:
remote.credentials = _remote_credentials
transfer_progress = remote.fetch()
if transfer_progress.received_objects:
self.up2date = False
def clone(repo_name, internal):
uniq_path = hashlib.sha224(repo_name).hexdigest()
uniq_path = hashlib.sha224(repo_name).hexdigest()
if os.path.isdir(os.getcwd() + '/clones/' + uniq_path):
shutil.rmtree(os.getcwd() + '/clones/' + uniq_path)
repo_url = 'https://github.com/%s.git' % (repo_name)
try:
clone_dir = os.getcwd() + '/clones/'
if not os.path.isdir(clone_dir):
os.makedirs(clone_dir)
repo_path = clone_dir + uniq_path
login_info = ''
if internal==True:
username = '******' #your-github-username-here
password = str(keyring.get_password('github', username))
login_info = git.UserPass(username, password)
git_obj = git.clone_repository(repo_url, repo_path, credentials=login_info)
return repo_path
except Exception, e:
print e
if str(e).find('Unexpected HTTP status code: 404'):
print "Repo doesn't exists"
return "Repo doesn't exists"
def init(self):
"""
Init plugin by loading manifest.json and listing available files from filesystem.
In case of Git repository, clone, fetch changes and checkout to target ref.
"""
local_repository = pygit2.discover_repository(self.basepath)
if local_repository:
self._repo = pygit2.Repository(local_repository)
self.checkout(self.repository)
elif self.repository:
self._repo = pygit2.clone_repository(self.repository,
self.basepath,
bare=False,
callbacks=self.callbacks)
self.checkout()
else:
# Target directory is not a Git repository, no need to checkout
pass
self.files = self._list_files()
if not self.files:
raise PluginException("Error loading plugin files",
"plugin-error-files")
self.config = self._load_config()
if not self.config:
raise PluginException("Error loading plugin config",
"plugin-error-config")
return self
def update_project(conn, project_id, repo_url, name, branch, last_updated, last_hash):
path = os.path.join(repo_dir, name)
repo = None
if os.path.exists(path):
print("fetching to '%s'" %(path))
# if project exists, then fetch it
#update_cmd = "GIT_DIR=%s git fetch" %(path)
#subprocess.call(update_cmd, shell=True)
repo = pygit2.Repository(path)
repo.remotes["origin"].fetch()
else:
print("cloning to '%s'" %(path))
# if project doesn't exist, then we clone it
#clone_cmd = "git clone --no-checkout --bare %s %s" %(repo_url, path)
#subprocess.call(clone_cmd, shell=True)
repo = pygit2.clone_repository(url=repo_url, path=path, bare=True, checkout_branch=None)
# get current hash
commit = repo.revparse_single("origin/%s" %(branch))
hash = str(commit.oid)
# save hash to db along with last_updated
conn.execute("UPDATE project set last_updated=CURRENT_TIMESTAMP, last_hash=? where id=?", (hash, project_id))
def __init__(self, package_name=None, clone_url=None, dest_path=None,
branch='master'):
self.repo_name = package_name
self.repo_url = clone_url
self.local_path = os.path.join(dest_path, package_name)
self.repo = None
# Load if it is an existing git repo
if os.path.exists(self.local_path):
try:
self.repo = pygit2.Repository(self.local_path)
self.repo.checkout('refs/heads/' + branch)
LOG.info("Found existent repository at destination path %s" % (
self.local_path))
except KeyError:
raise exception.RepositoryError(package=package_name,
repo_path=dest_path)
else:
LOG.info("Cloning into %s..." % self.local_path)
self.repo = pygit2.clone_repository(self.repo_url,
self.local_path,
checkout_branch=branch)
cmd = "cd %s ; git submodule init; git submodule update; cd %s" % (
self.local_path, os.getcwd())
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
output, error_output = p.communicate()
LOG.info("STDOUT: %s" % output)
LOG.info("STDERR: %s" % error_output)
def install_package(self, pkg_name, expression):
package_cache_path = path.join(APP_CACHE_DIR, pkg_name)
# Check if the package is cached
try:
repo = Repository(package_cache_path)
# TODO: check lastest version
# If lastest is cached, advance
# If no, prompt user
except KeyError as e:
repo = clone_repository(APP_REMOTE['url'], package_cache_path)
# Get suitable version for the expression
versions = utils.get_versions_cached(repo)
version = max_satisfy(versions=versions.keys(), expression='')
package_archive_name = '{0}-{1}.tar'.format(pkg_name, version)
package_archive_path = path.join(APP_ARCHIVE_DIR, package_archive_name)
# Create archive file
# If the file already exists, move forward
if path.isfile(package_archive_path):
with tarfile.open(package_archive_path, 'w') as archive:
repo.write_archive(archive=archive,
treeish=versions[version].tree_id)
# TODO: use strategy
# Extract archive to package dir
path_to_package = path.join(USER_PACKAGE_FOLDER, 'user', pkg_name)
tar = tarfile.open(package_archive_path)
tar.extractall(path=path_to_package)
tar.close()
def test_clone_repository_and_remote_callbacks(tmp_path):
src_repo_relpath = "./test/data/testrepo.git/"
repo_path = os.path.join(tmp_path, "clone-into")
url = pathname2url(os.path.realpath(src_repo_relpath))
if url.startswith('///'):
url = 'file:' + url
else:
url = 'file://' + url
def create_repository(path, bare):
return init_repository(path, bare)
# here we override the name
def create_remote(repo, name, url):
return repo.remotes.create("custom_remote", url)
repo = clone_repository(url,
repo_path,
repository=create_repository,
remote=create_remote)
assert not repo.is_empty
assert 'refs/remotes/custom_remote/master' in repo.listall_references()
assert b'refs/remotes/custom_remote/master' in repo.raw_listall_references(
)
assert repo.remotes["custom_remote"] is not None
def __init__(self, is_bare=False, clone_from_repo=None):
self.temp_dir = TemporaryDirectory()
if clone_from_repo:
self.repo = clone_repository(clone_from_repo.path,
self.temp_dir.name)
else:
self.repo = init_repository(self.temp_dir.name, is_bare)
def set_up_git_repo(self, name='test'):
""" Set up the git repo to play with. """
# Create a git repo to play with
gitrepo = os.path.join(self.path, 'repos', '%s.git' % name)
repo = pygit2.init_repository(gitrepo, bare=True)
newpath = tempfile.mkdtemp(prefix='pagure-other-test')
repopath = os.path.join(newpath, 'test')
clone_repo = pygit2.clone_repository(gitrepo, repopath)
# Create a file in that git repo
with open(os.path.join(repopath, 'sources'), 'w') as stream:
stream.write('foo\n bar')
clone_repo.index.add('sources')
clone_repo.index.write()
# Commits the files added
tree = clone_repo.index.write_tree()
author = pygit2.Signature('Alice Author', '*****@*****.**')
committer = pygit2.Signature('Cecil Committer', '*****@*****.**')
clone_repo.create_commit(
'refs/heads/master', # the name of the reference to update
author,
committer,
'Add sources file for testing',
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
[])
refname = 'refs/heads/master'
ori_remote = clone_repo.remotes[0]
PagureRepo.push(ori_remote, refname)
def clone(
project_dir: str,
workspace_dir: str,
vars: Dict = {},
**kwargs) -> None:
"""`handoff github clone -v organization=<github_org> repository=<github_repo> local_dir=<local_dir> force=False`
Clone remote repository. If local_dir is omitted, ./<repository> is used.
If force is set to True, an existing directory will be deleted before cloning.
"""
state = _get_state()
access_token = vars.get("access_token", state.get(GITHUB_ACCESS_TOKEN))
github = _get_github(access_token)
repo_name = vars["repository"]
org_name = vars.get("organization")
local_dir = vars.get("local_dir", "./" + repo_name)
if os.path.exists(local_dir):
if not vars.get("force", False):
raise Exception("The directory already exists.")
shutil.rmtree(local_dir)
git_url = vars.get("url", "https://github.com/" + str(org_name) + "/" + repo_name + ".git")
if vars.get("use_cli", False):
LOGGER.debug("Running git CLI")
git_url = git_url.replace("https://", f"https://{access_token}:x-oauth-basic@")
git_path = os.environ.get("GIT_PATH", "git")
os.system(f"{git_path} clone {git_url}")
else:
repo_clone = pygit2.clone_repository(
git_url, local_dir, callbacks=_get_callbacks(access_token))
return {"status": "success", "repository": repo_name}
def create_repo(repo_path, remote_url, creds):
if os.path.exists(repo_path):
logger.info('Cleaning up repo path...')
shutil.rmtree(repo_path)
repo = clone_repository(remote_url, repo_path, callbacks=creds)
return repo
def clean_git(obj, repo, repofolder, objtype='ticket'):
""" Update the given issue remove it from its git.
"""
if not repofolder:
return
# Get the fork
repopath = os.path.join(repofolder, repo.path)
# Clone the repo into a temp folder
newpath = tempfile.mkdtemp(prefix='pagure-')
new_repo = pygit2.clone_repository(repopath, newpath)
file_path = os.path.join(newpath, obj.uid)
# Get the current index
index = new_repo.index
# Are we adding files
if not os.path.exists(file_path):
shutil.rmtree(newpath)
return
# Remove the file
os.unlink(file_path)
# Add the changes to the index
index.remove(obj.uid)
# See if there is a parent to this commit
parent = None
try:
parent = new_repo.head.get_object().oid
except pygit2.GitError:
pass
parents = []
if parent:
parents.append(parent)
# Author/commiter will always be this one
author = pygit2.Signature(name='pagure', email='pagure')
# Actually commit
new_repo.create_commit('refs/heads/master', author, author,
'Removed %s %s: %s' % (objtype, obj.uid, obj.title),
new_repo.index.write_tree(), parents)
index.write()
# Push to origin
ori_remote = new_repo.remotes[0]
master_ref = new_repo.lookup_reference('HEAD').resolve()
refname = '%s:%s' % (master_ref.name, master_ref.name)
PagureRepo.push(ori_remote, refname)
# Remove the clone
shutil.rmtree(newpath)
def traeremotolocal(url, rutadest):
"""
:param url: url del repositorio git remoto a clonar
:param rutadest: ruta local limpia y vuelca el clonado.
:return: pygit2.Repository del repo,
:raises: Excepciones si rutadest ya existe. GitError, ValueError u
otras si falla el clonado.
"""
try:
# basedir = os.path.dirname(os.path.abspath(rutadest))
# if os.path.exists(basedir):
# os.chdir(basedir)
if os.path.exists(rutadest):
# mejor elevar excepcion y se haga fuera y no aqui
#shutil.rmtree(rutadest)
raise Exception('{1} no debe existir para clonar hay el repo.\n'
'lanza "rm -r {1}"'.format(rutadest))
else:
os.makedirs(rutadest)
except Exception as excp:
raise
try:
# Lo traemos no bare.
repo = clone_repository(url, rutadest, bare=False)
# repo.describe()
except (pygit2.GitError, ValueError, Exception) as excp:
raise
return repo
def lambda_handler(event, context):
outputbucket = event['context']['output-bucket']
repo_url = event['body-json']['repository']['clone_url']
full_name = event['body-json']['repository']['full_name']
repo_branch = event['body-json']['ref'].replace('refs/heads/', '')
repo_path = '/tmp/%s' % full_name
clone_repository(repo_url, repo_path, checkout_branch=repo_branch)
zipfile = zip_repo(repo_path, full_name)
push_s3(zipfile, full_name, outputbucket)
shutil.rmtree(repo_path)
os.remove(zipfile)
return 'Successfully updated %s' % full_name