def get_fetch_info_from_stderr(repo, proc, progress):
# skip first line as it is some remote info we are not interested in
output = IterableList('name')
# lines which are no progress are fetch info lines
# this also waits for the command to finish
# Skip some progress lines that don't provide relevant information
fetch_info_lines = list()
for line in digest_process_messages(proc.stderr, progress):
if line.startswith('From') or line.startswith('remote: Total'):
continue
elif line.startswith('warning:'):
print >> sys.stderr, line
continue
elif line.startswith('fatal:'):
raise GitCommandError(("Error when fetching: %s" % line, ), 2)
# END handle special messages
fetch_info_lines.append(line)
# END for each line
# read head information
fp = open(join(repo.git_dir, 'FETCH_HEAD'), 'r')
fetch_head_info = fp.readlines()
fp.close()
assert len(fetch_info_lines) == len(fetch_head_info)
output.extend(
CmdFetchInfo._from_line(repo, err_line, fetch_line)
for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info))
finalize_process(proc)
return output
def test_push_branch_with_retry(self, mock_sleep, mock_git):
mock_repo = mock.Mock()
mock_repo.active_branch = mock.Mock()
mock_repo.active_branch.name = 'womp'
mock_repo.git = mock.Mock()
mock_repo.git.checkout = mock.Mock()
mock_git.Repo = mock.Mock(return_value=mock_repo)
mock_ret_f = mock.Mock()
mock_ret_f.flags = 1
mock_ret_f.ERROR = 0
mock_ret_f.summary = "mock failure summary"
mock_ret_ok = mock.Mock()
mock_ret_ok.flags = 0
mock_ret_ok.ERROR = 1
mock_repo.remotes = mock.Mock()
mock_repo.remotes.origin = mock.Mock()
mock_repo.remotes.origin.push = mock.Mock(
side_effect=[[mock_ret_f], [mock_ret_f], [mock_ret_ok]])
ghc = GitHubContext('REPO')
ghc.push_branch_with_retry('womp', attempts=3, cooloff=2)
self.assertEqual(mock_repo.remotes.origin.push.call_count, 3)
mock_repo.remotes.origin.push = mock.Mock(
side_effect=GitCommandError('B', 128))
self.assertRaises(RuntimeError,
ghc.push_branch_with_retry,
'womp',
attempts=3,
cooloff=2)
def test_pull_repo_exc_reraise(repo_mock):
exc_msg = 'something else'
repo_mock.clone_from.side_effect = GitCommandError('foobar', exc_msg)
with pytest.raises(GitCommandError) as exc:
clocme.pull_repo(MOCK_REPO)
assert repo_mock.clone_from.called
assert not repo_mock.called
assert 'foobar' in exc.value.command
assert 'something else' in exc.value.status
def test_pull_repo_existing_repo_found(repo_mock):
exc_msg = 'already exists and is not an empty directory'
repo_mock.clone_from.side_effect = GitCommandError('clone_from', exc_msg)
clocme.pull_repo(MOCK_REPO)
assert repo_mock.clone_from.called
assert MOCK_REPO in str(repo_mock.clone_from.call_args)
assert clocme.COPY_PATH in str(repo_mock.clone_from.call_args)
assert repo_mock.called
assert repo_mock.call_args[0][0].startswith(clocme.COPY_PATH)
assert repo_mock.call_args[0][0].endswith(MOCK_REPO.split('/')[-1])
def test_GitCommandError(self, init_args):
argv, cause = init_args
c = GitCommandError(argv, cause)
s = str(c)
self.assertIn(argv[0], s)
if cause:
self.assertIn(' failed due to: ', s)
self.assertIn(str(cause), s)
else:
self.assertIn(' failed!', s)
def test_update_failed_because_offline(
self, fake_warn, fake_repo, local_folder_exists, *_
):
from git.exc import GitCommandError
repo = MagicMock(autospec=True)
fake_repo.return_value = repo
repo.git.pull.side_effect = [GitCommandError("a", "b")]
git_clone(self.require_with_reference)
fake_warn.assert_called_with("Unable to run git commands. Offline?")
def handle_stderr(proc, iter_checked_out_files):
stderr = proc.stderr.read()
if not stderr:
return
# line contents:
stderr = stderr.decode(defenc)
# git-checkout-index: this already exists
failed_files = list()
failed_reasons = list()
unknown_lines = list()
endings = (' already exists', ' is not in the cache',
' does not exist at stage', ' is unmerged')
for line in stderr.splitlines():
if not line.startswith(
"git checkout-index: ") and not line.startswith(
"git-checkout-index: "):
is_a_dir = " is a directory"
unlink_issue = "unable to unlink old '"
already_exists_issue = ' already exists, no checkout' # created by entry.c:checkout_entry(...)
if line.endswith(is_a_dir):
failed_files.append(line[:-len(is_a_dir)])
failed_reasons.append(is_a_dir)
elif line.startswith(unlink_issue):
failed_files.append(
line[len(unlink_issue):line.rfind("'")])
failed_reasons.append(unlink_issue)
elif line.endswith(already_exists_issue):
failed_files.append(line[:-len(already_exists_issue)])
failed_reasons.append(already_exists_issue)
else:
unknown_lines.append(line)
continue
# END special lines parsing
for e in endings:
if line.endswith(e):
failed_files.append(line[20:-len(e)])
failed_reasons.append(e)
break
# END if ending matches
# END for each possible ending
# END for each line
if unknown_lines:
raise GitCommandError(("git-checkout-index", ), 128, stderr)
if failed_files:
valid_files = list(
set(iter_checked_out_files) - set(failed_files))
raise CheckoutError(
"Some files could not be checked out from the index due to local modifications",
failed_files, valid_files, failed_reasons)
def test_merge_branch_conflict(self, mock_git):
"""test merge branch"""
mock_repo = mock.Mock()
mock_repo.git = mock.Mock()
mock_repo.git.merge = mock.Mock(
side_effect=GitCommandError(mock.Mock(), mock.Mock(), mock.Mock()))
mock_repo.active_branch = mock.Mock()
mock_repo.active_branch.name = "ACTIVE"
mock_repo.index = mock.Mock()
mock_repo.index.unmerged_blobs = mock.Mock(return_value={
'file1': [(1, "BLOB1")],
'file2': [(2, "BLOB2")]
})
mock_git.Repo = mock.Mock(return_value=mock_repo)
ghc = GitHubContext('REPO')
self.assertRaises(GitCommandError, ghc.merge_branch, 'develop')
def test_clone_invalid_repo(self):
"""
Invalid repo raises error.
"""
with patch.object(Git, 'execute'):
to_dir = join(self.workingdir, 'a')
Git.execute.side_effect = GitCommandError(['command'],
128,
stderr='bad command')
with self.assertRaises(GitCloneError) as gce:
clone('http://github.com/user/repo', to_dir)
self.assertIn("'command' returned exit status 128: bad command",
gce.exception)
def test_collection_post_error(self, mock_method):
mock_method.side_effect = GitCommandError('git clone',
'Boom!',
stderr='mocked response')
request = testing.DummyRequest({})
request.validated = {
'repo_url': 'git://example.org/bar.git',
'repo_name': None
}
request.errors = Errors()
resource = RepositoryResource(request)
resource.collection_post()
[error] = request.errors
self.assertEqual(error['location'], 'body')
self.assertEqual(error['name'], 'repo_url')
self.assertEqual(error['description'], 'mocked response')
def test_handles_git_python_exceptions(self):
"""
If the fetch to retrieve new information results in an exception.
"""
with patch('jig.gitutils.remote.git') as git:
git.Repo.side_effect = AttributeError
self.assertTrue(remote_has_updates(self.local_workingdir))
with patch('jig.gitutils.remote.git') as git:
git.Repo.side_effect = GitCommandError(None, None)
self.assertTrue(remote_has_updates(self.local_workingdir))
with patch('jig.gitutils.remote.git') as git:
git.Repo.side_effect = AssertionError
self.assertTrue(remote_has_updates(self.local_workingdir))
def test_switch_to_branch_new_exists():
mock_repo = MagicMock()
mock_git = MagicMock()
mock_checkout = MagicMock()
mock_branch = MagicMock()
mock_is_dirty = MagicMock(return_value=False)
mock_branch.side_effect = GitCommandError('Branch exists', status=0)
type(mock_repo).git = mock_git
type(mock_repo).is_dirty = mock_is_dirty
type(mock_git).checkout = mock_checkout
type(mock_git).branch = mock_branch
# exising branch -> checkout
branch_name = 'qweeewqwqw'
switch_to_branch(mock_repo, branch_name, True)
mock_checkout.assert_called_with(branch_name)
mock_branch.assert_called_with(branch_name)
def test_push_branch(self, mock_git):
mock_repo = mock.Mock()
mock_repo.active_branch = mock.Mock()
mock_repo.active_branch.name = 'womp'
mock_repo.head = "HEAD"
mock_repo.git = mock.Mock()
mock_repo.git.checkout = mock.Mock()
mock_git.Repo = mock.Mock(return_value=mock_repo)
mock_repo.remotes = mock.Mock()
mock_ret = mock.Mock()
mock_ret.flags = 1000
mock_repo.remotes.origin = mock.Mock()
mock_repo.remotes.origin.push = mock.Mock(return_value=[mock_ret])
ghc = GitHubContext('REPO')
ghc.push_branch('womp')
self.failUnless(mock_repo.remotes.origin.push.called)
mock_repo.remotes.origin.push.assert_has_calls([mock.call('HEAD')])
mock_repo.remotes.origin.push = mock.Mock(
side_effect=GitCommandError('A', 128))
self.assertRaises(RuntimeError, ghc.push_branch, 'womp2')
def test_has_uncommitted_changes_throws_error(self):
cg = CivisGit()
cg.repo = MagicMock(spec=Repo)
cg.repo().index.diff = MagicMock(
side_effect=GitCommandError('diff', 'failed'))
self.assertRaises(CivisGitError, lambda: cg.has_uncommitted_changes())
def execute(self,
command,
istream=None,
with_extended_output=False,
with_exceptions=True,
as_process=False,
output_stream=None,
stdout_as_string=True,
kill_after_timeout=None,
with_stdout=True,
universal_newlines=False,
shell=None,
env=None,
max_chunk_size=io.DEFAULT_BUFFER_SIZE,
callback=None,
**subprocess_kwargs):
"""
Handles executing the command on the shell and consumes and returns
the returned information (stdout)
``command``
The command argument list to execute
``istream``
Standard input filehandle passed to subprocess.Popen.
``with_keep_cwd``
Whether to use the current working directory from os.getcwd().
GitPython uses get_work_tree() as its working directory by
default and get_git_dir() for bare repositories.
``with_extended_output``
Whether to return a (status, stdout, stderr) tuple.
``with_exceptions``
Whether to raise an exception when git returns a non-zero status.
``with_raw_output``
Whether to avoid stripping off trailing whitespace.
``callback``
User supplied callback to handle data processing
Returns
str(output) # extended_output = False (Default)
tuple(int(status), str(output)) # extended_output = True
callback # callback != None
"""
if GIT_PYTHON_TRACE and not GIT_PYTHON_TRACE == 'full':
print(' '.join(command))
# Allow the user to have the command executed in their working dir.
if with_keep_cwd or self.git_dir is None:
cwd = os.getcwd()
else:
cwd = self.git_dir
# Start the process
proc = subprocess.Popen(command,
cwd=cwd,
stdin=istream,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
**extra)
# if the user supplied a callback, use that instead.
if callback:
return (callback(proc))
# Wait for the process to return
try:
stdout_value = proc.stdout.read()
stderr_value = proc.stderr.read()
status = proc.wait()
finally:
proc.stdout.close()
proc.stderr.close()
# Strip off trailing whitespace by default
if not with_raw_output:
stdout_value = stdout_value.rstrip()
stderr_value = stderr_value.rstrip()
if with_exceptions and status != 0:
raise GitCommandError(command, status, stderr_value)
if GIT_PYTHON_TRACE == 'full':
if stderr_value:
print("%s -> %d: '%s' !! '%s'" %
(command, status, stdout_value, stderr_value))
elif stdout_value:
print("%s -> %d: '%s'" % (command, status, stdout_value))
else:
print("%s -> %d" % (command, status))
# Allow access to the command's status code
if with_extended_output:
return (status, stdout_value, stderr_value)
else:
return stdout_value
def test_generic_check_fail(self, sys_exit_mock, print_error_mock):
commit_range = MagicMock()
commit_range.__iter__.side_effect = GitCommandError("test", 123)
_generic_check(commit_range, None, check_merge_commits=False)
print_error_mock.assert_called_once()
sys_exit_mock.assert_called_once_with(ERROR_EXIT_CODE)
def test_clone_repository_throws_error(self, repo_clone):
repo_clone.side_effect = GitCommandError('clone', 'failed')
self.assertRaises(CivisGitError, lambda: CivisGit().clone_repository())
def checkout(self,
paths=None,
force=False,
fprogress=lambda *args: None,
**kwargs):
"""Checkout the given paths or all files from the version known to the index into
the working tree.
:note: Be sure you have written pending changes using the ``write`` method
in case you have altered the enties dictionary directly
:param paths:
If None, all paths in the index will be checked out. Otherwise an iterable
of relative or absolute paths or a single path pointing to files or directories
in the index is expected.
:param force:
If True, existing files will be overwritten even if they contain local modifications.
If False, these will trigger a CheckoutError.
:param fprogress:
see Index.add_ for signature and explanation.
The provided progress information will contain None as path and item if no
explicit paths are given. Otherwise progress information will be send
prior and after a file has been checked out
:param kwargs:
Additional arguments to be pasesd to git-checkout-index
:return:
iterable yielding paths to files which have been checked out and are
guaranteed to match the version stored in the index
:raise CheckoutError:
If at least one file failed to be checked out. This is a summary,
hence it will checkout as many files as it can anyway.
If one of files or directories do not exist in the index
( as opposed to the original git command who ignores them ).
Raise GitCommandError if error lines could not be parsed - this truly is
an exceptional state
.. note:: The checkout is limited to checking out the files in the
index. Files which are not in the index anymore and exist in
the working tree will not be deleted. This behaviour is fundamentally
different to *head.checkout*, i.e. if you want git-checkout like behaviour,
use head.checkout instead of index.checkout.
"""
args = ["--index"]
if force:
args.append("--force")
def handle_stderr(proc, iter_checked_out_files):
stderr = proc.stderr.read()
if not stderr:
return
# line contents:
# git-checkout-index: this already exists
failed_files = list()
failed_reasons = list()
unknown_lines = list()
endings = (' already exists', ' is not in the cache',
' does not exist at stage', ' is unmerged')
for line in stderr.splitlines():
if not line.startswith(
"git checkout-index: ") and not line.startswith(
"git-checkout-index: "):
is_a_dir = " is a directory"
unlink_issue = "unable to unlink old '"
already_exists_issue = ' already exists, no checkout' # created by entry.c:checkout_entry(...)
if line.endswith(is_a_dir):
failed_files.append(line[:-len(is_a_dir)])
failed_reasons.append(is_a_dir)
elif line.startswith(unlink_issue):
failed_files.append(
line[len(unlink_issue):line.rfind("'")])
failed_reasons.append(unlink_issue)
elif line.endswith(already_exists_issue):
failed_files.append(line[:-len(already_exists_issue)])
failed_reasons.append(already_exists_issue)
else:
unknown_lines.append(line)
continue
# END special lines parsing
for e in endings:
if line.endswith(e):
failed_files.append(line[20:-len(e)])
failed_reasons.append(e)
break
# END if ending matches
# END for each possible ending
# END for each line
if unknown_lines:
raise GitCommandError(("git-checkout-index", ), 128, stderr)
if failed_files:
valid_files = list(
set(iter_checked_out_files) - set(failed_files))
raise CheckoutError(
"Some files could not be checked out from the index due to local modifications",
failed_files, valid_files, failed_reasons)
# END stderr handler
if paths is None:
args.append("--all")
kwargs['as_process'] = 1
fprogress(None, False, None)
proc = self.repo.git.checkout_index(*args, **kwargs)
proc.wait()
fprogress(None, True, None)
rval_iter = (e.path for e in self.entries.itervalues())
handle_stderr(proc, rval_iter)
return rval_iter
else:
if isinstance(paths, basestring):
paths = [paths]
# make sure we have our entries loaded before we start checkout_index
# which will hold a lock on it. We try to get the lock as well during
# our entries initialization
self.entries
args.append("--stdin")
kwargs['as_process'] = True
kwargs['istream'] = subprocess.PIPE
proc = self.repo.git.checkout_index(args, **kwargs)
make_exc = lambda: GitCommandError(
("git-checkout-index", ) + tuple(args), 128, proc.stderr.read(
))
checked_out_files = list()
for path in paths:
co_path = to_native_path_linux(self._to_relative_path(path))
# if the item is not in the index, it could be a directory
path_is_directory = False
try:
self.entries[(co_path, 0)]
except KeyError:
dir = co_path
if not dir.endswith('/'):
dir += '/'
for entry in self.entries.itervalues():
if entry.path.startswith(dir):
p = entry.path
self._write_path_to_stdin(proc,
p,
p,
make_exc,
fprogress,
read_from_stdout=False)
checked_out_files.append(p)
path_is_directory = True
# END if entry is in directory
# END for each entry
# END path exception handlnig
if not path_is_directory:
self._write_path_to_stdin(proc,
co_path,
path,
make_exc,
fprogress,
read_from_stdout=False)
checked_out_files.append(co_path)
# END path is a file
# END for each path
self._flush_stdin_and_wait(proc, ignore_stdout=True)
handle_stderr(proc, checked_out_files)
return checked_out_files
# END paths handling
assert "Should not reach this point"
def __init__(self, path, url=None, runner=None, create=True):
"""Creates representation of git repository at `path`.
If `url` is given, a clone is created at `path`.
Can also be used to create a git repository at `path`.
Parameters
----------
path: str
path to the git repository; In case it's not an absolute path,
it's relative to PWD
url: str
url to the to-be-cloned repository. Requires a valid git url
according to:
http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS .
create: bool
if true, creates a git repository at `path` if there is none. Also
creates `path`, if it doesn't exist.
If set to false, an exception is raised in case `path` doesn't exist
or doesn't contain a git repository.
"""
self.path = abspath(normpath(path))
self.cmd_call_wrapper = runner or Runner(cwd=self.path)
# TODO: Concept of when to set to "dry".
# Includes: What to do in gitrepo class?
# Now: setting "dry" means to give a dry-runner to constructor.
# => Do it similar in gitrepo/dataset.
# Still we need a concept of when to set it and whether this
# should be a single instance collecting everything or more
# fine grained.
# TODO: somehow do more extensive checks that url and path don't point to the
# same location
if url is not None and not (url == path):
# TODO: What to do, in case url is given, but path exists already?
# Just rely on whatever clone_from() does, independently on value
# of create argument?
try:
lgr.debug("Git clone from {0} to {1}".format(url, path))
self.cmd_call_wrapper(git.Repo.clone_from, url, path)
lgr.debug("Git clone completed")
# TODO: more arguments possible: ObjectDB etc.
except GitCommandError as e:
# log here but let caller decide what to do
lgr.error(str(e))
raise
except ValueError as e:
if git.__version__ == '1.0.2' and e.message == "I/O operation on closed file":
# bug https://github.com/gitpython-developers/GitPython/issues/383
raise GitCommandError(
"clone has failed, telling ya",
999, # good number
stdout="%s already exists" if exists(path) else "")
raise # reraise original
if create and not exists(opj(path, '.git')):
try:
lgr.debug(
"Initialize empty Git repository at {0}".format(path))
self.repo = self.cmd_call_wrapper(git.Repo.init,
path,
True,
odbt=default_git_odbt)
except GitCommandError as e:
lgr.error(str(e))
raise
else:
try:
self.repo = self.cmd_call_wrapper(Repo, path)
lgr.debug("Using existing Git repository at {0}".format(path))
except (GitCommandError, NoSuchPathError,
InvalidGitRepositoryError) as e:
lgr.error("%s: %s" % (type(e), str(e)))
raise