def get_fetch_info_from_stderr(repo, proc, progress): # skip first line as it is some remote info we are not interested in output = IterableList('name') # lines which are no progress are fetch info lines # this also waits for the command to finish # Skip some progress lines that don't provide relevant information fetch_info_lines = list() for line in digest_process_messages(proc.stderr, progress): if line.startswith('From') or line.startswith('remote: Total'): continue elif line.startswith('warning:'): print >> sys.stderr, line continue elif line.startswith('fatal:'): raise GitCommandError(("Error when fetching: %s" % line, ), 2) # END handle special messages fetch_info_lines.append(line) # END for each line # read head information fp = open(join(repo.git_dir, 'FETCH_HEAD'), 'r') fetch_head_info = fp.readlines() fp.close() assert len(fetch_info_lines) == len(fetch_head_info) output.extend( CmdFetchInfo._from_line(repo, err_line, fetch_line) for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info)) finalize_process(proc) return output
def test_push_branch_with_retry(self, mock_sleep, mock_git): mock_repo = mock.Mock() mock_repo.active_branch = mock.Mock() mock_repo.active_branch.name = 'womp' mock_repo.git = mock.Mock() mock_repo.git.checkout = mock.Mock() mock_git.Repo = mock.Mock(return_value=mock_repo) mock_ret_f = mock.Mock() mock_ret_f.flags = 1 mock_ret_f.ERROR = 0 mock_ret_f.summary = "mock failure summary" mock_ret_ok = mock.Mock() mock_ret_ok.flags = 0 mock_ret_ok.ERROR = 1 mock_repo.remotes = mock.Mock() mock_repo.remotes.origin = mock.Mock() mock_repo.remotes.origin.push = mock.Mock( side_effect=[[mock_ret_f], [mock_ret_f], [mock_ret_ok]]) ghc = GitHubContext('REPO') ghc.push_branch_with_retry('womp', attempts=3, cooloff=2) self.assertEqual(mock_repo.remotes.origin.push.call_count, 3) mock_repo.remotes.origin.push = mock.Mock( side_effect=GitCommandError('B', 128)) self.assertRaises(RuntimeError, ghc.push_branch_with_retry, 'womp', attempts=3, cooloff=2)
def test_pull_repo_exc_reraise(repo_mock): exc_msg = 'something else' repo_mock.clone_from.side_effect = GitCommandError('foobar', exc_msg) with pytest.raises(GitCommandError) as exc: clocme.pull_repo(MOCK_REPO) assert repo_mock.clone_from.called assert not repo_mock.called assert 'foobar' in exc.value.command assert 'something else' in exc.value.status
def test_pull_repo_existing_repo_found(repo_mock): exc_msg = 'already exists and is not an empty directory' repo_mock.clone_from.side_effect = GitCommandError('clone_from', exc_msg) clocme.pull_repo(MOCK_REPO) assert repo_mock.clone_from.called assert MOCK_REPO in str(repo_mock.clone_from.call_args) assert clocme.COPY_PATH in str(repo_mock.clone_from.call_args) assert repo_mock.called assert repo_mock.call_args[0][0].startswith(clocme.COPY_PATH) assert repo_mock.call_args[0][0].endswith(MOCK_REPO.split('/')[-1])
def test_GitCommandError(self, init_args): argv, cause = init_args c = GitCommandError(argv, cause) s = str(c) self.assertIn(argv[0], s) if cause: self.assertIn(' failed due to: ', s) self.assertIn(str(cause), s) else: self.assertIn(' failed!', s)
def test_update_failed_because_offline( self, fake_warn, fake_repo, local_folder_exists, *_ ): from git.exc import GitCommandError repo = MagicMock(autospec=True) fake_repo.return_value = repo repo.git.pull.side_effect = [GitCommandError("a", "b")] git_clone(self.require_with_reference) fake_warn.assert_called_with("Unable to run git commands. Offline?")
def handle_stderr(proc, iter_checked_out_files): stderr = proc.stderr.read() if not stderr: return # line contents: stderr = stderr.decode(defenc) # git-checkout-index: this already exists failed_files = list() failed_reasons = list() unknown_lines = list() endings = (' already exists', ' is not in the cache', ' does not exist at stage', ' is unmerged') for line in stderr.splitlines(): if not line.startswith( "git checkout-index: ") and not line.startswith( "git-checkout-index: "): is_a_dir = " is a directory" unlink_issue = "unable to unlink old '" already_exists_issue = ' already exists, no checkout' # created by entry.c:checkout_entry(...) if line.endswith(is_a_dir): failed_files.append(line[:-len(is_a_dir)]) failed_reasons.append(is_a_dir) elif line.startswith(unlink_issue): failed_files.append( line[len(unlink_issue):line.rfind("'")]) failed_reasons.append(unlink_issue) elif line.endswith(already_exists_issue): failed_files.append(line[:-len(already_exists_issue)]) failed_reasons.append(already_exists_issue) else: unknown_lines.append(line) continue # END special lines parsing for e in endings: if line.endswith(e): failed_files.append(line[20:-len(e)]) failed_reasons.append(e) break # END if ending matches # END for each possible ending # END for each line if unknown_lines: raise GitCommandError(("git-checkout-index", ), 128, stderr) if failed_files: valid_files = list( set(iter_checked_out_files) - set(failed_files)) raise CheckoutError( "Some files could not be checked out from the index due to local modifications", failed_files, valid_files, failed_reasons)
def test_merge_branch_conflict(self, mock_git): """test merge branch""" mock_repo = mock.Mock() mock_repo.git = mock.Mock() mock_repo.git.merge = mock.Mock( side_effect=GitCommandError(mock.Mock(), mock.Mock(), mock.Mock())) mock_repo.active_branch = mock.Mock() mock_repo.active_branch.name = "ACTIVE" mock_repo.index = mock.Mock() mock_repo.index.unmerged_blobs = mock.Mock(return_value={ 'file1': [(1, "BLOB1")], 'file2': [(2, "BLOB2")] }) mock_git.Repo = mock.Mock(return_value=mock_repo) ghc = GitHubContext('REPO') self.assertRaises(GitCommandError, ghc.merge_branch, 'develop')
def test_clone_invalid_repo(self): """ Invalid repo raises error. """ with patch.object(Git, 'execute'): to_dir = join(self.workingdir, 'a') Git.execute.side_effect = GitCommandError(['command'], 128, stderr='bad command') with self.assertRaises(GitCloneError) as gce: clone('http://github.com/user/repo', to_dir) self.assertIn("'command' returned exit status 128: bad command", gce.exception)
def test_collection_post_error(self, mock_method): mock_method.side_effect = GitCommandError('git clone', 'Boom!', stderr='mocked response') request = testing.DummyRequest({}) request.validated = { 'repo_url': 'git://example.org/bar.git', 'repo_name': None } request.errors = Errors() resource = RepositoryResource(request) resource.collection_post() [error] = request.errors self.assertEqual(error['location'], 'body') self.assertEqual(error['name'], 'repo_url') self.assertEqual(error['description'], 'mocked response')
def test_handles_git_python_exceptions(self): """ If the fetch to retrieve new information results in an exception. """ with patch('jig.gitutils.remote.git') as git: git.Repo.side_effect = AttributeError self.assertTrue(remote_has_updates(self.local_workingdir)) with patch('jig.gitutils.remote.git') as git: git.Repo.side_effect = GitCommandError(None, None) self.assertTrue(remote_has_updates(self.local_workingdir)) with patch('jig.gitutils.remote.git') as git: git.Repo.side_effect = AssertionError self.assertTrue(remote_has_updates(self.local_workingdir))
def test_switch_to_branch_new_exists(): mock_repo = MagicMock() mock_git = MagicMock() mock_checkout = MagicMock() mock_branch = MagicMock() mock_is_dirty = MagicMock(return_value=False) mock_branch.side_effect = GitCommandError('Branch exists', status=0) type(mock_repo).git = mock_git type(mock_repo).is_dirty = mock_is_dirty type(mock_git).checkout = mock_checkout type(mock_git).branch = mock_branch # exising branch -> checkout branch_name = 'qweeewqwqw' switch_to_branch(mock_repo, branch_name, True) mock_checkout.assert_called_with(branch_name) mock_branch.assert_called_with(branch_name)
def test_push_branch(self, mock_git): mock_repo = mock.Mock() mock_repo.active_branch = mock.Mock() mock_repo.active_branch.name = 'womp' mock_repo.head = "HEAD" mock_repo.git = mock.Mock() mock_repo.git.checkout = mock.Mock() mock_git.Repo = mock.Mock(return_value=mock_repo) mock_repo.remotes = mock.Mock() mock_ret = mock.Mock() mock_ret.flags = 1000 mock_repo.remotes.origin = mock.Mock() mock_repo.remotes.origin.push = mock.Mock(return_value=[mock_ret]) ghc = GitHubContext('REPO') ghc.push_branch('womp') self.failUnless(mock_repo.remotes.origin.push.called) mock_repo.remotes.origin.push.assert_has_calls([mock.call('HEAD')]) mock_repo.remotes.origin.push = mock.Mock( side_effect=GitCommandError('A', 128)) self.assertRaises(RuntimeError, ghc.push_branch, 'womp2')
def test_has_uncommitted_changes_throws_error(self): cg = CivisGit() cg.repo = MagicMock(spec=Repo) cg.repo().index.diff = MagicMock( side_effect=GitCommandError('diff', 'failed')) self.assertRaises(CivisGitError, lambda: cg.has_uncommitted_changes())
def execute(self, command, istream=None, with_extended_output=False, with_exceptions=True, as_process=False, output_stream=None, stdout_as_string=True, kill_after_timeout=None, with_stdout=True, universal_newlines=False, shell=None, env=None, max_chunk_size=io.DEFAULT_BUFFER_SIZE, callback=None, **subprocess_kwargs): """ Handles executing the command on the shell and consumes and returns the returned information (stdout) ``command`` The command argument list to execute ``istream`` Standard input filehandle passed to subprocess.Popen. ``with_keep_cwd`` Whether to use the current working directory from os.getcwd(). GitPython uses get_work_tree() as its working directory by default and get_git_dir() for bare repositories. ``with_extended_output`` Whether to return a (status, stdout, stderr) tuple. ``with_exceptions`` Whether to raise an exception when git returns a non-zero status. ``with_raw_output`` Whether to avoid stripping off trailing whitespace. ``callback`` User supplied callback to handle data processing Returns str(output) # extended_output = False (Default) tuple(int(status), str(output)) # extended_output = True callback # callback != None """ if GIT_PYTHON_TRACE and not GIT_PYTHON_TRACE == 'full': print(' '.join(command)) # Allow the user to have the command executed in their working dir. if with_keep_cwd or self.git_dir is None: cwd = os.getcwd() else: cwd = self.git_dir # Start the process proc = subprocess.Popen(command, cwd=cwd, stdin=istream, stderr=subprocess.PIPE, stdout=subprocess.PIPE, **extra) # if the user supplied a callback, use that instead. if callback: return (callback(proc)) # Wait for the process to return try: stdout_value = proc.stdout.read() stderr_value = proc.stderr.read() status = proc.wait() finally: proc.stdout.close() proc.stderr.close() # Strip off trailing whitespace by default if not with_raw_output: stdout_value = stdout_value.rstrip() stderr_value = stderr_value.rstrip() if with_exceptions and status != 0: raise GitCommandError(command, status, stderr_value) if GIT_PYTHON_TRACE == 'full': if stderr_value: print("%s -> %d: '%s' !! '%s'" % (command, status, stdout_value, stderr_value)) elif stdout_value: print("%s -> %d: '%s'" % (command, status, stdout_value)) else: print("%s -> %d" % (command, status)) # Allow access to the command's status code if with_extended_output: return (status, stdout_value, stderr_value) else: return stdout_value
def test_generic_check_fail(self, sys_exit_mock, print_error_mock): commit_range = MagicMock() commit_range.__iter__.side_effect = GitCommandError("test", 123) _generic_check(commit_range, None, check_merge_commits=False) print_error_mock.assert_called_once() sys_exit_mock.assert_called_once_with(ERROR_EXIT_CODE)
def test_clone_repository_throws_error(self, repo_clone): repo_clone.side_effect = GitCommandError('clone', 'failed') self.assertRaises(CivisGitError, lambda: CivisGit().clone_repository())
def checkout(self, paths=None, force=False, fprogress=lambda *args: None, **kwargs): """Checkout the given paths or all files from the version known to the index into the working tree. :note: Be sure you have written pending changes using the ``write`` method in case you have altered the enties dictionary directly :param paths: If None, all paths in the index will be checked out. Otherwise an iterable of relative or absolute paths or a single path pointing to files or directories in the index is expected. :param force: If True, existing files will be overwritten even if they contain local modifications. If False, these will trigger a CheckoutError. :param fprogress: see Index.add_ for signature and explanation. The provided progress information will contain None as path and item if no explicit paths are given. Otherwise progress information will be send prior and after a file has been checked out :param kwargs: Additional arguments to be pasesd to git-checkout-index :return: iterable yielding paths to files which have been checked out and are guaranteed to match the version stored in the index :raise CheckoutError: If at least one file failed to be checked out. This is a summary, hence it will checkout as many files as it can anyway. If one of files or directories do not exist in the index ( as opposed to the original git command who ignores them ). Raise GitCommandError if error lines could not be parsed - this truly is an exceptional state .. note:: The checkout is limited to checking out the files in the index. Files which are not in the index anymore and exist in the working tree will not be deleted. This behaviour is fundamentally different to *head.checkout*, i.e. if you want git-checkout like behaviour, use head.checkout instead of index.checkout. """ args = ["--index"] if force: args.append("--force") def handle_stderr(proc, iter_checked_out_files): stderr = proc.stderr.read() if not stderr: return # line contents: # git-checkout-index: this already exists failed_files = list() failed_reasons = list() unknown_lines = list() endings = (' already exists', ' is not in the cache', ' does not exist at stage', ' is unmerged') for line in stderr.splitlines(): if not line.startswith( "git checkout-index: ") and not line.startswith( "git-checkout-index: "): is_a_dir = " is a directory" unlink_issue = "unable to unlink old '" already_exists_issue = ' already exists, no checkout' # created by entry.c:checkout_entry(...) if line.endswith(is_a_dir): failed_files.append(line[:-len(is_a_dir)]) failed_reasons.append(is_a_dir) elif line.startswith(unlink_issue): failed_files.append( line[len(unlink_issue):line.rfind("'")]) failed_reasons.append(unlink_issue) elif line.endswith(already_exists_issue): failed_files.append(line[:-len(already_exists_issue)]) failed_reasons.append(already_exists_issue) else: unknown_lines.append(line) continue # END special lines parsing for e in endings: if line.endswith(e): failed_files.append(line[20:-len(e)]) failed_reasons.append(e) break # END if ending matches # END for each possible ending # END for each line if unknown_lines: raise GitCommandError(("git-checkout-index", ), 128, stderr) if failed_files: valid_files = list( set(iter_checked_out_files) - set(failed_files)) raise CheckoutError( "Some files could not be checked out from the index due to local modifications", failed_files, valid_files, failed_reasons) # END stderr handler if paths is None: args.append("--all") kwargs['as_process'] = 1 fprogress(None, False, None) proc = self.repo.git.checkout_index(*args, **kwargs) proc.wait() fprogress(None, True, None) rval_iter = (e.path for e in self.entries.itervalues()) handle_stderr(proc, rval_iter) return rval_iter else: if isinstance(paths, basestring): paths = [paths] # make sure we have our entries loaded before we start checkout_index # which will hold a lock on it. We try to get the lock as well during # our entries initialization self.entries args.append("--stdin") kwargs['as_process'] = True kwargs['istream'] = subprocess.PIPE proc = self.repo.git.checkout_index(args, **kwargs) make_exc = lambda: GitCommandError( ("git-checkout-index", ) + tuple(args), 128, proc.stderr.read( )) checked_out_files = list() for path in paths: co_path = to_native_path_linux(self._to_relative_path(path)) # if the item is not in the index, it could be a directory path_is_directory = False try: self.entries[(co_path, 0)] except KeyError: dir = co_path if not dir.endswith('/'): dir += '/' for entry in self.entries.itervalues(): if entry.path.startswith(dir): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(p) path_is_directory = True # END if entry is in directory # END for each entry # END path exception handlnig if not path_is_directory: self._write_path_to_stdin(proc, co_path, path, make_exc, fprogress, read_from_stdout=False) checked_out_files.append(co_path) # END path is a file # END for each path self._flush_stdin_and_wait(proc, ignore_stdout=True) handle_stderr(proc, checked_out_files) return checked_out_files # END paths handling assert "Should not reach this point"
def __init__(self, path, url=None, runner=None, create=True): """Creates representation of git repository at `path`. If `url` is given, a clone is created at `path`. Can also be used to create a git repository at `path`. Parameters ---------- path: str path to the git repository; In case it's not an absolute path, it's relative to PWD url: str url to the to-be-cloned repository. Requires a valid git url according to: http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS . create: bool if true, creates a git repository at `path` if there is none. Also creates `path`, if it doesn't exist. If set to false, an exception is raised in case `path` doesn't exist or doesn't contain a git repository. """ self.path = abspath(normpath(path)) self.cmd_call_wrapper = runner or Runner(cwd=self.path) # TODO: Concept of when to set to "dry". # Includes: What to do in gitrepo class? # Now: setting "dry" means to give a dry-runner to constructor. # => Do it similar in gitrepo/dataset. # Still we need a concept of when to set it and whether this # should be a single instance collecting everything or more # fine grained. # TODO: somehow do more extensive checks that url and path don't point to the # same location if url is not None and not (url == path): # TODO: What to do, in case url is given, but path exists already? # Just rely on whatever clone_from() does, independently on value # of create argument? try: lgr.debug("Git clone from {0} to {1}".format(url, path)) self.cmd_call_wrapper(git.Repo.clone_from, url, path) lgr.debug("Git clone completed") # TODO: more arguments possible: ObjectDB etc. except GitCommandError as e: # log here but let caller decide what to do lgr.error(str(e)) raise except ValueError as e: if git.__version__ == '1.0.2' and e.message == "I/O operation on closed file": # bug https://github.com/gitpython-developers/GitPython/issues/383 raise GitCommandError( "clone has failed, telling ya", 999, # good number stdout="%s already exists" if exists(path) else "") raise # reraise original if create and not exists(opj(path, '.git')): try: lgr.debug( "Initialize empty Git repository at {0}".format(path)) self.repo = self.cmd_call_wrapper(git.Repo.init, path, True, odbt=default_git_odbt) except GitCommandError as e: lgr.error(str(e)) raise else: try: self.repo = self.cmd_call_wrapper(Repo, path) lgr.debug("Using existing Git repository at {0}".format(path)) except (GitCommandError, NoSuchPathError, InvalidGitRepositoryError) as e: lgr.error("%s: %s" % (type(e), str(e))) raise