def _get_push_info(self, proc, progress):
progress = to_progress_instance(progress)
# read progress information from stderr
# we hope stdout can hold all the data, it should ...
# read the lines manually as it will use carriage returns between the messages
# to override the previous one. This is why we read the bytes manually
progress_handler = progress.new_message_handler()
output = IterableList('name')
def stdout_handler(line):
try:
output.append(PushInfo._from_line(self, line))
except ValueError:
# If an error happens, additional info is given which we parse below.
pass
handle_process_output(proc,
stdout_handler,
progress_handler,
finalizer=None,
decode_streams=False)
stderr_text = progress.error_lines and '\n'.join(
progress.error_lines) or ''
try:
proc.wait(stderr=stderr_text)
except Exception:
if not output:
raise
elif stderr_text:
log.warning("Error lines received while fetching: %s",
stderr_text)
return output
def _get_push_info(self, proc, progress):
progress = to_progress_instance(progress)
# read progress information from stderr
# we hope stdout can hold all the data, it should ...
# read the lines manually as it will use carriage returns between the messages
# to override the previous one. This is why we read the bytes manually
progress_handler = progress.new_message_handler()
output = IterableList('name')
def stdout_handler(line):
try:
output.append(PushInfo._from_line(self, line))
except ValueError:
# if an error happens, additional info is given which we cannot parse
pass
# END exception handling
# END for each line
try:
handle_process_output(proc, stdout_handler, progress_handler, finalize_process)
except Exception:
if len(output) == 0:
raise
return output
def run_commit_hook(name, index):
"""Run the commit hook of the given name. Silently ignores hooks that do not exist.
:param name: name of hook, like 'pre-commit'
:param index: IndexFile instance
:raises HookExecutionError: """
hp = hook_path(name, index.repo.git_dir)
if not os.access(hp, os.X_OK):
return
env = os.environ.copy()
env['GIT_INDEX_FILE'] = safe_decode(index.path) if PY3 else safe_encode(index.path)
env['GIT_EDITOR'] = ':'
try:
cmd = subprocess.Popen(hp,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=index.repo.working_dir,
close_fds=is_posix,
creationflags=PROC_CREATIONFLAGS,)
except Exception as ex:
raise HookExecutionError(hp, ex)
else:
stdout = []
stderr = []
handle_process_output(cmd, stdout.append, stderr.append, finalize_process)
stdout = ''.join(stdout)
stderr = ''.join(stderr)
if cmd.returncode != 0:
stdout = force_text(stdout, defenc)
stderr = force_text(stderr, defenc)
raise HookExecutionError(hp, cmd.returncode, stdout, stderr)
def _index_from_raw_format(cls, repo, proc):
"""Create a new DiffIndex from the given stream which must be in raw format.
:return: git.DiffIndex"""
# handles
# :100644 100644 687099101... 37c5e30c8... M .gitignore
index = DiffIndex()
def handle_diff_line(line):
line = line.decode(defenc)
if not line.startswith(":"):
return
meta, _, path = line[1:].partition('\t')
old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(
None, 4)
# Change type can be R100
# R: status letter
# 100: score (in case of copy and rename)
change_type = _change_type[0]
score_str = ''.join(_change_type[1:])
score = int(score_str) if score_str.isdigit() else None
path = path.strip()
a_path = path.encode(defenc)
b_path = path.encode(defenc)
deleted_file = False
new_file = False
rename_from = None
rename_to = None
# NOTE: We cannot conclude from the existence of a blob to change type
# as diffs with the working do not have blobs yet
if change_type == 'D':
b_blob_id = None
deleted_file = True
elif change_type == 'A':
a_blob_id = None
new_file = True
elif change_type == 'R':
a_path, b_path = path.split('\t', 1)
a_path = a_path.encode(defenc)
b_path = b_path.encode(defenc)
rename_from, rename_to = a_path, b_path
elif change_type == 'T':
# Nothing to do
pass
# END add/remove handling
diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode,
new_mode, new_file, deleted_file, rename_from,
rename_to, '', change_type, score)
index.append(diff)
handle_process_output(proc,
handle_diff_line,
None,
finalize_process,
decode_streams=False)
return index
def _get_push_info(self, proc, progress):
progress = to_progress_instance(progress)
# read progress information from stderr
# we hope stdout can hold all the data, it should ...
# read the lines manually as it will use carriage returns between the messages
# to override the previous one. This is why we read the bytes manually
progress_handler = progress.new_message_handler()
output = IterableList('name')
def stdout_handler(line):
try:
output.append(PushInfo._from_line(self, line))
except ValueError:
# If an error happens, additional info is given which we parse below.
pass
handle_process_output(proc, stdout_handler, progress_handler, finalizer=None, decode_streams=False)
stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or ''
try:
proc.wait(stderr=stderr_text)
except Exception:
if not output:
raise
elif stderr_text:
log.warning("Error lines received while fetching: %s", stderr_text)
return output
def test_handle_process_output(self):
from git.cmd import handle_process_output
line_count = 5002
count = [None, 0, 0]
def counter_stdout(line):
count[1] += 1
def counter_stderr(line):
count[2] += 1
proc = subprocess.Popen([
sys.executable,
fixture_path('cat_file.py'),
str(fixture_path('issue-301_stderr'))
],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
handle_process_output(proc, counter_stdout, counter_stderr,
lambda proc: proc.wait())
assert count[1] == line_count
assert count[2] == line_count
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
# special handling for windows for path at which the clone should be
# created.
# tilde '~' will be expanded to the HOME no matter where the ~ occours. Hence
# we at least give a proper error instead of letting git fail
prev_cwd = None
prev_path = None
odbt = kwargs.pop('odbt', odb_default_type)
if os.name == 'nt':
if '~' in path:
raise OSError("Git cannot handle the ~ character in path %r correctly" % path)
# on windows, git will think paths like c: are relative and prepend the
# current working dir ( before it fails ). We temporarily adjust the working
# dir to make this actually work
match = re.match("(\w:[/\\\])(.*)", path)
if match:
prev_cwd = os.getcwd()
prev_path = path
drive, rest_of_path = match.groups()
os.chdir(drive)
path = rest_of_path
kwargs['with_keep_cwd'] = True
# END cwd preparation
# END windows handling
try:
proc = git.clone(url, path, with_extended_output=True, as_process=True,
v=True, **add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, progress.new_message_handler(), finalize_process)
else:
(stdout, stderr) = proc.communicate()
finalize_process(proc, stderr=stderr)
# end handle progress
finally:
if prev_cwd is not None:
os.chdir(prev_cwd)
path = prev_path
# END reset previous working dir
# END bad windows handling
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not os.path.isabs(path) and git.working_dir:
path = join(git._working_dir, path)
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
repo = cls(os.path.abspath(path), odbt=odbt)
if repo.remotes:
writer = repo.remotes[0].config_writer
writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/"))
# PY3: be sure cleanup is performed and lock is released
writer.release()
# END handle remote repo
return repo
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
if progress is not None:
progress = to_progress_instance(progress)
odbt = kwargs.pop('odbt', odb_default_type)
# when pathlib.Path or other classbased path is passed
if not isinstance(path, str):
path = str(path)
## A bug win cygwin's Git, when `--bare` or `--separate-git-dir`
# it prepends the cwd or(?) the `url` into the `path, so::
# git clone --bare /cygwin/d/foo.git C:\\Work
# becomes::
# git clone --bare /cygwin/d/foo.git /cygwin/d/C:\\Work
#
clone_path = (Git.polish_url(path)
if Git.is_cygwin() and 'bare' in kwargs else path)
sep_dir = kwargs.get('separate_git_dir')
if sep_dir:
kwargs['separate_git_dir'] = Git.polish_url(sep_dir)
proc = git.clone(Git.polish_url(url),
clone_path,
with_extended_output=True,
as_process=True,
v=True,
universal_newlines=True,
**add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc,
None,
progress.new_message_handler(),
finalize_process,
decode_streams=False)
else:
(stdout, stderr) = proc.communicate()
log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''),
stdout)
finalize_process(proc, stderr=stderr)
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not osp.isabs(path) and git.working_dir:
path = osp.join(git._working_dir, path)
repo = cls(path, odbt=odbt)
# retain env values that were passed to _clone()
repo.git.update_environment(**git.environment())
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo
def test_handle_process_output(self):
from git.cmd import handle_process_output
line_count = 5002
count = [None, 0, 0]
def counter_stdout(line):
count[1] += 1
def counter_stderr(line):
count[2] += 1
cmdline = [sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr'))]
proc = subprocess.Popen(cmdline,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
creationflags=cmd.PROC_CREATIONFLAGS,
)
handle_process_output(proc, counter_stdout, counter_stderr, finalize_process)
self.assertEqual(count[1], line_count)
self.assertEqual(count[2], line_count)
def _index_from_patch_format(cls, repo, proc):
"""Create a new DiffIndex from the given text which must be in patch format
:param repo: is the repository we are operating on - it is required
:param stream: result of 'git diff' as a stream (supporting file protocol)
:return: git.DiffIndex """
## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise.
text = []
handle_process_output(proc,
text.append,
None,
finalize_process,
decode_streams=False)
# for now, we have to bake the stream
text = b''.join(text)
index = DiffIndex()
previous_header = None
header = None
for _header in cls.re_header.finditer(text):
a_path_fallback, b_path_fallback, \
old_mode, new_mode, \
rename_from, rename_to, \
new_file_mode, deleted_file_mode, copied_file_name, \
a_blob_id, b_blob_id, b_mode, \
a_path, b_path = _header.groups()
new_file, deleted_file, copied_file = \
bool(new_file_mode), bool(deleted_file_mode), bool(copied_file_name)
a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback)
b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback)
# Our only means to find the actual text is to see what has not been matched by our regex,
# and then retro-actively assign it to our index
if previous_header is not None:
index[-1].diff = text[previous_header.end():_header.start()]
# end assign actual diff
# Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid
# We just use the one mode we should have parsed
a_mode = old_mode or deleted_file_mode or (
a_path and (b_mode or new_mode or new_file_mode))
b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode)
index.append(
Diff(repo, a_path, b_path, a_blob_id
and a_blob_id.decode(defenc), b_blob_id
and b_blob_id.decode(defenc), a_mode
and a_mode.decode(defenc), b_mode
and b_mode.decode(defenc), new_file, deleted_file,
copied_file, rename_from, rename_to, None, None, None))
previous_header = _header
header = _header
# end for each header we parse
if index:
index[-1].diff = text[header.end():]
# end assign last diff
return index
def _get_push_info(self, proc, progress):
# read progress information from stderr
# we hope stdout can hold all the data, it should ...
# read the lines manually as it will use carriage returns between the messages
# to override the previous one. This is why we read the bytes manually
# TODO: poll() on file descriptors to know what to read next, process streams concurrently
progress_handler = progress.new_message_handler()
output = IterableList('name')
def stdout_handler(line):
try:
output.append(PushInfo._from_line(self, line))
except ValueError:
# if an error happens, additional info is given which we cannot parse
pass
# END exception handling
# END for each line
try:
handle_process_output(proc, stdout_handler, progress_handler,
finalize_process)
except Exception:
if len(output) == 0:
raise
return output
def _index_from_name_only_format(cls, repo, proc):
"""Create a new DiffIndex from the given text which must be in name only format
:param repo: is the repository we are operating on - it is required
:param stream: result of 'git diff' as a stream (supporting file protocol)
:return: git.DiffIndex """
index = DiffIndex()
def handle_diff_line_name_only(lines):
lines = lines.decode(defenc)
for line in lines.split('\x00'):
path = line.strip()
if len(path) == 0:
continue
if cls.is_first:
cls.is_first = False
continue
a_path = path.encode(defenc)
b_path = path.encode(defenc)
index.append(
Diff(repo, a_path, b_path, None, None, None, None, False,
False, None, None, None, '', None, None))
handle_process_output(proc,
handle_diff_line_name_only,
None,
finalize_process,
decode_streams=False)
return index
def test_handle_process_output(self):
from git.cmd import handle_process_output
line_count = 5002
count = [None, 0, 0]
def counter_stdout(line):
count[1] += 1
def counter_stderr(line):
count[2] += 1
cmdline = [
sys.executable,
fixture_path('cat_file.py'),
str(fixture_path('issue-301_stderr'))
]
proc = subprocess.Popen(
cmdline,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
creationflags=cmd.PROC_CREATIONFLAGS,
)
handle_process_output(proc, counter_stdout, counter_stderr,
finalize_process)
self.assertEqual(count[1], line_count)
self.assertEqual(count[2], line_count)
def run_commit_hook(name, index, *args):
"""Run the commit hook of the given name. Silently ignores hooks that do not exist.
:param name: name of hook, like 'pre-commit'
:param index: IndexFile instance
:param args: arguments passed to hook file
:raises HookExecutionError: """
hp = hook_path(name, index.repo.git_dir)
if not os.access(hp, os.X_OK):
return
env = os.environ.copy()
env['GIT_INDEX_FILE'] = safe_decode(index.path) if PY3 else safe_encode(index.path)
env['GIT_EDITOR'] = ':'
try:
cmd = subprocess.Popen([hp] + list(args),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=index.repo.working_dir,
close_fds=is_posix,
creationflags=PROC_CREATIONFLAGS,)
except Exception as ex:
raise HookExecutionError(hp, ex)
else:
stdout = []
stderr = []
handle_process_output(cmd, stdout.append, stderr.append, finalize_process)
stdout = ''.join(stdout)
stderr = ''.join(stderr)
if cmd.returncode != 0:
stdout = force_text(stdout, defenc)
stderr = force_text(stderr, defenc)
raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
def _get_push_info(self, proc: 'Git.AutoInterrupt',
progress: Union[Callable[..., Any], RemoteProgress, None],
kill_after_timeout: Union[None, float] = None) -> PushInfoList:
progress = to_progress_instance(progress)
# read progress information from stderr
# we hope stdout can hold all the data, it should ...
# read the lines manually as it will use carriage returns between the messages
# to override the previous one. This is why we read the bytes manually
progress_handler = progress.new_message_handler()
output: PushInfoList = PushInfoList()
def stdout_handler(line: str) -> None:
try:
output.append(PushInfo._from_line(self, line))
except ValueError:
# If an error happens, additional info is given which we parse below.
pass
handle_process_output(proc, stdout_handler, progress_handler, finalizer=None, decode_streams=False,
kill_after_timeout=kill_after_timeout)
stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or ''
try:
proc.wait(stderr=stderr_text)
except Exception as e:
# This is different than fetch (which fails if there is any std_err
# even if there is an output)
if not output:
raise
elif stderr_text:
log.warning("Error lines received while fetching: %s", stderr_text)
output.error = e
return output
def _get_fetch_info_from_stderr(self, proc, progress):
progress = to_progress_instance(progress)
# skip first line as it is some remote info we are not interested in
output = IterableList('name')
# lines which are no progress are fetch info lines
# this also waits for the command to finish
# Skip some progress lines that don't provide relevant information
fetch_info_lines = []
# Basically we want all fetch info lines which appear to be in regular form, and thus have a
# command character. Everything else we ignore,
cmds = set(FetchInfo._flag_map.keys())
progress_handler = progress.new_message_handler()
handle_process_output(proc,
None,
progress_handler,
finalizer=None,
decode_streams=False)
stderr_text = progress.error_lines and '\n'.join(
progress.error_lines) or ''
proc.wait(stderr=stderr_text)
if stderr_text:
log.warning("Error lines received while fetching: %s", stderr_text)
for line in progress.other_lines:
line = force_text(line)
for cmd in cmds:
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
fetch_info_lines.append(line)
continue
# read head information
with open(osp.join(self.repo.common_dir, 'FETCH_HEAD'), 'rb') as fp:
fetch_head_info = [l.decode(defenc) for l in fp.readlines()]
l_fil = len(fetch_info_lines)
l_fhi = len(fetch_head_info)
if l_fil != l_fhi:
msg = "Fetch head lines do not match lines provided via progress information\n"
msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n"
msg += "Will ignore extra progress lines or fetch head lines."
msg %= (l_fil, l_fhi)
log.debug(msg)
log.debug("info lines: " + str(fetch_info_lines))
log.debug("head info : " + str(fetch_head_info))
if l_fil < l_fhi:
fetch_head_info = fetch_head_info[:l_fil]
else:
fetch_info_lines = fetch_info_lines[:l_fhi]
# end truncate correct list
# end sanity check + sanitization
output.extend(
FetchInfo._from_line(self.repo, err_line, fetch_line)
for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info))
return output
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
# special handling for windows for path at which the clone should be
# created.
# tilde '~' will be expanded to the HOME no matter where the ~ occours. Hence
# we at least give a proper error instead of letting git fail
prev_cwd = None
prev_path = None
odbt = kwargs.pop('odbt', odb_default_type)
if os.name == 'nt':
if '~' in path:
raise OSError("Git cannot handle the ~ character in path %r correctly" % path)
# on windows, git will think paths like c: are relative and prepend the
# current working dir ( before it fails ). We temporarily adjust the working
# dir to make this actually work
match = re.match("(\w:[/\\\])(.*)", path)
if match:
prev_cwd = os.getcwd()
prev_path = path
drive, rest_of_path = match.groups()
os.chdir(drive)
path = rest_of_path
kwargs['with_keep_cwd'] = True
# END cwd preparation
# END windows handling
try:
proc = git.clone(url, path, with_extended_output=True, as_process=True,
v=True, **add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, progress.new_message_handler(), finalize_process)
else:
finalize_process(proc)
# end handle progress
finally:
if prev_cwd is not None:
os.chdir(prev_cwd)
path = prev_path
# END reset previous working dir
# END bad windows handling
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not os.path.isabs(path) and git.working_dir:
path = join(git._working_dir, path)
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
repo = cls(os.path.abspath(path), odbt=odbt)
if repo.remotes:
writer = repo.remotes[0].config_writer
writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/"))
# PY3: be sure cleanup is performed and lock is released
writer.release()
# END handle remote repo
return repo
def _index_from_patch_format(cls, repo, proc):
"""Create a new DiffIndex from the given text which must be in patch format
:param repo: is the repository we are operating on - it is required
:param stream: result of 'git diff' as a stream (supporting file protocol)
:return: git.DiffIndex """
## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise.
text = []
handle_process_output(proc, text.append, None, finalize_process, decode_streams=False)
# for now, we have to bake the stream
text = b''.join(text)
index = DiffIndex()
previous_header = None
for header in cls.re_header.finditer(text):
a_path_fallback, b_path_fallback, \
old_mode, new_mode, \
rename_from, rename_to, \
new_file_mode, deleted_file_mode, \
a_blob_id, b_blob_id, b_mode, \
a_path, b_path = header.groups()
new_file, deleted_file = bool(new_file_mode), bool(deleted_file_mode)
a_path = cls._pick_best_path(a_path, rename_from, a_path_fallback)
b_path = cls._pick_best_path(b_path, rename_to, b_path_fallback)
# Our only means to find the actual text is to see what has not been matched by our regex,
# and then retro-actively assign it to our index
if previous_header is not None:
index[-1].diff = text[previous_header.end():header.start()]
# end assign actual diff
# Make sure the mode is set if the path is set. Otherwise the resulting blob is invalid
# We just use the one mode we should have parsed
a_mode = old_mode or deleted_file_mode or (a_path and (b_mode or new_mode or new_file_mode))
b_mode = b_mode or new_mode or new_file_mode or (b_path and a_mode)
index.append(Diff(repo,
a_path,
b_path,
a_blob_id and a_blob_id.decode(defenc),
b_blob_id and b_blob_id.decode(defenc),
a_mode and a_mode.decode(defenc),
b_mode and b_mode.decode(defenc),
new_file, deleted_file,
rename_from,
rename_to,
None, None))
previous_header = header
# end for each header we parse
if index:
index[-1].diff = text[header.end():]
# end assign last diff
return index
def _get_fetch_info_from_stderr(self, proc, progress):
progress = to_progress_instance(progress)
# skip first line as it is some remote info we are not interested in
output = IterableList('name')
# lines which are no progress are fetch info lines
# this also waits for the command to finish
# Skip some progress lines that don't provide relevant information
fetch_info_lines = []
# Basically we want all fetch info lines which appear to be in regular form, and thus have a
# command character. Everything else we ignore,
cmds = set(FetchInfo._flag_map.keys())
progress_handler = progress.new_message_handler()
handle_process_output(proc, None, progress_handler, finalizer=None, decode_streams=False)
stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or ''
proc.wait(stderr=stderr_text)
if stderr_text:
log.warning("Error lines received while fetching: %s", stderr_text)
for line in progress.other_lines:
line = force_text(line)
for cmd in cmds:
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
fetch_info_lines.append(line)
continue
# read head information
with open(osp.join(self.repo.common_dir, 'FETCH_HEAD'), 'rb') as fp:
fetch_head_info = [l.decode(defenc) for l in fp.readlines()]
l_fil = len(fetch_info_lines)
l_fhi = len(fetch_head_info)
if l_fil != l_fhi:
msg = "Fetch head lines do not match lines provided via progress information\n"
msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n"
msg += "Will ignore extra progress lines or fetch head lines."
msg %= (l_fil, l_fhi)
log.debug(msg)
log.debug("info lines: " + str(fetch_info_lines))
log.debug("head info : " + str(fetch_head_info))
if l_fil < l_fhi:
fetch_head_info = fetch_head_info[:l_fil]
else:
fetch_info_lines = fetch_info_lines[:l_fhi]
# end truncate correct list
# end sanity check + sanitization
output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line)
for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info))
return output
def _index_from_raw_format(cls, repo, proc):
"""Create a new DiffIndex from the given stream which must be in raw format.
:return: git.DiffIndex"""
# handles
# :100644 100644 687099101... 37c5e30c8... M .gitignore
index = DiffIndex()
def handle_diff_line(line):
line = line.decode(defenc)
if not line.startswith(":"):
return
meta, _, path = line[1:].partition('\t')
old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split(
None, 4)
path = path.strip()
a_path = path.encode(defenc)
b_path = path.encode(defenc)
deleted_file = False
new_file = False
rename_from = None
rename_to = None
# NOTE: We cannot conclude from the existence of a blob to change type
# as diffs with the working do not have blobs yet
if change_type == 'D':
b_blob_id = None
deleted_file = True
elif change_type == 'A':
a_blob_id = None
new_file = True
elif change_type[
0] == 'R': # parses RXXX, where XXX is a confidence value
a_path, b_path = path.split('\t', 1)
a_path = a_path.encode(defenc)
b_path = b_path.encode(defenc)
rename_from, rename_to = a_path, b_path
# END add/remove handling
diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode,
new_mode, new_file, deleted_file, rename_from,
rename_to, '', change_type)
index.append(diff)
handle_process_output(proc,
handle_diff_line,
None,
finalize_process,
decode_streams=False)
return index
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
if progress is not None:
progress = to_progress_instance(progress)
odbt = kwargs.pop('odbt', odb_default_type)
# when pathlib.Path or other classbased path is passed
if not isinstance(path, str):
path = str(path)
## A bug win cygwin's Git, when `--bare` or `--separate-git-dir`
# it prepends the cwd or(?) the `url` into the `path, so::
# git clone --bare /cygwin/d/foo.git C:\\Work
# becomes::
# git clone --bare /cygwin/d/foo.git /cygwin/d/C:\\Work
#
clone_path = (Git.polish_url(path)
if Git.is_cygwin() and 'bare' in kwargs
else path)
sep_dir = kwargs.get('separate_git_dir')
if sep_dir:
kwargs['separate_git_dir'] = Git.polish_url(sep_dir)
proc = git.clone(Git.polish_url(url), clone_path, with_extended_output=True, as_process=True,
v=True, universal_newlines=True, **add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, progress.new_message_handler(), finalize_process, decode_streams=False)
else:
(stdout, stderr) = proc.communicate()
log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''), stdout)
finalize_process(proc, stderr=stderr)
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not osp.isabs(path) and git.working_dir:
path = osp.join(git._working_dir, path)
repo = cls(path, odbt=odbt)
# retain env values that were passed to _clone()
repo.git.update_environment(**git.environment())
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo
def _index_from_raw_format(cls, repo: 'Repo', proc: TBD) -> DiffIndex:
"""Create a new DiffIndex from the given stream which must be in raw format.
:return: git.DiffIndex"""
# handles
# :100644 100644 687099101... 37c5e30c8... M .gitignore
index = DiffIndex()
handle_process_output(
proc,
lambda bytes: cls._handle_diff_line(bytes, repo, index),
None,
finalize_process,
decode_streams=False)
return index
def _get_fetch_info_from_stderr(self, proc, progress):
# skip first line as it is some remote info we are not interested in
output = IterableList('name')
# lines which are no progress are fetch info lines
# this also waits for the command to finish
# Skip some progress lines that don't provide relevant information
fetch_info_lines = list()
# Basically we want all fetch info lines which appear to be in regular form, and thus have a
# command character. Everything else we ignore,
cmds = set(PushInfo._flag_map.keys()) & set(FetchInfo._flag_map.keys())
progress_handler = progress.new_message_handler()
def my_progress_handler(line):
for pline in progress_handler(line):
if line.startswith('fatal:') or line.startswith('error:'):
raise GitCommandError(("Error when fetching: %s" % line, ),
2)
# END handle special messages
for cmd in cmds:
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
fetch_info_lines.append(line)
continue
# end find command code
# end for each comand code we know
# end for each line progress didn't handle
# end
# We are only interested in stderr here ...
handle_process_output(proc, None, my_progress_handler,
finalize_process)
# read head information
fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb')
fetch_head_info = [l.decode(defenc) for l in fp.readlines()]
fp.close()
# NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it.
assert len(fetch_info_lines) >= len(
fetch_head_info), "len(%s) <= len(%s)" % (fetch_head_info,
fetch_info_lines)
output.extend(
FetchInfo._from_line(self.repo, err_line, fetch_line)
for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info))
return output
def _index_from_raw_format(cls, repo, proc):
"""Create a new DiffIndex from the given stream which must be in raw format.
:return: git.DiffIndex"""
# handles
# :100644 100644 687099101... 37c5e30c8... M .gitignore
index = DiffIndex()
def handle_diff_line(line):
line = line.decode(defenc)
if not line.startswith(":"):
return
meta, _, path = line[1:].partition('\t')
old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split(None, 4)
path = path.strip()
a_path = path.encode(defenc)
b_path = path.encode(defenc)
deleted_file = False
new_file = False
rename_from = None
rename_to = None
# NOTE: We cannot conclude from the existence of a blob to change type
# as diffs with the working do not have blobs yet
if change_type == 'D':
b_blob_id = None
deleted_file = True
elif change_type == 'A':
a_blob_id = None
new_file = True
elif change_type[0] == 'R': # parses RXXX, where XXX is a confidence value
a_path, b_path = path.split('\t', 1)
a_path = a_path.encode(defenc)
b_path = b_path.encode(defenc)
rename_from, rename_to = a_path, b_path
# END add/remove handling
diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode,
new_file, deleted_file, rename_from, rename_to, '', change_type)
index.append(diff)
handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False)
return index
def run_commit_hook(name: str, index: 'IndexFile', *args: str) -> None:
"""Run the commit hook of the given name. Silently ignores hooks that do not exist.
:param name: name of hook, like 'pre-commit'
:param index: IndexFile instance
:param args: arguments passed to hook file
:raises HookExecutionError: """
hp = hook_path(name, index.repo.git_dir)
if not os.access(hp, os.X_OK):
return None
env = os.environ.copy()
env['GIT_INDEX_FILE'] = safe_decode(str(index.path))
env['GIT_EDITOR'] = ':'
cmd = [hp]
try:
if is_win and not _has_file_extension(hp):
# Windows only uses extensions to determine how to open files
# (doesn't understand shebangs). Try using bash to run the hook.
relative_hp = Path(hp).relative_to(
index.repo.working_dir).as_posix()
cmd = ["bash.exe", relative_hp]
cmd = subprocess.Popen(
cmd + list(args),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=index.repo.working_dir,
close_fds=is_posix,
creationflags=PROC_CREATIONFLAGS,
)
except Exception as ex:
raise HookExecutionError(hp, ex) from ex
else:
stdout_list: List[str] = []
stderr_list: List[str] = []
handle_process_output(cmd, stdout_list.append, stderr_list.append,
finalize_process)
stdout = ''.join(stdout_list)
stderr = ''.join(stderr_list)
if cmd.returncode != 0:
stdout = force_text(stdout, defenc)
stderr = force_text(stderr, defenc)
raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
def _get_fetch_info_from_stderr(self, proc, progress):
# skip first line as it is some remote info we are not interested in
output = IterableList('name')
# lines which are no progress are fetch info lines
# this also waits for the command to finish
# Skip some progress lines that don't provide relevant information
fetch_info_lines = list()
# Basically we want all fetch info lines which appear to be in regular form, and thus have a
# command character. Everything else we ignore,
cmds = set(PushInfo._flag_map.keys()) & set(FetchInfo._flag_map.keys())
progress_handler = progress.new_message_handler()
def my_progress_handler(line):
for pline in progress_handler(line):
if line.startswith('fatal:') or line.startswith('error:'):
raise GitCommandError(("Error when fetching: %s" % line,), 2)
# END handle special messages
for cmd in cmds:
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
fetch_info_lines.append(line)
continue
# end find command code
# end for each comand code we know
# end for each line progress didn't handle
# end
# We are only interested in stderr here ...
handle_process_output(proc, None, my_progress_handler, finalize_process)
# read head information
fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb')
fetch_head_info = [l.decode(defenc) for l in fp.readlines()]
fp.close()
# NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it.
assert len(fetch_info_lines) >= len(fetch_head_info), "len(%s) <= len(%s)" % (fetch_head_info,
fetch_info_lines)
output.extend(FetchInfo._from_line(self.repo, err_line, fetch_line)
for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info))
return output
def _get_push_info(self, proc, progress):
# read progress information from stderr
# we hope stdout can hold all the data, it should ...
# read the lines manually as it will use carriage returns between the messages
# to override the previous one. This is why we read the bytes manually
# TODO: poll() on file descriptors to know what to read next, process streams concurrently
progress_handler = progress.new_message_handler()
output = IterableList('name')
def stdout_handler(line):
try:
output.append(PushInfo._from_line(self, line))
except ValueError:
# if an error happens, additional info is given which we cannot parse
pass
# END exception handling
# END for each line
handle_process_output(proc, stdout_handler, progress_handler, finalize_process)
return output
def test_handle_process_output(self):
from git.cmd import handle_process_output
line_count = 5002
count = [None, 0, 0]
def counter_stdout(line):
count[1] += 1
def counter_stderr(line):
count[2] += 1
proc = subprocess.Popen([sys.executable, fixture_path('cat_file.py'), str(fixture_path('issue-301_stderr'))],
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
handle_process_output(proc, counter_stdout, counter_stderr, lambda proc: proc.wait())
assert count[1] == line_count
assert count[2] == line_count
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
if progress is not None:
progress = to_progress_instance(progress)
odbt = kwargs.pop('odbt', odb_default_type)
proc = git.clone(url,
path,
with_extended_output=True,
as_process=True,
v=True,
**add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, progress.new_message_handler(),
finalize_process)
else:
(stdout, stderr
) = proc.communicate() # FIXME: Will block of outputs are big!
log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''),
stdout)
finalize_process(proc, stderr=stderr)
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not os.path.isabs(path) and git.working_dir:
path = join(git._working_dir, path)
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
repo = cls(path, odbt=odbt)
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo
def run_commit_hook(name: str, index: 'IndexFile', *args: str) -> None:
"""Run the commit hook of the given name. Silently ignores hooks that do not exist.
:param name: name of hook, like 'pre-commit'
:param index: IndexFile instance
:param args: arguments passed to hook file
:raises HookExecutionError: """
hp = hook_path(name, index.repo.git_dir)
if not os.access(hp, os.X_OK):
return None
env = os.environ.copy()
env['GIT_INDEX_FILE'] = safe_decode(str(index.path))
env['GIT_EDITOR'] = ':'
try:
cmd = subprocess.Popen(
[hp] + list(args),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=index.repo.working_dir,
close_fds=is_posix,
creationflags=PROC_CREATIONFLAGS,
)
except Exception as ex:
raise HookExecutionError(hp, ex) from ex
else:
stdout_list = [] # type: List[str]
stderr_list = [] # type: List[str]
handle_process_output(cmd, stdout_list.append, stderr_list.append,
finalize_process)
stdout = ''.join(stdout_list)
stderr = ''.join(stderr_list)
if cmd.returncode != 0:
stdout = force_text(stdout, defenc)
stderr = force_text(stderr, defenc)
raise HookExecutionError(hp, cmd.returncode, stderr, stdout)
def _get_fetch_info_from_stderr(self, proc: 'Git.AutoInterrupt',
progress: Union[Callable[..., Any], RemoteProgress, None],
kill_after_timeout: Union[None, float] = None,
) -> IterableList['FetchInfo']:
progress = to_progress_instance(progress)
# skip first line as it is some remote info we are not interested in
output: IterableList['FetchInfo'] = IterableList('name')
# lines which are no progress are fetch info lines
# this also waits for the command to finish
# Skip some progress lines that don't provide relevant information
fetch_info_lines = []
# Basically we want all fetch info lines which appear to be in regular form, and thus have a
# command character. Everything else we ignore,
cmds = set(FetchInfo._flag_map.keys())
progress_handler = progress.new_message_handler()
handle_process_output(proc, None, progress_handler, finalizer=None, decode_streams=False,
kill_after_timeout=kill_after_timeout)
stderr_text = progress.error_lines and '\n'.join(progress.error_lines) or ''
proc.wait(stderr=stderr_text)
if stderr_text:
log.warning("Error lines received while fetching: %s", stderr_text)
for line in progress.other_lines:
line = force_text(line)
for cmd in cmds:
if len(line) > 1 and line[0] == ' ' and line[1] == cmd:
fetch_info_lines.append(line)
continue
# read head information
fetch_head = SymbolicReference(self.repo, "FETCH_HEAD")
with open(fetch_head.abspath, 'rb') as fp:
fetch_head_info = [line.decode(defenc) for line in fp.readlines()]
l_fil = len(fetch_info_lines)
l_fhi = len(fetch_head_info)
if l_fil != l_fhi:
msg = "Fetch head lines do not match lines provided via progress information\n"
msg += "length of progress lines %i should be equal to lines in FETCH_HEAD file %i\n"
msg += "Will ignore extra progress lines or fetch head lines."
msg %= (l_fil, l_fhi)
log.debug(msg)
log.debug(b"info lines: " + str(fetch_info_lines).encode("UTF-8"))
log.debug(b"head info: " + str(fetch_head_info).encode("UTF-8"))
if l_fil < l_fhi:
fetch_head_info = fetch_head_info[:l_fil]
else:
fetch_info_lines = fetch_info_lines[:l_fhi]
# end truncate correct list
# end sanity check + sanitization
for err_line, fetch_line in zip(fetch_info_lines, fetch_head_info):
try:
output.append(FetchInfo._from_line(self.repo, err_line, fetch_line))
except ValueError as exc:
log.debug("Caught error while parsing line: %s", exc)
log.warning("Git informed while fetching: %s", err_line.strip())
return output
