def pump_stream(cmdline, name, stream, is_decode, handler):
try:
for line in stream:
if handler:
if is_decode:
line = line.decode(defenc)
handler(line)
except Exception as ex:
log.error("Pumping %r of cmd(%s) failed due to: %r", name,
remove_password_if_present(cmdline), ex)
raise CommandError(['<%s-pump>' % name] +
remove_password_if_present(cmdline), ex) from ex
finally:
stream.close()
def wait(
self,
stderr=b''
): # TODO: Bad choice to mimic `proc.wait()` but with different args.
"""Wait for the process and return its status code.
:param stderr: Previously read value of stderr, in case stderr is already closed.
:warn: may deadlock if output or error pipes are used and not handled separately.
:raise GitCommandError: if the return status is not 0"""
if stderr is None:
stderr = b''
stderr = force_bytes(data=stderr, encoding='utf-8')
status = self.proc.wait()
def read_all_from_possibly_closed_stream(stream):
try:
return stderr + force_bytes(stream.read())
except ValueError:
return stderr or b''
if status != 0:
errstr = read_all_from_possibly_closed_stream(self.proc.stderr)
log.debug('AutoInterrupt wait stderr: %r' % (errstr, ))
raise GitCommandError(remove_password_if_present(self.args),
status, errstr)
# END status handling
return status
def pump_stream(cmdline: str, name: str, stream: Union[BinaryIO, TextIO], is_decode: bool,
handler: Union[None, Callable[[Union[bytes, str]], None]]) -> None:
try:
for line in stream:
if handler:
if is_decode:
assert isinstance(line, bytes)
line_str = line.decode(defenc)
handler(line_str)
else:
handler(line)
except Exception as ex:
log.error("Pumping %r of cmd(%s) failed due to: %r", name, remove_password_if_present(cmdline), ex)
raise CommandError(['<%s-pump>' % name] + remove_password_if_present(cmdline), ex) from ex
finally:
stream.close()
def test_remove_password_from_command_line(self):
password = "******"
url_with_pass = "******".format(
password)
url_without_pass = "******"
cmd_1 = ["git", "clone", "-v", url_with_pass]
cmd_2 = ["git", "clone", "-v", url_without_pass]
cmd_3 = ["no", "url", "in", "this", "one"]
redacted_cmd_1 = remove_password_if_present(cmd_1)
assert password not in " ".join(redacted_cmd_1)
# Check that we use a copy
assert cmd_1 is not redacted_cmd_1
assert password in " ".join(cmd_1)
assert cmd_2 == remove_password_if_present(cmd_2)
assert cmd_3 == remove_password_if_present(cmd_3)
def _clone(cls, git: 'Git', url: PathLike, path: PathLike, odb_default_type: Type[GitCmdObjectDB],
progress: Optional[Callable], multi_options: Optional[List[str]] = None, **kwargs: Any
) -> 'Repo':
odbt = kwargs.pop('odbt', odb_default_type)
# when pathlib.Path or other classbased path is passed
if not isinstance(path, str):
path = str(path)
## A bug win cygwin's Git, when `--bare` or `--separate-git-dir`
# it prepends the cwd or(?) the `url` into the `path, so::
# git clone --bare /cygwin/d/foo.git C:\\Work
# becomes::
# git clone --bare /cygwin/d/foo.git /cygwin/d/C:\\Work
#
clone_path = (Git.polish_url(path)
if Git.is_cygwin() and 'bare' in kwargs
else path)
sep_dir = kwargs.get('separate_git_dir')
if sep_dir:
kwargs['separate_git_dir'] = Git.polish_url(sep_dir)
multi = None
if multi_options:
multi = ' '.join(multi_options).split(' ')
proc = git.clone(multi, Git.polish_url(url), clone_path, with_extended_output=True, as_process=True,
v=True, universal_newlines=True, **add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, to_progress_instance(progress).new_message_handler(),
finalize_process, decode_streams=False)
else:
(stdout, stderr) = proc.communicate()
cmdline = getattr(proc, 'args', '')
cmdline = remove_password_if_present(cmdline)
log.debug("Cmd(%s)'s unused stdout: %s", cmdline, stdout)
finalize_process(proc, stderr=stderr)
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not osp.isabs(path):
path = osp.join(git._working_dir, path) if git._working_dir is not None else path
repo = cls(path, odbt=odbt)
# retain env values that were passed to _clone()
repo.git.update_environment(**git.environment())
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo
def execute(self,
command,
istream=None,
with_extended_output=False,
with_exceptions=True,
as_process=False,
output_stream=None,
stdout_as_string=True,
kill_after_timeout=None,
with_stdout=True,
universal_newlines=False,
shell=None,
env=None,
max_chunk_size=io.DEFAULT_BUFFER_SIZE,
**subprocess_kwargs):
"""Handles executing the command on the shell and consumes and returns
the returned information (stdout)
:param command:
The command argument list to execute.
It should be a string, or a sequence of program arguments. The
program to execute is the first item in the args sequence or string.
:param istream:
Standard input filehandle passed to subprocess.Popen.
:param with_extended_output:
Whether to return a (status, stdout, stderr) tuple.
:param with_exceptions:
Whether to raise an exception when git returns a non-zero status.
:param as_process:
Whether to return the created process instance directly from which
streams can be read on demand. This will render with_extended_output and
with_exceptions ineffective - the caller will have
to deal with the details himself.
It is important to note that the process will be placed into an AutoInterrupt
wrapper that will interrupt the process once it goes out of scope. If you
use the command in iterators, you should pass the whole process instance
instead of a single stream.
:param output_stream:
If set to a file-like object, data produced by the git command will be
output to the given stream directly.
This feature only has any effect if as_process is False. Processes will
always be created with a pipe due to issues with subprocess.
This merely is a workaround as data will be copied from the
output pipe to the given output stream directly.
Judging from the implementation, you shouldn't use this flag !
:param stdout_as_string:
if False, the commands standard output will be bytes. Otherwise, it will be
decoded into a string using the default encoding (usually utf-8).
The latter can fail, if the output contains binary data.
:param env:
A dictionary of environment variables to be passed to `subprocess.Popen`.
:param max_chunk_size:
Maximum number of bytes in one chunk of data passed to the output_stream in
one invocation of write() method. If the given number is not positive then
the default value is used.
:param subprocess_kwargs:
Keyword arguments to be passed to subprocess.Popen. Please note that
some of the valid kwargs are already set by this method, the ones you
specify may not be the same ones.
:param with_stdout: If True, default True, we open stdout on the created process
:param universal_newlines:
if True, pipes will be opened as text, and lines are split at
all known line endings.
:param shell:
Whether to invoke commands through a shell (see `Popen(..., shell=True)`).
It overrides :attr:`USE_SHELL` if it is not `None`.
:param kill_after_timeout:
To specify a timeout in seconds for the git command, after which the process
should be killed. This will have no effect if as_process is set to True. It is
set to None by default and will let the process run until the timeout is
explicitly specified. This feature is not supported on Windows. It's also worth
noting that kill_after_timeout uses SIGKILL, which can have negative side
effects on a repository. For example, stale locks in case of git gc could
render the repository incapable of accepting changes until the lock is manually
removed.
:return:
* str(output) if extended_output = False (Default)
* tuple(int(status), str(stdout), str(stderr)) if extended_output = True
if output_stream is True, the stdout value will be your output stream:
* output_stream if extended_output = False
* tuple(int(status), output_stream, str(stderr)) if extended_output = True
Note git is executed with LC_MESSAGES="C" to ensure consistent
output regardless of system language.
:raise GitCommandError:
:note:
If you add additional keyword arguments to the signature of this method,
you must update the execute_kwargs tuple housed in this module."""
# Remove password for the command if present
redacted_command = remove_password_if_present(command)
if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full'
or as_process):
log.info(' '.join(redacted_command))
# Allow the user to have the command executed in their working dir.
cwd = self._working_dir or os.getcwd()
# Start the process
inline_env = env
env = os.environ.copy()
# Attempt to force all output to plain ascii english, which is what some parsing code
# may expect.
# According to stackoverflow (http://goo.gl/l74GC8), we are setting LANGUAGE as well
# just to be sure.
env["LANGUAGE"] = "C"
env["LC_ALL"] = "C"
env.update(self._environment)
if inline_env is not None:
env.update(inline_env)
if is_win:
cmd_not_found_exception = OSError
if kill_after_timeout:
raise GitCommandError(
redacted_command,
'"kill_after_timeout" feature is not supported on Windows.'
)
else:
if sys.version_info[0] > 2:
cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable
else:
cmd_not_found_exception = OSError
# end handle
stdout_sink = (PIPE if with_stdout else getattr(
subprocess, 'DEVNULL', None) or open(os.devnull, 'wb'))
istream_ok = "None"
if istream:
istream_ok = "<valid stream>"
log.debug(
"Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)",
redacted_command, cwd, universal_newlines, shell, istream_ok)
try:
proc = Popen(
command,
env=env,
cwd=cwd,
bufsize=-1,
stdin=istream,
stderr=PIPE,
stdout=stdout_sink,
shell=shell is not None and shell or self.USE_SHELL,
close_fds=is_posix, # unsupported on windows
universal_newlines=universal_newlines,
creationflags=PROC_CREATIONFLAGS,
**subprocess_kwargs)
except cmd_not_found_exception as err:
raise GitCommandNotFound(redacted_command, err) from err
if as_process:
return self.AutoInterrupt(proc, command)
def _kill_process(pid):
""" Callback method to kill a process. """
p = Popen(['ps', '--ppid', str(pid)],
stdout=PIPE,
creationflags=PROC_CREATIONFLAGS)
child_pids = []
for line in p.stdout:
if len(line.split()) > 0:
local_pid = (line.split())[0]
if local_pid.isdigit():
child_pids.append(int(local_pid))
try:
# Windows does not have SIGKILL, so use SIGTERM instead
sig = getattr(signal, 'SIGKILL', signal.SIGTERM)
os.kill(pid, sig)
for child_pid in child_pids:
try:
os.kill(child_pid, sig)
except OSError:
pass
kill_check.set(
) # tell the main routine that the process was killed
except OSError:
# It is possible that the process gets completed in the duration after timeout
# happens and before we try to kill the process.
pass
return
# end
if kill_after_timeout:
kill_check = threading.Event()
watchdog = threading.Timer(kill_after_timeout,
_kill_process,
args=(proc.pid, ))
# Wait for the process to return
status = 0
stdout_value = b''
stderr_value = b''
newline = "\n" if universal_newlines else b"\n"
try:
if output_stream is None:
if kill_after_timeout:
watchdog.start()
stdout_value, stderr_value = proc.communicate()
if kill_after_timeout:
watchdog.cancel()
if kill_check.isSet():
stderr_value = (
'Timeout: the command "%s" did not complete in %d '
'secs.' %
(" ".join(redacted_command), kill_after_timeout))
if not universal_newlines:
stderr_value = stderr_value.encode(defenc)
# strip trailing "\n"
if stdout_value.endswith(newline):
stdout_value = stdout_value[:-1]
if stderr_value.endswith(newline):
stderr_value = stderr_value[:-1]
status = proc.returncode
else:
max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE
stream_copy(proc.stdout, output_stream, max_chunk_size)
stdout_value = proc.stdout.read()
stderr_value = proc.stderr.read()
# strip trailing "\n"
if stderr_value.endswith(newline):
stderr_value = stderr_value[:-1]
status = proc.wait()
# END stdout handling
finally:
proc.stdout.close()
proc.stderr.close()
if self.GIT_PYTHON_TRACE == 'full':
cmdstr = " ".join(redacted_command)
def as_text(stdout_value):
return not output_stream and safe_decode(
stdout_value) or '<OUTPUT_STREAM>'
# end
if stderr_value:
log.info("%s -> %d; stdout: '%s'; stderr: '%s'", cmdstr,
status, as_text(stdout_value),
safe_decode(stderr_value))
elif stdout_value:
log.info("%s -> %d; stdout: '%s'", cmdstr, status,
as_text(stdout_value))
else:
log.info("%s -> %d", cmdstr, status)
# END handle debug printing
if with_exceptions and status != 0:
raise GitCommandError(redacted_command, status, stderr_value,
stdout_value)
if isinstance(
stdout_value,
bytes) and stdout_as_string: # could also be output_stream
stdout_value = safe_decode(stdout_value)
# Allow access to the command's status code
if with_extended_output:
return (status, stdout_value, safe_decode(stderr_value))
else:
return stdout_value