def _run_legacy(
hook_type: str,
hook_dir: str,
args: Sequence[str],
) -> Tuple[int, bytes]:
if os.environ.get('PRE_COMMIT_RUNNING_LEGACY'):
raise SystemExit(
f"bug: pre-commit's script is installed in migration mode\n"
f'run `pre-commit install -f --hook-type {hook_type}` to fix '
f'this\n\n'
f'Please report this bug at '
f'https://github.com/pre-commit/pre-commit/issues', )
if hook_type == 'pre-push':
stdin = sys.stdin.buffer.read()
else:
stdin = b''
# not running in legacy mode
legacy_hook = os.path.join(hook_dir, f'{hook_type}.legacy')
if not os.access(legacy_hook, os.X_OK):
return 0, stdin
with envcontext((('PRE_COMMIT_RUNNING_LEGACY', '1'), )):
cmd = normalize_cmd((legacy_hook, *args))
return subprocess.run(cmd, input=stdin).returncode, stdin
def cmd_output_b(*cmd, **kwargs):
retcode = kwargs.pop('retcode', 0)
popen_kwargs = {
'stdin': subprocess.PIPE,
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
}
# py2/py3 on windows are more strict about the types here
cmd = tuple(five.n(arg) for arg in cmd)
kwargs['env'] = {
five.n(key): five.n(value)
for key, value in kwargs.pop('env', {}).items()
} or None
popen_kwargs.update(kwargs)
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
returncode, stdout_b, stderr_b = e.to_output()
else:
proc = subprocess.Popen(cmd, **popen_kwargs)
stdout_b, stderr_b = proc.communicate()
returncode = proc.returncode
if retcode is not None and retcode != returncode:
raise CalledProcessError(
returncode,
cmd,
retcode,
output=(stdout_b, stderr_b),
)
return returncode, stdout_b, stderr_b
def cmd_output_p(
*cmd: str,
retcode: Optional[int] = 0,
**kwargs: Any,
) -> Tuple[int, bytes, Optional[bytes]]:
assert retcode is None
assert kwargs['stderr'] == subprocess.STDOUT, kwargs['stderr']
_setdefault_kwargs(kwargs)
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()
with open(os.devnull) as devnull, Pty() as pty:
assert pty.r is not None
kwargs.update({'stdin': devnull, 'stdout': pty.w, 'stderr': pty.w})
proc = subprocess.Popen(cmd, **kwargs)
pty.close_w()
buf = b''
while True:
try:
bts = os.read(pty.r, 4096)
except OSError as e:
if e.errno == errno.EIO:
bts = b''
else:
raise
else:
buf += bts
if not bts:
break
return proc.wait(), buf, None
def cmd_output_p(*cmd, **kwargs):
assert kwargs.pop('retcode') is None
assert kwargs['stderr'] == subprocess.STDOUT, kwargs['stderr']
cmd, kwargs = _cmd_kwargs(*cmd, **kwargs)
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()
with open(os.devnull) as devnull, Pty() as pty:
kwargs.update({'stdin': devnull, 'stdout': pty.w, 'stderr': pty.w})
proc = subprocess.Popen(cmd, **kwargs)
pty.close_w()
buf = b''
while True:
try:
bts = os.read(pty.r, 4096)
except OSError as e:
if e.errno == errno.EIO:
bts = b''
else:
raise
else:
buf += bts
if not bts:
break
return proc.wait(), buf, None
def xargs(cmd, varargs, **kwargs):
"""A simplified implementation of xargs.
color: Make a pty if on a platform that supports it
target_concurrency: Target number of partitions to run concurrently
"""
color = kwargs.pop('color', False)
target_concurrency = kwargs.pop('target_concurrency', 1)
max_length = kwargs.pop('_max_length', _get_platform_max_length())
cmd_fn = cmd_output_p if color else cmd_output_b
retcode = 0
stdout = b''
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()[:2]
partitions = partition(cmd, varargs, target_concurrency, max_length)
def run_cmd_partition(run_cmd):
return cmd_fn(*run_cmd,
retcode=None,
stderr=subprocess.STDOUT,
**kwargs)
threads = min(len(partitions), target_concurrency)
with _thread_mapper(threads) as thread_map:
results = thread_map(run_cmd_partition, partitions)
for proc_retcode, proc_out, _ in results:
retcode = max(retcode, proc_retcode)
stdout += proc_out
return retcode, stdout
def cmd_output(*cmd, **kwargs):
retcode = kwargs.pop('retcode', 0)
encoding = kwargs.pop('encoding', 'UTF-8')
__popen = kwargs.pop('__popen', subprocess.Popen)
popen_kwargs = {
'stdin': subprocess.PIPE,
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
}
# py2/py3 on windows are more strict about the types here
cmd = tuple(five.n(arg) for arg in cmd)
kwargs['env'] = dict(
(five.n(key), five.n(value))
for key, value in kwargs.pop('env', {}).items()
) or None
cmd = parse_shebang.normalize_cmd(cmd)
popen_kwargs.update(kwargs)
proc = __popen(cmd, **popen_kwargs)
stdout, stderr = proc.communicate()
if encoding is not None and stdout is not None:
stdout = stdout.decode(encoding)
if encoding is not None and stderr is not None:
stderr = stderr.decode(encoding)
returncode = proc.returncode
if retcode is not None and retcode != returncode:
raise CalledProcessError(
returncode, cmd, retcode, output=(stdout, stderr),
)
return proc.returncode, stdout, stderr
def test_xargs_propagate_kwargs_to_cmd():
env = {'PRE_COMMIT_TEST_VAR': 'Pre commit is awesome'}
cmd = ('bash', '-c', 'echo $PRE_COMMIT_TEST_VAR', '--')
cmd = parse_shebang.normalize_cmd(cmd)
ret, stdout = xargs.xargs(cmd, ('1', ), env=env)
assert ret == 0
assert b'Pre commit is awesome' in stdout
def test_xargs_propagate_kwargs_to_cmd():
env = {'PRE_COMMIT_TEST_VAR': 'Pre commit is awesome'}
cmd = ('bash', '-c', 'echo $PRE_COMMIT_TEST_VAR', '--')
cmd = parse_shebang.normalize_cmd(cmd)
ret, stdout, _ = xargs.xargs(cmd, ('1',), env=env)
assert ret == 0
assert b'Pre commit is awesome' in stdout
def xargs(
cmd: Tuple[str, ...],
varargs: Sequence[str],
*,
color: bool = False,
target_concurrency: int = 1,
_max_length: int = _get_platform_max_length(),
**kwargs: Any,
) -> Tuple[int, bytes]:
"""A simplified implementation of xargs.
color: Make a pty if on a platform that supports it
target_concurrency: Target number of partitions to run concurrently
"""
cmd_fn = cmd_output_p if color else cmd_output_b
retcode = 0
stdout = b''
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()[:2]
# on windows, batch files have a separate length limit than windows itself
if (sys.platform == 'win32' and cmd[0].lower().endswith(
('.bat', '.cmd'))): # pragma: win32 cover
# this is implementation details but the command gets translated into
# full/path/to/cmd.exe /c *cmd
cmd_exe = parse_shebang.find_executable('cmd.exe')
# 1024 is additionally subtracted to give headroom for further
# expansion inside the batch file
_max_length = 8192 - len(cmd_exe) - len(' /c ') - 1024
partitions = partition(cmd, varargs, target_concurrency, _max_length)
def run_cmd_partition(
run_cmd: Tuple[str, ...], ) -> Tuple[int, bytes, Optional[bytes]]:
return cmd_fn(
*run_cmd,
retcode=None,
stderr=subprocess.STDOUT,
**kwargs,
)
threads = min(len(partitions), target_concurrency)
with _thread_mapper(threads) as thread_map:
results = thread_map(run_cmd_partition, partitions)
for proc_retcode, proc_out, _ in results:
retcode = max(retcode, proc_retcode)
stdout += proc_out
return retcode, stdout
def xargs(cmd, varargs, **kwargs):
"""A simplified implementation of xargs.
color: Make a pty if on a platform that supports it
negate: Make nonzero successful and zero a failure
target_concurrency: Target number of partitions to run concurrently
"""
color = kwargs.pop('color', False)
negate = kwargs.pop('negate', False)
target_concurrency = kwargs.pop('target_concurrency', 1)
max_length = kwargs.pop('_max_length', _get_platform_max_length())
cmd_fn = cmd_output_p if color else cmd_output_b
retcode = 0
stdout = b''
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()[:2]
partitions = partition(cmd, varargs, target_concurrency, max_length)
def run_cmd_partition(run_cmd):
return cmd_fn(*run_cmd,
retcode=None,
stderr=subprocess.STDOUT,
**kwargs)
threads = min(len(partitions), target_concurrency)
with _thread_mapper(threads) as thread_map:
results = thread_map(run_cmd_partition, partitions)
for proc_retcode, proc_out, _ in results:
# This is *slightly* too clever so I'll explain it.
# First the xor boolean table:
# T | F |
# +-------+
# T | F | T |
# --+-------+
# F | T | F |
# --+-------+
# When negate is True, it has the effect of flipping the return
# code. Otherwise, the returncode is unchanged.
retcode |= bool(proc_retcode) ^ negate
stdout += proc_out
return retcode, stdout
def test_xargs_concurrency():
bash_cmd = parse_shebang.normalize_cmd(('bash', '-c'))
print_pid = ('sleep 0.5 && echo $$',)
start = time.time()
ret, stdout = xargs.xargs(
bash_cmd, print_pid * 5,
target_concurrency=5,
_max_length=len(' '.join(bash_cmd + print_pid)) + 1,
)
elapsed = time.time() - start
assert ret == 0
pids = stdout.splitlines()
assert len(pids) == 5
# It would take 0.5*5=2.5 seconds to run all of these in serial, so if it
# takes less, they must have run concurrently.
assert elapsed < 2.5
def test_xargs_concurrency():
bash_cmd = parse_shebang.normalize_cmd(('bash', '-c'))
print_pid = ('sleep 0.5 && echo $$',)
start = time.time()
ret, stdout, _ = xargs.xargs(
bash_cmd, print_pid * 5,
target_concurrency=5,
_max_length=len(' '.join(bash_cmd + print_pid)) + 1,
)
elapsed = time.time() - start
assert ret == 0
pids = stdout.splitlines()
assert len(pids) == 5
# It would take 0.5*5=2.5 seconds ot run all of these in serial, so if it
# takes less, they must have run concurrently.
assert elapsed < 2.5
def cmd_output_b(*cmd, **kwargs):
retcode = kwargs.pop('retcode', 0)
cmd, kwargs = _cmd_kwargs(*cmd, **kwargs)
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
returncode, stdout_b, stderr_b = e.to_output()
else:
proc = subprocess.Popen(cmd, **kwargs)
stdout_b, stderr_b = proc.communicate()
returncode = proc.returncode
if retcode is not None and retcode != returncode:
raise CalledProcessError(returncode, cmd, retcode, stdout_b, stderr_b)
return returncode, stdout_b, stderr_b
def xargs(
cmd: Tuple[str, ...],
varargs: Sequence[str],
*,
color: bool = False,
target_concurrency: int = 1,
_max_length: int = _get_platform_max_length(),
**kwargs: Any,
) -> Tuple[int, bytes]:
"""A simplified implementation of xargs.
color: Make a pty if on a platform that supports it
target_concurrency: Target number of partitions to run concurrently
"""
cmd_fn = cmd_output_p if color else cmd_output_b
retcode = 0
stdout = b""
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()[:2]
partitions = partition(cmd, varargs, target_concurrency, _max_length)
def run_cmd_partition(
run_cmd: Tuple[str, ...], ) -> Tuple[int, bytes, Optional[bytes]]:
return cmd_fn(
*run_cmd,
retcode=None,
stderr=subprocess.STDOUT,
**kwargs,
)
threads = min(len(partitions), target_concurrency)
with _thread_mapper(threads) as thread_map:
results = thread_map(run_cmd_partition, partitions)
for proc_retcode, proc_out, _ in results:
retcode = max(retcode, proc_retcode)
stdout += proc_out
return retcode, stdout
def xargs(cmd, varargs, **kwargs):
"""A simplified implementation of xargs.
negate: Make nonzero successful and zero a failure
target_concurrency: Target number of partitions to run concurrently
"""
negate = kwargs.pop('negate', False)
target_concurrency = kwargs.pop('target_concurrency', 1)
retcode = 0
stdout = b''
stderr = b''
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
return e.to_output()
partitions = partition(cmd, varargs, target_concurrency, **kwargs)
def run_cmd_partition(run_cmd):
return cmd_output(*run_cmd, encoding=None, retcode=None)
threads = min(len(partitions), target_concurrency)
with _thread_mapper(threads) as thread_map:
results = thread_map(run_cmd_partition, partitions)
for proc_retcode, proc_out, proc_err in results:
# This is *slightly* too clever so I'll explain it.
# First the xor boolean table:
# T | F |
# +-------+
# T | F | T |
# --+-------+
# F | T | F |
# --+-------+
# When negate is True, it has the effect of flipping the return
# code. Otherwise, the returncode is unchanged.
retcode |= bool(proc_retcode) ^ negate
stdout += proc_out
stderr += proc_err
return retcode, stdout, stderr
def cmd_output_b(
*cmd: str,
retcode: Optional[int] = 0,
**kwargs: Any,
) -> Tuple[int, bytes, Optional[bytes]]:
_setdefault_kwargs(kwargs)
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
returncode, stdout_b, stderr_b = e.to_output()
else:
proc = subprocess.Popen(cmd, **kwargs)
stdout_b, stderr_b = proc.communicate()
returncode = proc.returncode
if retcode is not None and retcode != returncode:
raise CalledProcessError(returncode, cmd, retcode, stdout_b, stderr_b)
return returncode, stdout_b, stderr_b
def cmd_output(*cmd, **kwargs):
retcode = kwargs.pop('retcode', 0)
encoding = kwargs.pop('encoding', 'UTF-8')
__popen = kwargs.pop('__popen', subprocess.Popen)
popen_kwargs = {
'stdin': subprocess.PIPE,
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
}
# py2/py3 on windows are more strict about the types here
cmd = tuple(five.n(arg) for arg in cmd)
kwargs['env'] = dict(
(five.n(key), five.n(value))
for key, value in kwargs.pop('env', {}).items()) or None
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
returncode, stdout, stderr = (-1, e.args[0].encode('UTF-8'), b'')
else:
popen_kwargs.update(kwargs)
proc = __popen(cmd, **popen_kwargs)
stdout, stderr = proc.communicate()
if encoding is not None and stdout is not None:
stdout = stdout.decode(encoding)
if encoding is not None and stderr is not None:
stderr = stderr.decode(encoding)
returncode = proc.returncode
if retcode is not None and retcode != returncode:
raise CalledProcessError(
returncode,
cmd,
retcode,
output=(stdout, stderr),
)
return returncode, stdout, stderr
def cmd_output(*cmd, **kwargs):
retcode = kwargs.pop('retcode', 0)
encoding = kwargs.pop('encoding', 'UTF-8')
popen_kwargs = {
'stdin': subprocess.PIPE,
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
}
# py2/py3 on windows are more strict about the types here
cmd = tuple(five.n(arg) for arg in cmd)
kwargs['env'] = {
five.n(key): five.n(value)
for key, value in kwargs.pop('env', {}).items()
} or None
try:
cmd = parse_shebang.normalize_cmd(cmd)
except parse_shebang.ExecutableNotFoundError as e:
returncode, stdout, stderr = e.to_output()
else:
popen_kwargs.update(kwargs)
proc = subprocess.Popen(cmd, **popen_kwargs)
stdout, stderr = proc.communicate()
returncode = proc.returncode
if encoding is not None and stdout is not None:
stdout = stdout.decode(encoding)
if encoding is not None and stderr is not None:
stderr = stderr.decode(encoding)
if retcode is not None and retcode != returncode:
raise CalledProcessError(
returncode, cmd, retcode, output=(stdout, stderr),
)
return returncode, stdout, stderr
def test_normalize_cmd_shebang(in_tmpdir):
echo = _echo_exe().replace(os.sep, '/')
path = write_executable(echo)
assert parse_shebang.normalize_cmd((path,)) == (echo, path)
def test_normalize_cmd_trivial():
cmd = (_echo_exe(), 'hi')
assert parse_shebang.normalize_cmd(cmd) == cmd
def test_normalize_cmd_PATH():
cmd = ('echo', '--version')
expected = (_echo_exe(), '--version')
assert parse_shebang.normalize_cmd(cmd) == expected
def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir):
python = distutils.spawn.find_executable('python')
path = write_executable('/usr/bin/env python')
with bin_on_path():
ret = parse_shebang.normalize_cmd(('run', ))
assert ret == (python, os.path.abspath(path))
def test_normalize_cmd_trivial():
cmd = (distutils.spawn.find_executable('echo'), 'hi')
assert parse_shebang.normalize_cmd(cmd) == cmd
def test_normalize_cmd_shebang(in_tmpdir):
python = distutils.spawn.find_executable('python')
path = write_executable(python.replace(os.sep, '/'))
assert parse_shebang.normalize_cmd((path, )) == (python, path)
def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir):
python = distutils.spawn.find_executable('python')
path = write_executable(python.replace(os.sep, '/'))
with bin_on_path():
ret = parse_shebang.normalize_cmd(('run', ))
assert ret == (python, os.path.abspath(path))
)
def test_argument_too_long():
with pytest.raises(xargs.ArgumentTooLongError):
xargs.partition(('a' * 5,), ('a' * 5,), 1, _max_length=10)
def test_xargs_smoke():
ret, out, err = xargs.xargs(('echo',), ('hello', 'world'))
assert ret == 0
assert out.replace(b'\r\n', b'\n') == b'hello world\n'
assert err == b''
exit_cmd = parse_shebang.normalize_cmd(('bash', '-c', 'exit $1', '--'))
# Abuse max_length to control the exit code
max_length = len(' '.join(exit_cmd)) + 3
def test_xargs_negate():
ret, _, _ = xargs.xargs(
exit_cmd, ('1',), negate=True, _max_length=max_length,
)
assert ret == 0
ret, _, _ = xargs.xargs(
exit_cmd, ('1', '0'), negate=True, _max_length=max_length,
)
assert ret == 1
def test_normalize_cmd_PATH():
cmd = ('python', '--version')
expected = (distutils.spawn.find_executable('python'), '--version')
assert parse_shebang.normalize_cmd(cmd) == expected
('foo', ) + ('A', ) * 2,
)
def test_argument_too_long():
with pytest.raises(xargs.ArgumentTooLongError):
xargs.partition(('a' * 5, ), ('a' * 5, ), 1, _max_length=10)
def test_xargs_smoke():
ret, out = xargs.xargs(('echo', ), ('hello', 'world'))
assert ret == 0
assert out.replace(b'\r\n', b'\n') == b'hello world\n'
exit_cmd = parse_shebang.normalize_cmd(('bash', '-c', 'exit $1', '--'))
# Abuse max_length to control the exit code
max_length = len(' '.join(exit_cmd)) + 3
def test_xargs_negate():
ret, _ = xargs.xargs(
exit_cmd,
('1', ),
negate=True,
_max_length=max_length,
)
assert ret == 0
ret, _ = xargs.xargs(
exit_cmd,
def test_normalize_cmd_shebang(in_tmpdir):
echo = distutils.spawn.find_executable('echo').replace(os.sep, '/')
path = write_executable(echo)
assert parse_shebang.normalize_cmd((path, )) == (echo, path)
def test_normalize_cmd_trivial():
cmd = (distutils.spawn.find_executable('echo'), 'hi')
assert parse_shebang.normalize_cmd(cmd) == cmd
def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir):
python = distutils.spawn.find_executable('python')
path = write_executable('/usr/bin/env python')
with bin_on_path():
ret = parse_shebang.normalize_cmd(('run',))
assert ret == (python, os.path.abspath(path))
def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir):
python = distutils.spawn.find_executable('python')
path = write_executable(python.replace(os.sep, '/'))
with bin_on_path():
ret = parse_shebang.normalize_cmd(('run',))
assert ret == (python, os.path.abspath(path))
def test_normalize_cmd_shebang(in_tmpdir):
python = distutils.spawn.find_executable('python')
path = write_executable(python.replace(os.sep, '/'))
assert parse_shebang.normalize_cmd((path,)) == (python, path)
def test_normalize_cmd_PATH_shebang_full_path(in_tmpdir):
echo = _echo_exe().replace(os.sep, '/')
path = write_executable(echo)
with bin_on_path():
ret = parse_shebang.normalize_cmd(('run',))
assert ret == (echo, os.path.abspath(path))
def test_normalize_cmd_PATH_shebang_PATH(in_tmpdir):
echo = _echo_exe()
path = write_executable('/usr/bin/env echo')
with bin_on_path():
ret = parse_shebang.normalize_cmd(('run',))
assert ret == (echo, os.path.abspath(path))
def test_normalize_cmd_PATH():
cmd = ('python', '--version')
expected = (distutils.spawn.find_executable('python'), '--version')
assert parse_shebang.normalize_cmd(cmd) == expected
