def fake_getent_passwd(*popenargs, **kwargs):
if get_command(popenargs) == 'getent' and get_sub_command(popenargs) == 'passwd' and \
get_command_parameter(popenargs, 'passwd') == 'john':
return subprocess.CompletedProcess(
"fakerun",
0,
stdout='john:x:1000:1000:John Doe,,,:/home/john:/bin/bash\n')
else:
return subprocess.run(*popenargs, **kwargs)
def test_run_tarsnap_command_fsck_error(self, run):
"""Test error asking to run --fsck."""
run.return_value = subprocess.CompletedProcess(
self.TARSNAP_COMMAND, 1, b"Sequence no mismatch, please run --fsck to correct")
with self.assertRaises(tarsnap.TarsnapFsckException):
tarsnap.run_tarsnap_command(self.TARSNAP_COMMAND)
self.run_basic_checks(run)
def test_multipath_show_paths(self, m_run):
mp_out = MP_SEP.join([random_string() for x in range(0, 8)])
cp = subprocess.CompletedProcess(args=['foo'], returncode=0,
stdout=mp_out.encode('utf-8'),
stderr="")
m_run.return_value = cp
expected_result = [multipath.MPath(*mp_out.split(MP_SEP))._asdict()]
result = multipath.multipath_show_paths()
self.assertEqual(expected_result, result)
def test_run_tarsnap_command_error_case(self, run):
"""Test for a miscellaneous failure."""
run.return_value = subprocess.CompletedProcess(
self.TARSNAP_COMMAND, 1, b"Permission Denied: /etc/keyfile")
with self.assertRaises(tarsnap.TarsnapException):
tarsnap.run_tarsnap_command(self.TARSNAP_COMMAND)
self.run_basic_checks(run)
def test_get_current_commit(caplog, tmpdir):
with caplog.at_level(logging.WARNING):
target_dir = tmpdir.mkdir("target_dir")
get_current_commit(target_dir)
for record in caplog.records:
assert "Failed getting current git commit" in record
flexmock(subprocess).should_receive("run").and_return(
subprocess.CompletedProcess(1, 0, "some_hash"))
assert get_current_commit(".") == "some_hash"
def _check_output(self, container, cmd, split=True):
"""Check execution of a command in a container"""
if split:
cmd = cmd.split()
exit_code, stdout, stderr = container.execute(cmd)
logger.debug(f"Exit code {exit_code} from {cmd}")
subprocess.CompletedProcess(cmd, exit_code, stdout,
stderr).check_returncode()
return stdout
def test_run_codenarc_failure_code(default_jar_versions):
"""Test that run_codenarc raises an error if CodeNarc failed to run."""
with patch("subprocess.run") as subprocess_mock:
subprocess_mock.return_value = subprocess.CompletedProcess(
args="", returncode=1, stdout=MOCK_CODENARC_SUMMARY)
with pytest.raises(ValueError):
run_codenarc(args=parse_args(
args=[], default_jar_versions=default_jar_versions))
def subprocess_sbatch(*args, **kwargs):
job_id = 1234
out_file = f'slurm-{job_id}.out'
with open(out_file, 'w') as fid:
fid.write('Hello!!!\n')
stdout = f'Submitted batch job {job_id}'
return subprocess.CompletedProcess(*args,
returncode=1,
stdout=stdout.encode('utf-8'))
def test_get_directories_to_analyze(self, find_global_and_local_root, run) -> None:
original_directory = "/"
find_global_and_local_root.return_value = find_directories.FoundRoot(
Path("base")
)
arguments = mock_arguments(source_directories=["base"])
configuration = mock_configuration()
handler = commands.Reporting(
arguments, original_directory, configuration, AnalysisDirectory("base")
)
run.return_value = subprocess.CompletedProcess(
args=[],
returncode=0,
stdout="\n".join(
[
"external/a/.pyre_configuration.local",
"external/b/c/.pyre_configuration.local",
]
).encode("utf-8"),
)
with patch("builtins.open", mock_open(read_data="{}")):
self.assertEqual(handler._get_directories_to_analyze(), {"base"})
configuration.local_configuration = "a/b/.pyre_configuration.local"
handler = commands.Reporting(
arguments, original_directory, configuration, AnalysisDirectory("base")
)
self.assertEqual(handler._get_directories_to_analyze(), {"base"})
configuration.local_configuration = "a/b/.pyre_configuration.local"
arguments = mock_arguments(source_directories=[])
handler = commands.Reporting(
arguments,
original_directory,
configuration,
AnalysisDirectory("base", filter_paths={"a/b"}),
)
self.assertEqual(handler._get_directories_to_analyze(), {"a/b"})
# With no local configuration, no filter paths, and a shared analysis
# directory, fall back on the pyre root (current directory).
configuration.local_configuration = None
handler = commands.Reporting(
arguments,
original_directory,
configuration,
SharedAnalysisDirectory(
[],
["//target/name"],
project_root="source_directory",
filter_paths=set(),
),
)
with patch.object(os, "getcwd", return_value="source_directory"):
self.assertEqual(
handler._get_directories_to_analyze(), {"source_directory"}
)
def fake_lxc_exec_command(*popenargs, **kwargs):
if get_command(popenargs).endswith('lxc') and get_sub_command(popenargs) == 'exec':
if get_command_parameter(popenargs, '--') == 'true':
cmd = ['bash', '-c', '>&2 echo -e "lxc command failed" ; exit 1']
return subprocess.run(cmd, **kwargs)
else:
return subprocess.CompletedProcess("fakerun", 0, '')
else:
return subprocess.run(*popenargs, **kwargs)
def fake_ansible_playbook_run(*popenargs, **kwargs):
if get_command(popenargs) == 'ansible-playbook':
assert 'lxd' == get_command_parameter(popenargs, '--connection')
verify_inventory(get_command_parameter(popenargs, '--inventory'))
verify_extra_vars(get_command_parameter(popenargs, '--extra-vars').lstrip('@'))
# TODO: verify --user for ssh connection
return subprocess.CompletedProcess("fakerun", 0, '')
else:
return subprocess.run(*popenargs, **kwargs)
def patched_subprocess_run(docker_compose_command,
capture_output=True,
input=None):
# Print out the docker-compose command to perform test validation
logger.error(
f"PYTEST_PATCHED_DOCKER_COMPOSE_COMMAND=[{docker_compose_command}]"
)
# Always succeed - we don't care if the command should have failed or not
return subprocess.CompletedProcess(returncode=0, args=None)
async def run(*args, **kwargs):
kwargs.setdefault('stdout', subprocess.PIPE)
kwargs.setdefault('stderr', subprocess.PIPE)
check = kwargs.pop('check', True)
proc = await asyncio.create_subprocess_exec(*args, **kwargs)
stdout, stderr = await proc.communicate()
cp = subprocess.CompletedProcess(args, proc.returncode, stdout=stdout, stderr=stderr)
if check:
cp.check_returncode()
return cp
def test_run_command(caplog):
cmd = ["run", "this"]
log_level = logging.INFO
additional_env = {"SOME_ADDITIONAL_ENV": "value"}
env = copy.deepcopy(os.environ)
env.update(additional_env)
combine_out_err = False
flexmock(subprocess).should_receive("run").\
with_args(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, text=True, env=env).\
and_return(subprocess.CompletedProcess(cmd, 0, "stdout", "stderr"))
res = run_command(cmd,
log_level=log_level,
additional_env=additional_env,
combine_out_err=combine_out_err)
assert res.returncode == 0
assert res.stdout == "stdout"
assert res.stderr == "stderr"
flexmock(subprocess).should_receive("run").\
with_args(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, text=True, env=env).\
and_raise(subprocess.CalledProcessError(1, cmd, "stdout", "stderr"))
with pytest.raises(subprocess.CalledProcessError):
run_command(cmd,
log_level=log_level,
additional_env=additional_env,
combine_out_err=combine_out_err)
# test that secrets are not logged
set_log(log)
caplog.clear()
caplog.set_level(logging.DEBUG, logger=log.name)
secret = "abcdefg"
cmd = ["run", {"secret": True, "item": secret}]
flexmock(subprocess).should_receive("run").\
with_args(["run", "abcdefg"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, text=True, env=env).\
and_return(subprocess.CompletedProcess(cmd, 0, "stdout", "stderr"))
res = run_command(cmd,
log_level=log_level,
additional_env=additional_env,
combine_out_err=combine_out_err)
assert secret not in caplog.text
assert REDACTED_OUT_SECRET in caplog.text
def build(self, nodes: List[Node]) -> (subprocess.CompletedProcess, float):
# Early exit if no targets were provided
if not nodes:
G_LOGGER.debug(f"No targets specified, skipping build.")
return subprocess.CompletedProcess(args=[], returncode=0, stdout=b"", stderr=b"No targets specified"), 0
paths = [node.path for node in nodes]
cmd = ["rbuild", "--threads", str(multiprocessing.cpu_count()), f"{self.config_file}"] + paths
G_LOGGER.verbose(f"Build command: {' '.join(cmd)}\nTarget file paths: {paths}")
return utils.time_subprocess(cmd)
def test_repo_init_spec_name_collision(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
None, 0, _FAKE_REPO_INFO)
with self.assertRaises(ValueError):
cipd_utils.Repo("/repo/root",
spec={
"foo": "same_alias",
"baz": "same_alias"
})
def _run_input_lines(cmd, input_lines, *, kwargs):
popen = subprocess.Popen(cmd, stdin=subprocess.PIPE, **kwargs)
stdin_write = popen.stdin.write
for line in input_lines:
stdin_write(line)
stdout, stderr = popen.communicate()
return subprocess.CompletedProcess(popen.args, popen.returncode,
stdout=stdout, stderr=stderr)
def _started_to_completed(self,
stdout='',
stderr='') -> subprocess.CompletedProcess:
assert self.started_proc is not None, "process must be started before calling this method"
returncode = self.started_proc.returncode
assert returncode is not None, "only call this if started process has terminated"
args = self.started_proc.args
self.completed_proc = subprocess.CompletedProcess(
args, returncode, stdout, stderr)
return self.completed_proc
def test_restart_notebook_server(lifecycle_config, mocker, caplog):
completed_process = subprocess.CompletedProcess(args=[], returncode=0)
mocker.patch("notebook.lifecycle_config.subprocess.run",
return_value=completed_process)
with caplog.at_level(logging.INFO):
lifecycle_config.restart_notebook_server()
assert "Restarting Jupyter Server" in caplog.text
assert "Failed" not in caplog.text
def run(self, command, cwd=None):
"""Run a command in the shell for this client."""
self.print_divider(
f'{"Running" if not self.dry_run else "Would run"} command "{command}"'
)
command = command.split(" ")
# Dry runs should pretend the command succeeded.
if self.dry_run:
return subprocess.CompletedProcess(command, 0)
return subprocess.run(command, cwd=cwd if cwd else self.cwd)
def run_mock(args, stdout):
if args[0] == "host":
if host_good:
return subprocess.CompletedProcess(
args="", returncode=0, stdout=host_ret_good
)
else:
return subprocess.CompletedProcess(
args="", returncode=1, stdout=host_ret_bad
)
if args[0] == "scontrol":
if scontrol_good:
return subprocess.CompletedProcess(
args="", returncode=0, stdout=scontrol_ret_good
)
else:
return subprocess.CompletedProcess(
args="", returncode=1, stdout=scontrol_ret_bad
)
def test_generate_docs(self):
package_dir = os.path.join(self.temp_dir, 'test_package')
out_dir = os.path.join(self.temp_dir, 'out')
prebuilts = os.path.join(self.temp_dir, 'prebuilts')
with mock.patch.object(gen_reference_docs.tempfile,
'TemporaryDirectory') as mock_dir:
with mock.patch.object(gen_reference_docs.subprocess,
'run') as mock_run:
with mock.patch.object(gen_reference_docs.os,
'environ') as mock_env:
mock_run.return_value = subprocess.CompletedProcess(
args='', returncode=0)
mock_env.return_value = {}
mock_dir.return_value.__enter__.return_value = 'fakedir'
result = gen_reference_docs.generate_docs(
package_dir=package_dir,
out_dir=out_dir,
dart_prebuilt_dir=prebuilts)
self.assertEqual(result, 0)
mock_run.assert_has_calls([
mock.call(
[os.path.join(prebuilts, 'dart'), 'pub', 'get'],
cwd=package_dir,
env={
"PUB_CACHE": "fakedir",
"HOME": "fakedir"
},
capture_output=True,
universal_newlines=True),
mock.call([
os.path.join(prebuilts, 'dart'), 'pub', 'global',
'activate', 'dartdoc'
],
cwd=package_dir,
env={
"PUB_CACHE": "fakedir",
"HOME": "fakedir"
},
capture_output=True,
universal_newlines=True),
# TODO(fxb/93159): Re-enable `dart doc` after it is known
# how to incorporate the following dropped flags. Once done,
# we can get rid of this `dart pub global activate dartdoc`
# workaround.
mock.call([
os.path.join(prebuilts, 'dart'), 'pub', 'global',
'run', 'dartdoc', '--auto-include-dependencies',
'--exclude-packages', 'Dart,logging', '--output',
os.path.join(out_dir, 'dartdoc'), '--format', 'md'
],
cwd=package_dir,
env={"PUB_CACHE": "fakedir"},
capture_output=True,
universal_newlines=True),
])
def run_playbook(self, playbook, syncheck=False, checkmode=False):
"""Run an ansible playbook, optionally in syntax check mode or
with --check --diff"""
playbook = os.path.abspath(playbook)
inventory = os.path.abspath(self.inventory)
my_ansible_playbook_cmd = copy.deepcopy(self.ansible_playbook_cmd)
if syncheck:
my_ansible_playbook_cmd.extend(['--syntax-check'])
if checkmode:
my_ansible_playbook_cmd.extend(['--check', '--diff'])
my_ansible_playbook_cmd.extend(['--inventory', inventory, playbook])
my_env = os.environ.copy()
my_rundate = strftime(self.date_format, gmtime())
my_runtime = strftime(self.time_format, gmtime())
my_logdir = ('/var/log/ansible/playbook/' +
os.path.basename(playbook) + '/' + my_rundate + '/')
Path(my_logdir).mkdir(parents=True, exist_ok=True)
my_env['ANSIBLE_LOG_PATH'] = (my_logdir + os.path.basename(playbook) +
'.' + my_rundate + '.' + my_runtime +
'.log')
#my_env['ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS'] = 'silently'
self.logger.info("Running '%s' and logging to '%s'",
' '.join(my_ansible_playbook_cmd),
str(my_env['ANSIBLE_LOG_PATH']))
try:
ret = subprocess.run(my_ansible_playbook_cmd,
env=my_env,
timeout=self.timeout,
check=False,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
except subprocess.TimeoutExpired:
# create a dummy completedProcess obj with a bad return code
ret = subprocess.CompletedProcess(my_ansible_playbook_cmd, 255)
# killall ansible-playbook procs to tidy up after
# killing the main one
killedprocs = killall(playtokill=' '.join(my_ansible_playbook_cmd))
self.logger.error("Timed out waiting %s seconds for '%s'",
self.timeout, ' '.join(my_ansible_playbook_cmd))
for proc in killedprocs:
self.logger.warning("KILLED '%s' due to timeout",
' '.join(proc['cmdline']))
return ret
if ret.returncode == 0:
self.logger.info("ansible-playbook %s return code: %s",
str(playbook), str(ret.returncode))
return ret
## should only log as an error if return code not 0
self.logger.error(
"ansible-playbook %s did not complete, return code: %s",
str(playbook), str(ret.returncode))
return ret
def raise_error(self) -> None:
"""Raise and error from the subprocess in a clean way."""
if self.completed():
if isinstance(self.__process, subprocess.Popen) and isinstance(
self.return_code, int):
result = subprocess.CompletedProcess(self.__process.args,
self.return_code,
self.std_out,
self.std_err)
result.check_returncode()
def subprocess_intercept(args, **kwargs):
if len(args) > 1 and args[1].endswith('run.py'):
import sys
with patch.object(sys, 'argv', args):
spec = importlib.util.spec_from_file_location("__main__", args[1])
runner_script = importlib.util.module_from_spec(spec)
returncode = spec.loader.exec_module(runner_script)
return subprocess.CompletedProcess(args, returncode)
else:
return original_impl(args, **kwargs)
def test_error_recovered(self):
"""Test for a case where first error is recoverable, we recover by --fsck and then do backup successfully"""
exception = tarsnap.TarsnapFsckException(
subprocess.CompletedProcess(self.BACKUP_COMMAND, 1, stdout="Please run --fsck"))
calls = [
mock.call(self.BACKUP_COMMAND),
mock.call(self.FSCK_COMMAND),
mock.call(self.BACKUP_COMMAND)
]
self.do_test_backup([exception, None, None], calls, True)
def test_get_latest_template_version_w_git_ssh(mocker, stdout, stderr, expected):
"""Tests temple.update._get_latest_template_version_w_git_ssh"""
ls_remote_return = subprocess.CompletedProcess([], returncode=0, stdout=stdout,
stderr=stderr)
mock_shell = mocker.patch('temple.utils.shell',
autospec=True,
return_value=ls_remote_return)
assert temple.update._get_latest_template_version_w_git_ssh('t') == expected
cmd = 'git ls-remote t | grep HEAD | cut -f1'
mock_shell.assert_called_once_with(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def test_valid_latency(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=[],
returncode=0,
stdout="PING www.google.com (216.58.199.36) 56(84) bytes of data.\n64 bytes from syd09s12-in-f4.1e100.net (216.58.199.36): icmp_seq=1 ttl=55 time=7.07 ms\n64 bytes from syd09s12-in-f4.1e100.net (216.58.199.36): icmp_seq=2 ttl=55 time=6.68 ms\n64 bytes from syd09s12-in-f4.1e100.net (216.58.199.36): icmp_seq=3 ttl=55 time=6.21 ms\n64 bytes from syd09s12-in-f4.1e100.net (216.58.199.36): icmp_seq=4 ttl=55 time=6.51 ms\n\n--- www.google.com ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 7ms\nrtt min/avg/max/mdev = 6.211/6.617/7.069/0.315 ms\n",
stderr="",
)
self.assertEqual(
self.valid_latency,
self.measurement._get_latency_results("validfakehost.com")[0],
)
def test_latency_invalid_regex(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=[],
returncode=0,
stdout="\nrtt min/avg/max/mdev = [BAD REGEX] ms\n",
stderr="",
)
self.assertEqual(
self.invalid_regex,
self.measurement._get_latency_results("validfakehost.com")[0],
)
def test_invalid_latency(self, mock_run):
mock_run.return_value = subprocess.CompletedProcess(
args=[],
returncode=1,
stdout="\nrtt min/avg/max/mdev = 5.484/6.133/7.133/0.611 ms\n",
stderr="the ping messed up!",
)
self.assertEqual(
self.invalid_latency,
self.measurement._get_latency_results("validfakehost.com")[0],
)
