def python3(self, code: str) -> CompletedProcess:
"""Execute Python 3 code in an isolated environment and return the completed process."""
with NamedTemporaryFile() as nsj_log:
args = (self.nsjail_binary, "--config", NSJAIL_CFG, "--log",
nsj_log.name, f"--cgroup_mem_max={MEM_MAX}",
"--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent),
"--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name,
"--cgroup_pids_max=1", "--cgroup_pids_mount",
str(CGROUP_PIDS_PARENT.parent), "--cgroup_pids_parent",
CGROUP_PIDS_PARENT.name, "--", self.python_binary, "-Iqu",
"-c", code)
msg = "Executing code..."
if DEBUG:
msg = f"{msg[:-3]}:\n{textwrap.indent(code, ' ')}"
log.info(msg)
try:
result = subprocess.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
except ValueError:
return CompletedProcess(args, None,
"ValueError: embedded null byte", None)
log_lines = nsj_log.read().decode("utf-8").splitlines()
if not log_lines and result.returncode == 255:
# NsJail probably failed to parse arguments so log output will still be in stdout
log_lines = result.stdout.splitlines()
self._parse_log(log_lines)
return result
def patched_run_command(mocker):
m = mocker.patch("molecule.util.run_command")
m.return_value = CompletedProcess(
args="foo", returncode=0, stdout="patched-run-command-stdout", stderr=""
)
return m
def _execute(self, *args, **kwargs):
params = [
'{}={}'.format(k, v) for k, v in self.config.params.items()
]
command = ' '.join(params + [executable] + list(args))
click.echo(command)
return CompletedProcess([], 0)
def test_create_index_failure(self, mocked_subprocess):
# Initialize mock and test objects
mocked_subprocess.return_value = CompletedProcess(
[], 1, stdout=None, stderr="Error: something went wrong.")
batch, gtf_file, fasta_file = init_objects()
processor_job = ProcessorJob.create_job_and_relationships(
batches=[batch])
# Mock out the job_context with everything the function under
# test will expect
job_context = utils.start_job({
"job": processor_job,
"job_id": processor_job.id,
"gtf_file": gtf_file,
"gtf_file_path": "dummy",
"fasta_file": fasta_file,
"fasta_file_path": "dummy",
"genes_to_transcripts_path": "dummy"
})
job_context = transcriptome_index._set_job_prefix(job_context)
# The function being tested.
job_context = transcriptome_index._create_index(job_context)
self.assertFalse(job_context["success"])
self.assertEqual(
processor_job.failure_reason,
("Shell call to rsem-prepare-reference failed because: "
"Error: something went wrong."))
self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def run_subprocess(
args: List[str],
log_output_live: bool,
cwd: Optional[Union[bytes, str]] = None) -> CompletedProcess:
"""
Run a command in a subprocess.
Args:
args: See `subprocess.run`.
log_output_live: If `True`, log output live. If `True`, stderr is
merged into stdout in the return value.
cwd: See `subprocess.run`.
Returns:
See `subprocess.run`.
Raises:
CalledProcessError: See `subprocess.run`.
"""
# It is hard to log output of both stdout and stderr live unless we
# combine them.
# See http://stackoverflow.com/a/18423003.
if log_output_live:
process_stderr = STDOUT
else:
process_stderr = PIPE
with Popen(
args=args,
cwd=cwd,
stdout=PIPE,
stderr=process_stderr,
) as process:
try:
if log_output_live:
stdout = b''
stderr = b''
for line in process.stdout:
LOGGER.debug(line)
stdout += line
# Without this, `.poll()` will return None on some
# systems.
# See https://stackoverflow.com/a/33563376.
process.communicate()
else:
stdout, stderr = process.communicate()
except: # noqa: B001 pragma: no cover
# We clean up if there is an error while getting the output.
# This may not happen while running tests so we ignore coverage.
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode > 0:
LOGGER.info(str(stderr))
raise CalledProcessError(retcode,
args,
output=stdout,
stderr=stderr)
return CompletedProcess(args, retcode, stdout, stderr)
def run(self, *popenargs, stdout: BufferedIOBase = None, stderr: BufferedIOBase = None, input=None, timeout=None,
check=False, **kwargs) -> CompletedProcess:
if input is not None:
if 'stdin' in kwargs:
raise ValueError('stdin and input arguments may not both be used.')
kwargs['stdin'] = PIPE
with self.Popen(stdout=PIPE, stderr=PIPE, *popenargs, **kwargs) as process:
try:
stdout, stderr = process.communicate(input, timeout=timeout)
except TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
raise TimeoutExpired(process.args, timeout, output=stdout,
stderr=stderr)
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if check and retcode:
raise CalledProcessError(retcode, process.args,
output=stdout, stderr=stderr)
return CompletedProcess(process.args, retcode, stdout, stderr)
def test_create_tag(self, mock_run):
mock_run.return_value = CompletedProcess('', returncode=0)
create_tag('/fake/path', '2.0.0', 'new tag: {}')
args, kwargs = mock_run.call_args
assert args == (['git', 'tag', '-a', '2.0.0', '-m',
'new tag: 2.0.0'], )
assert kwargs == {'cwd': '/fake/path'}
def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs):
"""Run a process like ``ocrmypdf.subprocess.run``, and poll stderr.
Every line of produced by stderr will be forwarded to the callback function.
The intended use is monitoring progress of subprocesses that output their
own progress indicators. In addition, each line will be logged if debug
logging is enabled.
Requires stderr to be opened in text mode for ease of handling errors. In
addition the expected encoding= and errors= arguments should be set. Note
that if stdout is already set up, it need not be binary.
"""
args, env, process_log, text = _fix_process_args(args, env, kwargs)
assert text, "Must use text=True"
proc = Popen(args, env=env, **kwargs)
lines = []
while proc.poll() is None:
for msg in iter(proc.stderr.readline, ''):
if process_log.isEnabledFor(logging.DEBUG):
process_log.debug(msg.strip())
callback(msg)
lines.append(msg)
stderr = ''.join(lines)
if check and proc.returncode != 0:
raise CalledProcessError(proc.returncode,
args,
output=None,
stderr=stderr)
return CompletedProcess(args, proc.returncode, None, stderr=stderr)
def test_check_and_log_raises_exit(self, mock_logging):
test_output = CompletedProcess(args='',
returncode=1,
stdout=b'',
stderr=b'ERROR:": HTTP Error 429"\n')
with self.assertRaises(SystemExit):
self.youtube_dl_service.check_and_log_download_output(test_output)
def test_check_and_log_without_error(self):
test_output = CompletedProcess(args='',
returncode=1,
stdout=b'Download success')
flag = self.youtube_dl_service.check_and_log_download_output(
test_output)
self.assertFalse(flag)
def run() -> Union[CompletedProcess, int]:
"""
This is the pretf CLI tool entrypoint.
"""
cmd, args, _, _ = util.parse_args()
if cmd == "version":
print(f"Pretf v{__version__}")
if cmd in {"", "0.12upgrade", "fmt", "help", "version"}:
skip = True
elif cmd == "workspace" and args and args[0] == "show":
skip = True
else:
skip = False
if skip:
return workflow.execute_terraform(verbose=False)
try:
workflow_path = util.find_workflow_path()
if workflow_path:
if workflow_path.name == "pretf.py":
log.bad(
"workflow: pretf.py is deprecated, rename it to pretf.workflow.py"
)
return workflow.custom(workflow_path)
else:
return workflow.default()
except (log.bad, log.ok):
pass
except FunctionNotFoundError as error:
log.bad(error)
except RequiredFilesNotFoundError as error:
log.bad(f"required files not found: {' '.join(error.name_patterns)}")
candidates = error.get_candidates()
if candidates:
log.bad("try changing directory to:")
for path in candidates:
log.bad(f"* {path}")
except VariableError as error:
if hasattr(error, "errors"):
for error in error.errors:
log.bad(error)
else:
log.bad(error)
return CompletedProcess(args=sys.argv, returncode=1)
def test_podman_exists1(self, mocked_run):
mocked_run.return_value = CompletedProcess(
args=["podman", "--version"],
returncode=0,
stdout="podman version 2.0.6\n",
stderr="",
)
podman_exists()
def __call__(self, args, capture_output, check, env):
self._args = args
self._env = env
if self._return_code == 0:
return CompletedProcess(args,
self._return_code,
stdout=self._stdout)
raise CalledProcessError(self._return_code, args, stderr=self._stderr)
def test_check_and_log_with_yt_errors(self, mock_logging):
test_output = CompletedProcess(args='',
returncode=1,
stdout=b'',
stderr=b'HTTP Error 404: Not Found"\n')
flag = self.youtube_dl_service.check_and_log_download_output(
test_output)
self.assertTrue(flag)
def test_check_call(self):
comp_proc = CompletedProcess(None, 0, stdout='stdout')
with mock.patch('builtins.open', mock.mock_open()) as mopen, \
mock.patch('subprocess.run', return_value=comp_proc): # noqa
with log.LogFile.open('pkgdir', 'package') as logfile:
logfile.check_call(['cmd', '--arg'])
output = ''.join(i[-2][0] for i in mopen().write.mock_calls)
self.assertEqual(output, '$ cmd --arg\nstdout\n')
def test_delete_fixations_group(monkeypatch, client):
monkeypatch.setattr(
'subprocess.run',
lambda command, check, shell: CompletedProcess(['some_command'],
returncode=0))
response = client.delete('/groups/Fixations')
assert response.status_code == 400
assert b'The fixations group cannot be deleted' in response.data
def test_post_image_route_with_file(monkeypatch, client):
subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST"])
monkeypatch.setattr('werkzeug.datastructures.FileStorage.save', lambda save_path, filename: "" )
monkeypatch.setattr('subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0))
data = {'file': (BytesIO(b"Testing: \x00\x01"), 'test_file.png')}
response = client.post('/GRP_TST/image', data=data)
assert response.status_code == 201
assert b'Success: Image saved.' in response.data
def test_delete_image_is_a_directory_exception(client, monkeypatch):
subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST"])
subprocess.call(["touch", "elephant_vending_machine/static/img/GRP_TST/blank.jpg"])
monkeypatch.setattr('subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0))
monkeypatch.setattr('os.remove', lambda file: (_ for _ in ()).throw(IsADirectoryError))
response = client.delete('/image/GRP_TST/blank.jpg')
assert response.status_code == 400
assert json.loads(response.data)['message'] == 'blank.jpg exists, but is a directory and not a file. Deletion failed.'
def test_failureToBoot( self ):
''' raise exception if device fails to boot '''
global failCommand
failCommand = lpcTarInfo
self._bootMock.return_value = CompletedProcess( args = [], returncode = 1 )
self._lpcMock.side_effect = failOnCommand
with self.assertRaises(bootError):
deviceBooted()
async def __aenter__(self):
if self.create:
shutil.rmtree(self.envname, ignore_errors=True)
result = await async_run(self._create_args())
if result.returncode != 0:
return (None, result)
return (self, result)
return (self, CompletedProcess([], 0, b'', b''))
def sh(s, capture_output=True, capture_error=False, exception_on_retcode=None):
debug = u.Debug(False)
if exception_on_retcode is None: # set default
if capture_output or capture_error:
exception_on_retcode = True
else:
exception_on_retcode = False
if s == '': # Make a fake completedprocess that looks fine, since the '' program should always return empty strings and success.
return SHVal(
CompletedProcess(args='',
returncode=0,
stdout=('' if capture_output else None),
stderr=('' if capture_error else None)))
s += f'\necho $?\001$PWD > {u.pwd_file}' # Note that this will only change the directory if the whole script finishes
stdout = sp.PIPE if capture_output else None
stderr = sp.PIPE if capture_error else None
#/bin/bash -O expand_aliases -i -c 'ls'
#res = sp.run(['/bin/bash',u.src_path+'backend.sh',s],stdout=stdout)
try:
#res = sp.run(['/bin/bash','-O','expand_aliases','-O','checkwinsize','-l','-c',s],stdout=stdout)
res = esrun(
[
'/bin/bash', '-O', 'expand_aliases', '-O', 'checkwinsize',
'-l', '-c', s
],
stdout=stdout,
stderr=stderr,
debug=debug,
text=True
) # replace \r\n with \n, replace \r with \n, decode with utf-8. Basically as long as output is text and not arbitrary binary data then this should be used.
except KeyboardInterrupt: # should not happen, tho with race conditions it may
raise ValueError(
f"esrun() was interrupted at a bad time and was unable to recover. Command may or may not have executed, but stdout, stderr, and error code were unable to be recovered."
)
#if capture_output or capture_error:
# if they requested the output and we can't give it to them then their logic is at risk, so we shouldn't return an empty string instead we should raise an error.
# raise ValueError(f"output of sh() was requested via capture_* but unable to provide it because of ctrl-c during sh setup or teardown")
#return SHVal(None, exception_on_retcode)
try:
with open(u.pwd_file, 'r') as f:
returncode, new_dir = f.read().strip().split('\001')
res.returncode = int(
returncode
) # without this the completeprocess return code is always just 0 since the /bin/bash process succeeds even tho the child (which is our actual process) did not succeed
os.remove(u.pwd_file)
os.chdir(new_dir)
except OSError:
pass # common case if pwd_file is not created bc the sh script terminated early. This is fine.
ret = SHVal(res, exception_on_retcode)
global _prev_shval
_prev_shval = ret
return ret
def mock_run(*args, **kwargs):
if args[0] == ['uci', 'get', f'network.{interface_name}']:
return CompletedProcess(args=args[0], returncode=0)
elif args[0] == ['uci', 'get', f'network.{interface_name}.addresses']:
return CompletedProcess(
args=args[0],
returncode=0,
stdout='192.168.1.2/24 192.168.2.2/24'.encode())
elif args[0] == [
'uci', 'get', f'network.{interface_name}.private_key'
]:
return CompletedProcess(args=args[0],
returncode=0,
stdout='privatekey'.encode())
elif args[0] == [
'uci', 'get', f'network.{interface_name}.listen_port'
]:
if listen_port:
return CompletedProcess(args=args[0],
returncode=0,
stdout=f'{listen_port}'.encode())
else:
return CompletedProcess(args=args[0],
returncode=1,
stdout=''.encode())
elif args[0] == ['uci', 'get', f'network.{interface_name}.fwmark']:
if fwmark:
return CompletedProcess(args=args[0],
returncode=0,
stdout=f'{fwmark}'.encode())
else:
return CompletedProcess(args=args[0],
returncode=1,
stdout=''.encode())
def test_reset_password_dryrun(self, mocked_run):
"""It should not reset the users password."""
mocked_run.return_value = CompletedProcess(
(), 0, stdout=USERLIST_OUTPUT.encode())
self.ipmi_dry_run.reset_password("root", "a" * 16)
mocked_run.called_once_with(IPMITOOL_BASE + ["user", "list", "1"],
env=ENV,
stdout=PIPE,
check=True)
def test_get_testbed_instance(self, subprocess):
output = "sample header\ninstance1\tsample info1\ninstance2\tsample info2"
subprocess.return_value = CompletedProcess("anything", 0, output, None)
result = parser.get_instance_name()
self.assertEqual(result, "instance1")
result = parser.get_instance_name(1)
self.assertEqual(result, "instance2")
def test_image_copy_to_group_happy_path(client, monkeypatch):
subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST"])
subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST2"])
subprocess.call(["touch", "elephant_vending_machine/static/img/GRP_TST/blank.jpg"])
monkeypatch.setattr('subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0))
data = {"name": "GRP_TST2"}
response = client.post("/GRP_TST/blank.jpg/copy", data=data)
assert response.status_code == 200
assert json.loads(response.data)['message'] == "File blank.jpg was successfully copied to group 'GRP_TST2'."
def test_delete_group_happy_path(monkeypatch, client):
monkeypatch.setattr(
'subprocess.run',
lambda command, check, shell: CompletedProcess(['some_command'],
returncode=0))
subprocess.call(["mkdir", "elephant_vending_machine/static/img/test"])
response = client.delete('/groups/test')
assert response.status_code == 200
assert b'Group test was successfully deleted.' in response.data
def test_post_group_route_happy_path(monkeypatch, client):
monkeypatch.setattr(
'subprocess.run',
lambda command, check, shell: CompletedProcess(['some_command'],
returncode=0))
data = {'name': 'test'}
response = client.post('/groups', data=data)
assert response.status_code == 201
assert b'Success: Group created.'
def mock_subprocess(monkeypatch):
ret_run = CompletedProcess(['arg0', 'arg1', 'arg3'], 42, 'res_stdout',
'res_stderr')
ret_popen = object()
mock_run = Mock(return_value=ret_run)
mock_popen = Mock(return_value=ret_popen)
monkeypatch.setattr('subprocess.run', mock_run)
monkeypatch.setattr('subprocess.Popen', mock_popen)
return mock_run, mock_popen, ret_run, ret_popen
def runner(self, args: Sequence[str], capture: bool,
cwd: Path) -> CompletedProcess:
if self._check_args:
assert self._args == args
if self._cwd:
assert self._cwd == cwd
return CompletedProcess(args, self._rc,
self._get_content(self._stdout, capture),
self._get_content(self._stderr, capture))
def test_podman_exists2(self, mocked_run):
mocked_run.return_value = CompletedProcess(
args=["podman", "--version"],
returncode=127,
stdout="",
stderr="--version: podman: command not found\n",
)
with pytest.raises(OSError, match="podman package is not available!"):
podman_exists()
