def python3(self, code: str) -> CompletedProcess: """Execute Python 3 code in an isolated environment and return the completed process.""" with NamedTemporaryFile() as nsj_log: args = (self.nsjail_binary, "--config", NSJAIL_CFG, "--log", nsj_log.name, f"--cgroup_mem_max={MEM_MAX}", "--cgroup_mem_mount", str(CGROUP_MEMORY_PARENT.parent), "--cgroup_mem_parent", CGROUP_MEMORY_PARENT.name, "--cgroup_pids_max=1", "--cgroup_pids_mount", str(CGROUP_PIDS_PARENT.parent), "--cgroup_pids_parent", CGROUP_PIDS_PARENT.name, "--", self.python_binary, "-Iqu", "-c", code) msg = "Executing code..." if DEBUG: msg = f"{msg[:-3]}:\n{textwrap.indent(code, ' ')}" log.info(msg) try: result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) except ValueError: return CompletedProcess(args, None, "ValueError: embedded null byte", None) log_lines = nsj_log.read().decode("utf-8").splitlines() if not log_lines and result.returncode == 255: # NsJail probably failed to parse arguments so log output will still be in stdout log_lines = result.stdout.splitlines() self._parse_log(log_lines) return result
def patched_run_command(mocker): m = mocker.patch("molecule.util.run_command") m.return_value = CompletedProcess( args="foo", returncode=0, stdout="patched-run-command-stdout", stderr="" ) return m
def _execute(self, *args, **kwargs): params = [ '{}={}'.format(k, v) for k, v in self.config.params.items() ] command = ' '.join(params + [executable] + list(args)) click.echo(command) return CompletedProcess([], 0)
def test_create_index_failure(self, mocked_subprocess): # Initialize mock and test objects mocked_subprocess.return_value = CompletedProcess( [], 1, stdout=None, stderr="Error: something went wrong.") batch, gtf_file, fasta_file = init_objects() processor_job = ProcessorJob.create_job_and_relationships( batches=[batch]) # Mock out the job_context with everything the function under # test will expect job_context = utils.start_job({ "job": processor_job, "job_id": processor_job.id, "gtf_file": gtf_file, "gtf_file_path": "dummy", "fasta_file": fasta_file, "fasta_file_path": "dummy", "genes_to_transcripts_path": "dummy" }) job_context = transcriptome_index._set_job_prefix(job_context) # The function being tested. job_context = transcriptome_index._create_index(job_context) self.assertFalse(job_context["success"]) self.assertEqual( processor_job.failure_reason, ("Shell call to rsem-prepare-reference failed because: " "Error: something went wrong.")) self.assertFalse(os.path.isfile(batch.files[0].get_temp_pre_path()))
def run_subprocess( args: List[str], log_output_live: bool, cwd: Optional[Union[bytes, str]] = None) -> CompletedProcess: """ Run a command in a subprocess. Args: args: See `subprocess.run`. log_output_live: If `True`, log output live. If `True`, stderr is merged into stdout in the return value. cwd: See `subprocess.run`. Returns: See `subprocess.run`. Raises: CalledProcessError: See `subprocess.run`. """ # It is hard to log output of both stdout and stderr live unless we # combine them. # See http://stackoverflow.com/a/18423003. if log_output_live: process_stderr = STDOUT else: process_stderr = PIPE with Popen( args=args, cwd=cwd, stdout=PIPE, stderr=process_stderr, ) as process: try: if log_output_live: stdout = b'' stderr = b'' for line in process.stdout: LOGGER.debug(line) stdout += line # Without this, `.poll()` will return None on some # systems. # See https://stackoverflow.com/a/33563376. process.communicate() else: stdout, stderr = process.communicate() except: # noqa: B001 pragma: no cover # We clean up if there is an error while getting the output. # This may not happen while running tests so we ignore coverage. process.kill() process.wait() raise retcode = process.poll() if retcode > 0: LOGGER.info(str(stderr)) raise CalledProcessError(retcode, args, output=stdout, stderr=stderr) return CompletedProcess(args, retcode, stdout, stderr)
def run(self, *popenargs, stdout: BufferedIOBase = None, stderr: BufferedIOBase = None, input=None, timeout=None, check=False, **kwargs) -> CompletedProcess: if input is not None: if 'stdin' in kwargs: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE with self.Popen(stdout=PIPE, stderr=PIPE, *popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired: process.kill() stdout, stderr = process.communicate() raise TimeoutExpired(process.args, timeout, output=stdout, stderr=stderr) except: process.kill() process.wait() raise retcode = process.poll() if check and retcode: raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) return CompletedProcess(process.args, retcode, stdout, stderr)
def test_create_tag(self, mock_run): mock_run.return_value = CompletedProcess('', returncode=0) create_tag('/fake/path', '2.0.0', 'new tag: {}') args, kwargs = mock_run.call_args assert args == (['git', 'tag', '-a', '2.0.0', '-m', 'new tag: 2.0.0'], ) assert kwargs == {'cwd': '/fake/path'}
def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs): """Run a process like ``ocrmypdf.subprocess.run``, and poll stderr. Every line of produced by stderr will be forwarded to the callback function. The intended use is monitoring progress of subprocesses that output their own progress indicators. In addition, each line will be logged if debug logging is enabled. Requires stderr to be opened in text mode for ease of handling errors. In addition the expected encoding= and errors= arguments should be set. Note that if stdout is already set up, it need not be binary. """ args, env, process_log, text = _fix_process_args(args, env, kwargs) assert text, "Must use text=True" proc = Popen(args, env=env, **kwargs) lines = [] while proc.poll() is None: for msg in iter(proc.stderr.readline, ''): if process_log.isEnabledFor(logging.DEBUG): process_log.debug(msg.strip()) callback(msg) lines.append(msg) stderr = ''.join(lines) if check and proc.returncode != 0: raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr) return CompletedProcess(args, proc.returncode, None, stderr=stderr)
def test_check_and_log_raises_exit(self, mock_logging): test_output = CompletedProcess(args='', returncode=1, stdout=b'', stderr=b'ERROR:": HTTP Error 429"\n') with self.assertRaises(SystemExit): self.youtube_dl_service.check_and_log_download_output(test_output)
def test_check_and_log_without_error(self): test_output = CompletedProcess(args='', returncode=1, stdout=b'Download success') flag = self.youtube_dl_service.check_and_log_download_output( test_output) self.assertFalse(flag)
def run() -> Union[CompletedProcess, int]: """ This is the pretf CLI tool entrypoint. """ cmd, args, _, _ = util.parse_args() if cmd == "version": print(f"Pretf v{__version__}") if cmd in {"", "0.12upgrade", "fmt", "help", "version"}: skip = True elif cmd == "workspace" and args and args[0] == "show": skip = True else: skip = False if skip: return workflow.execute_terraform(verbose=False) try: workflow_path = util.find_workflow_path() if workflow_path: if workflow_path.name == "pretf.py": log.bad( "workflow: pretf.py is deprecated, rename it to pretf.workflow.py" ) return workflow.custom(workflow_path) else: return workflow.default() except (log.bad, log.ok): pass except FunctionNotFoundError as error: log.bad(error) except RequiredFilesNotFoundError as error: log.bad(f"required files not found: {' '.join(error.name_patterns)}") candidates = error.get_candidates() if candidates: log.bad("try changing directory to:") for path in candidates: log.bad(f"* {path}") except VariableError as error: if hasattr(error, "errors"): for error in error.errors: log.bad(error) else: log.bad(error) return CompletedProcess(args=sys.argv, returncode=1)
def test_podman_exists1(self, mocked_run): mocked_run.return_value = CompletedProcess( args=["podman", "--version"], returncode=0, stdout="podman version 2.0.6\n", stderr="", ) podman_exists()
def __call__(self, args, capture_output, check, env): self._args = args self._env = env if self._return_code == 0: return CompletedProcess(args, self._return_code, stdout=self._stdout) raise CalledProcessError(self._return_code, args, stderr=self._stderr)
def test_check_and_log_with_yt_errors(self, mock_logging): test_output = CompletedProcess(args='', returncode=1, stdout=b'', stderr=b'HTTP Error 404: Not Found"\n') flag = self.youtube_dl_service.check_and_log_download_output( test_output) self.assertTrue(flag)
def test_check_call(self): comp_proc = CompletedProcess(None, 0, stdout='stdout') with mock.patch('builtins.open', mock.mock_open()) as mopen, \ mock.patch('subprocess.run', return_value=comp_proc): # noqa with log.LogFile.open('pkgdir', 'package') as logfile: logfile.check_call(['cmd', '--arg']) output = ''.join(i[-2][0] for i in mopen().write.mock_calls) self.assertEqual(output, '$ cmd --arg\nstdout\n')
def test_delete_fixations_group(monkeypatch, client): monkeypatch.setattr( 'subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0)) response = client.delete('/groups/Fixations') assert response.status_code == 400 assert b'The fixations group cannot be deleted' in response.data
def test_post_image_route_with_file(monkeypatch, client): subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST"]) monkeypatch.setattr('werkzeug.datastructures.FileStorage.save', lambda save_path, filename: "" ) monkeypatch.setattr('subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0)) data = {'file': (BytesIO(b"Testing: \x00\x01"), 'test_file.png')} response = client.post('/GRP_TST/image', data=data) assert response.status_code == 201 assert b'Success: Image saved.' in response.data
def test_delete_image_is_a_directory_exception(client, monkeypatch): subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST"]) subprocess.call(["touch", "elephant_vending_machine/static/img/GRP_TST/blank.jpg"]) monkeypatch.setattr('subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0)) monkeypatch.setattr('os.remove', lambda file: (_ for _ in ()).throw(IsADirectoryError)) response = client.delete('/image/GRP_TST/blank.jpg') assert response.status_code == 400 assert json.loads(response.data)['message'] == 'blank.jpg exists, but is a directory and not a file. Deletion failed.'
def test_failureToBoot( self ): ''' raise exception if device fails to boot ''' global failCommand failCommand = lpcTarInfo self._bootMock.return_value = CompletedProcess( args = [], returncode = 1 ) self._lpcMock.side_effect = failOnCommand with self.assertRaises(bootError): deviceBooted()
async def __aenter__(self): if self.create: shutil.rmtree(self.envname, ignore_errors=True) result = await async_run(self._create_args()) if result.returncode != 0: return (None, result) return (self, result) return (self, CompletedProcess([], 0, b'', b''))
def sh(s, capture_output=True, capture_error=False, exception_on_retcode=None): debug = u.Debug(False) if exception_on_retcode is None: # set default if capture_output or capture_error: exception_on_retcode = True else: exception_on_retcode = False if s == '': # Make a fake completedprocess that looks fine, since the '' program should always return empty strings and success. return SHVal( CompletedProcess(args='', returncode=0, stdout=('' if capture_output else None), stderr=('' if capture_error else None))) s += f'\necho $?\001$PWD > {u.pwd_file}' # Note that this will only change the directory if the whole script finishes stdout = sp.PIPE if capture_output else None stderr = sp.PIPE if capture_error else None #/bin/bash -O expand_aliases -i -c 'ls' #res = sp.run(['/bin/bash',u.src_path+'backend.sh',s],stdout=stdout) try: #res = sp.run(['/bin/bash','-O','expand_aliases','-O','checkwinsize','-l','-c',s],stdout=stdout) res = esrun( [ '/bin/bash', '-O', 'expand_aliases', '-O', 'checkwinsize', '-l', '-c', s ], stdout=stdout, stderr=stderr, debug=debug, text=True ) # replace \r\n with \n, replace \r with \n, decode with utf-8. Basically as long as output is text and not arbitrary binary data then this should be used. except KeyboardInterrupt: # should not happen, tho with race conditions it may raise ValueError( f"esrun() was interrupted at a bad time and was unable to recover. Command may or may not have executed, but stdout, stderr, and error code were unable to be recovered." ) #if capture_output or capture_error: # if they requested the output and we can't give it to them then their logic is at risk, so we shouldn't return an empty string instead we should raise an error. # raise ValueError(f"output of sh() was requested via capture_* but unable to provide it because of ctrl-c during sh setup or teardown") #return SHVal(None, exception_on_retcode) try: with open(u.pwd_file, 'r') as f: returncode, new_dir = f.read().strip().split('\001') res.returncode = int( returncode ) # without this the completeprocess return code is always just 0 since the /bin/bash process succeeds even tho the child (which is our actual process) did not succeed os.remove(u.pwd_file) os.chdir(new_dir) except OSError: pass # common case if pwd_file is not created bc the sh script terminated early. This is fine. ret = SHVal(res, exception_on_retcode) global _prev_shval _prev_shval = ret return ret
def mock_run(*args, **kwargs): if args[0] == ['uci', 'get', f'network.{interface_name}']: return CompletedProcess(args=args[0], returncode=0) elif args[0] == ['uci', 'get', f'network.{interface_name}.addresses']: return CompletedProcess( args=args[0], returncode=0, stdout='192.168.1.2/24 192.168.2.2/24'.encode()) elif args[0] == [ 'uci', 'get', f'network.{interface_name}.private_key' ]: return CompletedProcess(args=args[0], returncode=0, stdout='privatekey'.encode()) elif args[0] == [ 'uci', 'get', f'network.{interface_name}.listen_port' ]: if listen_port: return CompletedProcess(args=args[0], returncode=0, stdout=f'{listen_port}'.encode()) else: return CompletedProcess(args=args[0], returncode=1, stdout=''.encode()) elif args[0] == ['uci', 'get', f'network.{interface_name}.fwmark']: if fwmark: return CompletedProcess(args=args[0], returncode=0, stdout=f'{fwmark}'.encode()) else: return CompletedProcess(args=args[0], returncode=1, stdout=''.encode())
def test_reset_password_dryrun(self, mocked_run): """It should not reset the users password.""" mocked_run.return_value = CompletedProcess( (), 0, stdout=USERLIST_OUTPUT.encode()) self.ipmi_dry_run.reset_password("root", "a" * 16) mocked_run.called_once_with(IPMITOOL_BASE + ["user", "list", "1"], env=ENV, stdout=PIPE, check=True)
def test_get_testbed_instance(self, subprocess): output = "sample header\ninstance1\tsample info1\ninstance2\tsample info2" subprocess.return_value = CompletedProcess("anything", 0, output, None) result = parser.get_instance_name() self.assertEqual(result, "instance1") result = parser.get_instance_name(1) self.assertEqual(result, "instance2")
def test_image_copy_to_group_happy_path(client, monkeypatch): subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST"]) subprocess.call(["mkdir", "elephant_vending_machine/static/img/GRP_TST2"]) subprocess.call(["touch", "elephant_vending_machine/static/img/GRP_TST/blank.jpg"]) monkeypatch.setattr('subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0)) data = {"name": "GRP_TST2"} response = client.post("/GRP_TST/blank.jpg/copy", data=data) assert response.status_code == 200 assert json.loads(response.data)['message'] == "File blank.jpg was successfully copied to group 'GRP_TST2'."
def test_delete_group_happy_path(monkeypatch, client): monkeypatch.setattr( 'subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0)) subprocess.call(["mkdir", "elephant_vending_machine/static/img/test"]) response = client.delete('/groups/test') assert response.status_code == 200 assert b'Group test was successfully deleted.' in response.data
def test_post_group_route_happy_path(monkeypatch, client): monkeypatch.setattr( 'subprocess.run', lambda command, check, shell: CompletedProcess(['some_command'], returncode=0)) data = {'name': 'test'} response = client.post('/groups', data=data) assert response.status_code == 201 assert b'Success: Group created.'
def mock_subprocess(monkeypatch): ret_run = CompletedProcess(['arg0', 'arg1', 'arg3'], 42, 'res_stdout', 'res_stderr') ret_popen = object() mock_run = Mock(return_value=ret_run) mock_popen = Mock(return_value=ret_popen) monkeypatch.setattr('subprocess.run', mock_run) monkeypatch.setattr('subprocess.Popen', mock_popen) return mock_run, mock_popen, ret_run, ret_popen
def runner(self, args: Sequence[str], capture: bool, cwd: Path) -> CompletedProcess: if self._check_args: assert self._args == args if self._cwd: assert self._cwd == cwd return CompletedProcess(args, self._rc, self._get_content(self._stdout, capture), self._get_content(self._stderr, capture))
def test_podman_exists2(self, mocked_run): mocked_run.return_value = CompletedProcess( args=["podman", "--version"], returncode=127, stdout="", stderr="--version: podman: command not found\n", ) with pytest.raises(OSError, match="podman package is not available!"): podman_exists()