def _exec_command_on_client_blocking(self, command):
"""
:type command: str
:rtype: Response
"""
escaped_command = self._escaped_ssh_command(command)
self._logger.debug('SSH popen blocking [{}:{}]: {}'.format(self.user, self.host, escaped_command))
proc = Popen_with_delayed_expansion(escaped_command, shell=True, stdout=PIPE, stderr=PIPE)
output, error = proc.communicate()
return Response(raw_output=output, raw_error=error, returncode=proc.returncode)
def _exec_command_on_client_blocking(self, command):
"""
:param command:
:type command: str
:return:
:rtype: Response
"""
proc = Popen_with_delayed_expansion(command, shell=True, stdout=PIPE, stderr=PIPE)
self._logger.debug('popen blocking [{}:{}]: {}'.format(self.user, self.host, command))
output, error = proc.communicate()
return Response(raw_output=output, raw_error=error, returncode=proc.returncode)
def _copy_on_client(self, source, destination):
"""
:type source: str
:type destination: str
:rtype: Response
"""
# Avoid any ssh known_hosts prompts.
command = 'scp -o StrictHostKeyChecking=no {} {}:{}'.format(source, self._host_string(), destination)
self._logger.debug('SCP popen blocking [{}:{}]: {}'.format(self.user, self.host, command))
proc = Popen_with_delayed_expansion(command, shell=True, stdout=PIPE, stderr=PIPE)
output, error = proc.communicate()
return Response(raw_output=output, raw_error=error, returncode=proc.returncode)
def _copy_on_client(self, source, destination):
"""
:type source: str
:type destination: str
:rtype: Response
"""
# Avoid any ssh known_hosts prompts.
command = 'scp -o StrictHostKeyChecking=no {} {}:{}'.format(
source, self._host_string(), destination)
self._logger.debug('SCP popen blocking [{}:{}]: {}'.format(
self.user, self.host, command))
proc = Popen_with_delayed_expansion(command,
shell=True,
stdout=PIPE,
stderr=PIPE)
output, error = proc.communicate()
return Response(raw_output=output,
raw_error=error,
returncode=proc.returncode)
def _exec_command_on_client_async(self, command):
"""
:param command:
:return:
:rtype: Response
"""
# todo investigate why this assignment is required for launching async operations using Popen
self._logger.debug('popen async [{}:{}]: {}'.format(self.user, self.host, command))
Popen_with_delayed_expansion(command, shell=True, stdout=DEVNULL, stderr=DEVNULL)
return EmptyResponse()
def test_Popen_with_deplayed_expansion(self, input_cmd, os_name,
expected_final_cmd):
# Arrange
mock_os = self.patch('app.util.process_utils.os')
mock_os.name = os_name
mock_subprocess_popen = self.patch('subprocess.Popen')
# Act
Popen_with_delayed_expansion(input_cmd)
# Assert
mock_subprocess_popen.assert_called_once_with(expected_final_cmd)
def _run_service(self, cmd, service_url=None):
"""
Runs a service with a specified shell cmd
:param service_url:
:param cmd: shell command for running the service as a background process
:return:
"""
print('running cmd: {}'.format(cmd))
if service_url is not None and self.is_up(service_url):
return
Popen_with_delayed_expansion(cmd, stdout=DEVNULL)
if service_url is not None and not self.is_up(service_url, timeout=10):
raise ServiceRunError("Failed to run service on {}.".format(service_url))
def _exec_command_on_client_async(self, command):
"""
:type command: str
:rtype: Response
"""
escaped_command = self._escaped_ssh_command(command)
self._logger.debug('SSH popen async [{}:{}]: {}'.format(
self.user, self.host, escaped_command))
Popen_with_delayed_expansion(escaped_command,
shell=True,
stdout=DEVNULL,
stderr=DEVNULL)
return EmptyResponse()
def test_Popen_with_deplayed_expansion(self, input_cmd, os_name,
expected_final_cmd):
# Arrange
mock_os = self.patch('app.util.process_utils.os')
mock_os.name = os_name
mock_subprocess_popen = self.patch('subprocess.Popen')
# Act
Popen_with_delayed_expansion(input_cmd)
# Assert
self.assertEqual(len(mock_subprocess_popen.call_args_list), 1)
(cmd, ), _ = mock_subprocess_popen.call_args
self.assertEqual(cmd, expected_final_cmd)
def async_delete(path):
"""
Asynchronously delete a file or a directory. This functionality is handy for deleting large directories.
For example, in ClusterRunner, deleting the results directory can take up to half an hour. With async_delete(),
the delete call will return almost immediately.
Under the covers, this is implemented by first synchronously renaming the file or directory to a unique name
inside of a temporary directory. After that, we launch an asynchronous process that deletes that file/directory.
This asynchronous process does not die when the invoking process (this process) exits/dies.
:param path: the absolute path to the file or directory to asynchronously delete
:type path: str
"""
new_temp_path = tempfile.mkdtemp(prefix='async_delete_directory')
shutil.move(path, new_temp_path)
# TODO: make the following command cross-platform.
Popen_with_delayed_expansion(['rm', '-rf', new_temp_path])
def execute_command_in_project(self, command, extra_environment_vars=None, timeout=None, **popen_kwargs):
"""
Execute a command in the context of the project
:param command: the shell command to execute
:type command: string
:param extra_environment_vars: additional environment variables to set for command execution
:type extra_environment_vars: dict[str, str]
:param timeout: A maximum number of seconds before the process is terminated, or None for no timeout
:type timeout: int | None
:param popen_kwargs: additional keyword arguments to pass through to subprocess.Popen
:type popen_kwargs: dict[str, mixed]
:return: a tuple of (the string output from the command, the exit code of the command)
:rtype: (string, int)
"""
environment_setter = self.shell_environment_command(extra_environment_vars)
command = self.command_in_project('{} {}'.format(environment_setter, command))
self._logger.debug('Executing command in project: {}', command)
# Redirect output to files instead of using pipes to avoid: https://github.com/box/ClusterRunner/issues/57
stdout_file = TemporaryFile()
stderr_file = TemporaryFile()
pipe = Popen_with_delayed_expansion(
command,
shell=True,
stdout=stdout_file,
stderr=stderr_file,
start_new_session=True, # Starts a new process group (so we can kill it without killing clusterrunner).
**popen_kwargs
)
clusterrunner_error_msgs = []
command_completed = False
timeout_time = time.time() + (timeout or float('inf'))
# Wait for the command to complete, but also periodically check the kill event flag to see if we should
# terminate the process prematurely.
while not command_completed and not self._kill_event.is_set() and time.time() < timeout_time:
try:
pipe.wait(timeout=1)
command_completed = True # wait() didn't raise TimeoutExpired, so process has finished executing.
except TimeoutExpired:
continue
except Exception as ex: # pylint: disable=broad-except
error_message = 'Exception while waiting for process to finish.'
self._logger.exception(error_message)
clusterrunner_error_msgs.append(
'ClusterRunner: {} ({}: "{}")'.format(error_message, type(ex).__name__, ex))
break
if not command_completed:
# We've been signaled to terminate subprocesses, so terminate them. But we still collect stdout and stderr.
# We must kill the entire process group since shell=True launches 'sh -c "cmd"' and just killing the pid
# will kill only "sh" and not its child processes.
# Note: We may lose buffered output from the subprocess that hasn't been flushed before termination.
self._logger.warning('Terminating PID: {}, Command: "{}"', pipe.pid, command)
try:
# todo: os.killpg sends a SIGTERM to all processes in the process group. If the immediate child process
# ("sh") dies but its child processes do not, we will leave them running orphaned.
try:
os.killpg(pipe.pid, signal.SIGTERM)
except AttributeError:
self._logger.warning('os.killpg is not available. This is expected if ClusterRunner is running'
'on Windows. Using os.kill instead.')
os.kill(pipe.pid, signal.SIGTERM)
except (PermissionError, ProcessLookupError) as ex: # os.killpg will raise if process has already ended
self._logger.warning('Attempted to kill process group (pgid: {}) but raised {}: "{}".',
pipe.pid, type(ex).__name__, ex)
try:
pipe.wait()
except Exception as ex: # pylint: disable=broad-except
error_message = 'Exception while waiting for terminated process to finish.'
self._logger.exception(error_message)
clusterrunner_error_msgs.append(
'ClusterRunner: {} ({}: "{}")'.format(error_message, type(ex).__name__, ex))
stdout, stderr = [self._read_file_contents_and_close(f) for f in [stdout_file, stderr_file]]
exit_code = pipe.returncode
if exit_code != 0:
max_log_length = 300
logged_stdout, logged_stderr = stdout, stderr
if len(stdout) > max_log_length:
logged_stdout = '{}... (total stdout length: {})'.format(stdout[:max_log_length], len(stdout))
if len(stderr) > max_log_length:
logged_stderr = '{}... (total stderr length: {})'.format(stderr[:max_log_length], len(stderr))
# Note we are intentionally not logging at error or warning level here. Interpreting a non-zero return code
# as a failure is context-dependent, so we can't make that determination here.
self._logger.notice(
'Command exited with non-zero exit code.\nCommand: {}\nExit code: {}\nStdout: {}\nStderr: {}\n',
command, exit_code, logged_stdout, logged_stderr)
else:
self._logger.debug('Command completed with exit code {}.', exit_code)
exit_code = exit_code if exit_code is not None else -1 # Make sure we always return an int.
combined_command_output = '\n'.join([stdout, stderr] + clusterrunner_error_msgs)
return combined_command_output, exit_code
def execute_command_in_project(self,
command,
extra_environment_vars=None,
timeout=None,
**popen_kwargs):
"""
Execute a command in the context of the project
:param command: the shell command to execute
:type command: string
:param extra_environment_vars: additional environment variables to set for command execution
:type extra_environment_vars: dict[str, str]
:param timeout: A maximum number of seconds before the process is terminated, or None for no timeout
:type timeout: int | None
:param popen_kwargs: additional keyword arguments to pass through to subprocess.Popen
:type popen_kwargs: dict[str, mixed]
:return: a tuple of (the string output from the command, the exit code of the command)
:rtype: (string, int)
"""
environment_setter = self.shell_environment_command(
extra_environment_vars)
command = self.command_in_project('{} {}'.format(
environment_setter, command))
self._logger.debug('Executing command in project: {}', command)
# Redirect output to files instead of using pipes to avoid: https://github.com/box/ClusterRunner/issues/57
stdout_file = TemporaryFile()
stderr_file = TemporaryFile()
pipe = Popen_with_delayed_expansion(
command,
shell=True,
stdout=stdout_file,
stderr=stderr_file,
start_new_session=
True, # Starts a new process group (so we can kill it without killing clusterrunner).
**popen_kwargs)
clusterrunner_error_msgs = []
command_completed = False
timeout_time = time.time() + (timeout or float('inf'))
# Wait for the command to complete, but also periodically check the kill event flag to see if we should
# terminate the process prematurely.
while not command_completed and not self._kill_event.is_set(
) and time.time() < timeout_time:
try:
pipe.wait(timeout=1)
command_completed = True # wait() didn't raise TimeoutExpired, so process has finished executing.
except TimeoutExpired:
continue
except Exception as ex: # pylint: disable=broad-except
error_message = 'Exception while waiting for process to finish.'
self._logger.exception(error_message)
clusterrunner_error_msgs.append(
'ClusterRunner: {} ({}: "{}")'.format(
error_message,
type(ex).__name__, ex))
break
if not command_completed:
# We've been signaled to terminate subprocesses, so terminate them. But we still collect stdout and stderr.
# We must kill the entire process group since shell=True launches 'sh -c "cmd"' and just killing the pid
# will kill only "sh" and not its child processes.
# Note: We may lose buffered output from the subprocess that hasn't been flushed before termination.
self._logger.warning('Terminating PID: {}, Command: "{}"',
pipe.pid, command)
try:
# todo: os.killpg sends a SIGTERM to all processes in the process group. If the immediate child process
# ("sh") dies but its child processes do not, we will leave them running orphaned.
try:
os.killpg(pipe.pid, signal.SIGTERM)
except AttributeError:
self._logger.warning(
'os.killpg is not available. This is expected if ClusterRunner is running'
'on Windows. Using os.kill instead.')
os.kill(pipe.pid, signal.SIGTERM)
except (
PermissionError, ProcessLookupError
) as ex: # os.killpg will raise if process has already ended
self._logger.warning(
'Attempted to kill process group (pgid: {}) but raised {}: "{}".',
pipe.pid,
type(ex).__name__, ex)
try:
pipe.wait()
except Exception as ex: # pylint: disable=broad-except
error_message = 'Exception while waiting for terminated process to finish.'
self._logger.exception(error_message)
clusterrunner_error_msgs.append(
'ClusterRunner: {} ({}: "{}")'.format(
error_message,
type(ex).__name__, ex))
stdout, stderr = [
self._read_file_contents_and_close(f)
for f in [stdout_file, stderr_file]
]
exit_code = pipe.returncode
if exit_code != 0:
max_log_length = 300
logged_stdout, logged_stderr = stdout, stderr
if len(stdout) > max_log_length:
logged_stdout = '{}... (total stdout length: {})'.format(
stdout[:max_log_length], len(stdout))
if len(stderr) > max_log_length:
logged_stderr = '{}... (total stderr length: {})'.format(
stderr[:max_log_length], len(stderr))
# Note we are intentionally not logging at error or warning level here. Interpreting a non-zero return code
# as a failure is context-dependent, so we can't make that determination here.
self._logger.notice(
'Command exited with non-zero exit code.\nCommand: {}\nExit code: {}\nStdout: {}\nStderr: {}\n',
command, exit_code, logged_stdout, logged_stderr)
else:
self._logger.debug('Command completed with exit code {}.',
exit_code)
exit_code = exit_code if exit_code is not None else -1 # Make sure we always return an int.
combined_command_output = '\n'.join([stdout, stderr] +
clusterrunner_error_msgs)
return combined_command_output, exit_code