def test_get_metadata_dir_by_name(self):
pm = ProcessManager(self.NAME, metadata_base_dir=self.BUILDROOT)
self.assertEqual(
ProcessManager._get_metadata_dir_by_name(self.NAME,
self.BUILDROOT),
os.path.join(self.BUILDROOT, pm.host_fingerprint, self.NAME),
)
def __init__(self,
identity,
workdir,
nailgun_classpath,
distribution,
ins=None,
connect_timeout=10,
connect_attempts=5):
Executor.__init__(self, distribution=distribution)
ProcessManager.__init__(self,
name=identity,
process_name=self._PROCESS_NAME)
if not isinstance(workdir, string_types):
raise ValueError(
'Workdir must be a path string, not: {workdir}'.format(
workdir=workdir))
self._identity = identity
self._workdir = workdir
self._ng_stdout = os.path.join(workdir, 'stdout')
self._ng_stderr = os.path.join(workdir, 'stderr')
self._nailgun_classpath = maybe_list(nailgun_classpath)
self._ins = ins
self._connect_timeout = connect_timeout
self._connect_attempts = connect_attempts
def setUp(self):
super(TestProcessManager, self).setUp()
# N.B. We pass in `metadata_base_dir` here because ProcessManager (itself a non-task/non-
# subsystem) depends on an initialized `GlobalOptions` subsystem for the value of
# `--pants-subprocessdir` in the default case. This is normally provided by subsystem
# dependencies in a typical pants run (and integration tests), but not in unit tests.
# Thus, passing this parameter here short-circuits the subsystem-reliant path for the
# purposes of unit testing without requiring adhoc subsystem initialization.
self.pm = ProcessManager('test', metadata_base_dir=self.subprocess_dir)
def _parse_pid_from_output(self, output):
try:
# Parse the watchman pidfile from the cli output (in JSON).
return json.loads(output)['pid']
except ValueError:
# JSON parse failure.
self._logger.critical('invalid output from watchman!\n{output!s}'.format(output=output))
raise ProcessManager.InvalidCommandOutput(output)
except KeyError:
# Key access error on 'pid' - bad output from watchman.
self._logger.critical('no pid from watchman!')
raise ProcessManager.InvalidCommandOutput(output)
def __init__(self, work_dir, log_level=1, watchman_path=None):
ProcessManager.__init__(self, name='watchman', process_name='watchman', socket_type=str)
self._logger = logging.getLogger(__name__)
self._log_level = log_level
self.watchman_path = self._resolve_watchman_path(watchman_path)
# TODO(kwlzn): should these live in .pids or .pants.d? i.e. should watchman survive clean-all?
self._work_dir = os.path.join(work_dir, self.name)
self._state_file = os.path.join(self._work_dir, '{}.state'.format(self.name))
self._log_file = os.path.join(self._work_dir, '{}.log'.format(self.name))
self._sock_file = os.path.join(self._work_dir, '{}.sock'.format(self.name))
self._watchman_client = None
def __init__(self, identity, workdir, nailgun_classpath, distribution, ins=None,
connect_timeout=10, connect_attempts=5):
Executor.__init__(self, distribution=distribution)
ProcessManager.__init__(self, name=identity, process_name=self._PROCESS_NAME)
if not isinstance(workdir, string_types):
raise ValueError('Workdir must be a path string, not: {workdir}'.format(workdir=workdir))
self._identity = identity
self._workdir = workdir
self._ng_stdout = os.path.join(workdir, 'stdout')
self._ng_stderr = os.path.join(workdir, 'stderr')
self._nailgun_classpath = maybe_list(nailgun_classpath)
self._ins = ins
self._connect_timeout = connect_timeout
self._connect_attempts = connect_attempts
def test_process_name_setproctitle_integration(
tmpdir_factory: TempdirFactory) -> None:
# NB: This test forks and then loads this module: we declare it so that it is inferred.
import setproctitle # noqa: F401
buildroot = tmpdir_factory.mktemp("buildroot")
manager_name = "Bob"
process_name = f"{manager_name} [{buildroot}]"
metadata_base_dir = tmpdir_factory.mktemp(".pids")
subprocess.check_call(
args=[
sys.executable,
"-c",
dedent(f"""\
from setproctitle import setproctitle as set_process_title
from pants.pantsd.process_manager import ProcessManager
set_process_title({process_name!r})
pm = ProcessManager({manager_name!r}, metadata_base_dir="{metadata_base_dir}")
pm.write_pid()
pm.write_process_name()
"""),
],
env=dict(PYTHONPATH=os.pathsep.join(sys.path)),
)
pm = ProcessManager(manager_name, metadata_base_dir=str(metadata_base_dir))
assert process_name == pm.process_name
def setUp(self):
super(TestProcessManager, self).setUp()
# N.B. We pass in `metadata_base_dir` here because ProcessManager (itself a non-task/non-
# subsystem) depends on an initialized `GlobalOptions` subsystem for the value of
# `--pants-subprocessdir` in the default case. This is normally provided by subsystem
# dependencies in a typical pants run (and integration tests), but not in unit tests.
# Thus, passing this parameter here short-circuits the subsystem-reliant path for the
# purposes of unit testing without requiring adhoc subsystem initialization.
self.pm = ProcessManager('test', metadata_base_dir=self.subprocess_dir)
def __init__(self, work_dir, log_level=1, watchman_path=None):
ProcessManager.__init__(self,
name='watchman',
process_name='watchman',
socket_type=str)
self._logger = logging.getLogger(__name__)
self._log_level = log_level
self.watchman_path = self._resolve_watchman_path(watchman_path)
# TODO(kwlzn): should these live in .pids or .pants.d? i.e. should watchman survive clean-all?
self._work_dir = os.path.join(work_dir, self.name)
self._state_file = os.path.join(self._work_dir,
'{}.state'.format(self.name))
self._log_file = os.path.join(self._work_dir,
'{}.log'.format(self.name))
self._sock_file = os.path.join(self._work_dir,
'{}.sock'.format(self.name))
self._watchman_client = None
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='name', cmdline=['cmd', 'line'], status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_get_subprocess_output(self):
test_str = '333'
self.assertEqual(
self.pm.get_subprocess_output(['echo', '-n', test_str]), test_str)
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['i_do_not_exist'])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['false'])
def test_await_pid(self):
with mock.patch.object(ProcessManager,
'await_metadata_by_name') as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name, 'pid', 5, mock.ANY)
def test_await_socket(self):
with mock.patch.object(ProcessManager,
'await_metadata_by_name') as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(self.pm.name, 'socket', 5, mock.ANY)
def test_write_pid(self):
with mock.patch.object(ProcessManager,
'write_metadata_by_name') as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, 'pid', '31337')
def test_write_socket(self):
with mock.patch.object(ProcessManager,
'write_metadata_by_name') as mock_write:
self.pm.write_socket('/path/to/unix/socket')
mock_write.assert_called_once_with(self.pm.name, 'socket',
'/path/to/unix/socket')
def test_write_named_socket(self):
with mock.patch.object(ProcessManager,
'write_metadata_by_name') as mock_write:
self.pm.write_named_socket('pailgun', '31337')
mock_write.assert_called_once_with(self.pm.name, 'socket_pailgun',
'31337')
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process = mock.Mock(status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True):
with self.assertRaises(self.pm.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True), \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(
mock_kill.called,
'If we have no pid, kills should noop gracefully.')
@contextmanager
def setup_terminate(self):
with mock.patch.object(ProcessManager, '_kill', **PATCH_OPTS) as mock_kill, \
mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=True)
self.assertEqual(mock_kill.call_count,
len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self,
chk_pre=True,
chk_post_child=False,
chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent, \
mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child, \
mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge, \
mock.patch('os._exit', **PATCH_OPTS), \
mock.patch('os.chdir', **PATCH_OPTS), \
mock.patch('os.setsid', **PATCH_OPTS), \
mock.patch('os.waitpid', **PATCH_OPTS), \
mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent:
mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def setUp(self):
super().setUp()
self.pm = ProcessManager(self.NAME, metadata_base_dir=safe_mkdtemp())
self.pmm = self.pm
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager("test")
def test_process_properties(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name="name", cmdline=["cmd", "line"], status="status")
self.assertEqual(self.pm.cmdline, ["cmd", "line"])
self.assertEqual(self.pm.cmd, "cmd")
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline="")
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_get_subprocess_output(self):
test_str = "333"
self.assertEqual(self.pm.get_subprocess_output(["echo", "-n", test_str]), test_str)
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(["i_do_not_exist"])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(["false"])
def test_await_pid(self):
with mock.patch.object(ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name, "pid", 5, mock.ANY)
def test_await_socket(self):
with mock.patch.object(ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(self.pm.name, "socket", 5, mock.ANY)
def test_write_pid(self):
with mock.patch.object(ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, "pid", "31337")
def test_write_socket(self):
with mock.patch.object(ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_socket("/path/to/unix/socket")
mock_write.assert_called_once_with(self.pm.name, "socket", "/path/to/unix/socket")
def test_write_named_socket(self):
with mock.patch.object(ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_named_socket("pailgun", "31337")
mock_write.assert_called_once_with(self.pm.name, "socket_pailgun", "31337")
def test_as_process(self):
sentinel = 3333
with mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name="test", pid=3, status=psutil.STATUS_IDLE)
self.pm._process = mock.Mock(status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name="test", pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, "_as_process", **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name="not_test", pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = "test"
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with mock.patch.object(ProcessManager, "is_alive", return_value=True):
with self.assertRaises(self.pm.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with mock.patch.object(ProcessManager, "is_alive", return_value=True), mock.patch(
"pants.pantsd.process_manager.rm_rf"
) as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(mock_kill.called, "If we have no pid, kills should noop gracefully.")
@contextmanager
def setup_terminate(self):
with mock.patch.object(ProcessManager, "_kill", **PATCH_OPTS) as mock_kill, mock.patch.object(
ProcessManager, "is_alive", **PATCH_OPTS
) as mock_alive, mock.patch.object(ProcessManager, "purge_metadata", **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=0.1, purge=True)
self.assertEqual(mock_kill.call_count, len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self, chk_pre=True, chk_post_child=False, chk_post_parent=False):
with mock.patch.object(ProcessManager, "post_fork_parent", **PATCH_OPTS) as mock_post_parent, mock.patch.object(
ProcessManager, "post_fork_child", **PATCH_OPTS
) as mock_post_child, mock.patch.object(
ProcessManager, "pre_fork", **PATCH_OPTS
) as mock_pre, mock.patch.object(
ProcessManager, "purge_metadata", **PATCH_OPTS
) as mock_purge, mock.patch(
"os._exit", **PATCH_OPTS
), mock.patch(
"os.chdir", **PATCH_OPTS
), mock.patch(
"os.setsid", **PATCH_OPTS
), mock.patch(
"os.waitpid", **PATCH_OPTS
), mock.patch(
"os.fork", **PATCH_OPTS
) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre:
mock_pre.assert_called_once_with(self.pm)
if chk_post_child:
mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent:
mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
class TestProcessManager(unittest.TestCase):
NAME = "_test_"
TEST_KEY = "TEST"
TEST_VALUE = "300"
TEST_VALUE_INT = 300
BUILDROOT = "/mock_buildroot/"
SUBPROCESS_DIR = safe_mkdtemp()
def setUp(self):
super().setUp()
self.pm = ProcessManager(self.NAME, metadata_base_dir=safe_mkdtemp())
self.pmm = self.pm
def test_maybe_cast(self):
self.assertIsNone(self.pmm._maybe_cast(None, int))
self.assertEqual(self.pmm._maybe_cast("3333", int), 3333)
self.assertEqual(self.pmm._maybe_cast("ssss", int), "ssss")
def test_get_metadata_dir_by_name(self):
pm = ProcessManager(self.NAME, metadata_base_dir=self.BUILDROOT)
self.assertEqual(
ProcessManager._get_metadata_dir_by_name(self.NAME,
self.BUILDROOT),
os.path.join(self.BUILDROOT, pm.host_fingerprint, self.NAME),
)
def test_readwrite_metadata_by_name(self):
with temporary_dir() as tmpdir, unittest.mock.patch(
"pants.pantsd.process_manager.get_buildroot",
return_value=tmpdir):
self.pmm.write_metadata_by_name(self.TEST_KEY, self.TEST_VALUE)
self.assertEqual(self.pmm.read_metadata_by_name(self.TEST_KEY),
self.TEST_VALUE)
self.assertEqual(
self.pmm.read_metadata_by_name(self.TEST_KEY, int),
self.TEST_VALUE_INT)
@pytest.mark.skip(
reason="flaky: https://github.com/pantsbuild/pants/issues/6836")
def test_deadline_until(self):
with self.assertRaises(ProcessManager.Timeout):
with self.captured_logging(logging.INFO) as captured:
self.pmm._deadline_until(lambda: False,
"the impossible",
timeout=0.5,
info_interval=0.1)
self.assertTrue(
4 <= len(captured.infos()) <= 6,
f"Expected between 4 and 6 infos, got: {captured.infos()}",
)
def test_wait_for_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, "test.out")
safe_file_dump(test_filename, "test")
self.pmm._wait_for_file(test_filename,
"file to be created",
"file was created",
timeout=0.1)
def test_wait_for_file_timeout(self):
with temporary_dir() as td:
with self.assertRaises(ProcessManager.Timeout):
self.pmm._wait_for_file(
os.path.join(td, "non_existent_file"),
"file to be created",
"file was created",
timeout=0.1,
)
def test_await_metadata_by_name(self):
with temporary_dir() as tmpdir, unittest.mock.patch(
"pants.pantsd.process_manager.get_buildroot",
return_value=tmpdir):
self.pmm.write_metadata_by_name(self.TEST_KEY, self.TEST_VALUE)
self.assertEqual(
self.pmm.await_metadata_by_name(self.TEST_KEY,
"metadata to be created",
"metadata was created", 0.1),
self.TEST_VALUE,
)
def test_purge_metadata(self):
with unittest.mock.patch(
"pants.pantsd.process_manager.rm_rf") as mock_rm:
self.pmm.purge_metadata_by_name(self.NAME)
self.assertGreater(mock_rm.call_count, 0)
def test_purge_metadata_error(self):
with unittest.mock.patch(
"pants.pantsd.process_manager.rm_rf") as mock_rm:
mock_rm.side_effect = OSError(errno.EACCES,
os.strerror(errno.EACCES))
with self.assertRaises(ProcessManager.MetadataError):
self.pmm.purge_metadata_by_name(self.NAME)
self.assertGreater(mock_rm.call_count, 0)
def test_await_pid(self):
with unittest.mock.patch.object(
ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with("pid",
f"{self.NAME} to start",
f"{self.NAME} started",
5,
caster=unittest.mock.ANY)
def test_await_socket(self):
with unittest.mock.patch.object(
ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(
"socket",
f"{self.NAME} socket to be opened",
f"{self.NAME} socket opened",
5,
caster=unittest.mock.ANY,
)
def test_write_pid(self):
with unittest.mock.patch.object(
ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with("pid", "31337")
def test_write_socket(self):
with unittest.mock.patch.object(
ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_socket("/path/to/unix/socket")
mock_write.assert_called_once_with("socket", "/path/to/unix/socket")
def test_as_process(self):
sentinel = 3333
with unittest.mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm.write_pid(sentinel)
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with unittest.mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm.write_pid(fake_pid)
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
with self.assertRaises(self.pm.NotStarted):
self.pm._as_process()
def test_is_alive_neg(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="not_test", pid=3, status=psutil.STATUS_IDLE)
self.pm.write_process_name("test")
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_extra_check(self):
def extra_check(process):
return False
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_IDLE)
self.assertFalse(self.pm.is_alive(extra_check))
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with unittest.mock.patch.object(ProcessManager,
"is_alive",
return_value=True):
with self.assertRaises(ProcessManager.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with unittest.mock.patch.object(
ProcessManager, "is_alive",
return_value=True), unittest.mock.patch(
"pants.pantsd.process_manager.rm_rf") as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with unittest.mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm.write_pid(42)
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with unittest.mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(
mock_kill.called,
"If we have no pid, kills should noop gracefully.")
@contextmanager
def setup_terminate(self):
with unittest.mock.patch.object(
ProcessManager, "_kill",
**PATCH_OPTS) as mock_kill, unittest.mock.patch.object(
ProcessManager, "is_alive",
**PATCH_OPTS) as mock_alive, unittest.mock.patch.object(
ProcessManager, "purge_metadata",
**PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(ProcessManager.NonResponsiveProcess):
self.pm.terminate(kill_wait=0.1, purge=True)
self.assertEqual(mock_kill.call_count,
len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self,
chk_pre=True,
chk_post_child=False,
chk_post_parent=False):
with unittest.mock.patch.object(
ProcessManager, "post_fork_parent",
**PATCH_OPTS) as mock_post_parent, unittest.mock.patch.object(
ProcessManager, "post_fork_child", **
PATCH_OPTS) as mock_post_child, unittest.mock.patch.object(
ProcessManager, "pre_fork",
**PATCH_OPTS) as mock_pre, unittest.mock.patch.object(
ProcessManager, "purge_metadata",
**PATCH_OPTS) as mock_purge, unittest.mock.patch(
"os._exit", **PATCH_OPTS), unittest.mock.patch(
"os.chdir",
**PATCH_OPTS), unittest.mock.patch(
"os.setsid",
**PATCH_OPTS), unittest.mock.patch(
"os.waitpid",
**PATCH_OPTS), unittest.mock.patch(
"os.fork",
**PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre:
mock_pre.assert_called_once_with(self.pm)
if chk_post_child:
mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent:
mock_post_parent.assert_called_once_with(self.pm)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def setUp(self):
self.pm = ProcessManager('test')
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='name',
cmdline=['cmd', 'line'],
status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_maybe_cast(self):
self.assertIsNone(self.pm._maybe_cast(None, int))
self.assertEqual(self.pm._maybe_cast('3333', int), 3333)
self.assertEqual(self.pm._maybe_cast('ssss', int), 'ssss')
def test_readwrite_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
test_content = '3333'
self.pm._write_file(test_filename, test_content)
self.assertEqual(self.pm._read_file(test_filename), test_content)
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_deadline_until(self):
with self.assertRaises(self.pm.Timeout):
self.pm._deadline_until(lambda: False, timeout=.1)
def test_wait_for_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
self.pm._write_file(test_filename, 'test')
self.pm._wait_for_file(test_filename, timeout=.1)
def test_wait_for_file_timeout(self):
with temporary_dir() as td:
with self.assertRaises(self.pm.Timeout):
self.pm._wait_for_file(os.path.join(td, 'non_existent_file'), timeout=.1)
def test_await_pid(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.pid')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_pid_path', **PATCH_OPTS) as patched_pid:
patched_pid.return_value = test_filename
self.assertEqual(self.pm.await_pid(.1), 3333)
def test_await_socket(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.sock')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_socket_path', **PATCH_OPTS) as patched_socket:
patched_socket.return_value = test_filename
self.assertEqual(self.pm.await_socket(.1), 3333)
def test_maybe_init_metadata_dir(self):
with mock.patch('pants.pantsd.process_manager.safe_mkdir', **PATCH_OPTS) as mock_mkdir:
self.pm._maybe_init_metadata_dir()
mock_mkdir.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_abort(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
mock_alive.return_value = True
with self.assertRaises(AssertionError):
self.pm.purge_metadata()
def test_purge_metadata(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive, \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
mock_alive.return_value = False
self.pm.purge_metadata()
mock_rm_rf.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_alive_but_forced(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive, \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
mock_alive.return_value = True
self.pm.purge_metadata(force=True)
mock_rm_rf.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_metadata_error(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive, \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
mock_alive.return_value = False
mock_rm_rf.side_effect = OSError(errno.EACCES, os.strerror(errno.EACCES))
with self.assertRaises(ProcessManager.MetadataError):
self.pm.purge_metadata()
def test_get_metadata_dir(self):
self.assertEqual(self.pm.get_metadata_dir(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name))
def test_get_pid_path(self):
self.assertEqual(self.pm.get_pid_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'pid'))
def test_get_socket_path(self):
self.assertEqual(self.pm.get_socket_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'socket'))
def test_write_pid(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write, \
mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_pid(3333)
patched_write.assert_called_once_with(self.pm.get_pid_path(), '3333')
patched_init.assert_called_once_with()
def test_write_socket(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write, \
mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_socket(3333)
patched_write.assert_called_once_with(self.pm.get_socket_path(), '3333')
patched_init.assert_called_once_with()
def test_get_pid(self):
with mock.patch.object(ProcessManager, '_read_file', **PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm._get_pid(), 3333)
def test_get_socket(self):
with mock.patch.object(ProcessManager, '_read_file', **PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm._get_socket(), 3333)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process = mock.Mock(status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(mock_kill.called, 'If we have no pid, kills should noop gracefully.')
@contextmanager
def setup_terminate(self):
with mock.patch.object(ProcessManager, '_kill', **PATCH_OPTS) as mock_kill, \
mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=True)
self.assertEqual(mock_kill.call_count, len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
def test_get_subprocess_output(self):
test_str = '333'
self.assertEqual(self.pm.get_subprocess_output(['echo', '-n', test_str]), test_str)
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['i_do_not_exist'])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['false'])
@contextmanager
def mock_daemonize_context(self, chk_pre=True, chk_post_child=False, chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent, \
mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child, \
mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge, \
mock.patch('os.chdir', **PATCH_OPTS), \
mock.patch('os._exit', **PATCH_OPTS), \
mock.patch('os.setsid', **PATCH_OPTS), \
mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent: mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
class TestProcessManager(TestBase):
def setUp(self):
super(TestProcessManager, self).setUp()
# N.B. We pass in `metadata_base_dir` here because ProcessManager (itself a non-task/non-
# subsystem) depends on an initialized `GlobalOptions` subsystem for the value of
# `--pants-subprocessdir` in the default case. This is normally provided by subsystem
# dependencies in a typical pants run (and integration tests), but not in unit tests.
# Thus, passing this parameter here short-circuits the subsystem-reliant path for the
# purposes of unit testing without requiring adhoc subsystem initialization.
self.pm = ProcessManager('test', metadata_base_dir=self.subprocess_dir)
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='name',
cmdline=['cmd', 'line'],
status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_get_subprocess_output(self):
test_str = '333'
self.assertEqual(self.pm.get_subprocess_output(['echo', '-n', test_str]), test_str)
def test_get_subprocess_output_interleaved(self):
cmd_payload = 'import sys; ' + 'sys.stderr.write("9"); sys.stdout.write("3"); ' * 3
cmd = [sys.executable, '-c', cmd_payload]
self.assertEqual(self.pm.get_subprocess_output(cmd), '333')
self.assertEqual(self.pm.get_subprocess_output(cmd, ignore_stderr=False), '939393')
self.assertEqual(self.pm.get_subprocess_output(cmd, stderr=subprocess.STDOUT), '939393')
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['i_do_not_exist'])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['false'])
def test_await_pid(self):
with mock.patch.object(ProcessManager, 'await_metadata_by_name') as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name, 'pid', 5, mock.ANY)
def test_await_socket(self):
with mock.patch.object(ProcessManager, 'await_metadata_by_name') as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(self.pm.name, 'socket', 5, mock.ANY)
def test_write_pid(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, 'pid', '31337')
def test_write_socket(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_socket('/path/to/unix/socket')
mock_write.assert_called_once_with(self.pm.name, 'socket', '/path/to/unix/socket')
def test_write_named_socket(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_named_socket('pailgun', '31337')
mock_write.assert_called_once_with(self.pm.name, 'socket_pailgun', '31337')
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_extra_check(self):
def extra_check(process):
return False
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.assertFalse(self.pm.is_alive(extra_check))
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True):
with self.assertRaises(self.pm.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True), \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(mock_kill.called, 'If we have no pid, kills should noop gracefully.')
@contextmanager
def setup_terminate(self):
with mock.patch.object(ProcessManager, '_kill', **PATCH_OPTS) as mock_kill, \
mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=True)
self.assertEqual(mock_kill.call_count, len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self, chk_pre=True, chk_post_child=False, chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent, \
mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child, \
mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge, \
mock.patch('os._exit', **PATCH_OPTS), \
mock.patch('os.chdir', **PATCH_OPTS), \
mock.patch('os.setsid', **PATCH_OPTS), \
mock.patch('os.waitpid', **PATCH_OPTS), \
mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent: mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def setUp(self):
super().setUp()
self.pm = ProcessManager("test", metadata_base_dir=safe_mkdtemp())
def setUp(self):
self.pm = ProcessManager('test')
class TestProcessManager(unittest.TestCase):
def setUp(self):
super().setUp()
self.pm = ProcessManager("test", metadata_base_dir=safe_mkdtemp())
def test_await_pid(self):
with unittest.mock.patch.object(
ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name,
"pid",
"test to start",
"test started",
5,
caster=unittest.mock.ANY)
def test_await_socket(self):
with unittest.mock.patch.object(
ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(
self.pm.name,
"socket",
"test socket to be opened",
"test socket opened",
5,
caster=unittest.mock.ANY,
)
def test_write_pid(self):
with unittest.mock.patch.object(
ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, "pid", "31337")
def test_write_socket(self):
with unittest.mock.patch.object(
ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_socket("/path/to/unix/socket")
mock_write.assert_called_once_with(self.pm.name, "socket",
"/path/to/unix/socket")
def test_as_process(self):
sentinel = 3333
with unittest.mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm.write_pid(sentinel)
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with unittest.mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm.write_pid(fake_pid)
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
with self.assertRaises(self.pm.NotStarted):
self.pm._as_process()
def test_is_alive_neg(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="not_test", pid=3, status=psutil.STATUS_IDLE)
self.pm.write_process_name("test")
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_extra_check(self):
def extra_check(process):
return False
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_IDLE)
self.assertFalse(self.pm.is_alive(extra_check))
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with unittest.mock.patch.object(ProcessManager,
"is_alive",
return_value=True):
with self.assertRaises(ProcessManager.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with unittest.mock.patch.object(
ProcessManager, "is_alive",
return_value=True), unittest.mock.patch(
"pants.pantsd.process_manager.rm_rf") as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with unittest.mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm.write_pid(42)
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with unittest.mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(
mock_kill.called,
"If we have no pid, kills should noop gracefully.")
@contextmanager
def setup_terminate(self):
with unittest.mock.patch.object(
ProcessManager, "_kill",
**PATCH_OPTS) as mock_kill, unittest.mock.patch.object(
ProcessManager, "is_alive",
**PATCH_OPTS) as mock_alive, unittest.mock.patch.object(
ProcessManager, "purge_metadata",
**PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(ProcessManager.NonResponsiveProcess):
self.pm.terminate(kill_wait=0.1, purge=True)
self.assertEqual(mock_kill.call_count,
len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self,
chk_pre=True,
chk_post_child=False,
chk_post_parent=False):
with unittest.mock.patch.object(
ProcessManager, "post_fork_parent",
**PATCH_OPTS) as mock_post_parent, unittest.mock.patch.object(
ProcessManager, "post_fork_child", **
PATCH_OPTS) as mock_post_child, unittest.mock.patch.object(
ProcessManager, "pre_fork",
**PATCH_OPTS) as mock_pre, unittest.mock.patch.object(
ProcessManager, "purge_metadata",
**PATCH_OPTS) as mock_purge, unittest.mock.patch(
"os._exit", **PATCH_OPTS), unittest.mock.patch(
"os.chdir",
**PATCH_OPTS), unittest.mock.patch(
"os.setsid",
**PATCH_OPTS), unittest.mock.patch(
"os.waitpid",
**PATCH_OPTS), unittest.mock.patch(
"os.fork",
**PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre:
mock_pre.assert_called_once_with(self.pm)
if chk_post_child:
mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent:
mock_post_parent.assert_called_once_with(self.pm)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='name',
cmdline=['cmd', 'line'],
status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_maybe_cast(self):
self.assertIsNone(self.pm._maybe_cast(None, int))
self.assertEqual(self.pm._maybe_cast('3333', int), 3333)
self.assertEqual(self.pm._maybe_cast('ssss', int), 'ssss')
def test_readwrite_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
test_content = '3333'
self.pm._write_file(test_filename, test_content)
self.assertEqual(self.pm._read_file(test_filename), test_content)
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_wait_for_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
self.pm._write_file(test_filename, 'test')
self.pm._wait_for_file(test_filename, timeout=.1)
def test_wait_for_file_timeout(self):
with temporary_dir() as td:
with self.assertRaises(self.pm.Timeout):
self.pm._wait_for_file(os.path.join(td, 'non_existent_file'), timeout=.1)
def test_await_pid(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.pid')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_pid_path', **PATCH_OPTS) as patched_pid:
patched_pid.return_value = test_filename
self.assertEqual(self.pm.await_pid(.1), 3333)
def test_await_socket(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.sock')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_socket_path', **PATCH_OPTS) as patched_socket:
patched_socket.return_value = test_filename
self.assertEqual(self.pm.await_socket(.1), 3333)
def test_maybe_init_metadata_dir(self):
with mock.patch('pants.pantsd.process_manager.safe_mkdir', **PATCH_OPTS) as mock_mkdir:
self.pm._maybe_init_metadata_dir()
mock_mkdir.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_abort(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
mock_alive.return_value = True
with self.assertRaises(AssertionError):
self.pm._purge_metadata()
@mock.patch('pants.pantsd.process_manager.safe_delete')
def test_purge_metadata(self, *args):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
with mock.patch('os.path.exists') as mock_exists:
mock_alive.return_value = False
mock_exists.return_value = True
self.pm._purge_metadata()
def test_get_metadata_dir(self):
self.assertEqual(self.pm.get_metadata_dir(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name))
def test_get_pid_path(self):
self.assertEqual(self.pm.get_pid_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'pid'))
def test_get_socket_path(self):
self.assertEqual(self.pm.get_socket_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'socket'))
def test_write_pid(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write:
with mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_pid(3333)
patched_write.assert_called_once_with(self.pm.get_pid_path(), '3333')
patched_init.assert_called_once_with()
def test_write_socket(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write:
with mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_socket(3333)
patched_write.assert_called_once_with(self.pm.get_socket_path(), '3333')
patched_init.assert_called_once_with()
def test_get_pid(self):
with mock.patch.object(ProcessManager, '_read_file', **PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm._get_pid(), 3333)
def test_get_socket(self):
with mock.patch.object(ProcessManager, '_read_file', **PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm._get_socket(), 3333)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process = mock.Mock(status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(mock_kill.called, 'If we have no pid, kills should noop gracefully.')
def test_terminate(self):
with mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive:
with mock.patch('os.kill', **PATCH_OPTS):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=False)
def test_run_subprocess(self):
test_str = '333'
proc = self.pm.run_subprocess(['echo', test_str], stdout=subprocess.PIPE)
proc.wait()
self.assertEqual(proc.communicate()[0].strip(), test_str)
@contextmanager
def mock_daemonize_context(self, chk_pre=True, chk_post_child=False, chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent:
with mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child:
with mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre:
with mock.patch('os.chdir', **PATCH_OPTS):
with mock.patch('os._exit', **PATCH_OPTS):
with mock.patch('os.setsid', **PATCH_OPTS):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent: mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self, *args):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self, *args):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self, *args):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self, *args):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self, *args):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
class TestProcessManager(TestBase):
def setUp(self):
super().setUp()
# N.B. We pass in `metadata_base_dir` here because ProcessManager (itself a non-task/non-
# subsystem) depends on an initialized `GlobalOptions` subsystem for the value of
# `--pants-subprocessdir` in the default case. This is normally provided by subsystem
# dependencies in a typical pants run (and integration tests), but not in unit tests.
# Thus, passing this parameter here short-circuits the subsystem-reliant path for the
# purposes of unit testing without requiring adhoc subsystem initialization.
self.pm = ProcessManager("test", metadata_base_dir=self.subprocess_dir)
def test_process_properties(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="name", cmdline=["cmd", "line"], status="status")
self.assertEqual(self.pm.cmdline, ["cmd", "line"])
self.assertEqual(self.pm.cmd, "cmd")
def test_process_properties_cmd_indexing(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline="")
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_get_subprocess_output(self):
test_str = "333"
self.assertEqual(
self.pm.get_subprocess_output(["echo", "-n", test_str]), test_str)
def test_get_subprocess_output_interleaved(self):
cmd_payload = "import sys; " + (
'sys.stderr.write("9"); sys.stderr.flush(); sys.stdout.write("3"); sys.stdout.flush();'
* 3)
cmd = [sys.executable, "-c", cmd_payload]
self.assertEqual(self.pm.get_subprocess_output(cmd), "333")
self.assertEqual(
self.pm.get_subprocess_output(cmd, ignore_stderr=False), "939393")
self.assertEqual(
self.pm.get_subprocess_output(cmd, stderr=subprocess.STDOUT),
"939393")
def test_get_subprocess_output_interleaved_bash(self):
cmd_payload = 'printf "9">&2; printf "3";' * 3
cmd = ["/bin/bash", "-c", cmd_payload]
self.assertEqual(self.pm.get_subprocess_output(cmd), "333")
self.assertEqual(
self.pm.get_subprocess_output(cmd, ignore_stderr=False), "939393")
self.assertEqual(
self.pm.get_subprocess_output(cmd, stderr=subprocess.STDOUT),
"939393")
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(ProcessManager.ExecutionError):
self.pm.get_subprocess_output(["i_do_not_exist"])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(ProcessManager.ExecutionError):
self.pm.get_subprocess_output(["false"])
def test_await_pid(self):
with unittest.mock.patch.object(
ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name,
"pid",
"test to start",
"test started",
5,
caster=unittest.mock.ANY)
def test_await_socket(self):
with unittest.mock.patch.object(
ProcessManager, "await_metadata_by_name") as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(
self.pm.name,
"socket",
"test socket to be opened",
"test socket opened",
5,
caster=unittest.mock.ANY,
)
def test_write_pid(self):
with unittest.mock.patch.object(
ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, "pid", "31337")
def test_write_socket(self):
with unittest.mock.patch.object(
ProcessManager, "write_metadata_by_name") as mock_write:
self.pm.write_socket("/path/to/unix/socket")
mock_write.assert_called_once_with(self.pm.name, "socket",
"/path/to/unix/socket")
def test_as_process(self):
sentinel = 3333
with unittest.mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with unittest.mock.patch("psutil.Process", **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_is_alive_neg(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="not_test", pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = "test"
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_extra_check(self):
def extra_check(process):
return False
with unittest.mock.patch.object(ProcessManager, "_as_process",
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name="test", pid=3, status=psutil.STATUS_IDLE)
self.assertFalse(self.pm.is_alive(extra_check))
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with unittest.mock.patch.object(ProcessManager,
"is_alive",
return_value=True):
with self.assertRaises(ProcessManager.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with unittest.mock.patch.object(
ProcessManager, "is_alive",
return_value=True), unittest.mock.patch(
"pants.pantsd.process_manager.rm_rf") as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with unittest.mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with unittest.mock.patch("os.kill", **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(
mock_kill.called,
"If we have no pid, kills should noop gracefully.")
@contextmanager
def setup_terminate(self):
with unittest.mock.patch.object(
ProcessManager, "_kill",
**PATCH_OPTS) as mock_kill, unittest.mock.patch.object(
ProcessManager, "is_alive",
**PATCH_OPTS) as mock_alive, unittest.mock.patch.object(
ProcessManager, "purge_metadata",
**PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError("oops")
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=0.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(ProcessManager.NonResponsiveProcess):
self.pm.terminate(kill_wait=0.1, purge=True)
self.assertEqual(mock_kill.call_count,
len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self,
chk_pre=True,
chk_post_child=False,
chk_post_parent=False):
with unittest.mock.patch.object(
ProcessManager, "post_fork_parent",
**PATCH_OPTS) as mock_post_parent, unittest.mock.patch.object(
ProcessManager, "post_fork_child", **
PATCH_OPTS) as mock_post_child, unittest.mock.patch.object(
ProcessManager, "pre_fork",
**PATCH_OPTS) as mock_pre, unittest.mock.patch.object(
ProcessManager, "purge_metadata",
**PATCH_OPTS) as mock_purge, unittest.mock.patch(
"os._exit", **PATCH_OPTS), unittest.mock.patch(
"os.chdir",
**PATCH_OPTS), unittest.mock.patch(
"os.setsid",
**PATCH_OPTS), unittest.mock.patch(
"os.waitpid",
**PATCH_OPTS), unittest.mock.patch(
"os.fork",
**PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre:
mock_pre.assert_called_once_with(self.pm)
if chk_post_child:
mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent:
mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [1, 1
] # Simulate the original parent process.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def __init__(self, context=None, options=None): ProcessManager.__init__(self, name='reporting_server') self.context = context self.options = options
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_maybe_cast(self):
self.assertIsNone(self.pm._maybe_cast(None, int))
self.assertEqual(self.pm._maybe_cast('3333', int), 3333)
self.assertEqual(self.pm._maybe_cast('ssss', int), 'ssss')
def test_readwrite_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
test_content = '3333'
self.pm._write_file(test_filename, test_content)
self.assertEqual(self.pm._read_file(test_filename), test_content)
def test_as_process(self):
sentinel = 3333
with mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS):
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.assertEqual(self.pm.as_process(), sentinel)
def test_wait_for_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
self.pm._write_file(test_filename, 'test')
self.pm._wait_for_file(test_filename, timeout=.1)
def test_wait_for_file_timeout(self):
with temporary_dir() as td:
with self.assertRaises(self.pm.Timeout):
self.pm._wait_for_file(os.path.join(td, 'non_existent_file'), timeout=.1)
def test_await_pid(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.pid')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_pid_path', **PATCH_OPTS) as patched_pid:
patched_pid.return_value = test_filename
self.assertEqual(self.pm.await_pid(.1), 3333)
def test_await_socket(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.sock')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_socket_path', **PATCH_OPTS) as patched_socket:
patched_socket.return_value = test_filename
self.assertEqual(self.pm.await_socket(.1), 3333)
def test_maybe_init_metadata_dir(self):
with mock.patch('pants.pantsd.process_manager.safe_mkdir', **PATCH_OPTS) as mock_mkdir:
self.pm._maybe_init_metadata_dir()
mock_mkdir.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_abort(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
mock_alive.return_value = True
with self.assertRaises(AssertionError):
self.pm._purge_metadata()
@mock.patch('pants.pantsd.process_manager.safe_delete')
def test_purge_metadata(self, *args):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
with mock.patch('os.path.exists') as mock_exists:
mock_alive.return_value = False
mock_exists.return_value = True
self.pm._purge_metadata()
def test_get_metadata_dir(self):
self.assertEqual(self.pm.get_metadata_dir(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name))
def test_get_pid_path(self):
self.assertEqual(self.pm.get_pid_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'pid'))
def test_get_socket_path(self):
self.assertEqual(self.pm.get_socket_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'socket'))
def test_write_pid(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write:
with mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_pid(3333)
patched_write.assert_called_once_with(self.pm.get_pid_path(), '3333')
patched_init.assert_called_once_with()
def test_write_socket(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write:
with mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_socket(3333)
patched_write.assert_called_once_with(self.pm.get_socket_path(), '3333')
patched_init.assert_called_once_with()
def test_get_pid(self):
with mock.patch.object(ProcessManager, '_read_file', **PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm.get_pid(), 3333)
def test_get_socket(self):
with mock.patch.object(ProcessManager, '_read_file', **PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm.get_socket(), 3333)
def test_is_alive(self):
with mock.patch('psutil.pid_exists', **PATCH_OPTS) as mock_psutil:
mock_psutil.return_value = False
self.assertFalse(self.pm.is_alive())
mock_psutil.assert_called_once_with(None)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm.kill(0)
mock_kill.assert_called_once_with(None, 0)
def test_terminate(self):
with mock.patch('psutil.pid_exists', **PATCH_OPTS) as mock_exists:
with mock.patch('os.kill', **PATCH_OPTS):
mock_exists.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=False)
def test_run_subprocess(self):
test_str = '333'
proc = self.pm.run_subprocess(['echo', test_str], stdout=subprocess.PIPE)
proc.wait()
self.assertEqual(proc.communicate()[0].strip(), test_str)
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemonize_parent(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
# TODO(Kris Wilson): check that callbacks were called appropriately here and for daemon_spawn.
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemonize_child(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemonize_child_parent(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemon_spawn_parent(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemon_spawn_child(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='name', cmdline=['cmd', 'line'], status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_maybe_cast(self):
self.assertIsNone(self.pm._maybe_cast(None, int))
self.assertEqual(self.pm._maybe_cast('3333', int), 3333)
self.assertEqual(self.pm._maybe_cast('ssss', int), 'ssss')
def test_readwrite_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
test_content = '3333'
self.pm._write_file(test_filename, test_content)
self.assertEqual(self.pm._read_file(test_filename), test_content)
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_deadline_until(self):
with self.assertRaises(self.pm.Timeout):
self.pm._deadline_until(lambda: False, timeout=.1)
def test_wait_for_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
self.pm._write_file(test_filename, 'test')
self.pm._wait_for_file(test_filename, timeout=.1)
def test_wait_for_file_timeout(self):
with temporary_dir() as td:
with self.assertRaises(self.pm.Timeout):
self.pm._wait_for_file(os.path.join(td, 'non_existent_file'),
timeout=.1)
def test_await_pid(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.pid')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_pid_path',
**PATCH_OPTS) as patched_pid:
patched_pid.return_value = test_filename
self.assertEqual(self.pm.await_pid(.1), 3333)
def test_await_socket(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.sock')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_socket_path',
**PATCH_OPTS) as patched_socket:
patched_socket.return_value = test_filename
self.assertEqual(self.pm.await_socket(.1), 3333)
def test_maybe_init_metadata_dir(self):
with mock.patch('pants.pantsd.process_manager.safe_mkdir',
**PATCH_OPTS) as mock_mkdir:
self.pm._maybe_init_metadata_dir()
mock_mkdir.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_abort(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
mock_alive.return_value = True
with self.assertRaises(AssertionError):
self.pm._purge_metadata()
@mock.patch('pants.pantsd.process_manager.safe_delete')
def test_purge_metadata(self, *args):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive, \
mock.patch('os.path.exists') as mock_exists:
mock_alive.return_value = False
mock_exists.return_value = True
self.pm._purge_metadata()
def test_get_metadata_dir(self):
self.assertEqual(
self.pm.get_metadata_dir(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name))
def test_get_pid_path(self):
self.assertEqual(
self.pm.get_pid_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'pid'))
def test_get_socket_path(self):
self.assertEqual(
self.pm.get_socket_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'socket'))
def test_write_pid(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write, \
mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_pid(3333)
patched_write.assert_called_once_with(self.pm.get_pid_path(),
'3333')
patched_init.assert_called_once_with()
def test_write_socket(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write, \
mock.patch.object(ProcessManager, '_maybe_init_metadata_dir') as patched_init:
self.pm.write_socket(3333)
patched_write.assert_called_once_with(self.pm.get_socket_path(),
'3333')
patched_init.assert_called_once_with()
def test_get_pid(self):
with mock.patch.object(ProcessManager, '_read_file',
**PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm._get_pid(), 3333)
def test_get_socket(self):
with mock.patch.object(ProcessManager, '_read_file',
**PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm._get_socket(), 3333)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process = mock.Mock(status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process',
**PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(
name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(
mock_kill.called,
'If we have no pid, kills should noop gracefully.')
@contextmanager
def setup_terminate(self, assert_purged=True):
with mock.patch.object(ProcessManager, '_kill', **PATCH_OPTS) as mock_kill, \
mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive, \
mock.patch.object(ProcessManager, '_purge_metadata', **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive
if assert_purged:
mock_purge.assert_called_once_with(self.pm)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=.1)
def test_terminate(self):
with self.setup_terminate(assert_purged=False) as (mock_kill,
mock_alive):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=False)
def test_get_subprocess_output(self):
test_str = '333'
self.assertEqual(
self.pm.get_subprocess_output(['echo', '-n', test_str]), test_str)
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['i_do_not_exist'])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['false'])
@contextmanager
def mock_daemonize_context(self,
chk_pre=True,
chk_post_child=False,
chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent, \
mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child, \
mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre, \
mock.patch('os.chdir', **PATCH_OPTS), \
mock.patch('os._exit', **PATCH_OPTS), \
mock.patch('os.setsid', **PATCH_OPTS), \
mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent:
mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self, *args):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self, *args):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self, *args):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self, *args):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self, *args):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
class TestProcessManager(BaseTest):
def setUp(self):
super(TestProcessManager, self).setUp()
# N.B. We pass in `metadata_base_dir` here because ProcessManager (itself a non-task/non-
# subsystem) depends on an initialized `GlobalOptions` subsystem for the value of
# `--pants-subprocessdir` in the default case. This is normally provided by subsystem
# dependencies in a typical pants run (and integration tests), but not in unit tests.
# Thus, passing this parameter here short-circuits the subsystem-reliant path for the
# purposes of unit testing without requiring adhoc subsystem initialization.
self.pm = ProcessManager('test', metadata_base_dir=self.subprocess_dir)
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='name',
cmdline=['cmd', 'line'],
status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_get_subprocess_output(self):
test_str = '333'
self.assertEqual(self.pm.get_subprocess_output(['echo', '-n', test_str]), test_str)
def test_get_subprocess_output_interleaved(self):
cmd_payload = 'import sys; ' + 'sys.stderr.write("9"); sys.stdout.write("3"); ' * 3
cmd = [sys.executable, '-c', cmd_payload]
self.assertEqual(self.pm.get_subprocess_output(cmd), '333')
self.assertEqual(self.pm.get_subprocess_output(cmd, ignore_stderr=False), '939393')
self.assertEqual(self.pm.get_subprocess_output(cmd, stderr=subprocess.STDOUT), '939393')
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['i_do_not_exist'])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['false'])
def test_await_pid(self):
with mock.patch.object(ProcessManager, 'await_metadata_by_name') as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name, 'pid', 5, mock.ANY)
def test_await_socket(self):
with mock.patch.object(ProcessManager, 'await_metadata_by_name') as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(self.pm.name, 'socket', 5, mock.ANY)
def test_write_pid(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, 'pid', '31337')
def test_write_socket(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_socket('/path/to/unix/socket')
mock_write.assert_called_once_with(self.pm.name, 'socket', '/path/to/unix/socket')
def test_write_named_socket(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_named_socket('pailgun', '31337')
mock_write.assert_called_once_with(self.pm.name, 'socket_pailgun', '31337')
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_extra_check(self):
def extra_check(process):
return False
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.assertFalse(self.pm.is_alive(extra_check))
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True):
with self.assertRaises(self.pm.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True), \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(mock_kill.called, 'If we have no pid, kills should noop gracefully.')
@contextmanager
def setup_terminate(self):
with mock.patch.object(ProcessManager, '_kill', **PATCH_OPTS) as mock_kill, \
mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=True)
self.assertEqual(mock_kill.call_count, len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self, chk_pre=True, chk_post_child=False, chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent, \
mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child, \
mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge, \
mock.patch('os._exit', **PATCH_OPTS), \
mock.patch('os.chdir', **PATCH_OPTS), \
mock.patch('os.setsid', **PATCH_OPTS), \
mock.patch('os.waitpid', **PATCH_OPTS), \
mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent: mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()
def setUp(self):
self.pm = ProcessManager("test")
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_maybe_cast(self):
self.assertIsNone(self.pm._maybe_cast(None, int))
self.assertEqual(self.pm._maybe_cast('3333', int), 3333)
self.assertEqual(self.pm._maybe_cast('ssss', int), 'ssss')
def test_readwrite_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
test_content = '3333'
self.pm._write_file(test_filename, test_content)
self.assertEqual(self.pm._read_file(test_filename), test_content)
def test_as_process(self):
sentinel = 3333
with mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS):
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.assertEqual(self.pm.as_process(), sentinel)
def test_wait_for_file(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.out')
self.pm._write_file(test_filename, 'test')
self.pm._wait_for_file(test_filename, timeout=.1)
def test_wait_for_file_timeout(self):
with temporary_dir() as td:
with self.assertRaises(self.pm.Timeout):
self.pm._wait_for_file(os.path.join(td, 'non_existent_file'),
timeout=.1)
def test_await_pid(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.pid')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_pid_path',
**PATCH_OPTS) as patched_pid:
patched_pid.return_value = test_filename
self.assertEqual(self.pm.await_pid(.1), 3333)
def test_await_socket(self):
with temporary_dir() as td:
test_filename = os.path.join(td, 'test.sock')
self.pm._write_file(test_filename, '3333')
with mock.patch.object(ProcessManager, 'get_socket_path',
**PATCH_OPTS) as patched_socket:
patched_socket.return_value = test_filename
self.assertEqual(self.pm.await_socket(.1), 3333)
def test_maybe_init_metadata_dir(self):
with mock.patch('pants.pantsd.process_manager.safe_mkdir',
**PATCH_OPTS) as mock_mkdir:
self.pm._maybe_init_metadata_dir()
mock_mkdir.assert_called_once_with(self.pm.get_metadata_dir())
def test_purge_metadata_abort(self):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
mock_alive.return_value = True
with self.assertRaises(AssertionError):
self.pm._purge_metadata()
@mock.patch('pants.pantsd.process_manager.safe_delete')
def test_purge_metadata(self, *args):
with mock.patch.object(ProcessManager, 'is_alive') as mock_alive:
with mock.patch('os.path.exists') as mock_exists:
mock_alive.return_value = False
mock_exists.return_value = True
self.pm._purge_metadata()
def test_get_metadata_dir(self):
self.assertEqual(
self.pm.get_metadata_dir(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name))
def test_get_pid_path(self):
self.assertEqual(
self.pm.get_pid_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'pid'))
def test_get_socket_path(self):
self.assertEqual(
self.pm.get_socket_path(),
os.path.join(self.pm._buildroot, '.pids', self.pm._name, 'socket'))
def test_write_pid(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write:
with mock.patch.object(ProcessManager,
'_maybe_init_metadata_dir') as patched_init:
self.pm.write_pid(3333)
patched_write.assert_called_once_with(self.pm.get_pid_path(),
'3333')
patched_init.assert_called_once_with()
def test_write_socket(self):
with mock.patch.object(ProcessManager, '_write_file') as patched_write:
with mock.patch.object(ProcessManager,
'_maybe_init_metadata_dir') as patched_init:
self.pm.write_socket(3333)
patched_write.assert_called_once_with(
self.pm.get_socket_path(), '3333')
patched_init.assert_called_once_with()
def test_get_pid(self):
with mock.patch.object(ProcessManager, '_read_file',
**PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm.get_pid(), 3333)
def test_get_socket(self):
with mock.patch.object(ProcessManager, '_read_file',
**PATCH_OPTS) as patched_pm:
patched_pm.return_value = '3333'
self.assertEqual(self.pm.get_socket(), 3333)
def test_is_alive(self):
with mock.patch('psutil.pid_exists', **PATCH_OPTS) as mock_psutil:
mock_psutil.return_value = False
self.assertFalse(self.pm.is_alive())
mock_psutil.assert_called_once_with(None)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm.kill(0)
mock_kill.assert_called_once_with(None, 0)
def test_terminate(self):
with mock.patch('psutil.pid_exists', **PATCH_OPTS) as mock_exists:
with mock.patch('os.kill', **PATCH_OPTS):
mock_exists.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=False)
def test_run_subprocess(self):
test_str = '333'
proc = self.pm.run_subprocess(['echo', test_str],
stdout=subprocess.PIPE)
proc.wait()
self.assertEqual(proc.communicate()[0].strip(), test_str)
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemonize_parent(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
# TODO(Kris Wilson): check that callbacks were called appropriately here and for daemon_spawn.
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemonize_child(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemonize_child_parent(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemon_spawn_parent(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
@mock.patch('os.umask', **PATCH_OPTS)
@mock.patch('os.chdir', **PATCH_OPTS)
@mock.patch('os._exit', **PATCH_OPTS)
@mock.patch('os.setsid', **PATCH_OPTS)
def test_daemon_spawn_child(self, *args):
with mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def _normalize_watchman_path(self, watchman_path):
if not self._is_valid_executable(watchman_path):
raise ProcessManager.ExecutionError('invalid watchman binary at {}!'.format(watchman_path))
return os.path.abspath(watchman_path)
def __init__(self, context=None, options=None):
ProcessManager.__init__(self, name='reporting_server')
self.context = context
self.options = options
class TestProcessManager(unittest.TestCase):
def setUp(self):
self.pm = ProcessManager('test')
def test_process_properties(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='name',
cmdline=['cmd', 'line'],
status='status')
self.assertEqual(self.pm.cmdline, ['cmd', 'line'])
self.assertEqual(self.pm.cmd, 'cmd')
def test_process_properties_cmd_indexing(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(cmdline='')
self.assertEqual(self.pm.cmd, None)
def test_process_properties_none(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_asproc:
mock_asproc.return_value = None
self.assertEqual(self.pm.cmdline, None)
self.assertEqual(self.pm.cmd, None)
def test_get_subprocess_output(self):
test_str = '333'
self.assertEqual(self.pm.get_subprocess_output(['echo', '-n', test_str]), test_str)
def test_get_subprocess_output_oserror_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['i_do_not_exist'])
def test_get_subprocess_output_failure_exception(self):
with self.assertRaises(self.pm.ExecutionError):
self.pm.get_subprocess_output(['false'])
def test_await_pid(self):
with mock.patch.object(ProcessManager, 'await_metadata_by_name') as mock_await:
self.pm.await_pid(5)
mock_await.assert_called_once_with(self.pm.name, 'pid', 5, mock.ANY)
def test_await_socket(self):
with mock.patch.object(ProcessManager, 'await_metadata_by_name') as mock_await:
self.pm.await_socket(5)
mock_await.assert_called_once_with(self.pm.name, 'socket', 5, mock.ANY)
def test_write_pid(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_pid(31337)
mock_write.assert_called_once_with(self.pm.name, 'pid', '31337')
def test_write_socket(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_socket('/path/to/unix/socket')
mock_write.assert_called_once_with(self.pm.name, 'socket', '/path/to/unix/socket')
def test_write_named_socket(self):
with mock.patch.object(ProcessManager, 'write_metadata_by_name') as mock_write:
self.pm.write_named_socket('pailgun', '31337')
mock_write.assert_called_once_with(self.pm.name, 'socket_pailgun', '31337')
def test_as_process(self):
sentinel = 3333
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.return_value = sentinel
self.pm._pid = sentinel
self.assertEqual(self.pm._as_process(), sentinel)
def test_as_process_no_pid(self):
fake_pid = 3
with mock.patch('psutil.Process', **PATCH_OPTS) as mock_proc:
mock_proc.side_effect = psutil.NoSuchProcess(fake_pid)
self.pm._pid = fake_pid
with self.assertRaises(psutil.NoSuchProcess):
self.pm._as_process()
def test_as_process_none(self):
self.assertEqual(self.pm._as_process(), None)
def test_is_alive_neg(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = None
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_once_with(self.pm)
def test_is_alive(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.assertTrue(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_ZOMBIE)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_zombie_exception(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.side_effect = psutil.NoSuchProcess(0)
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_stale_pid(self):
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='not_test', pid=3, status=psutil.STATUS_IDLE)
self.pm._process_name = 'test'
self.assertFalse(self.pm.is_alive())
mock_as_process.assert_called_with(self.pm)
def test_is_alive_extra_check(self):
def extra_check(process):
return False
with mock.patch.object(ProcessManager, '_as_process', **PATCH_OPTS) as mock_as_process:
mock_as_process.return_value = fake_process(name='test', pid=3, status=psutil.STATUS_IDLE)
self.assertFalse(self.pm.is_alive(extra_check))
mock_as_process.assert_called_with(self.pm)
def test_purge_metadata_aborts(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True):
with self.assertRaises(self.pm.MetadataError):
self.pm.purge_metadata()
def test_purge_metadata_alive_but_forced(self):
with mock.patch.object(ProcessManager, 'is_alive', return_value=True), \
mock.patch('pants.pantsd.process_manager.rm_rf') as mock_rm_rf:
self.pm.purge_metadata(force=True)
self.assertGreater(mock_rm_rf.call_count, 0)
def test_kill(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._pid = 42
self.pm._kill(0)
mock_kill.assert_called_once_with(42, 0)
def test_kill_no_pid(self):
with mock.patch('os.kill', **PATCH_OPTS) as mock_kill:
self.pm._kill(0)
self.assertFalse(mock_kill.called, 'If we have no pid, kills should noop gracefully.')
@contextmanager
def setup_terminate(self):
with mock.patch.object(ProcessManager, '_kill', **PATCH_OPTS) as mock_kill, \
mock.patch.object(ProcessManager, 'is_alive', **PATCH_OPTS) as mock_alive, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge:
yield mock_kill, mock_alive, mock_purge
self.assertGreater(mock_alive.call_count, 0)
def test_terminate_quick_death(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_quick_death_no_purge(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_kill.side_effect = OSError('oops')
mock_alive.side_effect = [True, False]
self.pm.terminate(purge=False, kill_wait=.1)
self.assertEqual(mock_kill.call_count, 1)
self.assertEqual(mock_purge.call_count, 0)
def test_terminate_already_dead(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = False
self.pm.terminate(purge=True)
self.assertEqual(mock_kill.call_count, 0)
self.assertEqual(mock_purge.call_count, 1)
def test_terminate_no_kill(self):
with self.setup_terminate() as (mock_kill, mock_alive, mock_purge):
mock_alive.return_value = True
with self.assertRaises(self.pm.NonResponsiveProcess):
self.pm.terminate(kill_wait=.1, purge=True)
self.assertEqual(mock_kill.call_count, len(ProcessManager.KILL_CHAIN))
self.assertEqual(mock_purge.call_count, 0)
@contextmanager
def mock_daemonize_context(self, chk_pre=True, chk_post_child=False, chk_post_parent=False):
with mock.patch.object(ProcessManager, 'post_fork_parent', **PATCH_OPTS) as mock_post_parent, \
mock.patch.object(ProcessManager, 'post_fork_child', **PATCH_OPTS) as mock_post_child, \
mock.patch.object(ProcessManager, 'pre_fork', **PATCH_OPTS) as mock_pre, \
mock.patch.object(ProcessManager, 'purge_metadata', **PATCH_OPTS) as mock_purge, \
mock.patch('os._exit', **PATCH_OPTS), \
mock.patch('os.chdir', **PATCH_OPTS), \
mock.patch('os.setsid', **PATCH_OPTS), \
mock.patch('os.waitpid', **PATCH_OPTS), \
mock.patch('os.fork', **PATCH_OPTS) as mock_fork:
yield mock_fork
mock_purge.assert_called_once_with(self.pm)
if chk_pre: mock_pre.assert_called_once_with(self.pm)
if chk_post_child: mock_post_child.assert_called_once_with(self.pm)
if chk_post_parent: mock_post_parent.assert_called_once_with(self.pm)
def test_daemonize_parent(self):
with self.mock_daemonize_context() as mock_fork:
mock_fork.side_effect = [1, 1] # Simulate the parent.
self.pm.daemonize(write_pid=False)
def test_daemonize_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.side_effect = [0, 0] # Simulate the child.
self.pm.daemonize(write_pid=False)
def test_daemonize_child_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.side_effect = [0, 1] # Simulate the childs parent.
self.pm.daemonize(write_pid=False)
def test_daemon_spawn_parent(self):
with self.mock_daemonize_context(chk_post_parent=True) as mock_fork:
mock_fork.return_value = 1 # Simulate the parent.
self.pm.daemon_spawn()
def test_daemon_spawn_child(self):
with self.mock_daemonize_context(chk_post_child=True) as mock_fork:
mock_fork.return_value = 0 # Simulate the child.
self.pm.daemon_spawn()
def test_callbacks(self):
# For coverage.
self.pm.pre_fork()
self.pm.post_fork_child()
self.pm.post_fork_parent()