def setUp(self):
self.mock_handler_inst = unittest.mock.Mock()
# Add a fake environment for this to not timeout.
self.fake_environment = {"PANTSD_REQUEST_TIMEOUT_LIMIT": "-1"}
self.mock_handler_inst.parsed_request.return_value = (None, None, [], self.fake_environment)
self.mock_runner_factory = unittest.mock.Mock(
side_effect=Exception("this should never be called")
)
self.mock_handler_class = unittest.mock.Mock(return_value=self.mock_handler_inst)
self.lock = threading.RLock()
@contextmanager
def lock():
with self.lock:
yield
self.after_request_callback_calls = 0
def after_request_callback():
self.after_request_callback_calls += 1
with unittest.mock.patch.object(PailgunServer, "server_bind"), unittest.mock.patch.object(
PailgunServer, "server_activate"
):
self.server = PailgunServer(
server_address=("0.0.0.0", 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class,
lifecycle_lock=lock,
request_complete_callback=after_request_callback,
)
def setUp(self):
self.mock_handler_inst = mock.Mock()
self.mock_runner_factory = mock.Mock(
side_effect=Exception('this should never be called'))
self.mock_handler_class = mock.Mock(
return_value=self.mock_handler_inst)
self.lock = threading.RLock()
@contextmanager
def lock():
with self.lock:
yield
self.after_request_callback_calls = 0
def after_request_callback():
self.after_request_callback_calls += 1
with mock.patch.object(PailgunServer, 'server_bind'), \
mock.patch.object(PailgunServer, 'server_activate'):
self.server = PailgunServer(
server_address=('0.0.0.0', 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class,
lifecycle_lock=lock,
request_complete_callback=after_request_callback)
def setUp(self):
self.mock_handler_inst = mock.Mock()
self.mock_runner_factory = mock.Mock(side_effect=Exception('this should never be called'))
self.mock_handler_class = mock.Mock(return_value=self.mock_handler_inst)
with mock.patch.object(PailgunServer, 'server_bind'), \
mock.patch.object(PailgunServer, 'server_activate'):
self.server = PailgunServer(
server_address=('0.0.0.0', 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class
)
class TestPailgunServer(unittest.TestCase):
def setUp(self):
self.mock_handler_inst = mock.Mock()
self.mock_runner_factory = mock.Mock(side_effect=Exception('this should never be called'))
self.mock_handler_class = mock.Mock(return_value=self.mock_handler_inst)
self.lock = threading.RLock()
@contextmanager
def lock():
with self.lock:
yield
self.after_request_callback_calls = 0
def after_request_callback():
self.after_request_callback_calls += 1
with mock.patch.object(PailgunServer, 'server_bind'), \
mock.patch.object(PailgunServer, 'server_activate'):
self.server = PailgunServer(
server_address=('0.0.0.0', 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class,
lifecycle_lock=lock,
request_complete_callback=after_request_callback
)
@mock.patch.object(TCPServer, 'server_bind', **PATCH_OPTS)
def test_server_bind(self, mock_tcpserver_bind):
mock_sock = mock.Mock()
mock_sock.getsockname.return_value = ('0.0.0.0', 31337)
self.server.socket = mock_sock
self.server.server_bind()
self.assertEqual(self.server.server_port, 31337)
self.assertIs(mock_tcpserver_bind.called, True)
@mock.patch.object(PailgunServer, 'close_request', **PATCH_OPTS)
def test_process_request_thread(self, mock_close_request):
mock_request = mock.Mock()
self.server.process_request_thread(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
mock_close_request.assert_called_once_with(self.server, mock_request)
@mock.patch.object(PailgunServer, 'close_request', **PATCH_OPTS)
def test_process_request_calls_callback(self, mock_close_request):
mock_request = mock.Mock()
self.server.process_request_thread(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
assert(self.after_request_callback_calls == 1)
@mock.patch.object(PailgunServer, 'shutdown_request', **PATCH_OPTS)
def test_process_request_thread_error(self, mock_shutdown_request):
mock_request = mock.Mock()
self.mock_handler_inst.handle_request.side_effect = AttributeError('oops')
self.server.process_request_thread(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
self.assertIs(self.mock_handler_inst.handle_error.called, True)
mock_shutdown_request.assert_called_once_with(self.server, mock_request)
def setUp(self):
self.mock_handler_inst = mock.Mock()
self.mock_runner_factory = mock.Mock(side_effect=Exception('this should never be called'))
self.mock_handler_class = mock.Mock(return_value=self.mock_handler_inst)
self.lock = threading.RLock()
@contextmanager
def lock():
with self.lock:
yield
self.after_request_callback_calls = 0
def after_request_callback():
self.after_request_callback_calls += 1
with mock.patch.object(PailgunServer, 'server_bind'), \
mock.patch.object(PailgunServer, 'server_activate'):
self.server = PailgunServer(
server_address=('0.0.0.0', 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class,
lifecycle_lock=lock,
request_complete_callback=after_request_callback
)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
build_graph = None
self._logger.debug('execution commandline: %s', arguments)
if self._scheduler_service:
# N.B. This parses sys.argv by way of OptionsInitializer/OptionsBootstrapper prior to
# the main pants run to derive spec_roots for caching in the underlying scheduler.
spec_roots = self._parse_commandline_to_spec_roots(
args=arguments)
self._logger.debug('parsed spec_roots: %s', spec_roots)
try:
self._logger.debug('requesting BuildGraph from %s',
self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler for BuildGraph construction.
build_graph = self._scheduler_service.get_build_graph(
spec_roots)
except Exception:
self._logger.warning(
'encountered exception during SchedulerService.get_build_graph():\n%s',
traceback.format_exc())
return self._runner_class(sock, exiter, arguments, environment,
build_graph)
return PailgunServer(self._bind_addr, runner_factory)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
return self._runner_class(sock, exiter, arguments, environment)
return PailgunServer(self._bind_addr, runner_factory)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
return self._runner_class.create(sock, arguments, environment,
self._scheduler_service)
# Plumb the daemon's lifecycle lock to the `PailgunServer` to safeguard teardown.
@contextmanager
def lifecycle_lock():
with self.lifecycle_lock:
yield
return PailgunServer(self._bind_addr, runner_factory, lifecycle_lock)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
graph_helper = None
deferred_exc = None
self._logger.debug('execution commandline: %s', arguments)
options_bootstrapper = OptionsBootstrapper(args=arguments)
build_config = BuildConfigInitializer.get(options_bootstrapper)
options = OptionsInitializer.create(options_bootstrapper, build_config)
graph_helper, target_roots = None, None
try:
self._logger.debug('warming the product graph via %s', self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler.
graph_helper, target_roots = self._scheduler_service.warm_product_graph(
options,
self._target_roots_calculator
)
except Exception:
deferred_exc = sys.exc_info()
self._logger.warning(
'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
''.join(traceback.format_exception(*deferred_exc))
)
return self._runner_class(
sock,
exiter,
arguments,
environment,
target_roots,
graph_helper,
self.fork_lock,
deferred_exc
)
# Plumb the daemon's lifecycle lock to the `PailgunServer` to safeguard teardown.
@contextmanager
def lifecycle_lock():
with self.lifecycle_lock:
yield
return PailgunServer(self._bind_addr, runner_factory, lifecycle_lock)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
graph_helper = None
deferred_exc = None
# Capture the size of the graph prior to any warming, for stats.
preceding_graph_size = self._scheduler_service.product_graph_len()
self._logger.debug('resident graph size: %s', preceding_graph_size)
self._logger.debug('execution commandline: %s', arguments)
options, _ = OptionsInitializer(
OptionsBootstrapper(args=arguments)).setup(init_logging=False)
target_roots = self._target_roots_calculator.create(
options,
change_calculator=self._scheduler_service.change_calculator)
try:
self._logger.debug('warming the product graph via %s',
self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler.
graph_helper = self._scheduler_service.warm_product_graph(
target_roots)
except Exception:
deferred_exc = sys.exc_info()
self._logger.warning(
'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
''.join(traceback.format_exception(*deferred_exc)))
return self._runner_class(sock, exiter, arguments, environment,
target_roots, graph_helper,
self.fork_lock, preceding_graph_size,
deferred_exc)
# Plumb the daemon's lifecycle lock to the `PailgunServer` to safeguard teardown.
@contextmanager
def lifecycle_lock():
with self.lifecycle_lock:
yield
return PailgunServer(self._bind_addr, runner_factory, lifecycle_lock)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
exiter = self._exiter_class(sock)
graph_helper = None
deferred_exc = None
self._logger.debug('execution commandline: %s', arguments)
if self._scheduler_service:
# N.B. This parses sys.argv by way of OptionsInitializer/OptionsBootstrapper prior to
# the main pants run to derive target roots for caching in the underlying product graph.
target_roots = self._target_roots_class.create(
args=arguments,
change_calculator=self._scheduler_service.change_calculator
)
try:
self._logger.debug('warming the product graph via %s', self._scheduler_service)
# N.B. This call is made in the pre-fork daemon context for reach and reuse of the
# resident scheduler.
graph_helper = self._scheduler_service.warm_product_graph(target_roots)
except Exception:
deferred_exc = sys.exc_info()
self._logger.warning(
'encountered exception during SchedulerService.warm_product_graph(), deferring:\n%s',
''.join(traceback.format_exception(*deferred_exc))
)
return self._runner_class(sock, exiter, arguments, environment, graph_helper, deferred_exc)
@contextmanager
def context_lock():
"""This lock is used to safeguard Pailgun request handling against a fork() with the
scheduler lock held by another thread (e.g. the FSEventService thread), which can
lead to a pailgun deadlock.
"""
if self._scheduler_service:
with self._scheduler_service.locked():
yield
else:
yield
return PailgunServer(self._bind_addr, runner_factory, context_lock)
class TestPailgunServer(unittest.TestCase):
def setUp(self):
self.mock_handler_inst = mock.Mock()
self.mock_runner_factory = mock.Mock(
side_effect=Exception('this should never be called'))
self.mock_handler_class = mock.Mock(
return_value=self.mock_handler_inst)
self.lock = threading.RLock()
@contextmanager
def lock():
with self.lock:
yield
with mock.patch.object(PailgunServer, 'server_bind'), \
mock.patch.object(PailgunServer, 'server_activate'):
self.server = PailgunServer(
server_address=('0.0.0.0', 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class,
lifecycle_lock=lock)
@mock.patch.object(TCPServer, 'server_bind', **PATCH_OPTS)
def test_server_bind(self, mock_tcpserver_bind):
mock_sock = mock.Mock()
mock_sock.getsockname.return_value = ('0.0.0.0', 31337)
self.server.socket = mock_sock
self.server.server_bind()
self.assertEquals(self.server.server_port, 31337)
self.assertIs(mock_tcpserver_bind.called, True)
@mock.patch.object(PailgunServer, 'close_request', **PATCH_OPTS)
def test_process_request(self, mock_close_request):
mock_request = mock.Mock()
self.server.process_request(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
mock_close_request.assert_called_once_with(self.server, mock_request)
@mock.patch.object(PailgunServer, 'shutdown_request', **PATCH_OPTS)
def test_process_request_error(self, mock_shutdown_request):
mock_request = mock.Mock()
self.mock_handler_inst.handle_request.side_effect = AttributeError(
'oops')
self.server.process_request(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
self.assertIs(self.mock_handler_inst.handle_error.called, True)
mock_shutdown_request.assert_called_once_with(self.server,
mock_request)
def _setup_pailgun(self):
"""Sets up a PailgunServer instance."""
# Constructs and returns a runnable PantsRunner.
def runner_factory(sock, arguments, environment):
return self._runner_class.create(
sock,
arguments,
environment,
self.services,
self._scheduler_service,
)
# Plumb the daemon's lifecycle lock to the `PailgunServer` to safeguard teardown.
# This indirection exists to allow the server to be created before PantsService.setup
# has been called to actually initialize the `services` field.
@contextmanager
def lifecycle_lock():
with self.services.lifecycle_lock:
yield
return PailgunServer(self._bind_addr, runner_factory, lifecycle_lock)
class TestPailgunServer(unittest.TestCase):
def setUp(self):
self.mock_handler_inst = mock.Mock()
self.mock_runner_factory = mock.Mock(side_effect=Exception('this should never be called'))
self.mock_handler_class = mock.Mock(return_value=self.mock_handler_inst)
self.scheduler_lock = mock_context_lock
with mock.patch.object(PailgunServer, 'server_bind'), \
mock.patch.object(PailgunServer, 'server_activate'):
self.server = PailgunServer(
server_address=('0.0.0.0', 0),
runner_factory=self.mock_runner_factory,
context_lock=self.scheduler_lock,
handler_class=self.mock_handler_class
)
@mock.patch.object(TCPServer, 'server_bind', **PATCH_OPTS)
def test_server_bind(self, mock_tcpserver_bind):
mock_sock = mock.Mock()
mock_sock.getsockname.return_value = ('0.0.0.0', 31337)
self.server.socket = mock_sock
self.server.server_bind()
self.assertEquals(self.server.server_port, 31337)
self.assertIs(mock_tcpserver_bind.called, True)
@mock.patch.object(PailgunServer, 'close_request', **PATCH_OPTS)
def test_process_request(self, mock_close_request):
mock_request = mock.Mock()
self.server.process_request(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
mock_close_request.assert_called_once_with(self.server, mock_request)
@mock.patch.object(PailgunServer, 'shutdown_request', **PATCH_OPTS)
def test_process_request_error(self, mock_shutdown_request):
mock_request = mock.Mock()
self.mock_handler_inst.handle_request.side_effect = AttributeError('oops')
self.server.process_request(mock_request, ('1.2.3.4', 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
self.assertIs(self.mock_handler_inst.handle_error.called, True)
mock_shutdown_request.assert_called_once_with(self.server, mock_request)
class TestPailgunServer(unittest.TestCase):
def setUp(self):
self.mock_handler_inst = unittest.mock.Mock()
# Add a fake environment for this to not timeout.
self.fake_environment = {"PANTSD_REQUEST_TIMEOUT_LIMIT": "-1"}
self.mock_handler_inst.parsed_request.return_value = (None, None, [], self.fake_environment)
self.mock_runner_factory = unittest.mock.Mock(
side_effect=Exception("this should never be called")
)
self.mock_handler_class = unittest.mock.Mock(return_value=self.mock_handler_inst)
self.lock = threading.RLock()
@contextmanager
def lock():
with self.lock:
yield
self.after_request_callback_calls = 0
def after_request_callback():
self.after_request_callback_calls += 1
with unittest.mock.patch.object(PailgunServer, "server_bind"), unittest.mock.patch.object(
PailgunServer, "server_activate"
):
self.server = PailgunServer(
server_address=("0.0.0.0", 0),
runner_factory=self.mock_runner_factory,
handler_class=self.mock_handler_class,
lifecycle_lock=lock,
request_complete_callback=after_request_callback,
)
@unittest.mock.patch.object(TCPServer, "server_bind", **PATCH_OPTS)
def test_server_bind(self, mock_tcpserver_bind):
mock_sock = unittest.mock.Mock()
mock_sock.getsockname.return_value = ("0.0.0.0", 31337)
self.server.socket = mock_sock
self.server.server_bind()
self.assertEqual(self.server.server_port, 31337)
self.assertIs(mock_tcpserver_bind.called, True)
@unittest.mock.patch.object(PailgunServer, "close_request", **PATCH_OPTS)
def test_process_request_thread(self, mock_close_request):
mock_request = unittest.mock.Mock()
self.server.process_request_thread(mock_request, ("1.2.3.4", 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
mock_close_request.assert_called_once_with(self.server, mock_request)
@unittest.mock.patch.object(PailgunServer, "close_request", **PATCH_OPTS)
def test_process_request_calls_callback(self, mock_close_request):
mock_request = unittest.mock.Mock()
self.server.process_request_thread(mock_request, ("1.2.3.4", 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
assert self.after_request_callback_calls == 1
@unittest.mock.patch.object(PailgunServer, "shutdown_request", **PATCH_OPTS)
def test_process_request_thread_error(self, mock_shutdown_request):
mock_request = unittest.mock.Mock()
self.mock_handler_inst.handle_request.side_effect = AttributeError("oops")
self.server.process_request_thread(mock_request, ("1.2.3.4", 31338))
self.assertIs(self.mock_handler_inst.handle_request.called, True)
self.assertIs(self.mock_handler_inst.handle_error.called, True)
mock_shutdown_request.assert_called_once_with(self.server, mock_request)
def test_ensure_request_is_exclusive(self):
"""Launch many requests, assert that every one is trying to enter the critical section, and
assert that only one is doing so at a time."""
self.threads_to_start = 10
# Queues are thread safe (https://docs.python.org/2/library/queue.html)
self.thread_errors = Queue()
def threaded_assert_equal(one, other, message):
try:
self.assertEqual(one, other, message)
except AssertionError as error:
self.thread_errors.put(error)
self.threads_running_cond = threading.Condition()
self.threads_running = 0
def handle_thread_tried_to_handle_request():
"""Mark a thread as started, and block until every thread has been marked as
starting."""
self.threads_running_cond.acquire()
self.threads_running += 1
if self.threads_running == self.threads_to_start:
self.threads_running_cond.notify_all()
else:
while self.threads_running != self.threads_to_start:
self.threads_running_cond.wait()
threaded_assert_equal(
self.threads_running,
self.threads_to_start,
"This thread is unblocked before all the threads had started.",
)
self.threads_running_cond.release()
def handle_thread_finished():
"""Mark a thread as finished, and block until there are no more threads running."""
self.threads_running_cond.acquire()
self.threads_running -= 1
print(f"Handle_thread_finished, threads_running are {self.threads_running}")
if self.threads_running == 0:
self.threads_running_cond.notify_all()
else:
while self.threads_running != 0:
self.threads_running_cond.wait()
threaded_assert_equal(
self.threads_running,
0,
"handle_thread_finished exited when there still were threads running.",
)
self.threads_running_cond.release()
self.threads_handling_requests = 0
self.threads_handling_requests_lock = threading.Lock()
def handle_thread_starts_handling_request():
with self.threads_handling_requests_lock:
self.threads_handling_requests += 1
threaded_assert_equal(
self.threads_handling_requests, 1, "A thread is already handling a request!"
)
def check_only_one_thread_is_handling_a_request():
"""Assert that there's only ever one thread inside the lock."""
with self.threads_handling_requests_lock:
threaded_assert_equal(
self.threads_handling_requests, 1, "A thread is already handling a request!"
)
def handle_thread_finishing_handling_request():
"""Assert that I was the only thread handling a request."""
with self.threads_handling_requests_lock:
self.threads_handling_requests -= 1
threaded_assert_equal(
self.threads_handling_requests,
0,
"There were multiple threads handling a request when a thread finished",
)
# Wrap ensure_request_is_exclusive to notify when we acquire and release the lock.
def mock_ensure_request_is_exclusive(request_lock_under_test):
"""Wrap the lock under test.
Every thread that calls this function has reached the critical section.
"""
@contextmanager
def wrapper(environment, request):
# Assert that all threads are trying to handle a request.
handle_thread_tried_to_handle_request()
with request_lock_under_test(environment, request):
try:
# Assert that only one is allowed to handle a request.
print("Thread has entered the request handling code.")
handle_thread_starts_handling_request()
check_only_one_thread_is_handling_a_request()
yield
check_only_one_thread_is_handling_a_request()
print("Thread has exited the request handling code.")
finally:
# Account for a thread finishing a request.
handle_thread_finishing_handling_request()
# Notify that a thread is shutting down.
handle_thread_finished()
# At this point, we have asserted that all threads are finished.
return wrapper
self.server.ensure_request_is_exclusive = mock_ensure_request_is_exclusive(
self.server.ensure_request_is_exclusive
)
# Create as many mock threads as needed. Lauch all of them, and wait for all of them to finish.
mock_request = unittest.mock.Mock()
def create_request_thread(port):
return threading.Thread(
target=self.server.process_request_thread,
args=(mock_request, ("1.2.3.4", port)),
name=f"MockThread-{port}",
)
threads = [create_request_thread(0) for _ in range(0, self.threads_to_start)]
for thread in threads:
thread.start()
self.assertTrue(
self.thread_errors.empty(),
f"There were some errors in the threads:\n {self.thread_errors}",
)
for thread in threads:
# If this fails because it times out, it's definitely a legitimate error.
thread.join(10)
self.assertTrue(
self.thread_errors.empty(),
f"There were some errors in the threads:\n {self.thread_errors}",
)