def register_handler(self, name, metadata, callback):
"""Register subscriptions and their event handlers.
:param str name: the subscription name as used by watchman
:param dict metadata: a dictionary of metadata to be serialized and passed to the watchman
subscribe command. this should include the match expression as well
as any required callback fields.
:param func callback: the callback to execute on each matching filesystem event
"""
assert name not in self._handlers, 'duplicate handler name: {}'.format(name)
assert (
isinstance(metadata, dict) and 'fields' in metadata and 'expression' in metadata
), 'invalid handler metadata!'
self._handlers[name] = Watchman.EventHandler(name=name, metadata=metadata, callback=callback)
def __init__(
self, watchman: Watchman, scheduler: Scheduler, build_root: str,
):
"""
:param watchman: The Watchman instance as provided by the WatchmanLauncher subsystem.
:param session: A SchedulerSession to invalidate for.
:param build_root: The current build root.
"""
super().__init__()
self._logger = logging.getLogger(__name__)
self._watchman = watchman
self._build_root = os.path.realpath(build_root)
self._watchman_is_running = threading.Event()
self._scheduler_session = scheduler.new_session(
zipkin_trace_v2=False, build_id="fs_event_service_session"
)
self._handler = Watchman.EventHandler(
name=self.PANTS_ALL_FILES_SUBSCRIPTION_NAME,
metadata=dict(
fields=["name"],
# Request events for all file types.
# NB: Touching a file invalidates its parent directory due to:
# https://github.com/facebook/watchman/issues/305
# ...but if we were to skip watching directories, we'd still have to invalidate
# the parents of any changed files, and we wouldn't see creation/deletion of
# empty directories.
expression=[
"allof", # All of the below rules must be true to match.
["not", ["dirname", "dist", self.ZERO_DEPTH]], # Exclude the ./dist dir.
# N.B. 'wholename' ensures we match against the absolute ('x/y/z') vs base path ('z').
[
"not",
["pcre", r"^\..*", "wholename"],
], # Exclude files in hidden dirs (.pants.d etc).
["not", ["match", "*.pyc"]] # Exclude .pyc files.
# TODO(kwlzn): Make exclusions here optionable.
# Related: https://github.com/pantsbuild/pants/issues/2956
],
),
# NB: We stream events from Watchman in `self.run`, so we don't need a callback.
callback=lambda: None,
)
class TestWatchman(BaseTest):
PATCH_OPTS = dict(autospec=True, spec_set=True)
BUILD_ROOT = '/path/to/a/fake/build_root'
WATCHMAN_PATH = '/path/to/a/fake/watchman'
TEST_DIR = '/path/to/a/fake/test'
HANDLERS = [Watchman.EventHandler('test', {}, mock.Mock())]
def setUp(self):
super(TestWatchman, self).setUp()
with mock.patch.object(Watchman, '_is_valid_executable', **self.PATCH_OPTS) as mock_is_valid:
mock_is_valid.return_value = True
self.watchman = Watchman('/fake/path/to/watchman', self.subprocess_dir)
@property
def _watchman_dir(self):
return os.path.join(self.subprocess_dir, 'watchman')
@property
def _state_file(self):
return os.path.join(self._watchman_dir, 'watchman.state')
def test_client_property(self):
self.assertIsInstance(self.watchman.client, pywatchman.client)
def test_client_property_cached(self):
self.watchman._watchman_client = 1
self.assertEquals(self.watchman.client, 1)
def test_make_client(self):
self.assertIsInstance(self.watchman._make_client(), pywatchman.client)
def test_is_valid_executable(self):
self.assertTrue(self.watchman._is_valid_executable(sys.executable))
def test_resolve_watchman_path_provided_exception(self):
with self.assertRaises(Watchman.ExecutionError):
self.watchman = Watchman('/fake/path/to/watchman',
metadata_base_dir=self.subprocess_dir)
def test_maybe_init_metadata(self):
with mock.patch('pants.pantsd.watchman.safe_mkdir', **self.PATCH_OPTS) as mock_mkdir, \
mock.patch('pants.pantsd.watchman.safe_file_dump', **self.PATCH_OPTS) as mock_file_dump:
self.watchman._maybe_init_metadata()
mock_mkdir.assert_called_once_with(self._watchman_dir)
mock_file_dump.assert_called_once_with(self._state_file, '{}')
def test_construct_cmd(self):
output = self.watchman._construct_cmd(['cmd', 'parts', 'etc'],
'state_file',
'sock_file',
'log_file',
'log_level')
self.assertEquals(output, ['cmd',
'parts',
'etc',
'--no-save-state',
'--statefile=state_file',
'--sockname=sock_file',
'--logfile=log_file',
'--log-level',
'log_level'])
def test_parse_pid_from_output(self):
output = json.dumps(dict(pid=3))
self.assertEquals(self.watchman._parse_pid_from_output(output), 3)
def test_parse_pid_from_output_bad_output(self):
output = '{bad JSON.,/#!'
with self.assertRaises(self.watchman.InvalidCommandOutput):
self.watchman._parse_pid_from_output(output)
def test_parse_pid_from_output_no_pid(self):
output = json.dumps(dict(nothing=True))
with self.assertRaises(self.watchman.InvalidCommandOutput):
self.watchman._parse_pid_from_output(output)
def test_launch(self):
with mock.patch.object(Watchman, '_maybe_init_metadata') as mock_initmeta, \
mock.patch.object(Watchman, 'get_subprocess_output') as mock_getsubout, \
mock.patch.object(Watchman, 'write_pid') as mock_writepid, \
mock.patch.object(Watchman, 'write_socket') as mock_writesock:
mock_getsubout.return_value = json.dumps(dict(pid='3'))
self.watchman.launch()
assert mock_getsubout.called
mock_initmeta.assert_called_once_with()
mock_writepid.assert_called_once_with('3')
mock_writesock.assert_called_once_with(self.watchman._sock_file)
def test_watch_project(self):
self.watchman._watchman_client = mock.create_autospec(StreamableWatchmanClient, spec_set=True)
self.watchman.watch_project(self.TEST_DIR)
self.watchman._watchman_client.query.assert_called_once_with('watch-project', self.TEST_DIR)
@contextmanager
def setup_subscribed(self, iterable):
mock_client = mock.create_autospec(StreamableWatchmanClient, spec_set=True)
mock_client.stream_query.return_value = iter(iterable)
self.watchman._watchman_client = mock_client
yield mock_client
assert mock_client.stream_query.called
def test_subscribed_empty(self):
"""Test yielding when watchman reads timeout."""
with self.setup_subscribed([None]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEquals(list(out), [(None, None)])
def test_subscribed_response(self):
"""Test yielding on the watchman response to the initial subscribe command."""
with self.setup_subscribed([dict(subscribe='test')]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEquals(list(out), [(None, None)])
def test_subscribed_event(self):
"""Test yielding on a watchman event for a given subscription."""
test_event = dict(subscription='test3', msg='blah')
with self.setup_subscribed([test_event]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEquals(list(out), [('test3', test_event)])
def test_subscribed_unknown_event(self):
"""Test yielding on an unknown watchman event."""
with self.setup_subscribed([dict(unknown=True)]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEquals(list(out), [])
class TestWatchman(TestBase):
PATCH_OPTS = dict(autospec=True, spec_set=True)
BUILD_ROOT = "/path/to/a/fake/build_root"
WATCHMAN_PATH = "/path/to/a/fake/watchman"
TEST_DIR = "/path/to/a/fake/test"
HANDLERS = [Watchman.EventHandler("test", {}, unittest.mock.Mock())]
def setUp(self):
super().setUp()
with unittest.mock.patch.object(Watchman, "_is_valid_executable",
**self.PATCH_OPTS) as mock_is_valid:
mock_is_valid.return_value = True
self.watchman = Watchman("/fake/path/to/watchman",
self.subprocess_dir)
@property
def _watchman_dir(self):
return os.path.join(self.subprocess_dir, "watchman")
@property
def _state_file(self):
return os.path.join(self._watchman_dir, "watchman.state")
def test_client_property(self):
self.assertIsInstance(self.watchman.client, pywatchman.client)
def test_client_property_cached(self):
self.watchman._watchman_client = 1
self.assertEqual(self.watchman.client, 1)
def test_make_client(self):
self.assertIsInstance(self.watchman._make_client(), pywatchman.client)
def test_is_valid_executable(self):
self.assertTrue(self.watchman._is_valid_executable(sys.executable))
def test_resolve_watchman_path_provided_exception(self):
with self.assertRaises(Watchman.ExecutionError):
self.watchman = Watchman("/fake/path/to/watchman",
metadata_base_dir=self.subprocess_dir)
def test_maybe_init_metadata(self):
# TODO(#7106): is this the right path to patch?
with unittest.mock.patch(
"pants.pantsd.watchman.safe_mkdir",
**self.PATCH_OPTS) as mock_mkdir, unittest.mock.patch(
"pants.pantsd.watchman.safe_file_dump",
**self.PATCH_OPTS) as mock_file_dump:
self.watchman._maybe_init_metadata()
mock_mkdir.assert_called_once_with(self._watchman_dir)
mock_file_dump.assert_called_once_with(self._state_file,
b"{}",
mode="wb")
def test_construct_cmd(self):
output = self.watchman._construct_cmd(["cmd", "parts", "etc"],
"state_file", "sock_file",
"pid_file", "log_file",
"log_level")
self.assertEqual(
output,
[
"cmd",
"parts",
"etc",
"--no-save-state",
"--no-site-spawner",
"--statefile=state_file",
"--sockname=sock_file",
"--pidfile=pid_file",
"--logfile=log_file",
"--log-level",
"log_level",
],
)
def test_parse_pid_from_output(self):
output = json.dumps(dict(pid=3))
self.assertEqual(self.watchman._parse_pid_from_output(output), 3)
def test_parse_pid_from_output_bad_output(self):
output = "{bad JSON.,/#!"
with self.assertRaises(Watchman.InvalidCommandOutput):
self.watchman._parse_pid_from_output(output)
def test_parse_pid_from_output_no_pid(self):
output = json.dumps(dict(nothing=True))
with self.assertRaises(Watchman.InvalidCommandOutput):
self.watchman._parse_pid_from_output(output)
def test_launch(self):
with unittest.mock.patch.object(
Watchman, "_maybe_init_metadata"
) as mock_initmeta, unittest.mock.patch.object(
Watchman, "get_subprocess_output"
) as mock_getsubout, unittest.mock.patch.object(
Watchman,
"write_pid") as mock_writepid, unittest.mock.patch.object(
Watchman, "write_socket") as mock_writesock:
mock_getsubout.return_value = json.dumps(dict(pid="3"))
self.watchman.launch()
assert mock_getsubout.called
mock_initmeta.assert_called_once_with()
mock_writepid.assert_called_once_with("3")
mock_writesock.assert_called_once_with(self.watchman._sock_file)
def test_watch_project(self):
self.watchman._watchman_client = unittest.mock.create_autospec(
StreamableWatchmanClient, spec_set=True)
self.watchman.watch_project(self.TEST_DIR)
self.watchman._watchman_client.query.assert_called_once_with(
"watch-project", self.TEST_DIR)
@contextmanager
def setup_subscribed(self, iterable):
mock_client = unittest.mock.create_autospec(StreamableWatchmanClient,
spec_set=True)
mock_client.stream_query.return_value = iter(iterable)
self.watchman._watchman_client = mock_client
yield mock_client
assert mock_client.stream_query.called
def test_subscribed_empty(self):
"""Test yielding when watchman reads timeout."""
with self.setup_subscribed([None]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEqual(list(out), [(None, None)])
def test_subscribed_response(self):
"""Test yielding on the watchman response to the initial subscribe command."""
with self.setup_subscribed([dict(subscribe="test")]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEqual(list(out), [(None, None)])
def test_subscribed_event(self):
"""Test yielding on a watchman event for a given subscription."""
test_event = dict(subscription="test3", msg="blah")
with self.setup_subscribed([test_event]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEqual(list(out), [("test3", test_event)])
def test_subscribed_unknown_event(self):
"""Test yielding on an unknown watchman event."""
with self.setup_subscribed([dict(unknown=True)]):
out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS)
self.assertEqual(list(out), [])
