def register_handler(self, name, metadata, callback): """Register subscriptions and their event handlers. :param str name: the subscription name as used by watchman :param dict metadata: a dictionary of metadata to be serialized and passed to the watchman subscribe command. this should include the match expression as well as any required callback fields. :param func callback: the callback to execute on each matching filesystem event """ assert name not in self._handlers, 'duplicate handler name: {}'.format(name) assert ( isinstance(metadata, dict) and 'fields' in metadata and 'expression' in metadata ), 'invalid handler metadata!' self._handlers[name] = Watchman.EventHandler(name=name, metadata=metadata, callback=callback)
def __init__( self, watchman: Watchman, scheduler: Scheduler, build_root: str, ): """ :param watchman: The Watchman instance as provided by the WatchmanLauncher subsystem. :param session: A SchedulerSession to invalidate for. :param build_root: The current build root. """ super().__init__() self._logger = logging.getLogger(__name__) self._watchman = watchman self._build_root = os.path.realpath(build_root) self._watchman_is_running = threading.Event() self._scheduler_session = scheduler.new_session( zipkin_trace_v2=False, build_id="fs_event_service_session" ) self._handler = Watchman.EventHandler( name=self.PANTS_ALL_FILES_SUBSCRIPTION_NAME, metadata=dict( fields=["name"], # Request events for all file types. # NB: Touching a file invalidates its parent directory due to: # https://github.com/facebook/watchman/issues/305 # ...but if we were to skip watching directories, we'd still have to invalidate # the parents of any changed files, and we wouldn't see creation/deletion of # empty directories. expression=[ "allof", # All of the below rules must be true to match. ["not", ["dirname", "dist", self.ZERO_DEPTH]], # Exclude the ./dist dir. # N.B. 'wholename' ensures we match against the absolute ('x/y/z') vs base path ('z'). [ "not", ["pcre", r"^\..*", "wholename"], ], # Exclude files in hidden dirs (.pants.d etc). ["not", ["match", "*.pyc"]] # Exclude .pyc files. # TODO(kwlzn): Make exclusions here optionable. # Related: https://github.com/pantsbuild/pants/issues/2956 ], ), # NB: We stream events from Watchman in `self.run`, so we don't need a callback. callback=lambda: None, )
class TestWatchman(BaseTest): PATCH_OPTS = dict(autospec=True, spec_set=True) BUILD_ROOT = '/path/to/a/fake/build_root' WATCHMAN_PATH = '/path/to/a/fake/watchman' TEST_DIR = '/path/to/a/fake/test' HANDLERS = [Watchman.EventHandler('test', {}, mock.Mock())] def setUp(self): super(TestWatchman, self).setUp() with mock.patch.object(Watchman, '_is_valid_executable', **self.PATCH_OPTS) as mock_is_valid: mock_is_valid.return_value = True self.watchman = Watchman('/fake/path/to/watchman', self.subprocess_dir) @property def _watchman_dir(self): return os.path.join(self.subprocess_dir, 'watchman') @property def _state_file(self): return os.path.join(self._watchman_dir, 'watchman.state') def test_client_property(self): self.assertIsInstance(self.watchman.client, pywatchman.client) def test_client_property_cached(self): self.watchman._watchman_client = 1 self.assertEquals(self.watchman.client, 1) def test_make_client(self): self.assertIsInstance(self.watchman._make_client(), pywatchman.client) def test_is_valid_executable(self): self.assertTrue(self.watchman._is_valid_executable(sys.executable)) def test_resolve_watchman_path_provided_exception(self): with self.assertRaises(Watchman.ExecutionError): self.watchman = Watchman('/fake/path/to/watchman', metadata_base_dir=self.subprocess_dir) def test_maybe_init_metadata(self): with mock.patch('pants.pantsd.watchman.safe_mkdir', **self.PATCH_OPTS) as mock_mkdir, \ mock.patch('pants.pantsd.watchman.safe_file_dump', **self.PATCH_OPTS) as mock_file_dump: self.watchman._maybe_init_metadata() mock_mkdir.assert_called_once_with(self._watchman_dir) mock_file_dump.assert_called_once_with(self._state_file, '{}') def test_construct_cmd(self): output = self.watchman._construct_cmd(['cmd', 'parts', 'etc'], 'state_file', 'sock_file', 'log_file', 'log_level') self.assertEquals(output, ['cmd', 'parts', 'etc', '--no-save-state', '--statefile=state_file', '--sockname=sock_file', '--logfile=log_file', '--log-level', 'log_level']) def test_parse_pid_from_output(self): output = json.dumps(dict(pid=3)) self.assertEquals(self.watchman._parse_pid_from_output(output), 3) def test_parse_pid_from_output_bad_output(self): output = '{bad JSON.,/#!' with self.assertRaises(self.watchman.InvalidCommandOutput): self.watchman._parse_pid_from_output(output) def test_parse_pid_from_output_no_pid(self): output = json.dumps(dict(nothing=True)) with self.assertRaises(self.watchman.InvalidCommandOutput): self.watchman._parse_pid_from_output(output) def test_launch(self): with mock.patch.object(Watchman, '_maybe_init_metadata') as mock_initmeta, \ mock.patch.object(Watchman, 'get_subprocess_output') as mock_getsubout, \ mock.patch.object(Watchman, 'write_pid') as mock_writepid, \ mock.patch.object(Watchman, 'write_socket') as mock_writesock: mock_getsubout.return_value = json.dumps(dict(pid='3')) self.watchman.launch() assert mock_getsubout.called mock_initmeta.assert_called_once_with() mock_writepid.assert_called_once_with('3') mock_writesock.assert_called_once_with(self.watchman._sock_file) def test_watch_project(self): self.watchman._watchman_client = mock.create_autospec(StreamableWatchmanClient, spec_set=True) self.watchman.watch_project(self.TEST_DIR) self.watchman._watchman_client.query.assert_called_once_with('watch-project', self.TEST_DIR) @contextmanager def setup_subscribed(self, iterable): mock_client = mock.create_autospec(StreamableWatchmanClient, spec_set=True) mock_client.stream_query.return_value = iter(iterable) self.watchman._watchman_client = mock_client yield mock_client assert mock_client.stream_query.called def test_subscribed_empty(self): """Test yielding when watchman reads timeout.""" with self.setup_subscribed([None]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEquals(list(out), [(None, None)]) def test_subscribed_response(self): """Test yielding on the watchman response to the initial subscribe command.""" with self.setup_subscribed([dict(subscribe='test')]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEquals(list(out), [(None, None)]) def test_subscribed_event(self): """Test yielding on a watchman event for a given subscription.""" test_event = dict(subscription='test3', msg='blah') with self.setup_subscribed([test_event]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEquals(list(out), [('test3', test_event)]) def test_subscribed_unknown_event(self): """Test yielding on an unknown watchman event.""" with self.setup_subscribed([dict(unknown=True)]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEquals(list(out), [])
class TestWatchman(TestBase): PATCH_OPTS = dict(autospec=True, spec_set=True) BUILD_ROOT = "/path/to/a/fake/build_root" WATCHMAN_PATH = "/path/to/a/fake/watchman" TEST_DIR = "/path/to/a/fake/test" HANDLERS = [Watchman.EventHandler("test", {}, unittest.mock.Mock())] def setUp(self): super().setUp() with unittest.mock.patch.object(Watchman, "_is_valid_executable", **self.PATCH_OPTS) as mock_is_valid: mock_is_valid.return_value = True self.watchman = Watchman("/fake/path/to/watchman", self.subprocess_dir) @property def _watchman_dir(self): return os.path.join(self.subprocess_dir, "watchman") @property def _state_file(self): return os.path.join(self._watchman_dir, "watchman.state") def test_client_property(self): self.assertIsInstance(self.watchman.client, pywatchman.client) def test_client_property_cached(self): self.watchman._watchman_client = 1 self.assertEqual(self.watchman.client, 1) def test_make_client(self): self.assertIsInstance(self.watchman._make_client(), pywatchman.client) def test_is_valid_executable(self): self.assertTrue(self.watchman._is_valid_executable(sys.executable)) def test_resolve_watchman_path_provided_exception(self): with self.assertRaises(Watchman.ExecutionError): self.watchman = Watchman("/fake/path/to/watchman", metadata_base_dir=self.subprocess_dir) def test_maybe_init_metadata(self): # TODO(#7106): is this the right path to patch? with unittest.mock.patch( "pants.pantsd.watchman.safe_mkdir", **self.PATCH_OPTS) as mock_mkdir, unittest.mock.patch( "pants.pantsd.watchman.safe_file_dump", **self.PATCH_OPTS) as mock_file_dump: self.watchman._maybe_init_metadata() mock_mkdir.assert_called_once_with(self._watchman_dir) mock_file_dump.assert_called_once_with(self._state_file, b"{}", mode="wb") def test_construct_cmd(self): output = self.watchman._construct_cmd(["cmd", "parts", "etc"], "state_file", "sock_file", "pid_file", "log_file", "log_level") self.assertEqual( output, [ "cmd", "parts", "etc", "--no-save-state", "--no-site-spawner", "--statefile=state_file", "--sockname=sock_file", "--pidfile=pid_file", "--logfile=log_file", "--log-level", "log_level", ], ) def test_parse_pid_from_output(self): output = json.dumps(dict(pid=3)) self.assertEqual(self.watchman._parse_pid_from_output(output), 3) def test_parse_pid_from_output_bad_output(self): output = "{bad JSON.,/#!" with self.assertRaises(Watchman.InvalidCommandOutput): self.watchman._parse_pid_from_output(output) def test_parse_pid_from_output_no_pid(self): output = json.dumps(dict(nothing=True)) with self.assertRaises(Watchman.InvalidCommandOutput): self.watchman._parse_pid_from_output(output) def test_launch(self): with unittest.mock.patch.object( Watchman, "_maybe_init_metadata" ) as mock_initmeta, unittest.mock.patch.object( Watchman, "get_subprocess_output" ) as mock_getsubout, unittest.mock.patch.object( Watchman, "write_pid") as mock_writepid, unittest.mock.patch.object( Watchman, "write_socket") as mock_writesock: mock_getsubout.return_value = json.dumps(dict(pid="3")) self.watchman.launch() assert mock_getsubout.called mock_initmeta.assert_called_once_with() mock_writepid.assert_called_once_with("3") mock_writesock.assert_called_once_with(self.watchman._sock_file) def test_watch_project(self): self.watchman._watchman_client = unittest.mock.create_autospec( StreamableWatchmanClient, spec_set=True) self.watchman.watch_project(self.TEST_DIR) self.watchman._watchman_client.query.assert_called_once_with( "watch-project", self.TEST_DIR) @contextmanager def setup_subscribed(self, iterable): mock_client = unittest.mock.create_autospec(StreamableWatchmanClient, spec_set=True) mock_client.stream_query.return_value = iter(iterable) self.watchman._watchman_client = mock_client yield mock_client assert mock_client.stream_query.called def test_subscribed_empty(self): """Test yielding when watchman reads timeout.""" with self.setup_subscribed([None]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEqual(list(out), [(None, None)]) def test_subscribed_response(self): """Test yielding on the watchman response to the initial subscribe command.""" with self.setup_subscribed([dict(subscribe="test")]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEqual(list(out), [(None, None)]) def test_subscribed_event(self): """Test yielding on a watchman event for a given subscription.""" test_event = dict(subscription="test3", msg="blah") with self.setup_subscribed([test_event]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEqual(list(out), [("test3", test_event)]) def test_subscribed_unknown_event(self): """Test yielding on an unknown watchman event.""" with self.setup_subscribed([dict(unknown=True)]): out = self.watchman.subscribed(self.BUILD_ROOT, self.HANDLERS) self.assertEqual(list(out), [])