def _queue_dirs_modified(self, dirs_modified, ref_snapshot, new_snapshot): """ Queues events for directory modifications by scanning the directory for changes. A scan is a comparison between two snapshots of the same directory taken at two different times. This also determines whether files or directories were created, which updated the modified timestamp for the directory. """ if dirs_modified: for dir_modified in dirs_modified: self.queue_event(DirModifiedEvent(dir_modified)) diff_events = new_snapshot - ref_snapshot for file_created in diff_events.files_created: self.queue_event(FileCreatedEvent(file_created)) for directory_created in diff_events.dirs_created: self.queue_event(DirCreatedEvent(directory_created))
def test_modify(): touch(p('a')) start_watching() touch(p('a')) # Because the tests run so fast then on macOS it is almost certain that # we receive a coalesced event from fseventsd here, which triggers an # additional file created event and dir modified event here. if platform.is_darwin(): expect_event(FileCreatedEvent(p('a'))) expect_event(DirModifiedEvent(p())) expect_event(FileModifiedEvent(p('a'))) if platform.is_linux(): event = event_queue.get(timeout=5)[0] assert event.src_path == p('a') assert isinstance(event, FileClosedEvent)
def start_watching(path=None, use_full_emitter=False, recursive=True): # todo: check if other platforms expect the trailing slash (e.g. `p('')`) path = p() if path is None else path global emitter if platform.is_linux() and use_full_emitter: emitter = InotifyFullEmitter(event_queue, ObservedWatch(path, recursive=recursive)) else: emitter = Emitter(event_queue, ObservedWatch(path, recursive=recursive)) emitter.start() if platform.is_darwin(): # FSEvents _may_ report the event for the creation of `tmpdir`, # however, we're racing with fseventd there - if other filesystem # events happened _after_ `tmpdir` was created, but _before_ we # created the emitter then we won't get this event. # As such, let's create a sentinel event that tells us that we are # good to go. sentinel_file = os.path.join( path, '.sentinel' if isinstance(path, str) else '.sentinel'.encode()) touch(sentinel_file) sentinel_events = [ FileCreatedEvent(sentinel_file), DirModifiedEvent(path), FileModifiedEvent(sentinel_file) ] next_sentinel_event = sentinel_events.pop(0) now = time.monotonic() while time.monotonic() <= now + 30.0: try: event = event_queue.get(timeout=0.5)[0] if event == next_sentinel_event: if not sentinel_events: break next_sentinel_event = sentinel_events.pop(0) except Empty: pass time.sleep(0.1) else: assert False, "Sentinel event never arrived!"
def queue_events(self, timeout): # We don't want to hit the disk continuously. # timeout behaves like an interval for polling emitters. if self.stopped_event.wait(timeout): return with self._lock: if not self.should_keep_running(): return # Get event diff between fresh snapshot and previous snapshot. # Update snapshot. try: new_snapshot = self._take_snapshot() except OSError as e: self.queue_event(DirDeletedEvent(self.watch.path)) self.stop() return except Exception as e: raise e events = DirectorySnapshotDiff(self._snapshot, new_snapshot) self._snapshot = new_snapshot # Files. for src_path in events.files_deleted: self.queue_event(FileDeletedEvent(src_path)) for src_path in events.files_modified: self.queue_event(FileModifiedEvent(src_path)) for src_path in events.files_created: self.queue_event(FileCreatedEvent(src_path)) for src_path, dest_path in events.files_moved: self.queue_event(FileMovedEvent(src_path, dest_path)) # Directories. for src_path in events.dirs_deleted: self.queue_event(DirDeletedEvent(src_path)) for src_path in events.dirs_modified: self.queue_event(DirModifiedEvent(src_path)) for src_path in events.dirs_created: self.queue_event(DirCreatedEvent(src_path)) for src_path, dest_path in events.dirs_moved: self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout, full_events=False): # If "full_events" is true, then the method will report unmatched move events as seperate events # This behavior is by default only called by a InotifyFullEmitter with self._lock: event = self._inotify.read_event() if event is None: return if isinstance(event, tuple): move_from, move_to = event src_path = self._decode_path(move_from.src_path) dest_path = self._decode_path(move_to.src_path) cls = DirMovedEvent if move_from.is_directory else FileMovedEvent self.queue_event(cls(src_path, dest_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) self.queue_event(DirModifiedEvent(os.path.dirname(dest_path))) if move_from.is_directory and self.watch.is_recursive: for sub_event in generate_sub_moved_events(src_path, dest_path): self.queue_event(sub_event) return src_path = self._decode_path(event.src_path) if event.is_moved_to: if full_events: cls = DirMovedEvent if event.is_directory else FileMovedEvent self.queue_event(cls(None, src_path)) else: cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) if event.is_directory and self.watch.is_recursive: for sub_event in generate_sub_created_events(src_path): self.queue_event(sub_event) elif event.is_attrib: cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(src_path)) elif event.is_modify: cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(src_path)) elif event.is_delete or (event.is_moved_from and not full_events): cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) elif event.is_moved_from and full_events: cls = DirMovedEvent if event.is_directory else FileMovedEvent self.queue_event(cls(src_path, None)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) elif event.is_create: cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
def test_file_system_event_handler_dispatch(): dir_del_event = DirDeletedEvent('/path/blah.py') file_del_event = FileDeletedEvent('/path/blah.txt') dir_cre_event = DirCreatedEvent('/path/blah.py') file_cre_event = FileCreatedEvent('/path/blah.txt') dir_mod_event = DirModifiedEvent('/path/blah.py') file_mod_event = FileModifiedEvent('/path/blah.txt') dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah') file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah') all_events = [ dir_mod_event, dir_del_event, dir_cre_event, dir_mov_event, file_mod_event, file_del_event, file_cre_event, file_mov_event, ] class TestableEventHandler(FileSystemEventHandler): def on_any_event(self, event): assert True def on_modified(self, event): assert event.event_type == EVENT_TYPE_MODIFIED def on_deleted(self, event): assert event.event_type == EVENT_TYPE_DELETED def on_moved(self, event): assert event.event_type == EVENT_TYPE_MOVED def on_created(self, event): assert event.event_type == EVENT_TYPE_CREATED handler = TestableEventHandler() for event in all_events: assert not event.is_synthetic handler.dispatch(event)
def queue_events(self, timeout): events = self._fsevents.read_events() if events is None: return i = 0 while i < len(events): event = events[i] # For some reason the create and remove flags are sometimes also # set for rename and modify type events, so let those take # precedence. if event.is_renamed: # Internal moves appears to always be consecutive in the same # buffer and have IDs differ by exactly one (while others # don't) making it possible to pair up the two events coming # from a singe move operation. (None of this is documented!) # Otherwise, guess whether file was moved in or out. #TODO: handle id wrapping if (i+1 < len(events) and events[i+1].is_renamed and events[i+1].event_id == event.event_id + 1): cls = DirMovedEvent if event.is_directory else FileMovedEvent self.queue_event(cls(event.path, events[i+1].path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) self.queue_event(DirModifiedEvent(os.path.dirname(events[i+1].path))) i += 1 elif os.path.exists(event.path): cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) else: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) #TODO: generate events for tree elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod : cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(event.path)) elif event.is_created: cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) elif event.is_removed: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) i += 1
def _queue_events_except_renames_and_dir_modifications(self, event_list): """ Queues events from the kevent list returned from the call to :meth:`select.kqueue.control`. .. NOTE:: Queues only the deletions, file modifications, attribute modifications. The other events, namely, file creation, directory modification, file rename, directory rename, directory creation, etc. are determined by comparing directory snapshots. """ files_renamed = set() dirs_renamed = set() dirs_modified = set() for kev in event_list: descriptor = self._descriptors.get_for_fd(kev.ident) src_path = descriptor.path if is_deleted(kev): if descriptor.is_directory: self.queue_event(DirDeletedEvent(src_path)) else: self.queue_event(FileDeletedEvent(src_path)) elif is_attrib_modified(kev): if descriptor.is_directory: self.queue_event(DirModifiedEvent(src_path)) else: self.queue_event(FileModifiedEvent(src_path)) elif is_modified(kev): if descriptor.is_directory: dirs_modified.add(src_path) else: self.queue_event(FileModifiedEvent(src_path)) elif is_renamed(kev): if descriptor.is_directory: dirs_renamed.add(src_path) else: files_renamed.add(src_path) return (files_renamed, dirs_renamed, dirs_modified)
def start(self, *args, **kwargs): previous_snapshots = dict() if os.path.exists(self._filename): with open(self._filename, 'rb') as f: previous_snapshots = pickle.load(f) for watcher, handlers in self._handlers.items(): try: path = watcher.path curr_snap = DirectorySnapshot(path) pre_snap = previous_snapshots.get(path, _EmptySnapshot(path)) diff = DirectorySnapshotDiff(pre_snap, curr_snap) for handler in handlers: # Dispatch files modifications for new_path in diff.files_created: handler.dispatch(FileCreatedEvent(new_path)) for del_path in diff.files_deleted: handler.dispatch(FileDeletedEvent(del_path)) for mod_path in diff.files_modified: handler.dispatch(FileModifiedEvent(mod_path)) for src_path, mov_path in diff.files_moved: handler.dispatch(FileMovedEvent(src_path, mov_path)) # Dispatch directories modifications for new_dir in diff.dirs_created: handler.dispatch(DirCreatedEvent(new_dir)) for del_dir in diff.dirs_deleted: handler.dispatch(DirDeletedEvent(del_dir)) for mod_dir in diff.dirs_modified: handler.dispatch(DirModifiedEvent(mod_dir)) for src_path, mov_path in diff.dirs_moved: handler.dispatch(DirMovedEvent(src_path, mov_path)) except PermissionError as e: print(e) Observer.start(self, *args, **kwargs)
class TestConfigFileEventHandler(unittest.TestCase): def setUp(self) -> None: self.callback = mock.Mock(return_value=None, autospec=True) self.config_logger = logging.getLogger("test_config") self.test_config_event_handler = ConfigFileEventHandler( self.config_logger, self.callback, ["*.ini"]) def test_initialised(self): self.assertIsNotNone(self.test_config_event_handler) @parameterized.expand([ (FileCreatedEvent("test_src/test.ini"), ), (DirCreatedEvent("test_src/"), ), ]) def test_on_created_triggers_callback(self, event: FileSystemEvent): self.test_config_event_handler.on_created(event) self.callback.assert_called_once_with(event) @parameterized.expand([ (FileModifiedEvent("test_src/test.ini"), ), (DirModifiedEvent("test_src/"), ), ]) def test_on_modified_triggers_callback(self, event: FileSystemEvent): self.test_config_event_handler.on_modified(event) self.callback.assert_called_once_with(event) @parameterized.expand([ (FileDeletedEvent("test_src/test.ini"), ), (DirDeletedEvent("test_src/"), ), ]) def test_on_deleted_triggers_callback(self, event: FileSystemEvent): self.test_config_event_handler.on_deleted(event) self.callback.assert_called_once_with(event)
def test_separate_consecutive_moves(): mkdir(p('dir1')) mkfile(p('dir1', 'a')) mkfile(p('b')) start_watching(p('dir1')) mv(p('dir1', 'a'), p('c')) mv(p('b'), p('dir1', 'd')) dir_modif = DirModifiedEvent(p('dir1')) a_deleted = FileDeletedEvent(p('dir1', 'a')) d_created = FileCreatedEvent(p('dir1', 'd')) expected_events = [a_deleted, dir_modif, d_created, dir_modif] if platform.is_windows(): expected_events = [a_deleted, d_created] if platform.is_bsd(): # Due to the way kqueue works, we can't really order # 'Created' and 'Deleted' events in time, so creation queues first expected_events = [d_created, a_deleted, dir_modif, dir_modif] for expected_event in expected_events: expect_event(expected_event)
def queue_events(self, timeout): # We don't want to hit the disk continuously. # timeout behaves like an interval for polling emitters. time.sleep(timeout) with self._lock: if not self._snapshot: return # Get event diff between fresh snapshot and previous snapshot. # Update snapshot. new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive) events = DirectorySnapshotDiff(self._snapshot, new_snapshot) self._snapshot = new_snapshot # Files. for src_path in events.files_deleted: self.queue_event(FileDeletedEvent(src_path)) for src_path in events.files_modified: self.queue_event(FileModifiedEvent(src_path)) for src_path in events.files_created: self.queue_event(FileCreatedEvent(src_path)) for src_path, dest_path in events.files_moved: self.queue_event(FileMovedEvent(src_path, dest_path)) # Directories. for src_path in events.dirs_deleted: self.queue_event(DirDeletedEvent(src_path)) for src_path in events.dirs_modified: self.queue_event(DirModifiedEvent(src_path)) for src_path in events.dirs_created: self.queue_event(DirCreatedEvent(src_path)) for src_path, dest_path in events.dirs_moved: self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout): events = self._fsevents.read_events() if events is None: return i = 0 while i < len(events): event = events[i] if event.is_renamed: if (i+1 < len(events) and events[i+1].is_renamed and events[i+1].event_id == event.event_id + 1): cls = DirMovedEvent if event.is_directory else FileMovedEvent self.queue_event(cls(event.path, events[i+1].path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) self.queue_event(DirModifiedEvent(os.path.dirname(events[i+1].path))) i += 1 elif os.path.exists(event.path): cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) else: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) #TODO: generate events for tree elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod : cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(event.path)) elif event.is_created: cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) elif event.is_removed: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(event.path)) self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) i += 1
def test___init__(self): event = DirModifiedEvent(path_1) assert_equal(path_1, event.src_path) assert_equal(EVENT_TYPE_MODIFIED, event.event_type) assert_true(event.is_directory)
def test___repr__(self): event = DirModifiedEvent(path_1) self.assertEqual("<DirModifiedEvent: src_path=%s>" % path_1, event.__repr__())
def test_dispatch(self): # Utilities. regexes = [r".*\.py", r".*\.txt"] ignore_regexes = [r".*\.pyc"] def assert_regexes(handler, event): if has_attribute(event, "dest_path"): paths = [event.src_path, event.dest_path] else: paths = [event.src_path] filtered_paths = set() for p in paths: if any(r.match(p) for r in handler.regexes): filtered_paths.add(p) self.assertTrue(filtered_paths) dir_del_event_match = DirDeletedEvent("/path/blah.py") dir_del_event_not_match = DirDeletedEvent("/path/foobar") dir_del_event_ignored = DirDeletedEvent("/path/foobar.pyc") file_del_event_match = FileDeletedEvent("/path/blah.txt") file_del_event_not_match = FileDeletedEvent("/path/foobar") file_del_event_ignored = FileDeletedEvent("/path/blah.pyc") dir_cre_event_match = DirCreatedEvent("/path/blah.py") dir_cre_event_not_match = DirCreatedEvent("/path/foobar") dir_cre_event_ignored = DirCreatedEvent("/path/foobar.pyc") file_cre_event_match = FileCreatedEvent("/path/blah.txt") file_cre_event_not_match = FileCreatedEvent("/path/foobar") file_cre_event_ignored = FileCreatedEvent("/path/blah.pyc") dir_mod_event_match = DirModifiedEvent("/path/blah.py") dir_mod_event_not_match = DirModifiedEvent("/path/foobar") dir_mod_event_ignored = DirModifiedEvent("/path/foobar.pyc") file_mod_event_match = FileModifiedEvent("/path/blah.txt") file_mod_event_not_match = FileModifiedEvent("/path/foobar") file_mod_event_ignored = FileModifiedEvent("/path/blah.pyc") dir_mov_event_match = DirMovedEvent("/path/blah.py", "/path/blah") dir_mov_event_not_match = DirMovedEvent("/path/foobar", "/path/blah") dir_mov_event_ignored = DirMovedEvent("/path/foobar.pyc", "/path/blah") file_mov_event_match = FileMovedEvent("/path/blah.txt", "/path/blah") file_mov_event_not_match = FileMovedEvent("/path/foobar", "/path/blah") file_mov_event_ignored = FileMovedEvent("/path/blah.pyc", "/path/blah") all_dir_events = [ dir_mod_event_match, dir_mod_event_not_match, dir_mod_event_ignored, dir_del_event_match, dir_del_event_not_match, dir_del_event_ignored, dir_cre_event_match, dir_cre_event_not_match, dir_cre_event_ignored, dir_mov_event_match, dir_mov_event_not_match, dir_mov_event_ignored, ] all_file_events = [ file_mod_event_match, file_mod_event_not_match, file_mod_event_ignored, file_del_event_match, file_del_event_not_match, file_del_event_ignored, file_cre_event_match, file_cre_event_not_match, file_cre_event_ignored, file_mov_event_match, file_mov_event_not_match, file_mov_event_ignored, ] all_events = all_file_events + all_dir_events def assert_check_directory(handler, event): self.assertFalse(handler.ignore_directories and event.is_directory) def assert_equal(a, b): self.assertEqual(a, b) class TestableEventHandler(RegexMatchingEventHandler): def on_any_event(self, event): assert_check_directory(self, event) def on_modified(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MODIFIED) assert_regexes(self, event) def on_deleted(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_DELETED) assert_regexes(self, event) def on_moved(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MOVED) assert_regexes(self, event) def on_created(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_CREATED) assert_regexes(self, event) no_dirs_handler = TestableEventHandler( regexes=regexes, ignore_regexes=ignore_regexes, ignore_directories=True ) handler = TestableEventHandler( regexes=regexes, ignore_regexes=ignore_regexes, ignore_directories=False ) for event in all_events: no_dirs_handler.dispatch(event) for event in all_events: handler.dispatch(event)
def test_dispatch(self): # Utilities. patterns = ["*.py", "*.txt"] ignore_patterns = ["*.pyc"] def assert_patterns(event): if has_attribute(event, "dest_path"): paths = [event.src_path, event.dest_path] else: paths = [event.src_path] filtered_paths = filter_paths( paths, included_patterns=patterns, excluded_patterns=ignore_patterns, case_sensitive=False, ) self.assertTrue(filtered_paths) dir_del_event_match = DirDeletedEvent("/path/blah.py") dir_del_event_not_match = DirDeletedEvent("/path/foobar") dir_del_event_ignored = DirDeletedEvent("/path/foobar.pyc") file_del_event_match = FileDeletedEvent("/path/blah.txt") file_del_event_not_match = FileDeletedEvent("/path/foobar") file_del_event_ignored = FileDeletedEvent("/path/blah.pyc") dir_cre_event_match = DirCreatedEvent("/path/blah.py") dir_cre_event_not_match = DirCreatedEvent("/path/foobar") dir_cre_event_ignored = DirCreatedEvent("/path/foobar.pyc") file_cre_event_match = FileCreatedEvent("/path/blah.txt") file_cre_event_not_match = FileCreatedEvent("/path/foobar") file_cre_event_ignored = FileCreatedEvent("/path/blah.pyc") dir_mod_event_match = DirModifiedEvent("/path/blah.py") dir_mod_event_not_match = DirModifiedEvent("/path/foobar") dir_mod_event_ignored = DirModifiedEvent("/path/foobar.pyc") file_mod_event_match = FileModifiedEvent("/path/blah.txt") file_mod_event_not_match = FileModifiedEvent("/path/foobar") file_mod_event_ignored = FileModifiedEvent("/path/blah.pyc") dir_mov_event_match = DirMovedEvent("/path/blah.py", "/path/blah") dir_mov_event_not_match = DirMovedEvent("/path/foobar", "/path/blah") dir_mov_event_ignored = DirMovedEvent("/path/foobar.pyc", "/path/blah") file_mov_event_match = FileMovedEvent("/path/blah.txt", "/path/blah") file_mov_event_not_match = FileMovedEvent("/path/foobar", "/path/blah") file_mov_event_ignored = FileMovedEvent("/path/blah.pyc", "/path/blah") all_dir_events = [ dir_mod_event_match, dir_mod_event_not_match, dir_mod_event_ignored, dir_del_event_match, dir_del_event_not_match, dir_del_event_ignored, dir_cre_event_match, dir_cre_event_not_match, dir_cre_event_ignored, dir_mov_event_match, dir_mov_event_not_match, dir_mov_event_ignored, ] all_file_events = [ file_mod_event_match, file_mod_event_not_match, file_mod_event_ignored, file_del_event_match, file_del_event_not_match, file_del_event_ignored, file_cre_event_match, file_cre_event_not_match, file_cre_event_ignored, file_mov_event_match, file_mov_event_not_match, file_mov_event_ignored, ] all_events = all_file_events + all_dir_events def assert_check_directory(handler, event): self.assertFalse(handler.ignore_directories and event.is_directory) def assert_equal(a, b): self.assertEqual(a, b) class TestableEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): assert_check_directory(self, event) def on_modified(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MODIFIED) assert_patterns(event) def on_deleted(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_DELETED) assert_patterns(event) def on_moved(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MOVED) assert_patterns(event) def on_created(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_CREATED) assert_patterns(event) no_dirs_handler = TestableEventHandler( patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=True ) handler = TestableEventHandler( patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=False ) for event in all_events: no_dirs_handler.dispatch(event) for event in all_events: handler.dispatch(event)
def test___init__(self): SLEEP_TIME = 0.4 self.emitter.start() sleep(SLEEP_TIME) mkdir(p('project')) sleep(SLEEP_TIME) mkdir(p('project', 'blah')) sleep(SLEEP_TIME) touch(p('afile')) sleep(SLEEP_TIME) touch(p('fromfile')) sleep(SLEEP_TIME) mv(p('fromfile'), p('project', 'tofile')) sleep(SLEEP_TIME) touch(p('afile')) sleep(SLEEP_TIME) mv(p('project', 'blah'), p('project', 'boo')) sleep(SLEEP_TIME) rm(p('project'), recursive=True) sleep(SLEEP_TIME) rm(p('afile')) sleep(SLEEP_TIME) self.emitter.stop() # What we need here for the tests to pass is a collection type # that is: # * unordered # * non-unique # A multiset! Python's collections.Counter class seems appropriate. expected = set([ DirModifiedEvent(p()), DirCreatedEvent(p('project')), DirModifiedEvent(p('project')), DirCreatedEvent(p('project', 'blah')), FileCreatedEvent(p('afile')), DirModifiedEvent(p()), FileCreatedEvent(p('fromfile')), DirModifiedEvent(p()), DirModifiedEvent(p()), FileModifiedEvent(p('afile')), DirModifiedEvent(p('project')), DirModifiedEvent(p()), FileDeletedEvent(p('project', 'tofile')), DirDeletedEvent(p('project', 'boo')), DirDeletedEvent(p('project')), DirModifiedEvent(p()), FileDeletedEvent(p('afile')), ]) expected.add(FileMovedEvent(p('fromfile'), p('project', 'tofile'))) expected.add(DirMovedEvent(p('project', 'blah'), p('project', 'boo'))) got = set() while True: try: event, _ = self.event_queue.get_nowait() got.add(event) except queue.Empty: break self.assertEqual(expected, got)
class KqueueEmitter(EventEmitter): def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT): EventEmitter.__init__(self, event_queue, watch, timeout) self._kq = select.kqueue() self._lock = threading.RLock() # A collection of KeventDescriptor. self._descriptors = KeventDescriptorSet() def walker_callback(path, stat_info, self=self): self._register_kevent(path, stat.S_ISDIR(stat_info.st_mode)) self._snapshot = DirectorySnapshot(watch.path, watch.is_recursive, walker_callback) def _register_kevent(self, path, is_directory): try: self._descriptors.add(path, is_directory) except OSError as e: if e.errno == errno.ENOENT: pass else: # All other errors are propagated. raise def _unregister_kevent(self, path): self._descriptors.remove(path) def queue_event(self, event): EventEmitter.queue_event(self, event) if event.event_type == EVENT_TYPE_CREATED: self._register_kevent(event.src_path, event.is_directory) elif event.event_type == EVENT_TYPE_MOVED: self._unregister_kevent(event.src_path) self._register_kevent(event.dest_path, event.is_directory) elif event.event_type == EVENT_TYPE_DELETED: self._unregister_kevent(event.src_path) def _queue_dirs_modified(self, if dirs_modified: for dir_modified in dirs_modified: self.queue_event(DirModifiedEvent(dir_modified)) diff_events = new_snapshot - ref_snapshot for file_created in diff_events.files_created: self.queue_event(FileCreatedEvent(file_created)) for directory_created in diff_events.dirs_created: self.queue_event(DirCreatedEvent(directory_created)) def _queue_events_except_renames_and_dir_modifications(self, event_list): """ Queues deletion """ files_renamed = set() dirs_renamed = set() dirs_modified = set() for kev in event_list: descriptor = self._descriptors.get_for_fd(kev.ident) src_path = descriptor.path if is_deleted(kev): if descriptor.is_directory: self.queue_event(DirDeletedEvent(src_path)) else: self.queue_event(FileDeletedEvent(src_path)) elif is_attrib_modified(kev): if descriptor.is_directory: self.queue_event(DirModifiedEvent(src_path)) else: self.queue_event(FileModifiedEvent(src_path)) elif is_modified(kev): if descriptor.is_directory: dirs_modified.add(src_path) else: self.queue_event(FileModifiedEvent(src_path)) elif is_renamed(kev): if descriptor.is_directory: dirs_renamed.add(src_path) else: files_renamed.add(src_path) return files_renamed, dirs_renamed, dirs_modified def _queue_renamed(self, src_path, is_directory, ref_snapshot, new_snapshot): try: ref_stat_info = ref_snapshot.stat_info(src_path) except KeyError: if is_directory: self.queue_event(DirCreatedEvent(src_path)) self.queue_event(DirDeletedEvent(src_path)) else: self.queue_event(FileCreatedEvent(src_path)) self.queue_event(FileDeletedEvent(src_path)) return try: dest_path = absolute_path( new_snapshot.path_for_inode(ref_stat_info.st_ino)) if is_directory: event = DirMovedEvent(src_path, dest_path) if self.watch.is_recursive: for sub_event in event.sub_moved_events(): self.queue_event(sub_event) self.queue_event(event) else: self.queue_event(FileMovedEvent(src_path, dest_path)) except KeyError: if is_directory: self.queue_event(DirDeletedEvent(src_path)) else: self.queue_event(FileDeletedEvent(src_path))
def test_dispatch(self): # Utilities. patterns = ['*.py', '*.txt'] ignore_patterns = ["*.pyc"] def assert_patterns(event): if has_attribute(event, 'dest_path'): paths = [event.src_path, event.dest_path] else: paths = [event.src_path] filtered_paths = filter_paths(paths, included_patterns=patterns, excluded_patterns=ignore_patterns, case_sensitive=False) assert_true(filtered_paths) dir_del_event_match = DirDeletedEvent('/path/blah.py') dir_del_event_not_match = DirDeletedEvent('/path/foobar') dir_del_event_ignored = DirDeletedEvent('/path/foobar.pyc') file_del_event_match = FileDeletedEvent('/path/blah.txt') file_del_event_not_match = FileDeletedEvent('/path/foobar') file_del_event_ignored = FileDeletedEvent('/path/blah.pyc') dir_cre_event_match = DirCreatedEvent('/path/blah.py') dir_cre_event_not_match = DirCreatedEvent('/path/foobar') dir_cre_event_ignored = DirCreatedEvent('/path/foobar.pyc') file_cre_event_match = FileCreatedEvent('/path/blah.txt') file_cre_event_not_match = FileCreatedEvent('/path/foobar') file_cre_event_ignored = FileCreatedEvent('/path/blah.pyc') dir_mod_event_match = DirModifiedEvent('/path/blah.py') dir_mod_event_not_match = DirModifiedEvent('/path/foobar') dir_mod_event_ignored = DirModifiedEvent('/path/foobar.pyc') file_mod_event_match = FileModifiedEvent('/path/blah.txt') file_mod_event_not_match = FileModifiedEvent('/path/foobar') file_mod_event_ignored = FileModifiedEvent('/path/blah.pyc') dir_mov_event_match = DirMovedEvent('/path/blah.py', '/path/blah') dir_mov_event_not_match = DirMovedEvent('/path/foobar', '/path/blah') dir_mov_event_ignored = DirMovedEvent('/path/foobar.pyc', '/path/blah') file_mov_event_match = FileMovedEvent('/path/blah.txt', '/path/blah') file_mov_event_not_match = FileMovedEvent('/path/foobar', '/path/blah') file_mov_event_ignored = FileMovedEvent('/path/blah.pyc', '/path/blah') all_dir_events = [ dir_mod_event_match, dir_mod_event_not_match, dir_mod_event_ignored, dir_del_event_match, dir_del_event_not_match, dir_del_event_ignored, dir_cre_event_match, dir_cre_event_not_match, dir_cre_event_ignored, dir_mov_event_match, dir_mov_event_not_match, dir_mov_event_ignored, ] all_file_events = [ file_mod_event_match, file_mod_event_not_match, file_mod_event_ignored, file_del_event_match, file_del_event_not_match, file_del_event_ignored, file_cre_event_match, file_cre_event_not_match, file_cre_event_ignored, file_mov_event_match, file_mov_event_not_match, file_mov_event_ignored, ] all_events = all_file_events + all_dir_events class TestableEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): assert_check_directory(self, event) def on_modified(self, event): assert_check_directory(self, event) assert_event_type(event, EVENT_TYPE_MODIFIED) assert_patterns(event) def on_deleted(self, event): assert_check_directory(self, event) assert_event_type(event, EVENT_TYPE_DELETED) assert_patterns(event) def on_moved(self, event): assert_check_directory(self, event) assert_event_type(event, EVENT_TYPE_MOVED) assert_patterns(event) def on_created(self, event): assert_check_directory(self, event) assert_event_type(event, EVENT_TYPE_CREATED) assert_patterns(event) no_dirs_handler = TestableEventHandler(patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=True) handler = TestableEventHandler(patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=False) for event in all_events: no_dirs_handler.dispatch(event) for event in all_events: handler.dispatch(event)
def queue_events(self, timeout): # We don't want to hit the disk continuously. # timeout behaves like an interval for polling emitters. if self.stopped_event.wait(timeout): return with self._lock: if not self.should_keep_running(): return # Get event diff between fresh snapshot and previous snapshot. # Update snapshot. new_snapshot = self._take_snapshot() events = DirectorySnapshotDiff(self._snapshot, new_snapshot) self._snapshot = new_snapshot # Files. for src_path in events.files_deleted: self._modifying_files[None][ src_path] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT # self.queue_event(FileDeletedEvent(src_path)) for src_path in events.files_modified: self._modifying_files[src_path][ src_path] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT for src_path in events.files_created: self._modifying_files[src_path][ None] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT # self.queue_event(FileCreatedEvent(src_path)) for src_path, dest_path in events.files_moved: self._modifying_files[dest_path] = self._modifying_files[ src_path] del self._modifying_files[src_path] for modifying_src_path in self._modifying_files[dest_path]: self._modifying_files[dest_path][ modifying_src_path] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT # self.queue_event(FileMovedEvent(src_path, dest_path)) deletion_list = [] for modifying_dest_path in self._modifying_files: for modifying_src_path in self._modifying_files[ modifying_dest_path]: self._modifying_files[modifying_dest_path][ modifying_src_path] -= 1 if self._modifying_files[modifying_dest_path][ modifying_src_path] == 0: logging.debug( "New Event: Src: %s, Dest: %s", (modifying_src_path, modifying_dest_path)) if modifying_dest_path is not None and modifying_src_path is not None: self.queue_event( FileDeletedEvent(modifying_src_path)) self.queue_event( FileCreatedEvent(modifying_dest_path)) elif modifying_dest_path is not None: self.queue_event( FileCreatedEvent(modifying_dest_path)) elif modifying_src_path is not None: self.queue_event( FileDeletedEvent(modifying_src_path)) deletion_list.append( (modifying_dest_path, modifying_src_path)) for (dest_path, src_path) in deletion_list: del self._modifying_files[dest_path][src_path] # Directories. for src_path in events.dirs_deleted: self.queue_event(DirDeletedEvent(src_path)) for src_path in events.dirs_modified: self.queue_event(DirModifiedEvent(src_path)) for src_path in events.dirs_created: self.queue_event(DirCreatedEvent(src_path)) for src_path, dest_path in events.dirs_moved: self.queue_event(DirMovedEvent(src_path, dest_path))
def test_dir_modified_event(): event = DirModifiedEvent(path_1) assert path_1 == event.src_path assert EVENT_TYPE_MODIFIED == event.event_type assert event.is_directory assert not event.is_synthetic
def _queue_renamed_event(self, src_event, src_path, dst_path, src_dirname, dst_dirname): cls = DirMovedEvent if src_event.is_directory else FileMovedEvent dst_path = self._encode_path(dst_path) self.queue_event(cls(src_path, dst_path)) self.queue_event(DirModifiedEvent(src_dirname)) self.queue_event(DirModifiedEvent(dst_dirname))
def test_directory_events_ignored(): restart_event = threading.Event() restarter = reloader.Restarter(restart_event) app_modified = DirModifiedEvent(src_path='./') restarter.on_any_event(app_modified) assert not restart_event.is_set()
def queue_events(self, timeout): with self._lock: events = self.native_events i = 0 while i < len(events): event = events[i] src_path = self._encode_path(event.path) # For some reason the create and remove flags are sometimes also # set for rename and modify type events, so let those take # precedence. if event.is_renamed: # Internal moves appears to always be consecutive in the same # buffer and have IDs differ by exactly one (while others # don't) making it possible to pair up the two events coming # from a singe move operation. (None of this is documented!) # Otherwise, guess whether file was moved in or out. # TODO: handle id wrapping if (i + 1 < len(events) and events[i + 1].is_renamed and events[i + 1].event_id == event.event_id + 1): cls = DirMovedEvent if event.is_directory else FileMovedEvent dst_path = self._encode_path(events[i + 1].path) self.queue_event(cls(src_path, dst_path)) self.queue_event( DirModifiedEvent(os.path.dirname(src_path))) self.queue_event( DirModifiedEvent(os.path.dirname(dst_path))) i += 1 elif os.path.exists(event.path): cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(src_path)) self.queue_event( DirModifiedEvent(os.path.dirname(src_path))) else: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(src_path)) self.queue_event( DirModifiedEvent(os.path.dirname(src_path))) # TODO: generate events for tree elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod: cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(src_path)) elif event.is_created: cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(src_path)) self.queue_event( DirModifiedEvent(os.path.dirname(src_path))) elif event.is_removed: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(src_path)) self.queue_event( DirModifiedEvent(os.path.dirname(src_path))) if src_path == self.watch.path: # this should not really occur, instead we expect # is_root_changed to be set self.stop() elif event.is_root_changed: # This will be set if root or any if its parents is renamed or # deleted. # TODO: find out new path and generate DirMovedEvent? self.queue_event(DirDeletedEvent(self.watch.path)) self.stop() i += 1
def test_behavior_readonly_public_attributes(self): event = DirModifiedEvent(path_1) for prop in list_attributes(event): self.assertRaises(AttributeError, setattr, event, prop, None)
def test___repr__(self): event = DirModifiedEvent(path_1) assert_equal("<DirModifiedEvent: src_path=%s>" % path_1, event.__repr__())
def test_behavior_readonly_public_attributes(self): event = DirModifiedEvent(path_1) assert_readonly_public_attributes(event)
def queue_events(self, timeout): for idx in xrange(len(self.pathnames)): event_path = absolute_path(self.pathnames[idx]) event_flags = self.flags[idx] if not self.watch.is_recursive and self.watch.path != event_path: return recursive_update = bool(event_flags & FSEventsStreamFlag.MustScanSubDirs) # try to build only partial snapshot new_snapshot = DirectorySnapshot(event_path, recursive_update) if recursive_update and self.watch.path == event_path: # no optimization is possible events = new_snapshot - self.snapshot self.snapshot = new_snapshot else: # partial comparison will be done previous_snapshot = self.snapshot.copy( event_path, recursive_update) # compare them events = new_snapshot - previous_snapshot if events.dirs_deleted or events.dirs_created or events.dirs_moved: # add files from deleted dir to previous snapshot previous_snapshot.add_entries( self.snapshot.copy_multiple( events.dirs_deleted, True)) # add files from created dir to new_snapshot, create a recursive snapshot of new dir for new_path in events.dirs_created: new_snapshot.add_entries( DirectorySnapshot(new_path, True)) previous_snapshot.add_entries( self.snapshot.copy_multiple([ old_path for (old_path, new_path) in events.dirs_moved ], True)) for old_path, new_path in events.dirs_moved: new_snapshot.add_entries( DirectorySnapshot(new_path, True)) # re-do diff events = new_snapshot - previous_snapshot # update last snapshot self.snapshot.remove_entries(previous_snapshot) self.snapshot.add_entries(new_snapshot) # Files. for src_path in events.files_deleted: self.queue_event(FileDeletedEvent(src_path)) for src_path in events.files_modified: self.queue_event(FileModifiedEvent(src_path)) for src_path in events.files_created: self.queue_event(FileCreatedEvent(src_path)) for src_path, dest_path in events.files_moved: self.queue_event(FileMovedEvent(src_path, dest_path)) # Directories. for src_path in events.dirs_deleted: self.queue_event(DirDeletedEvent(src_path)) for src_path in events.dirs_modified: self.queue_event(DirModifiedEvent(src_path)) for src_path in events.dirs_created: self.queue_event(DirCreatedEvent(src_path)) for src_path, dest_path in events.dirs_moved: self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout, events): if logger.getEffectiveLevel() <= logging.DEBUG: for event in events: flags = ", ".join(attr for attr in dir(event) if getattr(event, attr) is True) logger.debug(f"{event}: {flags}") while events: event = events.pop(0) src_path = self._encode_path(event.path) if event.is_renamed: dest_event = next(iter(e for e in events if e.is_renamed and e.inode == event.inode), None) if dest_event: # item was moved within the watched folder events.remove(dest_event) logger.debug("Destination event for rename is %s", dest_event) cls = DirMovedEvent if event.is_directory else FileMovedEvent dst_path = self._encode_path(dest_event.path) self.queue_event(cls(src_path, dst_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) self.queue_event(DirModifiedEvent(os.path.dirname(dst_path))) for sub_event in generate_sub_moved_events(src_path, dst_path): logger.debug("Generated sub event: %s", sub_event) self.queue_event(sub_event) elif os.path.exists(event.path): # item was moved into the watched folder cls = DirCreatedEvent if event.is_directory else FileCreatedEvent self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) for sub_event in generate_sub_created_events(src_path): self.queue_event(sub_event) else: # item was moved out of the watched folder cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) if event.is_created: cls = DirCreatedEvent if event.is_directory else FileCreatedEvent if not event.is_coalesced or ( event.is_coalesced and not event.is_renamed and not event.is_modified and not event.is_inode_meta_mod and not event.is_xattr_mod ): self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) if event.is_modified and not event.is_coalesced and os.path.exists(src_path): cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(src_path)) if event.is_inode_meta_mod or event.is_xattr_mod: if os.path.exists(src_path) and not event.is_coalesced: # NB: in the scenario of touch(file) -> rm(file) we can trigger this twice cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(src_path)) if event.is_removed: cls = DirDeletedEvent if event.is_directory else FileDeletedEvent if not event.is_coalesced or (event.is_coalesced and not os.path.exists(event.path)): self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(os.path.dirname(src_path))) if src_path == self.watch.path: # this should not really occur, instead we expect # is_root_changed to be set logger.debug("Stopping because root path was removed") self.stop() if event.is_root_changed: # This will be set if root or any of its parents is renamed or # deleted. # TODO: find out new path and generate DirMovedEvent? self.queue_event(DirDeletedEvent(self.watch.path)) logger.debug("Stopping because root path was changed") self.stop()
def _queue_deleted_event(self, event, src_path, dirname): cls = DirDeletedEvent if event.is_directory else FileDeletedEvent self.queue_event(cls(src_path)) self.queue_event(DirModifiedEvent(dirname))