def _queue_dirs_modified(self, dirs_modified, ref_snapshot,
new_snapshot):
"""
Queues events for directory modifications by scanning the directory
for changes.
A scan is a comparison between two snapshots of the same directory
taken at two different times. This also determines whether files
or directories were created, which updated the modified timestamp
for the directory.
"""
if dirs_modified:
for dir_modified in dirs_modified:
self.queue_event(DirModifiedEvent(dir_modified))
diff_events = new_snapshot - ref_snapshot
for file_created in diff_events.files_created:
self.queue_event(FileCreatedEvent(file_created))
for directory_created in diff_events.dirs_created:
self.queue_event(DirCreatedEvent(directory_created))
def test_modify():
touch(p('a'))
start_watching()
touch(p('a'))
# Because the tests run so fast then on macOS it is almost certain that
# we receive a coalesced event from fseventsd here, which triggers an
# additional file created event and dir modified event here.
if platform.is_darwin():
expect_event(FileCreatedEvent(p('a')))
expect_event(DirModifiedEvent(p()))
expect_event(FileModifiedEvent(p('a')))
if platform.is_linux():
event = event_queue.get(timeout=5)[0]
assert event.src_path == p('a')
assert isinstance(event, FileClosedEvent)
def start_watching(path=None, use_full_emitter=False, recursive=True):
# todo: check if other platforms expect the trailing slash (e.g. `p('')`)
path = p() if path is None else path
global emitter
if platform.is_linux() and use_full_emitter:
emitter = InotifyFullEmitter(event_queue,
ObservedWatch(path, recursive=recursive))
else:
emitter = Emitter(event_queue, ObservedWatch(path,
recursive=recursive))
emitter.start()
if platform.is_darwin():
# FSEvents _may_ report the event for the creation of `tmpdir`,
# however, we're racing with fseventd there - if other filesystem
# events happened _after_ `tmpdir` was created, but _before_ we
# created the emitter then we won't get this event.
# As such, let's create a sentinel event that tells us that we are
# good to go.
sentinel_file = os.path.join(
path,
'.sentinel' if isinstance(path, str) else '.sentinel'.encode())
touch(sentinel_file)
sentinel_events = [
FileCreatedEvent(sentinel_file),
DirModifiedEvent(path),
FileModifiedEvent(sentinel_file)
]
next_sentinel_event = sentinel_events.pop(0)
now = time.monotonic()
while time.monotonic() <= now + 30.0:
try:
event = event_queue.get(timeout=0.5)[0]
if event == next_sentinel_event:
if not sentinel_events:
break
next_sentinel_event = sentinel_events.pop(0)
except Empty:
pass
time.sleep(0.1)
else:
assert False, "Sentinel event never arrived!"
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except OSError as e:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
except Exception as e:
raise e
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout, full_events=False):
# If "full_events" is true, then the method will report unmatched move events as seperate events
# This behavior is by default only called by a InotifyFullEmitter
with self._lock:
event = self._inotify.read_event()
if event is None:
return
if isinstance(event, tuple):
move_from, move_to = event
src_path = self._decode_path(move_from.src_path)
dest_path = self._decode_path(move_to.src_path)
cls = DirMovedEvent if move_from.is_directory else FileMovedEvent
self.queue_event(cls(src_path, dest_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
self.queue_event(DirModifiedEvent(os.path.dirname(dest_path)))
if move_from.is_directory and self.watch.is_recursive:
for sub_event in generate_sub_moved_events(src_path, dest_path):
self.queue_event(sub_event)
return
src_path = self._decode_path(event.src_path)
if event.is_moved_to:
if full_events:
cls = DirMovedEvent if event.is_directory else FileMovedEvent
self.queue_event(cls(None, src_path))
else:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
if event.is_directory and self.watch.is_recursive:
for sub_event in generate_sub_created_events(src_path):
self.queue_event(sub_event)
elif event.is_attrib:
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
elif event.is_modify:
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
elif event.is_delete or (event.is_moved_from and not full_events):
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
elif event.is_moved_from and full_events:
cls = DirMovedEvent if event.is_directory else FileMovedEvent
self.queue_event(cls(src_path, None))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
elif event.is_create:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
def test_file_system_event_handler_dispatch():
dir_del_event = DirDeletedEvent('/path/blah.py')
file_del_event = FileDeletedEvent('/path/blah.txt')
dir_cre_event = DirCreatedEvent('/path/blah.py')
file_cre_event = FileCreatedEvent('/path/blah.txt')
dir_mod_event = DirModifiedEvent('/path/blah.py')
file_mod_event = FileModifiedEvent('/path/blah.txt')
dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah')
file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah')
all_events = [
dir_mod_event,
dir_del_event,
dir_cre_event,
dir_mov_event,
file_mod_event,
file_del_event,
file_cre_event,
file_mov_event,
]
class TestableEventHandler(FileSystemEventHandler):
def on_any_event(self, event):
assert True
def on_modified(self, event):
assert event.event_type == EVENT_TYPE_MODIFIED
def on_deleted(self, event):
assert event.event_type == EVENT_TYPE_DELETED
def on_moved(self, event):
assert event.event_type == EVENT_TYPE_MOVED
def on_created(self, event):
assert event.event_type == EVENT_TYPE_CREATED
handler = TestableEventHandler()
for event in all_events:
assert not event.is_synthetic
handler.dispatch(event)
def queue_events(self, timeout):
events = self._fsevents.read_events()
if events is None:
return
i = 0
while i < len(events):
event = events[i]
# For some reason the create and remove flags are sometimes also
# set for rename and modify type events, so let those take
# precedence.
if event.is_renamed:
# Internal moves appears to always be consecutive in the same
# buffer and have IDs differ by exactly one (while others
# don't) making it possible to pair up the two events coming
# from a singe move operation. (None of this is documented!)
# Otherwise, guess whether file was moved in or out.
#TODO: handle id wrapping
if (i+1 < len(events) and events[i+1].is_renamed and
events[i+1].event_id == event.event_id + 1):
cls = DirMovedEvent if event.is_directory else FileMovedEvent
self.queue_event(cls(event.path, events[i+1].path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
self.queue_event(DirModifiedEvent(os.path.dirname(events[i+1].path)))
i += 1
elif os.path.exists(event.path):
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
else:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
#TODO: generate events for tree
elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod :
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(event.path))
elif event.is_created:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
elif event.is_removed:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
i += 1
def _queue_events_except_renames_and_dir_modifications(self, event_list):
"""
Queues events from the kevent list returned from the call to
:meth:`select.kqueue.control`.
.. NOTE:: Queues only the deletions, file modifications,
attribute modifications. The other events, namely,
file creation, directory modification, file rename,
directory rename, directory creation, etc. are
determined by comparing directory snapshots.
"""
files_renamed = set()
dirs_renamed = set()
dirs_modified = set()
for kev in event_list:
descriptor = self._descriptors.get_for_fd(kev.ident)
src_path = descriptor.path
if is_deleted(kev):
if descriptor.is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
elif is_attrib_modified(kev):
if descriptor.is_directory:
self.queue_event(DirModifiedEvent(src_path))
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_modified(kev):
if descriptor.is_directory:
dirs_modified.add(src_path)
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_renamed(kev):
if descriptor.is_directory:
dirs_renamed.add(src_path)
else:
files_renamed.add(src_path)
return (files_renamed, dirs_renamed, dirs_modified)
def start(self, *args, **kwargs):
previous_snapshots = dict()
if os.path.exists(self._filename):
with open(self._filename, 'rb') as f:
previous_snapshots = pickle.load(f)
for watcher, handlers in self._handlers.items():
try:
path = watcher.path
curr_snap = DirectorySnapshot(path)
pre_snap = previous_snapshots.get(path, _EmptySnapshot(path))
diff = DirectorySnapshotDiff(pre_snap, curr_snap)
for handler in handlers:
# Dispatch files modifications
for new_path in diff.files_created:
handler.dispatch(FileCreatedEvent(new_path))
for del_path in diff.files_deleted:
handler.dispatch(FileDeletedEvent(del_path))
for mod_path in diff.files_modified:
handler.dispatch(FileModifiedEvent(mod_path))
for src_path, mov_path in diff.files_moved:
handler.dispatch(FileMovedEvent(src_path, mov_path))
# Dispatch directories modifications
for new_dir in diff.dirs_created:
handler.dispatch(DirCreatedEvent(new_dir))
for del_dir in diff.dirs_deleted:
handler.dispatch(DirDeletedEvent(del_dir))
for mod_dir in diff.dirs_modified:
handler.dispatch(DirModifiedEvent(mod_dir))
for src_path, mov_path in diff.dirs_moved:
handler.dispatch(DirMovedEvent(src_path, mov_path))
except PermissionError as e:
print(e)
Observer.start(self, *args, **kwargs)
class TestConfigFileEventHandler(unittest.TestCase):
def setUp(self) -> None:
self.callback = mock.Mock(return_value=None, autospec=True)
self.config_logger = logging.getLogger("test_config")
self.test_config_event_handler = ConfigFileEventHandler(
self.config_logger, self.callback, ["*.ini"])
def test_initialised(self):
self.assertIsNotNone(self.test_config_event_handler)
@parameterized.expand([
(FileCreatedEvent("test_src/test.ini"), ),
(DirCreatedEvent("test_src/"), ),
])
def test_on_created_triggers_callback(self, event: FileSystemEvent):
self.test_config_event_handler.on_created(event)
self.callback.assert_called_once_with(event)
@parameterized.expand([
(FileModifiedEvent("test_src/test.ini"), ),
(DirModifiedEvent("test_src/"), ),
])
def test_on_modified_triggers_callback(self, event: FileSystemEvent):
self.test_config_event_handler.on_modified(event)
self.callback.assert_called_once_with(event)
@parameterized.expand([
(FileDeletedEvent("test_src/test.ini"), ),
(DirDeletedEvent("test_src/"), ),
])
def test_on_deleted_triggers_callback(self, event: FileSystemEvent):
self.test_config_event_handler.on_deleted(event)
self.callback.assert_called_once_with(event)
def test_separate_consecutive_moves():
mkdir(p('dir1'))
mkfile(p('dir1', 'a'))
mkfile(p('b'))
start_watching(p('dir1'))
mv(p('dir1', 'a'), p('c'))
mv(p('b'), p('dir1', 'd'))
dir_modif = DirModifiedEvent(p('dir1'))
a_deleted = FileDeletedEvent(p('dir1', 'a'))
d_created = FileCreatedEvent(p('dir1', 'd'))
expected_events = [a_deleted, dir_modif, d_created, dir_modif]
if platform.is_windows():
expected_events = [a_deleted, d_created]
if platform.is_bsd():
# Due to the way kqueue works, we can't really order
# 'Created' and 'Deleted' events in time, so creation queues first
expected_events = [d_created, a_deleted, dir_modif, dir_modif]
for expected_event in expected_events:
expect_event(expected_event)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
time.sleep(timeout)
with self._lock:
if not self._snapshot:
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
events = self._fsevents.read_events()
if events is None:
return
i = 0
while i < len(events):
event = events[i]
if event.is_renamed:
if (i+1 < len(events) and events[i+1].is_renamed and
events[i+1].event_id == event.event_id + 1):
cls = DirMovedEvent if event.is_directory else FileMovedEvent
self.queue_event(cls(event.path, events[i+1].path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
self.queue_event(DirModifiedEvent(os.path.dirname(events[i+1].path)))
i += 1
elif os.path.exists(event.path):
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
else:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
#TODO: generate events for tree
elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod :
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(event.path))
elif event.is_created:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
elif event.is_removed:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(event.path))
self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
i += 1
def test___init__(self):
event = DirModifiedEvent(path_1)
assert_equal(path_1, event.src_path)
assert_equal(EVENT_TYPE_MODIFIED, event.event_type)
assert_true(event.is_directory)
def test___repr__(self):
event = DirModifiedEvent(path_1)
self.assertEqual("<DirModifiedEvent: src_path=%s>" % path_1,
event.__repr__())
def test_dispatch(self):
# Utilities.
regexes = [r".*\.py", r".*\.txt"]
ignore_regexes = [r".*\.pyc"]
def assert_regexes(handler, event):
if has_attribute(event, "dest_path"):
paths = [event.src_path, event.dest_path]
else:
paths = [event.src_path]
filtered_paths = set()
for p in paths:
if any(r.match(p) for r in handler.regexes):
filtered_paths.add(p)
self.assertTrue(filtered_paths)
dir_del_event_match = DirDeletedEvent("/path/blah.py")
dir_del_event_not_match = DirDeletedEvent("/path/foobar")
dir_del_event_ignored = DirDeletedEvent("/path/foobar.pyc")
file_del_event_match = FileDeletedEvent("/path/blah.txt")
file_del_event_not_match = FileDeletedEvent("/path/foobar")
file_del_event_ignored = FileDeletedEvent("/path/blah.pyc")
dir_cre_event_match = DirCreatedEvent("/path/blah.py")
dir_cre_event_not_match = DirCreatedEvent("/path/foobar")
dir_cre_event_ignored = DirCreatedEvent("/path/foobar.pyc")
file_cre_event_match = FileCreatedEvent("/path/blah.txt")
file_cre_event_not_match = FileCreatedEvent("/path/foobar")
file_cre_event_ignored = FileCreatedEvent("/path/blah.pyc")
dir_mod_event_match = DirModifiedEvent("/path/blah.py")
dir_mod_event_not_match = DirModifiedEvent("/path/foobar")
dir_mod_event_ignored = DirModifiedEvent("/path/foobar.pyc")
file_mod_event_match = FileModifiedEvent("/path/blah.txt")
file_mod_event_not_match = FileModifiedEvent("/path/foobar")
file_mod_event_ignored = FileModifiedEvent("/path/blah.pyc")
dir_mov_event_match = DirMovedEvent("/path/blah.py", "/path/blah")
dir_mov_event_not_match = DirMovedEvent("/path/foobar", "/path/blah")
dir_mov_event_ignored = DirMovedEvent("/path/foobar.pyc", "/path/blah")
file_mov_event_match = FileMovedEvent("/path/blah.txt", "/path/blah")
file_mov_event_not_match = FileMovedEvent("/path/foobar", "/path/blah")
file_mov_event_ignored = FileMovedEvent("/path/blah.pyc", "/path/blah")
all_dir_events = [
dir_mod_event_match,
dir_mod_event_not_match,
dir_mod_event_ignored,
dir_del_event_match,
dir_del_event_not_match,
dir_del_event_ignored,
dir_cre_event_match,
dir_cre_event_not_match,
dir_cre_event_ignored,
dir_mov_event_match,
dir_mov_event_not_match,
dir_mov_event_ignored,
]
all_file_events = [
file_mod_event_match,
file_mod_event_not_match,
file_mod_event_ignored,
file_del_event_match,
file_del_event_not_match,
file_del_event_ignored,
file_cre_event_match,
file_cre_event_not_match,
file_cre_event_ignored,
file_mov_event_match,
file_mov_event_not_match,
file_mov_event_ignored,
]
all_events = all_file_events + all_dir_events
def assert_check_directory(handler, event):
self.assertFalse(handler.ignore_directories and event.is_directory)
def assert_equal(a, b):
self.assertEqual(a, b)
class TestableEventHandler(RegexMatchingEventHandler):
def on_any_event(self, event):
assert_check_directory(self, event)
def on_modified(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_MODIFIED)
assert_regexes(self, event)
def on_deleted(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_DELETED)
assert_regexes(self, event)
def on_moved(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_MOVED)
assert_regexes(self, event)
def on_created(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_CREATED)
assert_regexes(self, event)
no_dirs_handler = TestableEventHandler(
regexes=regexes, ignore_regexes=ignore_regexes, ignore_directories=True
)
handler = TestableEventHandler(
regexes=regexes, ignore_regexes=ignore_regexes, ignore_directories=False
)
for event in all_events:
no_dirs_handler.dispatch(event)
for event in all_events:
handler.dispatch(event)
def test_dispatch(self):
# Utilities.
patterns = ["*.py", "*.txt"]
ignore_patterns = ["*.pyc"]
def assert_patterns(event):
if has_attribute(event, "dest_path"):
paths = [event.src_path, event.dest_path]
else:
paths = [event.src_path]
filtered_paths = filter_paths(
paths,
included_patterns=patterns,
excluded_patterns=ignore_patterns,
case_sensitive=False,
)
self.assertTrue(filtered_paths)
dir_del_event_match = DirDeletedEvent("/path/blah.py")
dir_del_event_not_match = DirDeletedEvent("/path/foobar")
dir_del_event_ignored = DirDeletedEvent("/path/foobar.pyc")
file_del_event_match = FileDeletedEvent("/path/blah.txt")
file_del_event_not_match = FileDeletedEvent("/path/foobar")
file_del_event_ignored = FileDeletedEvent("/path/blah.pyc")
dir_cre_event_match = DirCreatedEvent("/path/blah.py")
dir_cre_event_not_match = DirCreatedEvent("/path/foobar")
dir_cre_event_ignored = DirCreatedEvent("/path/foobar.pyc")
file_cre_event_match = FileCreatedEvent("/path/blah.txt")
file_cre_event_not_match = FileCreatedEvent("/path/foobar")
file_cre_event_ignored = FileCreatedEvent("/path/blah.pyc")
dir_mod_event_match = DirModifiedEvent("/path/blah.py")
dir_mod_event_not_match = DirModifiedEvent("/path/foobar")
dir_mod_event_ignored = DirModifiedEvent("/path/foobar.pyc")
file_mod_event_match = FileModifiedEvent("/path/blah.txt")
file_mod_event_not_match = FileModifiedEvent("/path/foobar")
file_mod_event_ignored = FileModifiedEvent("/path/blah.pyc")
dir_mov_event_match = DirMovedEvent("/path/blah.py", "/path/blah")
dir_mov_event_not_match = DirMovedEvent("/path/foobar", "/path/blah")
dir_mov_event_ignored = DirMovedEvent("/path/foobar.pyc", "/path/blah")
file_mov_event_match = FileMovedEvent("/path/blah.txt", "/path/blah")
file_mov_event_not_match = FileMovedEvent("/path/foobar", "/path/blah")
file_mov_event_ignored = FileMovedEvent("/path/blah.pyc", "/path/blah")
all_dir_events = [
dir_mod_event_match,
dir_mod_event_not_match,
dir_mod_event_ignored,
dir_del_event_match,
dir_del_event_not_match,
dir_del_event_ignored,
dir_cre_event_match,
dir_cre_event_not_match,
dir_cre_event_ignored,
dir_mov_event_match,
dir_mov_event_not_match,
dir_mov_event_ignored,
]
all_file_events = [
file_mod_event_match,
file_mod_event_not_match,
file_mod_event_ignored,
file_del_event_match,
file_del_event_not_match,
file_del_event_ignored,
file_cre_event_match,
file_cre_event_not_match,
file_cre_event_ignored,
file_mov_event_match,
file_mov_event_not_match,
file_mov_event_ignored,
]
all_events = all_file_events + all_dir_events
def assert_check_directory(handler, event):
self.assertFalse(handler.ignore_directories and event.is_directory)
def assert_equal(a, b):
self.assertEqual(a, b)
class TestableEventHandler(PatternMatchingEventHandler):
def on_any_event(self, event):
assert_check_directory(self, event)
def on_modified(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_MODIFIED)
assert_patterns(event)
def on_deleted(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_DELETED)
assert_patterns(event)
def on_moved(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_MOVED)
assert_patterns(event)
def on_created(self, event):
assert_check_directory(self, event)
assert_equal(event.event_type, EVENT_TYPE_CREATED)
assert_patterns(event)
no_dirs_handler = TestableEventHandler(
patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=True
)
handler = TestableEventHandler(
patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=False
)
for event in all_events:
no_dirs_handler.dispatch(event)
for event in all_events:
handler.dispatch(event)
def test___init__(self):
SLEEP_TIME = 0.4
self.emitter.start()
sleep(SLEEP_TIME)
mkdir(p('project'))
sleep(SLEEP_TIME)
mkdir(p('project', 'blah'))
sleep(SLEEP_TIME)
touch(p('afile'))
sleep(SLEEP_TIME)
touch(p('fromfile'))
sleep(SLEEP_TIME)
mv(p('fromfile'), p('project', 'tofile'))
sleep(SLEEP_TIME)
touch(p('afile'))
sleep(SLEEP_TIME)
mv(p('project', 'blah'), p('project', 'boo'))
sleep(SLEEP_TIME)
rm(p('project'), recursive=True)
sleep(SLEEP_TIME)
rm(p('afile'))
sleep(SLEEP_TIME)
self.emitter.stop()
# What we need here for the tests to pass is a collection type
# that is:
# * unordered
# * non-unique
# A multiset! Python's collections.Counter class seems appropriate.
expected = set([
DirModifiedEvent(p()),
DirCreatedEvent(p('project')),
DirModifiedEvent(p('project')),
DirCreatedEvent(p('project', 'blah')),
FileCreatedEvent(p('afile')),
DirModifiedEvent(p()),
FileCreatedEvent(p('fromfile')),
DirModifiedEvent(p()),
DirModifiedEvent(p()),
FileModifiedEvent(p('afile')),
DirModifiedEvent(p('project')),
DirModifiedEvent(p()),
FileDeletedEvent(p('project', 'tofile')),
DirDeletedEvent(p('project', 'boo')),
DirDeletedEvent(p('project')),
DirModifiedEvent(p()),
FileDeletedEvent(p('afile')),
])
expected.add(FileMovedEvent(p('fromfile'), p('project', 'tofile')))
expected.add(DirMovedEvent(p('project', 'blah'), p('project', 'boo')))
got = set()
while True:
try:
event, _ = self.event_queue.get_nowait()
got.add(event)
except queue.Empty:
break
self.assertEqual(expected, got)
class KqueueEmitter(EventEmitter):
def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
EventEmitter.__init__(self, event_queue, watch, timeout)
self._kq = select.kqueue()
self._lock = threading.RLock()
# A collection of KeventDescriptor.
self._descriptors = KeventDescriptorSet()
def walker_callback(path, stat_info, self=self):
self._register_kevent(path, stat.S_ISDIR(stat_info.st_mode))
self._snapshot = DirectorySnapshot(watch.path,
watch.is_recursive,
walker_callback)
def _register_kevent(self, path, is_directory):
try:
self._descriptors.add(path, is_directory)
except OSError as e:
if e.errno == errno.ENOENT:
pass
else:
# All other errors are propagated.
raise
def _unregister_kevent(self, path):
self._descriptors.remove(path)
def queue_event(self, event):
EventEmitter.queue_event(self, event)
if event.event_type == EVENT_TYPE_CREATED:
self._register_kevent(event.src_path, event.is_directory)
elif event.event_type == EVENT_TYPE_MOVED:
self._unregister_kevent(event.src_path)
self._register_kevent(event.dest_path, event.is_directory)
elif event.event_type == EVENT_TYPE_DELETED:
self._unregister_kevent(event.src_path)
def _queue_dirs_modified(self,
if dirs_modified:
for dir_modified in dirs_modified:
self.queue_event(DirModifiedEvent(dir_modified))
diff_events = new_snapshot - ref_snapshot
for file_created in diff_events.files_created:
self.queue_event(FileCreatedEvent(file_created))
for directory_created in diff_events.dirs_created:
self.queue_event(DirCreatedEvent(directory_created))
def _queue_events_except_renames_and_dir_modifications(self, event_list):
"""
Queues deletion
"""
files_renamed = set()
dirs_renamed = set()
dirs_modified = set()
for kev in event_list:
descriptor = self._descriptors.get_for_fd(kev.ident)
src_path = descriptor.path
if is_deleted(kev):
if descriptor.is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
elif is_attrib_modified(kev):
if descriptor.is_directory:
self.queue_event(DirModifiedEvent(src_path))
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_modified(kev):
if descriptor.is_directory:
dirs_modified.add(src_path)
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_renamed(kev):
if descriptor.is_directory:
dirs_renamed.add(src_path)
else:
files_renamed.add(src_path)
return files_renamed, dirs_renamed, dirs_modified
def _queue_renamed(self,
src_path,
is_directory,
ref_snapshot,
new_snapshot):
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
return
try:
dest_path = absolute_path(
new_snapshot.path_for_inode(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def test_dispatch(self):
# Utilities.
patterns = ['*.py', '*.txt']
ignore_patterns = ["*.pyc"]
def assert_patterns(event):
if has_attribute(event, 'dest_path'):
paths = [event.src_path, event.dest_path]
else:
paths = [event.src_path]
filtered_paths = filter_paths(paths,
included_patterns=patterns,
excluded_patterns=ignore_patterns,
case_sensitive=False)
assert_true(filtered_paths)
dir_del_event_match = DirDeletedEvent('/path/blah.py')
dir_del_event_not_match = DirDeletedEvent('/path/foobar')
dir_del_event_ignored = DirDeletedEvent('/path/foobar.pyc')
file_del_event_match = FileDeletedEvent('/path/blah.txt')
file_del_event_not_match = FileDeletedEvent('/path/foobar')
file_del_event_ignored = FileDeletedEvent('/path/blah.pyc')
dir_cre_event_match = DirCreatedEvent('/path/blah.py')
dir_cre_event_not_match = DirCreatedEvent('/path/foobar')
dir_cre_event_ignored = DirCreatedEvent('/path/foobar.pyc')
file_cre_event_match = FileCreatedEvent('/path/blah.txt')
file_cre_event_not_match = FileCreatedEvent('/path/foobar')
file_cre_event_ignored = FileCreatedEvent('/path/blah.pyc')
dir_mod_event_match = DirModifiedEvent('/path/blah.py')
dir_mod_event_not_match = DirModifiedEvent('/path/foobar')
dir_mod_event_ignored = DirModifiedEvent('/path/foobar.pyc')
file_mod_event_match = FileModifiedEvent('/path/blah.txt')
file_mod_event_not_match = FileModifiedEvent('/path/foobar')
file_mod_event_ignored = FileModifiedEvent('/path/blah.pyc')
dir_mov_event_match = DirMovedEvent('/path/blah.py', '/path/blah')
dir_mov_event_not_match = DirMovedEvent('/path/foobar', '/path/blah')
dir_mov_event_ignored = DirMovedEvent('/path/foobar.pyc', '/path/blah')
file_mov_event_match = FileMovedEvent('/path/blah.txt', '/path/blah')
file_mov_event_not_match = FileMovedEvent('/path/foobar', '/path/blah')
file_mov_event_ignored = FileMovedEvent('/path/blah.pyc', '/path/blah')
all_dir_events = [
dir_mod_event_match,
dir_mod_event_not_match,
dir_mod_event_ignored,
dir_del_event_match,
dir_del_event_not_match,
dir_del_event_ignored,
dir_cre_event_match,
dir_cre_event_not_match,
dir_cre_event_ignored,
dir_mov_event_match,
dir_mov_event_not_match,
dir_mov_event_ignored,
]
all_file_events = [
file_mod_event_match,
file_mod_event_not_match,
file_mod_event_ignored,
file_del_event_match,
file_del_event_not_match,
file_del_event_ignored,
file_cre_event_match,
file_cre_event_not_match,
file_cre_event_ignored,
file_mov_event_match,
file_mov_event_not_match,
file_mov_event_ignored,
]
all_events = all_file_events + all_dir_events
class TestableEventHandler(PatternMatchingEventHandler):
def on_any_event(self, event):
assert_check_directory(self, event)
def on_modified(self, event):
assert_check_directory(self, event)
assert_event_type(event, EVENT_TYPE_MODIFIED)
assert_patterns(event)
def on_deleted(self, event):
assert_check_directory(self, event)
assert_event_type(event, EVENT_TYPE_DELETED)
assert_patterns(event)
def on_moved(self, event):
assert_check_directory(self, event)
assert_event_type(event, EVENT_TYPE_MOVED)
assert_patterns(event)
def on_created(self, event):
assert_check_directory(self, event)
assert_event_type(event, EVENT_TYPE_CREATED)
assert_patterns(event)
no_dirs_handler = TestableEventHandler(patterns=patterns,
ignore_patterns=ignore_patterns,
ignore_directories=True)
handler = TestableEventHandler(patterns=patterns,
ignore_patterns=ignore_patterns,
ignore_directories=False)
for event in all_events:
no_dirs_handler.dispatch(event)
for event in all_events:
handler.dispatch(event)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self._modifying_files[None][
src_path] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT
# self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self._modifying_files[src_path][
src_path] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT
for src_path in events.files_created:
self._modifying_files[src_path][
None] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT
# self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self._modifying_files[dest_path] = self._modifying_files[
src_path]
del self._modifying_files[src_path]
for modifying_src_path in self._modifying_files[dest_path]:
self._modifying_files[dest_path][
modifying_src_path] = MistWatchdogPollingEmitter.MODIFYING_DELAY_COUNT
# self.queue_event(FileMovedEvent(src_path, dest_path))
deletion_list = []
for modifying_dest_path in self._modifying_files:
for modifying_src_path in self._modifying_files[
modifying_dest_path]:
self._modifying_files[modifying_dest_path][
modifying_src_path] -= 1
if self._modifying_files[modifying_dest_path][
modifying_src_path] == 0:
logging.debug(
"New Event: Src: %s, Dest: %s",
(modifying_src_path, modifying_dest_path))
if modifying_dest_path is not None and modifying_src_path is not None:
self.queue_event(
FileDeletedEvent(modifying_src_path))
self.queue_event(
FileCreatedEvent(modifying_dest_path))
elif modifying_dest_path is not None:
self.queue_event(
FileCreatedEvent(modifying_dest_path))
elif modifying_src_path is not None:
self.queue_event(
FileDeletedEvent(modifying_src_path))
deletion_list.append(
(modifying_dest_path, modifying_src_path))
for (dest_path, src_path) in deletion_list:
del self._modifying_files[dest_path][src_path]
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_dir_modified_event():
event = DirModifiedEvent(path_1)
assert path_1 == event.src_path
assert EVENT_TYPE_MODIFIED == event.event_type
assert event.is_directory
assert not event.is_synthetic
def _queue_renamed_event(self, src_event, src_path, dst_path, src_dirname, dst_dirname):
cls = DirMovedEvent if src_event.is_directory else FileMovedEvent
dst_path = self._encode_path(dst_path)
self.queue_event(cls(src_path, dst_path))
self.queue_event(DirModifiedEvent(src_dirname))
self.queue_event(DirModifiedEvent(dst_dirname))
def test_directory_events_ignored():
restart_event = threading.Event()
restarter = reloader.Restarter(restart_event)
app_modified = DirModifiedEvent(src_path='./')
restarter.on_any_event(app_modified)
assert not restart_event.is_set()
def queue_events(self, timeout):
with self._lock:
events = self.native_events
i = 0
while i < len(events):
event = events[i]
src_path = self._encode_path(event.path)
# For some reason the create and remove flags are sometimes also
# set for rename and modify type events, so let those take
# precedence.
if event.is_renamed:
# Internal moves appears to always be consecutive in the same
# buffer and have IDs differ by exactly one (while others
# don't) making it possible to pair up the two events coming
# from a singe move operation. (None of this is documented!)
# Otherwise, guess whether file was moved in or out.
# TODO: handle id wrapping
if (i + 1 < len(events) and events[i + 1].is_renamed
and events[i + 1].event_id == event.event_id + 1):
cls = DirMovedEvent if event.is_directory else FileMovedEvent
dst_path = self._encode_path(events[i + 1].path)
self.queue_event(cls(src_path, dst_path))
self.queue_event(
DirModifiedEvent(os.path.dirname(src_path)))
self.queue_event(
DirModifiedEvent(os.path.dirname(dst_path)))
i += 1
elif os.path.exists(event.path):
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(
DirModifiedEvent(os.path.dirname(src_path)))
else:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(src_path))
self.queue_event(
DirModifiedEvent(os.path.dirname(src_path)))
# TODO: generate events for tree
elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod:
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
elif event.is_created:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(
DirModifiedEvent(os.path.dirname(src_path)))
elif event.is_removed:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(src_path))
self.queue_event(
DirModifiedEvent(os.path.dirname(src_path)))
if src_path == self.watch.path:
# this should not really occur, instead we expect
# is_root_changed to be set
self.stop()
elif event.is_root_changed:
# This will be set if root or any if its parents is renamed or
# deleted.
# TODO: find out new path and generate DirMovedEvent?
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
i += 1
def test_behavior_readonly_public_attributes(self):
event = DirModifiedEvent(path_1)
for prop in list_attributes(event):
self.assertRaises(AttributeError, setattr, event, prop, None)
def test___repr__(self):
event = DirModifiedEvent(path_1)
assert_equal("<DirModifiedEvent: src_path=%s>" % path_1,
event.__repr__())
def test_behavior_readonly_public_attributes(self):
event = DirModifiedEvent(path_1)
assert_readonly_public_attributes(event)
def queue_events(self, timeout):
for idx in xrange(len(self.pathnames)):
event_path = absolute_path(self.pathnames[idx])
event_flags = self.flags[idx]
if not self.watch.is_recursive and self.watch.path != event_path:
return
recursive_update = bool(event_flags
& FSEventsStreamFlag.MustScanSubDirs)
# try to build only partial snapshot
new_snapshot = DirectorySnapshot(event_path, recursive_update)
if recursive_update and self.watch.path == event_path:
# no optimization is possible
events = new_snapshot - self.snapshot
self.snapshot = new_snapshot
else:
# partial comparison will be done
previous_snapshot = self.snapshot.copy(
event_path, recursive_update)
# compare them
events = new_snapshot - previous_snapshot
if events.dirs_deleted or events.dirs_created or events.dirs_moved:
# add files from deleted dir to previous snapshot
previous_snapshot.add_entries(
self.snapshot.copy_multiple(
events.dirs_deleted, True))
# add files from created dir to new_snapshot, create a recursive snapshot of new dir
for new_path in events.dirs_created:
new_snapshot.add_entries(
DirectorySnapshot(new_path, True))
previous_snapshot.add_entries(
self.snapshot.copy_multiple([
old_path
for (old_path, new_path) in events.dirs_moved
], True))
for old_path, new_path in events.dirs_moved:
new_snapshot.add_entries(
DirectorySnapshot(new_path, True))
# re-do diff
events = new_snapshot - previous_snapshot
# update last snapshot
self.snapshot.remove_entries(previous_snapshot)
self.snapshot.add_entries(new_snapshot)
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout, events):
if logger.getEffectiveLevel() <= logging.DEBUG:
for event in events:
flags = ", ".join(attr for attr in dir(event) if getattr(event, attr) is True)
logger.debug(f"{event}: {flags}")
while events:
event = events.pop(0)
src_path = self._encode_path(event.path)
if event.is_renamed:
dest_event = next(iter(e for e in events if e.is_renamed and e.inode == event.inode), None)
if dest_event:
# item was moved within the watched folder
events.remove(dest_event)
logger.debug("Destination event for rename is %s", dest_event)
cls = DirMovedEvent if event.is_directory else FileMovedEvent
dst_path = self._encode_path(dest_event.path)
self.queue_event(cls(src_path, dst_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
self.queue_event(DirModifiedEvent(os.path.dirname(dst_path)))
for sub_event in generate_sub_moved_events(src_path, dst_path):
logger.debug("Generated sub event: %s", sub_event)
self.queue_event(sub_event)
elif os.path.exists(event.path):
# item was moved into the watched folder
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
for sub_event in generate_sub_created_events(src_path):
self.queue_event(sub_event)
else:
# item was moved out of the watched folder
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
if event.is_created:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
if not event.is_coalesced or (
event.is_coalesced and not event.is_renamed and not event.is_modified and not
event.is_inode_meta_mod and not event.is_xattr_mod
):
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
if event.is_modified and not event.is_coalesced and os.path.exists(src_path):
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
if event.is_inode_meta_mod or event.is_xattr_mod:
if os.path.exists(src_path) and not event.is_coalesced:
# NB: in the scenario of touch(file) -> rm(file) we can trigger this twice
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
if event.is_removed:
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
if not event.is_coalesced or (event.is_coalesced and not os.path.exists(event.path)):
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
if src_path == self.watch.path:
# this should not really occur, instead we expect
# is_root_changed to be set
logger.debug("Stopping because root path was removed")
self.stop()
if event.is_root_changed:
# This will be set if root or any of its parents is renamed or
# deleted.
# TODO: find out new path and generate DirMovedEvent?
self.queue_event(DirDeletedEvent(self.watch.path))
logger.debug("Stopping because root path was changed")
self.stop()
def _queue_deleted_event(self, event, src_path, dirname):
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(dirname))
