def _queue_renamed(self,
src_path,
is_directory,
ref_snapshot,
new_snapshot):
"""
Compares information from two directory snapshots (one taken before
the rename operation and another taken right after) to determine the
destination path of the file system object renamed, and adds
appropriate events to the event queue.
"""
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
# Probably caught a temporary file/directory that was renamed
# and deleted. Fires a sequence of created and deleted events
# for the path.
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
# We don't process any further and bail out assuming
# the event represents deletion/creation instead of movement.
return
try:
dest_path = absolute_path(
new_snapshot.path(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
# TODO: Do we need to fire moved events for the items
# inside the directory tree? Does kqueue does this
# all by itself? Check this and then enable this code
# only if it doesn't already.
# A: It doesn't. So I've enabled this block.
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
# If the new snapshot does not have an inode for the
# old path, we haven't found the new name. Therefore,
# we mark it as deleted and remove unregister the path.
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def test_dispatch(self):
# Utilities.
dir_del_event = DirDeletedEvent('/path/blah.py')
file_del_event = FileDeletedEvent('/path/blah.txt')
dir_cre_event = DirCreatedEvent('/path/blah.py')
file_cre_event = FileCreatedEvent('/path/blah.txt')
dir_mod_event = DirModifiedEvent('/path/blah.py')
file_mod_event = FileModifiedEvent('/path/blah.txt')
dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah')
file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah')
all_events = [
dir_mod_event,
dir_del_event,
dir_cre_event,
dir_mov_event,
file_mod_event,
file_del_event,
file_cre_event,
file_mov_event,
]
handler = _TestableEventHandler()
for event in all_events:
handler.dispatch(event)
def queue_events(self, timeout):
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_delete_self():
mkdir(p('dir1'))
start_watching(p('dir1'))
rm(p('dir1'), True)
expect_event(DirDeletedEvent(p('dir1')))
emitter.join(5)
assert not emitter.is_alive()
def queue_events(self, timeout, full_events=False):
# If "full_events" is true, then the method will report unmatched move events as separate events
# This behavior is by default only called by a InotifyFullEmitter
with self._lock:
event = self._inotify.read_event()
if event is None:
return
if isinstance(event, tuple):
move_from, move_to = event
src_path = self._decode_path(move_from.src_path)
dest_path = self._decode_path(move_to.src_path)
cls = DirMovedEvent if move_from.is_directory else FileMovedEvent
self.queue_event(cls(src_path, dest_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
self.queue_event(DirModifiedEvent(os.path.dirname(dest_path)))
if move_from.is_directory and self.watch.is_recursive:
for sub_event in generate_sub_moved_events(src_path, dest_path):
self.queue_event(sub_event)
return
src_path = self._decode_path(event.src_path)
if event.is_moved_to:
if full_events:
cls = DirMovedEvent if event.is_directory else FileMovedEvent
self.queue_event(cls(None, src_path))
else:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
if event.is_directory and self.watch.is_recursive:
for sub_event in generate_sub_created_events(src_path):
self.queue_event(sub_event)
elif event.is_attrib:
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
elif event.is_modify:
cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
self.queue_event(cls(src_path))
elif event.is_delete or (event.is_moved_from and not full_events):
cls = DirDeletedEvent if event.is_directory else FileDeletedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
elif event.is_moved_from and full_events:
cls = DirMovedEvent if event.is_directory else FileMovedEvent
self.queue_event(cls(src_path, None))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
elif event.is_create:
cls = DirCreatedEvent if event.is_directory else FileCreatedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
elif event.is_close_write and not event.is_directory:
cls = FileClosedEvent
self.queue_event(cls(src_path))
self.queue_event(DirModifiedEvent(os.path.dirname(src_path)))
# elif event.is_close_nowrite and not event.is_directory:
# cls = FileClosedEvent
# self.queue_event(cls(src_path))
elif event.is_delete_self and src_path == self.watch.path:
self.queue_event(DirDeletedEvent(src_path))
self.stop()
def _watch_dir(self, path):
contents = _get_contents(path)
if contents is None:
return
self._watchlist.add(path)
for exists in self._loop(path):
current_contents = _get_contents(path)
if current_contents is None:
break
added_contents = current_contents - contents
for filename in added_contents:
filepath = os.path.join(path, filename)
self._add(filepath)
if os.path.isdir(filepath):
self.queue_event(DirCreatedEvent(filepath))
else:
self.queue_event(FileCreatedEvent(filepath))
contents = current_contents
if os.path.exists(path):
self.queue_event(DirModifiedEvent(path))
for filename in contents:
filepath = os.path.join(path, filename)
self._watchlist.discard(filepath)
self._watchlist.discard(path)
self.queue_event(DirDeletedEvent(path))
def check_from_snapshot(self, sub_folder=None, state_callback=(lambda status: None), use_transaction=True):
logging.info('Scanning for changes since last application launch')
if (not sub_folder and os.path.exists(self.basepath)) or (sub_folder and os.path.exists(self.basepath + sub_folder)):
previous_snapshot = SqlSnapshot(self.basepath, self.job_data_path, sub_folder)
if sub_folder:
local_path = self.basepath + os.path.normpath(sub_folder)
else:
local_path = self.basepath
state_callback(status=_('Walking through your local folder, please wait...'))
def listdir(dir_path):
try:
return os.listdir(dir_path)
except OSError as o:
logging.error(o)
return []
snapshot = DirectorySnapshot(local_path, recursive=True, listdir=listdir)
diff = SnapshotDiffStart(previous_snapshot, snapshot)
state_callback(status=_('Detected %i local changes...') % (len(diff.dirs_created) + len(diff.files_created)
+ len(diff.dirs_moved) + len(diff.dirs_deleted)
+ len(diff.files_moved) +
len(diff.files_modified) +
len(diff.files_deleted)))
if use_transaction:
self.event_handler.begin_transaction()
for path in diff.dirs_created:
if self.interrupt:
return
self.event_handler.on_created(DirCreatedEvent(path))
for path in diff.files_created:
if self.interrupt:
return
self.event_handler.on_created(FileCreatedEvent(path))
for path in diff.dirs_moved:
if self.interrupt:
return
self.event_handler.on_moved(DirMovedEvent(path[0], path[1]))
for path in diff.files_moved:
if self.interrupt:
return
self.event_handler.on_moved(FileMovedEvent(path[0], path[1]))
for path in diff.files_modified:
if self.interrupt:
return
self.event_handler.on_modified(FileModifiedEvent(path))
for path in diff.files_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(FileDeletedEvent(path))
for path in diff.dirs_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(DirDeletedEvent(path))
if use_transaction:
self.event_handler.end_transaction()
def _queue_events_except_renames_and_dir_modifications(self, event_list):
files_renamed = set()
dirs_renamed = set()
dirs_modified = set()
for kev in event_list:
descriptor = self._descriptors.get_for_fd(kev.ident)
src_path = descriptor.path
if is_deleted(kev):
if descriptor.is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
elif is_attrib_modified(kev):
if descriptor.is_directory:
self.queue_event(DirModifiedEvent(src_path))
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_modified(kev):
if descriptor.is_directory:
dirs_modified.add(src_path)
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_renamed(kev):
if descriptor.is_directory:
dirs_renamed.add(src_path)
else:
files_renamed.add(src_path)
return (files_renamed, dirs_renamed, dirs_modified)
def queue_events(self, timeout):
with self._lock:
if not self.watch.is_recursive\
and self.watch.path not in self.pathnames:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = new_snapshot - self.snapshot
self.snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_delete_watched_dir(self):
SLEEP_TIME = 0.4
self.emitter.start()
rm(p(''), recursive=True)
sleep(SLEEP_TIME)
self.emitter.stop()
# What we need here for the tests to pass is a collection type
# that is:
# * unordered
# * non-unique
# A multiset! Python's collections.Counter class seems appropriate.
expected = set([
DirDeletedEvent(os.path.dirname(p(''))),
])
got = set()
while True:
try:
event, _ = self.event_queue.get_nowait()
got.add(event)
except queue.Empty:
break
self.assertEqual(expected, got)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(DirMovedEvent(src_path, dest_path))
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(FileMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
with self._lock:
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
time.sleep(timeout)
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
time.sleep(timeout)
with self._lock:
if not self._snapshot:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def start(self, *args, **kwargs):
previous_snapshots = dict()
if os.path.exists(self._filename):
with open(self._filename) as f:
previous_snapshots = pickle.load(f)
for watcher, handlers in self._handlers.iteritems():
path = watcher.path
curr_snap = DirectorySnapshot(path)
pre_snap = previous_snapshots.get(path, _EmptySnapshot())
diff = DirectorySnapshotDiff(pre_snap, curr_snap)
for handler in handlers:
# Dispatch files modifications
for new_path in diff.files_created:
handler.dispatch(FileCreatedEvent(new_path))
for del_path in diff.files_deleted:
handler.dispatch(FileDeletedEvent(del_path))
for mod_path in diff.files_modified:
handler.dispatch(FileModifiedEvent(mod_path))
for mov_path in diff.files_moved:
handler.dispatch(FileMovedEvent(mov_path))
# Dispatch directories modifications
for new_dir in diff.dirs_created:
handler.dispatch(DirCreatedEvent(new_dir))
for del_dir in diff.dirs_deleted:
handler.dispatch(DirDeletedEvent(del_dir))
for mod_dir in diff.dirs_modified:
handler.dispatch(DirModifiedEvent(mod_dir))
for mov_dir in diff.dirs_moved:
handler.dispatch(DirMovedEvent(mov_dir))
Observer.start(self, *args, **kwargs)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except OSError as e:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
except Exception as e:
raise e
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
winapi_events = self._read_events()
with self._lock:
last_renamed_src_path = ""
for winapi_event in winapi_events:
src_path = os.path.join(self.watch.path, winapi_event.src_path)
if winapi_event.is_renamed_old:
last_renamed_src_path = src_path
elif winapi_event.is_renamed_new:
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(dest_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
# HACK: We introduce a forced delay before
# traversing the moved directory. This will read
# only file movement that finishes within this
# delay time.
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
# The following block of code may not
# obtain moved events for the entire tree if
# the I/O is not completed within the above
# delay time. So, it's not guaranteed to work.
# TODO: Come up with a better solution, possibly
# a way to wait for I/O to complete before
# queuing events.
for sub_moved_event in generate_sub_moved_events(
src_path, dest_path):
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
elif winapi_event.is_modified:
cls = DirModifiedEvent if os.path.isdir(
src_path) else FileModifiedEvent
self.queue_event(cls(src_path))
elif winapi_event.is_added:
isdir = os.path.isdir(src_path)
cls = DirCreatedEvent if isdir else FileCreatedEvent
self.queue_event(cls(src_path))
if isdir and self.watch.is_recursive:
# If a directory is moved from outside the watched folder to inside it
# we only get a created directory event out of it, not any events for its children
# so use the same hack as for file moves to get the child events
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
sub_events = generate_sub_created_events(src_path)
for sub_created_event in sub_events:
self.queue_event(sub_created_event)
elif winapi_event.is_removed:
self.queue_event(FileDeletedEvent(src_path))
elif winapi_event.is_removed_self:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
def test_type_changes(sync):
file_events = [
# keep as is
FileDeletedEvent(ipath(1)),
DirCreatedEvent(ipath(1)),
# keep as is
DirDeletedEvent(ipath(2)),
FileCreatedEvent(ipath(2)),
]
res = [
# keep as is
FileDeletedEvent(ipath(1)),
DirCreatedEvent(ipath(1)),
# keep as is
DirDeletedEvent(ipath(2)),
FileCreatedEvent(ipath(2)),
]
cleaned_events = sync._clean_local_events(file_events)
assert set(cleaned_events) == set(res)
def test_performance(sync):
# 10,000 nested deleted events (5,000 folders, 5,000 files)
file_events = [DirDeletedEvent(n * ipath(1)) for n in range(1, 5001)]
file_events += [
FileDeletedEvent(n * ipath(1) + ".txt") for n in range(1, 5001)
]
# 10,000 nested moved events (5,000 folders, 5,000 files)
file_events += [
DirMovedEvent(n * ipath(2), n * ipath(3)) for n in range(1, 5001)
]
file_events += [
FileMovedEvent(n * ipath(2) + ".txt",
n * ipath(3) + ".txt") for n in range(1, 5001)
]
# 4,995 unrelated created events
file_events += [FileCreatedEvent(ipath(n)) for n in range(5, 5001)]
res = [
DirDeletedEvent(ipath(1)),
DirMovedEvent(ipath(2), ipath(3)),
FileDeletedEvent(ipath(1) + ".txt"),
FileMovedEvent(ipath(2) + ".txt",
ipath(3) + ".txt"),
]
res += [FileCreatedEvent(ipath(n)) for n in range(5, 5001)]
cleaned_events = sync._clean_local_events(file_events)
assert set(cleaned_events) == set(res)
n_loops = 4
duration = timeit.timeit(lambda: sync._clean_local_events(file_events),
number=n_loops)
assert duration < 10 * n_loops
def test_nested_events(sync):
file_events = [
# convert to a single DirDeleted
DirDeletedEvent(ipath(1)),
FileDeletedEvent(ipath(1) + "/file1.txt"),
FileDeletedEvent(ipath(1) + "/file2.txt"),
DirDeletedEvent(ipath(1) + "/sub"),
FileDeletedEvent(ipath(1) + "/sub/file3.txt"),
# convert to a single DirMoved
DirMovedEvent(ipath(2), ipath(3)),
FileMovedEvent(ipath(2) + "/file1.txt", ipath(3) + "/file1.txt"),
FileMovedEvent(ipath(2) + "/file2.txt", ipath(3) + "/file2.txt"),
DirMovedEvent(ipath(2) + "/sub", ipath(3) + "/sub"),
FileMovedEvent(ipath(2) + "/sub/file3.txt", ipath(3) + "/sub/file3.txt"),
]
res = [
DirDeletedEvent(ipath(1)),
DirMovedEvent(ipath(2), ipath(3)),
]
cleaned_events = sync._clean_local_events(file_events)
assert set(cleaned_events) == set(res)
def _queue_renamed(self, src_path, is_directory, ref_snapshot,
new_snapshot):
"""
Compares information from two directory snapshots (one taken before
the rename operation and another taken right after) to determine the
destination path of the file system object renamed, and adds
appropriate events to the event queue.
"""
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
return
try:
dest_path = absolute_path(
new_snapshot.path_for_inode(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def _queue_events_except_renames_and_dir_modifications(
self, event_list):
"""
Queues events from the kevent list returned from the call to
:meth:`select.kqueue.control`.
.. NOTE:: Queues only the deletions, file modifications,
attribute modifications. The other events, namely,
file creation, directory modification, file rename,
directory rename, directory creation, etc. are
determined by comparing directory snapshots.
"""
files_renamed = set()
dirs_renamed = set()
dirs_modified = set()
for kev in event_list:
descriptor = self._descriptors.get_for_fd(kev.ident)
src_path = descriptor.path
if is_deleted(kev):
if descriptor.is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
elif is_attrib_modified(kev):
if descriptor.is_directory:
self.queue_event(DirModifiedEvent(src_path))
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_modified(kev):
if descriptor.is_directory:
# When a directory is modified, it may be due to
# sub-file/directory renames or new file/directory
# creation. We determine all this by comparing
# snapshots later.
dirs_modified.add(src_path)
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_renamed(kev):
# Kqueue does not specify the destination names for renames
# to, so we have to process these after taking a snapshot
# of the directory.
if descriptor.is_directory:
dirs_renamed.add(src_path)
else:
files_renamed.add(src_path)
return files_renamed, dirs_renamed, dirs_modified
def test_file_system_event_handler_dispatch():
dir_del_event = DirDeletedEvent('/path/blah.py')
file_del_event = FileDeletedEvent('/path/blah.txt')
dir_cre_event = DirCreatedEvent('/path/blah.py')
file_cre_event = FileCreatedEvent('/path/blah.txt')
file_cls_event = FileClosedEvent('/path/blah.txt')
dir_mod_event = DirModifiedEvent('/path/blah.py')
file_mod_event = FileModifiedEvent('/path/blah.txt')
dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah')
file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah')
all_events = [
dir_mod_event,
dir_del_event,
dir_cre_event,
dir_mov_event,
file_mod_event,
file_del_event,
file_cre_event,
file_mov_event,
file_cls_event,
]
class TestableEventHandler(FileSystemEventHandler):
def on_any_event(self, event):
assert True
def on_modified(self, event):
assert event.event_type == EVENT_TYPE_MODIFIED
def on_deleted(self, event):
assert event.event_type == EVENT_TYPE_DELETED
def on_moved(self, event):
assert event.event_type == EVENT_TYPE_MOVED
def on_created(self, event):
assert event.event_type == EVENT_TYPE_CREATED
def on_closed(self, event):
assert event.event_type == EVENT_TYPE_CLOSED
handler = TestableEventHandler()
for event in all_events:
assert not event.is_synthetic
handler.dispatch(event)
def test_logging_event_handler_dispatch():
class _TestableEventHandler(LoggingEventHandler):
def on_any_event(self, event):
assert True
def on_modified(self, event):
super(_TestableEventHandler, self).on_modified(event)
assert event.event_type == EVENT_TYPE_MODIFIED
def on_deleted(self, event):
super(_TestableEventHandler, self).on_deleted(event)
assert event.event_type == EVENT_TYPE_DELETED
def on_moved(self, event):
super(_TestableEventHandler, self).on_moved(event)
assert event.event_type == EVENT_TYPE_MOVED
def on_created(self, event):
super(_TestableEventHandler, self).on_created(event)
assert event.event_type == EVENT_TYPE_CREATED
# Utilities.
dir_del_event = DirDeletedEvent('/path/blah.py')
file_del_event = FileDeletedEvent('/path/blah.txt')
dir_cre_event = DirCreatedEvent('/path/blah.py')
file_cre_event = FileCreatedEvent('/path/blah.txt')
dir_mod_event = DirModifiedEvent('/path/blah.py')
file_mod_event = FileModifiedEvent('/path/blah.txt')
dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah')
file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah')
all_events = [
dir_mod_event,
dir_del_event,
dir_cre_event,
dir_mov_event,
file_mod_event,
file_del_event,
file_cre_event,
file_mov_event,
]
handler = _TestableEventHandler()
for event in all_events:
handler.dispatch(event)
def _gen_kqueue_events(self,
kev,
ref_snapshot,
new_snapshot):
"""
Generate events from the kevent list returned from the call to
:meth:`select.kqueue.control`.
.. NOTE:: kqueue only tells us about deletions, file modifications,
attribute modifications. The other events, namely,
file creation, directory modification, file rename,
directory rename, directory creation, etc. are
determined by comparing directory snapshots.
"""
descriptor = self._descriptors.get_for_fd(kev.ident)
src_path = descriptor.path
if is_renamed(kev):
# Kqueue does not specify the destination names for renames
# to, so we have to process these using the a snapshot
# of the directory.
for event in self._gen_renamed_events(src_path,
descriptor.is_directory,
ref_snapshot,
new_snapshot):
yield event
elif is_attrib_modified(kev):
if descriptor.is_directory:
yield DirModifiedEvent(src_path)
else:
yield FileModifiedEvent(src_path)
elif is_modified(kev):
if descriptor.is_directory:
if self.watch.is_recursive or self.watch.path == src_path:
# When a directory is modified, it may be due to
# sub-file/directory renames or new file/directory
# creation. We determine all this by comparing
# snapshots later.
yield DirModifiedEvent(src_path)
else:
yield FileModifiedEvent(src_path)
elif is_deleted(kev):
if descriptor.is_directory:
yield DirDeletedEvent(src_path)
else:
yield FileDeletedEvent(src_path)
def test_dispatch(self):
dir_del_event = DirDeletedEvent('/path/blah.py')
file_del_event = FileDeletedEvent('/path/blah.txt')
dir_cre_event = DirCreatedEvent('/path/blah.py')
file_cre_event = FileCreatedEvent('/path/blah.txt')
dir_mod_event = DirModifiedEvent('/path/blah.py')
file_mod_event = FileModifiedEvent('/path/blah.txt')
dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah')
file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah')
all_events = [
dir_mod_event,
dir_del_event,
dir_cre_event,
dir_mov_event,
file_mod_event,
file_del_event,
file_cre_event,
file_mov_event,
]
def assert_equal(a, b):
self.assertEqual(a, b)
class TestableEventHandler(FileSystemEventHandler):
def on_any_event(self, event):
assert True
def on_modified(self, event):
assert_equal(event.event_type, EVENT_TYPE_MODIFIED)
def on_deleted(self, event):
assert_equal(event.event_type, EVENT_TYPE_DELETED)
def on_moved(self, event):
assert_equal(event.event_type, EVENT_TYPE_MOVED)
def on_created(self, event):
assert_equal(event.event_type, EVENT_TYPE_CREATED)
handler = TestableEventHandler()
for event in all_events:
handler.dispatch(event)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except Exception as e: # OSError:
logging.debug('queue_events _take_snapshot: %s' % e)
# stop commented to do not stop the thread if an
# error happened (i.e. watch folder not accessible temporarily)
self.queue_event(FileSystemErrorEvent(self.watch.path, e))
#self.stop()
return
events = DirectorySnapshotDiff(self._snapshot if self._snapshot else new_snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def check_from_snapshot(self, sub_folder=None):
logging.info('Scanning for changes since last application launch')
if (not sub_folder and os.path.exists(self.basepath)) or (
sub_folder and os.path.exists(self.basepath + sub_folder)):
previous_snapshot = SqlSnapshot(self.basepath, self.job_data_path,
sub_folder)
if sub_folder:
local_path = self.basepath + os.path.normpath(sub_folder)
else:
local_path = self.basepath
snapshot = DirectorySnapshot(local_path, recursive=True)
diff = SnapshotDiffStart(previous_snapshot, snapshot)
for path in diff.dirs_created:
if self.interrupt:
return
self.event_handler.on_created(DirCreatedEvent(path))
for path in diff.files_created:
if self.interrupt:
return
self.event_handler.on_created(FileCreatedEvent(path))
for path in diff.dirs_moved:
if self.interrupt:
return
self.event_handler.on_moved(DirMovedEvent(path[0], path[1]))
for path in diff.files_moved:
if self.interrupt:
return
self.event_handler.on_moved(FileMovedEvent(path[0], path[1]))
for path in diff.files_modified:
if self.interrupt:
return
self.event_handler.on_modified(FileModifiedEvent(path))
for path in diff.files_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(FileDeletedEvent(path))
for path in diff.dirs_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(DirDeletedEvent(path))
def _queue_events_except_renames_and_dir_modifications(self, event_list):
"""
Queues events from the kevent list returned from the call to
:meth:`select.kqueue.control`.
.. NOTE:: Queues only the deletions, file modifications,
attribute modifications. The other events, namely,
file creation, directory modification, file rename,
directory rename, directory creation, etc. are
determined by comparing directory snapshots.
"""
files_renamed = set()
dirs_renamed = set()
dirs_modified = set()
for kev in event_list:
descriptor = self._descriptors.get_for_fd(kev.ident)
src_path = descriptor.path
if is_deleted(kev):
if descriptor.is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
elif is_attrib_modified(kev):
if descriptor.is_directory:
self.queue_event(DirModifiedEvent(src_path))
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_modified(kev):
if descriptor.is_directory:
dirs_modified.add(src_path)
else:
self.queue_event(FileModifiedEvent(src_path))
elif is_renamed(kev):
if descriptor.is_directory:
dirs_renamed.add(src_path)
else:
files_renamed.add(src_path)
return (files_renamed, dirs_renamed, dirs_modified)
def _determine_event_type(self, local, db):
if not local and not db:
raise LocalDBBothNone
if local and not isinstance(local, ProperPath):
raise TypeError
if db and (not (isinstance(db, Node) or isinstance(db, File))):
raise TypeError
event = None
if local:
# TODO: find better way of ignoring files we want to ignore
if local.name.startswith('.') or local.name.endswith('~'):
return event
if local and db:
if self._get_proper_path(local) != self._get_proper_path(db):
raise IncorrectLocalDBMatch
if isinstance(
db,
File) and db.is_file and self._make_hash(local) != db.hash:
event = FileModifiedEvent(
self._get_proper_path(
local).full_path) # create changed event
# folder modified event cannot happen. It will be a create and delete event.
elif local is None:
db_path = self._get_proper_path(db).full_path
if isinstance(db, File) and db.is_file:
event = FileDeletedEvent(db_path) # delete event for file
else:
event = DirDeletedEvent(db_path) # delete event for folder
elif db is None:
local_path = self._get_proper_path(local)
if local_path.is_dir:
event = DirCreatedEvent(local_path.full_path)
else:
event = FileCreatedEvent(local_path.full_path)
return event
def test___repr__(self):
event = DirDeletedEvent(path_1)
self.assertEqual("<DirDeletedEvent: src_path=%s>" % path_1,
event.__repr__())