def test_sub_moved_events(self):
mock_walker_path = [
('/path',
['ad', 'bd'],
['af', 'bf', 'cf']),
('/path/ad',
[],
['af', 'bf', 'cf']),
('/path/bd',
[],
['af', 'bf', 'cf']),
]
dest_path = '/path'
src_path = '/foobar'
expected_events = set([
DirMovedEvent('/foobar/ad', '/path/ad'),
DirMovedEvent('/foobar/bd', '/path/bd'),
FileMovedEvent('/foobar/af', '/path/af'),
FileMovedEvent('/foobar/bf', '/path/bf'),
FileMovedEvent('/foobar/cf', '/path/cf'),
FileMovedEvent('/foobar/ad/af', '/path/ad/af'),
FileMovedEvent('/foobar/ad/bf', '/path/ad/bf'),
FileMovedEvent('/foobar/ad/cf', '/path/ad/cf'),
FileMovedEvent('/foobar/bd/af', '/path/bd/af'),
FileMovedEvent('/foobar/bd/bf', '/path/bd/bf'),
FileMovedEvent('/foobar/bd/cf', '/path/bd/cf'),
])
dir_moved_event = DirMovedEvent(src_path, dest_path)
def _mock_os_walker(path):
for root, directories, filenames in mock_walker_path:
yield (root, directories, filenames)
calculated_events = set(dir_moved_event.sub_moved_events(_walker=_mock_os_walker))
assert_equal(expected_events, calculated_events)
def test_generate_sub_moved_events_for(self):
mock_walker_path = [
('/path', ['ad', 'bd'], ['af', 'bf', 'cf']),
('/path/ad', [], ['af', 'bf', 'cf']),
('/path/bd', [], ['af', 'bf', 'cf']),
]
dest_path = '/path'
src_path = '/foobar'
expected_events = set([
DirMovedEvent('/foobar/ad', '/path/ad'),
DirMovedEvent('/foobar/bd', '/path/bd'),
FileMovedEvent('/foobar/af', '/path/af'),
FileMovedEvent('/foobar/bf', '/path/bf'),
FileMovedEvent('/foobar/cf', '/path/cf'),
FileMovedEvent('/foobar/ad/af', '/path/ad/af'),
FileMovedEvent('/foobar/ad/bf', '/path/ad/bf'),
FileMovedEvent('/foobar/ad/cf', '/path/ad/cf'),
FileMovedEvent('/foobar/bd/af', '/path/bd/af'),
FileMovedEvent('/foobar/bd/bf', '/path/bd/bf'),
FileMovedEvent('/foobar/bd/cf', '/path/bd/cf'),
])
def _mock_os_walker(path):
for root, directories, filenames in mock_walker_path:
yield (root, directories, filenames)
calculated_events = set(
_generate_sub_moved_events_for(src_path,
dest_path,
_walker=_mock_os_walker))
assert_equal(expected_events, calculated_events)
def _queue_renamed(self, src_path, is_directory, ref_snapshot, new_snapshot):
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
return
try:
dest_path = absolute_path(new_snapshot.path_for_inode(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def _queue_renamed(self, src_path, is_directory, ref_snapshot,
new_snapshot):
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
return
try:
dest_path = absolute_path(
new_snapshot.path_for_inode(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def test_nested_events(sync):
file_events = [
# convert to a single DirDeleted
DirDeletedEvent(ipath(1)),
FileDeletedEvent(ipath(1) + "/file1.txt"),
FileDeletedEvent(ipath(1) + "/file2.txt"),
DirDeletedEvent(ipath(1) + "/sub"),
FileDeletedEvent(ipath(1) + "/sub/file3.txt"),
# convert to a single DirMoved
DirMovedEvent(ipath(2), ipath(3)),
FileMovedEvent(ipath(2) + "/file1.txt",
ipath(3) + "/file1.txt"),
FileMovedEvent(ipath(2) + "/file2.txt",
ipath(3) + "/file2.txt"),
DirMovedEvent(ipath(2) + "/sub",
ipath(3) + "/sub"),
FileMovedEvent(
ipath(2) + "/sub/file3.txt",
ipath(3) + "/sub/file3.txt"),
]
res = [
DirDeletedEvent(ipath(1)),
DirMovedEvent(ipath(2), ipath(3)),
]
cleaned_events = sync._clean_local_events(file_events)
assert set(cleaned_events) == set(res)
def test_move_nested_subdirectories():
mkdir(p('dir1/dir2/dir3'), parents=True)
mkfile(p('dir1/dir2/dir3', 'a'))
start_watching()
mv(p('dir1/dir2'), p('dir2'))
expect_event(DirMovedEvent(p('dir1', 'dir2'), p('dir2')))
expect_event(DirModifiedEvent(p('dir1')))
expect_event(DirModifiedEvent(p()))
expect_event(DirMovedEvent(p('dir1', 'dir2', 'dir3'), p('dir2', 'dir3')))
expect_event(FileMovedEvent(p('dir1', 'dir2', 'dir3', 'a'), p('dir2', 'dir3', 'a')))
if platform.is_bsd():
event = event_queue.get(timeout=5)[0]
assert p(event.src_path) == p()
assert isinstance(event, DirModifiedEvent)
event = event_queue.get(timeout=5)[0]
assert p(event.src_path) == p('dir1')
assert isinstance(event, DirModifiedEvent)
touch(p('dir2/dir3', 'a'))
event = event_queue.get(timeout=5)[0]
assert event.src_path == p('dir2/dir3', 'a')
assert isinstance(event, FileModifiedEvent)
def queue_events(self, timeout):
with self._lock:
dir_changes, nbytes = read_directory_changes(self._directory_handle, self._buffer, self.watch.is_recursive)
last_renamed_src_path = ''
for action, src_path in get_FILE_NOTIFY_INFORMATION(dir_changes, nbytes):
src_path = absolute_path(os.path.join(self.watch.path, src_path))
if action == FILE_ACTION_RENAMED_OLD_NAME:
last_renamed_src_path = src_path
elif action == FILE_ACTION_RENAMED_NEW_NAME:
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(dest_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
for sub_moved_event in event.sub_moved_events():
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
elif os.path.isdir(src_path):
event = DIR_ACTION_EVENT_MAP[action](src_path)
if isinstance(event, DirCreatedEvent):
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
sub_events = _generate_sub_created_events_for(src_path)
for sub_created_event in sub_events:
self.queue_event(sub_created_event)
self.queue_event(event)
else:
self.queue_event(FILE_ACTION_EVENT_MAP[action](src_path))
def test_performance(sync):
# 10,000 nested deleted events (5,000 folders, 5,000 files)
file_events = [DirDeletedEvent(n * ipath(1)) for n in range(1, 5001)]
file_events += [FileDeletedEvent(n * ipath(1) + ".txt") for n in range(1, 5001)]
# 10,000 nested moved events (5,000 folders, 5,000 files)
file_events += [DirMovedEvent(n * ipath(2), n * ipath(3)) for n in range(1, 5001)]
file_events += [
FileMovedEvent(n * ipath(2) + ".txt", n * ipath(3) + ".txt")
for n in range(1, 5001)
]
# 4,995 unrelated created events
file_events += [FileCreatedEvent(ipath(n)) for n in range(5, 5001)]
res = [
DirDeletedEvent(ipath(1)),
DirMovedEvent(ipath(2), ipath(3)),
FileDeletedEvent(ipath(1) + ".txt"),
FileMovedEvent(ipath(2) + ".txt", ipath(3) + ".txt"),
]
res += [FileCreatedEvent(ipath(n)) for n in range(5, 5001)]
cleaned_events = sync._clean_local_events(file_events)
assert set(cleaned_events) == set(res)
n_loops = 4
duration = timeit.timeit(
lambda: sync._clean_local_events(file_events), number=n_loops
)
assert duration < 10 * n_loops
def queue_events(self, timeout):
with self._lock:
dir_changes, nbytes = read_directory_changes(
self._directory_handle, self._buffer,
self.watch.is_recursive)
last_renamed_src_path = ""
for action, src_path in get_FILE_NOTIFY_INFORMATION(
dir_changes, nbytes):
src_path = absolute_path(
os.path.join(self.watch.path, src_path))
if action == FILE_ACTION_RENAMED_OLD_NAME:
last_renamed_src_path = src_path
elif action == FILE_ACTION_RENAMED_NEW_NAME:
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(dest_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
# HACK: We introduce a forced delay before
# traversing the moved directory. This will read
# only file movement that finishes within this
# delay time.
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
# The following block of code may not
# obtain moved events for the entire tree if
# the I/O is not completed within the above
# delay time. So, it's not guaranteed to work.
# TODO: Come up with a better solution, possibly
# a way to wait for I/O to complete before
# queuing events.
for sub_moved_event in event.sub_moved_events(
):
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(
FileMovedEvent(src_path, dest_path))
else:
if os.path.isdir(src_path):
event = DIR_ACTION_EVENT_MAP[action](src_path)
if isinstance(event, DirCreatedEvent):
# If a directory is moved from outside the watched folder to inside it
# we only get a created directory event out of it, not any events for its children
# so use the same hack as for file moves to get the child events
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
sub_events = _generate_sub_created_events_for(
src_path)
for sub_created_event in sub_events:
self.queue_event(sub_created_event)
self.queue_event(event)
else:
self.queue_event(
FILE_ACTION_EVENT_MAP[action](src_path))
def queue_events(self, timeout):
with self._lock:
dir_changes, nbytes = read_directory_changes(self._directory_handle,
self._buffer,
self.watch.is_recursive)
last_renamed_src_path = ""
for action, src_path in get_FILE_NOTIFY_INFORMATION(dir_changes,
nbytes):
src_path = absolute_path(os.path.join(self.watch.path,
src_path))
if action == FILE_ACTION_RENAMED_OLD_NAME:
continue
last_renamed_src_path = src_path
elif action == FILE_ACTION_RENAMED_NEW_NAME:
continue
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(dest_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
# HACK: We introduce a forced delay before
# traversing the moved directory. This will read
# only file movement that finishes within this
# delay time.
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
# The following block of code may not
# obtain moved events for the entire tree if
# the I/O is not completed within the above
# delay time. So, it's not guaranteed to work.
# TODO: Come up with a better solution, possibly
# a way to wait for I/O to complete before
# queuing events.
for sub_moved_event in event.sub_moved_events():
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
else:
if os.path.isdir(src_path):
continue
event = DIR_ACTION_EVENT_MAP[action](src_path)
if isinstance(event, DirCreatedEvent):
# If a directory is moved from outside the watched folder to inside it
# we only get a created directory event out of it, not any events for its children
# so use the same hack as for file moves to get the child events
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
sub_events = _generate_sub_created_events_for(src_path)
for sub_created_event in sub_events:
self.queue_event(sub_created_event)
self.queue_event(event)
else:
self.queue_event(FILE_ACTION_EVENT_MAP[action](src_path))
def _queue_renamed(self,
src_path,
is_directory,
ref_snapshot,
new_snapshot):
"""
Compares information from two directory snapshots (one taken before
the rename operation and another taken right after) to determine the
destination path of the file system object renamed, and adds
appropriate events to the event queue.
"""
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
# Probably caught a temporary file/directory that was renamed
# and deleted. Fires a sequence of created and deleted events
# for the path.
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
# We don't process any further and bail out assuming
# the event represents deletion/creation instead of movement.
return
try:
dest_path = absolute_path(
new_snapshot.path(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
# TODO: Do we need to fire moved events for the items
# inside the directory tree? Does kqueue does this
# all by itself? Check this and then enable this code
# only if it doesn't already.
# A: It doesn't. So I've enabled this block.
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
# If the new snapshot does not have an inode for the
# old path, we haven't found the new name. Therefore,
# we mark it as deleted and remove unregister the path.
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def _queue_renamed(self,
src_path,
is_directory,
ref_snapshot,
new_snapshot):
"""
Compares information from two directory snapshots (one taken before
the rename operation and another taken right after) to determine the
destination path of the file system object renamed, and adds
appropriate events to the event queue.
"""
try:
ref_stat_info = ref_snapshot.stat_info(src_path)
except KeyError:
# Probably caught a temporary file/directory that was renamed
# and deleted. Fires a sequence of created and deleted events
# for the path.
if is_directory:
self.queue_event(DirCreatedEvent(src_path))
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileCreatedEvent(src_path))
self.queue_event(FileDeletedEvent(src_path))
# We don't process any further and bail out assuming
# the event represents deletion/creation instead of movement.
return
try:
dest_path = absolute_path(
new_snapshot.path_for_inode(ref_stat_info.st_ino))
if is_directory:
event = DirMovedEvent(src_path, dest_path)
# TODO: Do we need to fire moved events for the items
# inside the directory tree? Does kqueue does this
# all by itself? Check this and then enable this code
# only if it doesn't already.
# A: It doesn't. So I've enabled this block.
if self.watch.is_recursive:
for sub_event in event.sub_moved_events():
self.queue_event(sub_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
except KeyError:
# If the new snapshot does not have an inode for the
# old path, we haven't found the new name. Therefore,
# we mark it as deleted and remove unregister the path.
if is_directory:
self.queue_event(DirDeletedEvent(src_path))
else:
self.queue_event(FileDeletedEvent(src_path))
def test_type_changes_difficult(sync):
file_events = [
# convert to FileDeleted -> DirCreated
FileModifiedEvent(ipath(1)),
FileDeletedEvent(ipath(1)),
FileCreatedEvent(ipath(1)),
FileDeletedEvent(ipath(1)),
DirCreatedEvent(ipath(1)),
# convert to FileDeleted(path1) -> DirCreated(path2)
FileModifiedEvent(ipath(2)),
FileDeletedEvent(ipath(2)),
FileCreatedEvent(ipath(2)),
FileDeletedEvent(ipath(2)),
DirCreatedEvent(ipath(2)),
DirMovedEvent(ipath(2), ipath(3)),
]
res = [
FileDeletedEvent(ipath(1)),
DirCreatedEvent(ipath(1)),
FileDeletedEvent(ipath(2)),
DirCreatedEvent(ipath(3)),
]
cleaned_events = sync._clean_local_events(file_events)
assert set(cleaned_events) == set(res)
def test___init__(self):
SLEEP_TIME = 0.4
self.emitter.start()
sleep(SLEEP_TIME)
mkdir(p('project'))
sleep(SLEEP_TIME)
mkdir(p('project', 'blah'))
sleep(SLEEP_TIME)
touch(p('afile'))
sleep(SLEEP_TIME)
touch(p('fromfile'))
sleep(SLEEP_TIME)
mv(p('fromfile'), p('project', 'tofile'))
sleep(SLEEP_TIME)
touch(p('afile'))
sleep(SLEEP_TIME)
mv(p('project', 'blah'), p('project', 'boo'))
sleep(SLEEP_TIME)
rm(p('project'), recursive=True)
sleep(SLEEP_TIME)
rm(p('afile'))
sleep(SLEEP_TIME)
self.emitter.stop()
# What we need here for the tests to pass is a collection type
# that is:
# * unordered
# * non-unique
# A multiset! Python's collections.Counter class seems appropriate.
expected = set([
DirModifiedEvent(p()),
DirCreatedEvent(p('project')),
DirModifiedEvent(p('project')),
DirCreatedEvent(p('project', 'blah')),
FileCreatedEvent(p('afile')),
DirModifiedEvent(p()),
FileCreatedEvent(p('fromfile')),
DirModifiedEvent(p()),
DirModifiedEvent(p()),
FileMovedEvent(p('fromfile'), p('project', 'tofile')),
FileModifiedEvent(p('afile')),
DirModifiedEvent(p('project')),
DirMovedEvent(p('project', 'blah'), p('project', 'boo')),
DirModifiedEvent(p()),
FileDeletedEvent(p('project', 'boo')),
DirDeletedEvent(p('project', 'boo')),
DirDeletedEvent(p('project')),
DirModifiedEvent(p()),
FileDeletedEvent(p('afile')),
])
got = set()
while True:
try:
event, _ = self.event_queue.get_nowait()
got.add(event)
except queue.Empty:
break
assert_equal(expected, got)
def check_from_snapshot(self, sub_folder=None, state_callback=(lambda status: None), use_transaction=True):
logging.info('Scanning for changes since last application launch')
if (not sub_folder and os.path.exists(self.basepath)) or (sub_folder and os.path.exists(self.basepath + sub_folder)):
previous_snapshot = SqlSnapshot(self.basepath, self.job_data_path, sub_folder)
if sub_folder:
local_path = self.basepath + os.path.normpath(sub_folder)
else:
local_path = self.basepath
state_callback(status=_('Walking through your local folder, please wait...'))
def listdir(dir_path):
try:
return os.listdir(dir_path)
except OSError as o:
logging.error(o)
return []
snapshot = DirectorySnapshot(local_path, recursive=True, listdir=listdir)
diff = SnapshotDiffStart(previous_snapshot, snapshot)
state_callback(status=_('Detected %i local changes...') % (len(diff.dirs_created) + len(diff.files_created)
+ len(diff.dirs_moved) + len(diff.dirs_deleted)
+ len(diff.files_moved) +
len(diff.files_modified) +
len(diff.files_deleted)))
if use_transaction:
self.event_handler.begin_transaction()
for path in diff.dirs_created:
if self.interrupt:
return
self.event_handler.on_created(DirCreatedEvent(path))
for path in diff.files_created:
if self.interrupt:
return
self.event_handler.on_created(FileCreatedEvent(path))
for path in diff.dirs_moved:
if self.interrupt:
return
self.event_handler.on_moved(DirMovedEvent(path[0], path[1]))
for path in diff.files_moved:
if self.interrupt:
return
self.event_handler.on_moved(FileMovedEvent(path[0], path[1]))
for path in diff.files_modified:
if self.interrupt:
return
self.event_handler.on_modified(FileModifiedEvent(path))
for path in diff.files_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(FileDeletedEvent(path))
for path in diff.dirs_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(DirDeletedEvent(path))
if use_transaction:
self.event_handler.end_transaction()
def queue_events(self, timeout):
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
with self._lock:
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
time.sleep(timeout)
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def start(self, *args, **kwargs):
previous_snapshots = dict()
if os.path.exists(self._filename):
with open(self._filename) as f:
previous_snapshots = pickle.load(f)
for watcher, handlers in self._handlers.iteritems():
path = watcher.path
curr_snap = DirectorySnapshot(path)
pre_snap = previous_snapshots.get(path, _EmptySnapshot())
diff = DirectorySnapshotDiff(pre_snap, curr_snap)
for handler in handlers:
# Dispatch files modifications
for new_path in diff.files_created:
handler.dispatch(FileCreatedEvent(new_path))
for del_path in diff.files_deleted:
handler.dispatch(FileDeletedEvent(del_path))
for mod_path in diff.files_modified:
handler.dispatch(FileModifiedEvent(mod_path))
for mov_path in diff.files_moved:
handler.dispatch(FileMovedEvent(mov_path))
# Dispatch directories modifications
for new_dir in diff.dirs_created:
handler.dispatch(DirCreatedEvent(new_dir))
for del_dir in diff.dirs_deleted:
handler.dispatch(DirDeletedEvent(del_dir))
for mod_dir in diff.dirs_modified:
handler.dispatch(DirModifiedEvent(mod_dir))
for mov_dir in diff.dirs_moved:
handler.dispatch(DirMovedEvent(mov_dir))
Observer.start(self, *args, **kwargs)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(DirMovedEvent(src_path, dest_path))
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(FileMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
time.sleep(timeout)
with self._lock:
if not self._snapshot:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_dispatch(self):
# Utilities.
dir_del_event = DirDeletedEvent('/path/blah.py')
file_del_event = FileDeletedEvent('/path/blah.txt')
dir_cre_event = DirCreatedEvent('/path/blah.py')
file_cre_event = FileCreatedEvent('/path/blah.txt')
dir_mod_event = DirModifiedEvent('/path/blah.py')
file_mod_event = FileModifiedEvent('/path/blah.txt')
dir_mov_event = DirMovedEvent('/path/blah.py', '/path/blah')
file_mov_event = FileMovedEvent('/path/blah.txt', '/path/blah')
all_events = [
dir_mod_event,
dir_del_event,
dir_cre_event,
dir_mov_event,
file_mod_event,
file_del_event,
file_cre_event,
file_mov_event,
]
handler = _TestableEventHandler()
for event in all_events:
handler.dispatch(event)
def queue_events(self, timeout):
with self._lock:
if not self.watch.is_recursive\
and self.watch.path not in self.pathnames:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = new_snapshot - self.snapshot
self.snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test___init__(event_queue, emitter):
emitter.start()
sleep(SLEEP_TIME)
mkdir(p('fromdir'))
sleep(SLEEP_TIME)
mv(p('fromdir'), p('todir'))
sleep(SLEEP_TIME)
emitter.stop()
# What we need here for the tests to pass is a collection type
# that is:
# * unordered
# * non-unique
# A multiset! Python's collections.Counter class seems appropriate.
expected = {
DirCreatedEvent(p('fromdir')),
DirMovedEvent(p('fromdir'), p('todir')),
}
got = set()
while True:
try:
event, _ = event_queue.get_nowait()
except Empty:
break
else:
got.add(event)
assert expected == got
def test___init__(self):
SLEEP_TIME = 1
self.emitter.start()
sleep(SLEEP_TIME)
mkdir(p('fromdir'))
sleep(SLEEP_TIME)
mv(p('fromdir'), p('todir'))
sleep(SLEEP_TIME)
self.emitter.stop()
# What we need here for the tests to pass is a collection type
# that is:
# * unordered
# * non-unique
# A multiset! Python's collections.Counter class seems appropriate.
expected = set([
DirCreatedEvent(p('fromdir')),
DirMovedEvent(p('fromdir'),p('todir')),
])
got = set()
while True:
try:
event, _ = self.event_queue.get_nowait()
got.add(event)
except queue.Empty:
break
print(got)
self.assertEqual(expected, got)
def test_recursive_off():
mkdir(p('dir1'))
start_watching(recursive=False)
touch(p('dir1', 'a'))
with pytest.raises(Empty):
event_queue.get(timeout=5)
mkfile(p('b'))
expect_event(FileCreatedEvent(p('b')))
if not platform.is_windows():
expect_event(DirModifiedEvent(p()))
if platform.is_linux():
expect_event(FileClosedEvent(p('b')))
# currently limiting these additional events to macOS only, see https://github.com/gorakhargosh/watchdog/pull/779
if platform.is_darwin():
mkdir(p('dir1', 'dir2'))
with pytest.raises(Empty):
event_queue.get(timeout=5)
mkfile(p('dir1', 'dir2', 'somefile'))
with pytest.raises(Empty):
event_queue.get(timeout=5)
mkdir(p('dir3'))
expect_event(DirModifiedEvent(p())) # the contents of the parent directory changed
mv(p('dir1', 'dir2', 'somefile'), p('somefile'))
expect_event(FileMovedEvent(p('dir1', 'dir2', 'somefile'), p('somefile')))
expect_event(DirModifiedEvent(p()))
mv(p('dir1', 'dir2'), p('dir2'))
expect_event(DirMovedEvent(p('dir1', 'dir2'), p('dir2')))
expect_event(DirModifiedEvent(p()))
def queue_events(self, timeout):
with self._lock:
dir_changes, nbytes = read_directory_changes(
self._directory_handle, self._buffer,
self.watch.is_recursive)
last_renamed_src_path = ""
for action, src_path in get_FILE_NOTIFY_INFORMATION(
dir_changes, nbytes):
src_path = absolute_path(
os.path.join(self.watch.path, src_path))
if action == FILE_ACTION_RENAMED_OLD_NAME:
last_renamed_src_path = src_path
elif action == FILE_ACTION_RENAMED_NEW_NAME:
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(src_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
# HACK: We introduce a forced delay before
# traversing the moved directory. This will read
# only file movement that finishes within this
# delay time.
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
# The following block of code may not
# obtain moved events for the entire tree if
# the I/O is not completed within the above
# delay time. So, it's not guaranteed to work.
# TODO: Come up with a better solution, possibly
# a way to wait for I/O to complete before
# queuing events.
for sub_moved_event in event.sub_moved_events(
):
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(
FileMovedEvent(src_path, dest_path))
else:
if os.path.isdir(src_path):
action_event_map = DIR_ACTION_EVENT_MAP
else:
action_event_map = FILE_ACTION_EVENT_MAP
self.queue_event(action_event_map[action](src_path))
def test_start_with_command_default_and_event(self):
"""Default command only execute when there's an event."""
from watchdog.events import DirMovedEvent
handler = AutoRunTrick()
event = DirMovedEvent('/source/path', '/dest/path')
handler.start(event=event)
expected = "directory '/source/path/' is moved to '/dest/path/'\n"
self.assertEqual(expected, self.mock_out.getvalue())
def queue_events(self, timeout):
with self._lock:
dir_changes, nbytes = read_directory_changes(self._directory_handle,
self._buffer,
self.watch.is_recursive)
last_renamed_src_path = ""
for action, src_path in get_FILE_NOTIFY_INFORMATION(dir_changes,
nbytes):
src_path = absolute_path(os.path.join(self.watch.path,
src_path))
if action == FILE_ACTION_RENAMED_OLD_NAME:
last_renamed_src_path = src_path
elif action == FILE_ACTION_RENAMED_NEW_NAME:
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(dest_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
# HACK: We introduce a forced delay before
# traversing the moved directory. This will read
# only file movement that finishes within this
# delay time.
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
# The following block of code may not
# obtain moved events for the entire tree if
# the I/O is not completed within the above
# delay time. So, it's not guaranteed to work.
# TODO: Come up with a better solution, possibly
# a way to wait for I/O to complete before
# queuing events.
for sub_moved_event in event.sub_moved_events():
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path,
dest_path))
else:
if os.path.isdir(src_path):
action_event_map = DIR_ACTION_EVENT_MAP
else:
action_event_map = FILE_ACTION_EVENT_MAP
self.queue_event(action_event_map[action](src_path))
def test_on_any_event(self):
from watchdog.events import DirMovedEvent
handler = AutoRunTrick(command='echo hello')
event = DirMovedEvent('/source/path', '/dest/path')
with patch('subprocess.Popen', new=self.PipePopen):
handler.on_any_event(event)
outs, _ = handler._process.communicate()
expected = b'hello\n'
self.assertEqual(outs, expected)
def test_command_default_dir_event_substitution(self):
from watchdog.events import DirMovedEvent
path = '/source/path'
dest = '/dest/path'
handler = AutoRunTrick()
event = DirMovedEvent(path, dest)
command = handler._substitute_command(event)
expected = "directory '/source/path/' is moved to '/dest/path/'"
self.assertEqual(expected, command)
def test_renaming_top_level_directory():
start_watching()
mkdir(p('a'))
expect_event(DirCreatedEvent(p('a')))
expect_event(DirModifiedEvent(p()))
mkdir(p('a', 'b'))
expect_event(DirCreatedEvent(p('a', 'b')))
expect_event(DirModifiedEvent(p('a')))
mv(p('a'), p('a2'))
expect_event(DirMovedEvent(p('a'), p('a2')))
expect_event(DirModifiedEvent(p()))
expect_event(DirModifiedEvent(p()))
expect_event(DirMovedEvent(p('a', 'b'), p('a2', 'b')))
if platform.is_bsd():
expect_event(DirModifiedEvent(p()))
open(p('a2', 'b', 'c'), 'a').close()
# DirModifiedEvent may emitted, but sometimes after waiting time is out.
events = []
while True:
events.append(event_queue.get(timeout=5)[0])
if event_queue.empty():
break
assert all([
isinstance(e, (FileCreatedEvent, FileMovedEvent, DirModifiedEvent,
FileClosedEvent)) for e in events
])
for event in events:
if isinstance(event, FileCreatedEvent):
assert event.src_path == p('a2', 'b', 'c')
elif isinstance(event, FileMovedEvent):
assert event.dest_path == p('a2', 'b', 'c')
assert event.src_path == p('a', 'b', 'c')
elif isinstance(event, DirModifiedEvent):
assert event.src_path == p('a2', 'b')
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except OSError as e:
if e.errno in DEFAULT_RESUMABLE_ERRNO:
return
elif e.errno == errno.ENOENT: # The monitor directory has been removed.
if not SUPPORT_ENOENT_RECOVERY:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
else:
raise
if self._snapshot.mount != new_snapshot.mount:
# The mount state of monitor directory has changed, such as a local directory is
# mounted or a NAS directory is unmounted.
if not SUPPORT_UNMOUNT_RECOVERY:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
winapi_events = self._read_events()
with self._lock:
last_renamed_src_path = ""
for winapi_event in winapi_events:
src_path = os.path.join(self.watch.path, winapi_event.src_path)
if winapi_event.is_renamed_old:
last_renamed_src_path = src_path
elif winapi_event.is_renamed_new:
dest_path = src_path
src_path = last_renamed_src_path
if os.path.isdir(dest_path):
event = DirMovedEvent(src_path, dest_path)
if self.watch.is_recursive:
# HACK: We introduce a forced delay before
# traversing the moved directory. This will read
# only file movement that finishes within this
# delay time.
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
# The following block of code may not
# obtain moved events for the entire tree if
# the I/O is not completed within the above
# delay time. So, it's not guaranteed to work.
# TODO: Come up with a better solution, possibly
# a way to wait for I/O to complete before
# queuing events.
for sub_moved_event in generate_sub_moved_events(
src_path, dest_path):
self.queue_event(sub_moved_event)
self.queue_event(event)
else:
self.queue_event(FileMovedEvent(src_path, dest_path))
elif winapi_event.is_modified:
cls = DirModifiedEvent if os.path.isdir(
src_path) else FileModifiedEvent
self.queue_event(cls(src_path))
elif winapi_event.is_added:
isdir = os.path.isdir(src_path)
cls = DirCreatedEvent if isdir else FileCreatedEvent
self.queue_event(cls(src_path))
if isdir and self.watch.is_recursive:
# If a directory is moved from outside the watched folder to inside it
# we only get a created directory event out of it, not any events for its children
# so use the same hack as for file moves to get the child events
time.sleep(WATCHDOG_TRAVERSE_MOVED_DIR_DELAY)
sub_events = generate_sub_created_events(src_path)
for sub_created_event in sub_events:
self.queue_event(sub_created_event)
elif winapi_event.is_removed:
self.queue_event(FileDeletedEvent(src_path))
elif winapi_event.is_removed_self:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
def test_sub_moved_events(self):
mock_walker_path = [
(absolute_path('/path'), ['ad', 'bd'], ['af', 'bf', 'cf']),
(absolute_path('/path/ad'), [], ['af', 'bf', 'cf']),
(absolute_path('/path/bd'), [], ['af', 'bf', 'cf']),
]
dest_path = absolute_path('/path')
src_path = absolute_path('/foobar')
expected_events = set([
DirMovedEvent(absolute_path('/foobar/ad'),
absolute_path('/path/ad')),
DirMovedEvent(absolute_path('/foobar/bd'),
absolute_path('/path/bd')),
FileMovedEvent(absolute_path('/foobar/af'),
absolute_path('/path/af')),
FileMovedEvent(absolute_path('/foobar/bf'),
absolute_path('/path/bf')),
FileMovedEvent(absolute_path('/foobar/cf'),
absolute_path('/path/cf')),
FileMovedEvent(absolute_path('/foobar/ad/af'),
absolute_path('/path/ad/af')),
FileMovedEvent(absolute_path('/foobar/ad/bf'),
absolute_path('/path/ad/bf')),
FileMovedEvent(absolute_path('/foobar/ad/cf'),
absolute_path('/path/ad/cf')),
FileMovedEvent(absolute_path('/foobar/bd/af'),
absolute_path('/path/bd/af')),
FileMovedEvent(absolute_path('/foobar/bd/bf'),
absolute_path('/path/bd/bf')),
FileMovedEvent(absolute_path('/foobar/bd/cf'),
absolute_path('/path/bd/cf')),
])
dir_moved_event = DirMovedEvent(src_path, dest_path)
def _mock_os_walker(path):
for root, directories, filenames in mock_walker_path:
yield (root, directories, filenames)
calculated_events = set(
dir_moved_event.sub_moved_events(_walker=_mock_os_walker))
self.assertEqual(expected_events, calculated_events)
def test___repr__(self):
event = DirMovedEvent(path_1, path_2)
self.assertEqual("<DirMovedEvent: src_path=%s, dest_path=%s>" %\
(path_1, path_2), event.__repr__())
