def test_replace_same_folder(self):
mkdir(p('dir1'))
touch(p('dir1', 'a'))
touch(p('dir1', 'b'))
snapBefore = DirectorySnapshot(temp_dir)
sleep(1)
mv(p('dir1', 'a'), p('dir1', 'b'))
snapAfter = DirectorySnapshot(temp_dir)
changes = snapAfter - snapBefore
expected = {
'files_deleted': [
p('dir1', 'b'),
],
'files_moved': [
(p('dir1', 'a'), p('dir1', 'b')),
],
'dirs_modified': [
p('dir1'),
]
}
self.verify(expected, changes)
def handle(self, *args, **options):
snapshots = {}
roots = self.find_static_roots()
if not roots:
self.write('No static roots has been found in your application')
return
for path in roots:
snapshots[path] = DirectorySnapshot(path)
start_message = 'Directories to watch:\n{}'.format('\n'.join(roots))
self.write(start_message)
events = ('files_created', 'files_deleted', 'files_modified',
'files_moved')
try:
while True:
for path in roots:
snapshot = DirectorySnapshot(path)
diff = DirectorySnapshotDiff(snapshot, snapshots[path])
if any(getattr(diff, event) for event in events):
snapshots[path] = snapshot
self.write('Changes detected')
call_command('collectstatic', interactive=False)
except KeyboardInterrupt:
pass
def test_replace_in_other_folder(self):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
touch(p('dir2', 'a'))
snapBefore = DirectorySnapshot(temp_dir)
sleep(1)
# This case didn't work until the associated changes in disnapshot.py
mv(p('dir1', 'a'), p('dir2', 'a'))
snapAfter = DirectorySnapshot(temp_dir)
changes = snapAfter - snapBefore
expected = {
'files_deleted': [
p('dir2', 'a'),
],
'files_moved': [
(p('dir1', 'a'), p('dir2', 'a')),
],
'dirs_modified': [
p('dir1'),
p('dir2'),
]
}
self.verify(expected, changes)
def test_permission_error(monkeypatch, p):
# Test that unreadable folders are not raising exceptions
mkdir(p('a', 'b', 'c'), parents=True)
ref = DirectorySnapshot(p(''))
def walk(self, root):
"""Generate a permission error on folder "a/b"."""
# Generate the permission error
if root.startswith(p('a', 'b')):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
# Mimic the original method
for entry in walk_orig(self, root):
yield entry
walk_orig = DirectorySnapshot.walk
monkeypatch.setattr(DirectorySnapshot, "walk", walk)
# Should NOT raise an OSError (EACCES)
new_snapshot = DirectorySnapshot(p(''))
monkeypatch.undo()
diff = DirectorySnapshotDiff(ref, new_snapshot)
assert repr(diff)
# Children of a/b/ are no more accessible and so removed in the new snapshot
assert diff.dirs_deleted == [(p('a', 'b', 'c'))]
def _start(self):
# Local aliases to avoid namespace lookups in self
timeout = self.timeout
handler_func = self.event_handler.on_created
path = self.path
is_recursive = self.is_recursive
# Take an initial snapshot
snapshot = DirectorySnapshot(path, recursive=is_recursive)
while True:
# The latest snapshot ..
new_snapshot = DirectorySnapshot(path, recursive=is_recursive)
# .. difference between the old and new will return, in particular, new or modified files ..
diff = DirectorySnapshotDiff(snapshot, new_snapshot)
for path_created in diff.files_created:
handler_func(FileCreatedEvent(path_created))
for path_modified in diff.files_modified:
handler_func(FileModifiedEvent(path_modified))
# .. a new snapshot which will be treated as the old one in the next iteration ..
snapshot = DirectorySnapshot(path, recursive=is_recursive)
sleep(timeout)
def test_ignore_device(monkeypatch, p):
# Create a file and take a snapshot.
touch(p('file'))
ref = DirectorySnapshot(p(''))
wait()
def inode(self, path):
# This function will always return a different device_id,
# even for the same file.
result = inode_orig(self, path)
inode.times += 1
return result[0], result[1] + inode.times
inode.times = 0
# Set the custom inode function.
inode_orig = DirectorySnapshot.inode
monkeypatch.setattr(DirectorySnapshot, 'inode', inode)
# If we make the diff of the same directory, since by default the
# DirectorySnapshotDiff compares the snapshots using the device_id (and it will
# be different), it thinks that the same file has been deleted and created again.
snapshot = DirectorySnapshot(p(''))
diff_with_device = DirectorySnapshotDiff(ref, snapshot)
assert diff_with_device.files_deleted == [(p('file'))]
assert diff_with_device.files_created == [(p('file'))]
# Otherwise, if we choose to ignore the device, the file will not be detected as
# deleted and re-created.
snapshot = DirectorySnapshot(p(''))
diff_without_device = DirectorySnapshotDiff(ref,
snapshot,
ignore_device=True)
assert diff_without_device.files_deleted == []
assert diff_without_device.files_created == []
def __init__(self,
event_queue,
watch,
timeout=DEFAULT_EMITTER_TIMEOUT):
EventEmitter.__init__(self, event_queue, watch, timeout)
self._lock = threading.Lock()
self.snapshot = DirectorySnapshot(os.path.realpath(watch.path),
watch.is_recursive)
def test_move_from(p):
mkdir(p("dir1"))
mkdir(p("dir2"))
touch(p("dir1", "a"))
ref = DirectorySnapshot(p("dir1"))
mv(p("dir1", "a"), p("dir2", "b"))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p("dir1")))
assert diff.files_deleted == [p("dir1", "a")]
def test_move_from(p):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
ref = DirectorySnapshot(p('dir1'))
mv(p('dir1', 'a'), p('dir2', 'b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('dir1')))
assert diff.files_deleted == [p('dir1', 'a')]
def test_dir_modify_on_move(p):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
ref = DirectorySnapshot(p(''))
wait()
mv(p('dir1', 'a'), p('dir2', 'b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('')))
assert set(diff.dirs_modified) == set([p('dir1'), p('dir2')])
def test_dir_modify_on_move(p):
mkdir(p("dir1"))
mkdir(p("dir2"))
touch(p("dir1", "a"))
ref = DirectorySnapshot(p(""))
wait()
mv(p("dir1", "a"), p("dir2", "b"))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p("")))
assert set(diff.dirs_modified) == set([p("dir1"), p("dir2")])
def test_detect_modify_for_moved_files(p):
touch(p('a'))
ref = DirectorySnapshot(p(''))
wait()
touch(p('a'))
mv(p('a'), p('b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('')))
assert diff.files_moved == [(p('a'), p('b'))]
assert diff.files_modified == [p('a')]
def test_move_internal(p):
mkdir(p("dir1"))
mkdir(p("dir2"))
touch(p("dir1", "a"))
ref = DirectorySnapshot(p(""))
mv(p("dir1", "a"), p("dir2", "b"))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p("")))
assert diff.files_moved == [(p("dir1", "a"), p("dir2", "b"))]
assert diff.files_created == []
assert diff.files_deleted == []
def test_move_internal(p):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
ref = DirectorySnapshot(p(''))
mv(p('dir1', 'a'), p('dir2', 'b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('')))
assert diff.files_moved == [(p('dir1', 'a'), p('dir2', 'b'))]
assert diff.files_created == []
assert diff.files_deleted == []
def queue_events(self, timeout):
with self._lock:
if not self.watch.is_recursive and self.watch.path not in self.pathnames:
return
new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive)
diff = new_snapshot - self.snapshot
# add metadata modified events which will be missed by regular diff
try:
ctime_files_modified = set()
for path in self.snapshot.paths & new_snapshot.paths:
if not self.snapshot.isdir(path):
if self.snapshot.inode(path) == new_snapshot.inode(path):
if (
self.snapshot.stat_info(path).st_ctime
!= new_snapshot.stat_info(path).st_ctime
):
ctime_files_modified.add(path)
files_modified = set(ctime_files_modified) | set(diff.files_modified)
except Exception as exc:
print(exc)
# replace cached snapshot
self.snapshot = new_snapshot
# Files.
for src_path in diff.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path, dest_path in diff.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in diff.files_created:
self.queue_event(FileCreatedEvent(src_path))
# Directories.
for src_path in diff.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in diff.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path, dest_path in diff.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
for src_path in diff.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
# free some memory
del diff
del files_modified
def main():
args = parser.parse_args()
root_path = args.root_path
course = Course(root_path)
snap1 = DirectorySnapshot(".")
snap2 = DirectorySnapshot(".")
diff = DirectorySnapshotDiff(snap1, snap2)
diff.
setup_watchers(course)
asyncore.loop()
def getLastSyncSnapshot(self):
try:
with open(join(self.backup_path, '.snapshot'), 'rb') as snapshot_file:
snapshot = pickle.load(snapshot_file)
except FileNotFoundError:
snapshot = DirectorySnapshot(self.dir_path, listdir=lambda _: [])
return snapshot
def queue_events(self, timeout):
with self._lock:
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
time.sleep(timeout)
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_folder_tree_local(m):
"""Tests the upload sync of a nested local folder structure."""
# test creating tree
shutil.copytree(resources + "/test_folder", m.test_folder_local + "/test_folder")
snap = DirectorySnapshot(resources + "/test_folder")
num_items = len([p for p in snap.paths if not m.sync.is_excluded(p)])
wait_for_idle(m, 10)
assert_synced(m)
assert_child_count(m, "/sync_tests", num_items)
# test deleting tree
delete(m.test_folder_local + "/test_folder")
wait_for_idle(m)
assert_synced(m)
assert_child_count(m, "/sync_tests", 0)
# check for fatal errors
assert not m.fatal_errors
def queue_events(self, timeout):
time.sleep(timeout)
with self._lock:
if not self._snapshot:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def on_created(self, event) -> None:
global last_snapshot
self.logger.info("Detected new files!")
if os.path.isdir(self.path):
curr_snapshot = DirectorySnapshot(self.path)
files = [
file for file in DirectorySnapshotDiff(
last_snapshot, curr_snapshot).files_created
]
last_snapshot = curr_snapshot
for file_path in files:
upload_file(
api=self.api,
file_path=file_path,
logger=self.logger,
remove=self.remove,
deduplicate_api=self.deduplicate_api,
)
else:
upload_file(
api=self.api,
file_path=event.src_path,
logger=self.logger,
remove=self.remove,
deduplicate_api=self.deduplicate_api,
)
def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT,
stat=default_stat, listdir=os.listdir):
EventEmitter.__init__(self, event_queue, watch, timeout)
self._snapshot = None
self._lock = threading.Lock()
self._take_snapshot = lambda: DirectorySnapshot(
self.watch.path, self.watch.is_recursive, stat=stat, listdir=listdir)
def queue_events(self, timeout):
with self._lock:
if not self.watch.is_recursive\
and self.watch.path not in self.pathnames:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = new_snapshot - self.snapshot
self.snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
with self._lock:
try:
event_list = self._read_events(timeout)
files_renamed, dirs_renamed, dirs_modified = self._queue_events_except_renames_and_dir_modifications(
event_list)
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
ref_snapshot = self._snapshot
self._snapshot = new_snapshot
if files_renamed or dirs_renamed or dirs_modified:
for src_path in files_renamed:
self._queue_renamed(src_path, False, ref_snapshot,
new_snapshot)
for src_path in dirs_renamed:
self._queue_renamed(src_path, True, ref_snapshot,
new_snapshot)
self._queue_dirs_modified(dirs_modified, ref_snapshot,
new_snapshot)
except OSError as e:
if e.errno == errno.EBADF:
pass
else:
raise
def checkSnapshot(self):
snapshot = DirectorySnapshot(self.aim_path)
diff = DirectorySnapshotDiff(self.snapshot, snapshot)
self.snapshot = snapshot
self.timer = None
# diff.files_created
# diff.files_deleted
# diff.files_modified
# diff.files_moved
# diff.dirs_modified
# diff.dirs_moved
# diff.dirs_deleted
# diff.dirs_created
if diff.files_modified:
print(diff.files_modified[0])
path_file = diff.files_modified[0]
last_line = self.get_last_line(path_file).decode('utf-8').rstrip()
print(last_line)
text_list = last_line.split(',')
result = {
'test_date': text_list[0],
'serial_no': text_list[4],
'test_attribute': text_list[5],
'rotate': text_list[6],
'dynamic_balance1': self.handler_dynamic_balance(text_list[7]),
'angle1': int(float(text_list[8]) + 0.5),
'dynamic_balance2': self.handler_dynamic_balance(text_list[9]),
'angle2': int(float(text_list[10]) + 0.5)
}
print(text_list)
print(result)
def start(self, *args, **kwargs):
previous_snapshots = dict()
if os.path.exists(self._filename):
with open(self._filename) as f:
previous_snapshots = pickle.load(f)
for watcher, handlers in self._handlers.iteritems():
path = watcher.path
curr_snap = DirectorySnapshot(path)
pre_snap = previous_snapshots.get(path, _EmptySnapshot())
diff = DirectorySnapshotDiff(pre_snap, curr_snap)
for handler in handlers:
# Dispatch files modifications
for new_path in diff.files_created:
handler.dispatch(FileCreatedEvent(new_path))
for del_path in diff.files_deleted:
handler.dispatch(FileDeletedEvent(del_path))
for mod_path in diff.files_modified:
handler.dispatch(FileModifiedEvent(mod_path))
for mov_path in diff.files_moved:
handler.dispatch(FileMovedEvent(mov_path))
# Dispatch directories modifications
for new_dir in diff.dirs_created:
handler.dispatch(DirCreatedEvent(new_dir))
for del_dir in diff.dirs_deleted:
handler.dispatch(DirDeletedEvent(del_dir))
for mod_dir in diff.dirs_modified:
handler.dispatch(DirModifiedEvent(mod_dir))
for mov_dir in diff.dirs_moved:
handler.dispatch(DirMovedEvent(mov_dir))
Observer.start(self, *args, **kwargs)
def queue_events(self, timeout):
"""
Queues events by reading them from a call to the blocking
:meth:`select.kqueue.control()` method.
:param timeout:
Blocking timeout for reading events.
:type timeout:
``float`` (seconds)
"""
with self._lock:
try:
event_list = self._read_events(timeout)
files_renamed, dirs_renamed, dirs_modified = self._queue_events_except_renames_and_dir_modifications(
event_list)
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
ref_snapshot = self._snapshot
self._snapshot = new_snapshot
if files_renamed or dirs_renamed or dirs_modified:
for src_path in files_renamed:
self._queue_renamed(src_path, False, ref_snapshot,
new_snapshot)
for src_path in dirs_renamed:
self._queue_renamed(src_path, True, ref_snapshot,
new_snapshot)
self._queue_dirs_modified(dirs_modified, ref_snapshot,
new_snapshot)
except OSError as e:
if e.errno == errno.EBADF:
pass
else:
raise
def test_should_save_state_list(self):
collection = LocalCollectionStore(path="")
expected_state_list = [
WatchState(
path="/a/path/1",
snapshot=EmptyDirectorySnapshot(path=self.FIXTURE_FILES_PATH)),
WatchState(path="/a/path/2",
snapshot=DirectorySnapshot(self.FIXTURE_FILES_PATH,
True,
stat=os.stat,
listdir=listdir)),
]
# WHEN
collection.set_state_list(expected_state_list)
# THEN
saved_file_list = collection.state_list()
self.assertEqual("/a/path/1", saved_file_list[0].path)
self.assertEqual("/a/path/2", saved_file_list[1].path)
self.assertEqualSnapshot(expected_state_list[0].snapshot,
saved_file_list[0].snapshot)
self.assertEqualSnapshot(expected_state_list[1].snapshot,
saved_file_list[1].snapshot)
def check_from_snapshot(self, sub_folder=None, state_callback=(lambda status: None), use_transaction=True):
logging.info('Scanning for changes since last application launch')
if (not sub_folder and os.path.exists(self.basepath)) or (sub_folder and os.path.exists(self.basepath + sub_folder)):
previous_snapshot = SqlSnapshot(self.basepath, self.job_data_path, sub_folder)
if sub_folder:
local_path = self.basepath + os.path.normpath(sub_folder)
else:
local_path = self.basepath
state_callback(status=_('Walking through your local folder, please wait...'))
def listdir(dir_path):
try:
return os.listdir(dir_path)
except OSError as o:
logging.error(o)
return []
snapshot = DirectorySnapshot(local_path, recursive=True, listdir=listdir)
diff = SnapshotDiffStart(previous_snapshot, snapshot)
state_callback(status=_('Detected %i local changes...') % (len(diff.dirs_created) + len(diff.files_created)
+ len(diff.dirs_moved) + len(diff.dirs_deleted)
+ len(diff.files_moved) +
len(diff.files_modified) +
len(diff.files_deleted)))
if use_transaction:
self.event_handler.begin_transaction()
for path in diff.dirs_created:
if self.interrupt:
return
self.event_handler.on_created(DirCreatedEvent(path))
for path in diff.files_created:
if self.interrupt:
return
self.event_handler.on_created(FileCreatedEvent(path))
for path in diff.dirs_moved:
if self.interrupt:
return
self.event_handler.on_moved(DirMovedEvent(path[0], path[1]))
for path in diff.files_moved:
if self.interrupt:
return
self.event_handler.on_moved(FileMovedEvent(path[0], path[1]))
for path in diff.files_modified:
if self.interrupt:
return
self.event_handler.on_modified(FileModifiedEvent(path))
for path in diff.files_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(FileDeletedEvent(path))
for path in diff.dirs_deleted:
if self.interrupt:
return
self.event_handler.on_deleted(DirDeletedEvent(path))
if use_transaction:
self.event_handler.end_transaction()
def __init__(self,
_path,
_queue: Queue,
interval=5,
recursive=True,
daemon=True,
log_path=None,
log_name=None):
"""
文件监视类
:param _path: 要监视的目录
:param interval: 时间间隔，即每interval检查一次变更
:param recursive: 是否递归监视
:param daemon: 守护线程
"""
super().__init__()
self.path = _path
self.interval = interval
self.snap = DirectorySnapshot(self.path, recursive=recursive)
self.daemon = daemon
self.stop_flag = Event()
self.logger = None
self.queue = _queue
self.thread = ConsecutiveThread(self.stop_flag,
self.get_diff,
daemon=True)
if log_path:
self.log_name = log_name if log_name is not None else "FileSupervisor-%08x" % (
int.from_bytes(os.urandom(4), byteorder='big'))
self.log_path = log_path
self.logger = Logger(self.log_name, self.log_path)
self.logger.register_log_function("modify", "MODIFY")
self.logger.register_log_function("create", "CREATE")
def queue_events(self, timeout):
for idx in xrange(len(self.pathnames)):
event_path = absolute_path(self.pathnames[idx])
event_flags = self.flags[idx]
if not self.watch.is_recursive and self.watch.path != event_path:
return
recursive_update = bool(event_flags & FSEventsStreamFlag.MustScanSubDirs)
# try to build only partial snapshot
new_snapshot = DirectorySnapshot(
event_path,
recursive_update
)
if recursive_update and self.watch.path == event_path:
# no optimization is possible
events = new_snapshot - self.snapshot
self.snapshot = new_snapshot
else:
# partial comparison will be done
previous_snapshot = self.snapshot.copy(event_path, recursive_update)
# compare them
events = new_snapshot - previous_snapshot
if events.dirs_deleted or events.dirs_created or events.dirs_moved:
# add files from deleted dir to previous snapshot
previous_snapshot.add_entries(self.snapshot.copy_multiple(events.dirs_deleted, True))
# add files from created dir to new_snapshot, create a recursive snapshot of new dir
for new_path in events.dirs_created:
new_snapshot.add_entries(DirectorySnapshot(new_path, True))
previous_snapshot.add_entries(
self.snapshot.copy_multiple(
[old_path for (old_path, new_path) in events.dirs_moved],
True
)
)
for old_path, new_path in events.dirs_moved:
new_snapshot.add_entries(DirectorySnapshot(new_path, True))
# re-do diff
events = new_snapshot - previous_snapshot
# update last snapshot
self.snapshot.remove_entries(previous_snapshot)
self.snapshot.add_entries(new_snapshot)
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT): EventEmitter.__init__(self, event_queue, watch, timeout) self._lock = threading.Lock() self.snapshot = DirectorySnapshot(os.path.realpath(watch.path), watch.is_recursive)
class FSEventsEmitter(EventEmitter):
"""
Mac OS X FSEvents Emitter class.
:param event_queue:
The event queue to fill with events.
:param watch:
A watch object representing the directory to monitor.
:type watch:
:class:`watchdog.observers.api.ObservedWatch`
:param timeout:
Read events blocking timeout (in seconds).
:type timeout:
``float``
"""
def __init__(self, event_queue, watch, timeout=DEFAULT_EMITTER_TIMEOUT):
EventEmitter.__init__(self, event_queue, watch, timeout)
self._lock = threading.Lock()
self.snapshot = DirectorySnapshot(os.path.realpath(watch.path), watch.is_recursive)
def on_thread_exit(self):
_fsevents.remove_watch(self.watch)
_fsevents.stop(self)
def queue_events(self, timeout):
for idx in xrange(len(self.pathnames)):
event_path = absolute_path(self.pathnames[idx])
event_flags = self.flags[idx]
if not self.watch.is_recursive and self.watch.path != event_path:
return
recursive_update = bool(event_flags & FSEventsStreamFlag.MustScanSubDirs)
# try to build only partial snapshot
new_snapshot = DirectorySnapshot(
event_path,
recursive_update
)
if recursive_update and self.watch.path == event_path:
# no optimization is possible
events = new_snapshot - self.snapshot
self.snapshot = new_snapshot
else:
# partial comparison will be done
previous_snapshot = self.snapshot.copy(event_path, recursive_update)
# compare them
events = new_snapshot - previous_snapshot
if events.dirs_deleted or events.dirs_created or events.dirs_moved:
# add files from deleted dir to previous snapshot
previous_snapshot.add_entries(self.snapshot.copy_multiple(events.dirs_deleted, True))
# add files from created dir to new_snapshot, create a recursive snapshot of new dir
for new_path in events.dirs_created:
new_snapshot.add_entries(DirectorySnapshot(new_path, True))
previous_snapshot.add_entries(
self.snapshot.copy_multiple(
[old_path for (old_path, new_path) in events.dirs_moved],
True
)
)
for old_path, new_path in events.dirs_moved:
new_snapshot.add_entries(DirectorySnapshot(new_path, True))
# re-do diff
events = new_snapshot - previous_snapshot
# update last snapshot
self.snapshot.remove_entries(previous_snapshot)
self.snapshot.add_entries(new_snapshot)
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def run(self):
try:
def callback(pathnames, flags):
with self._lock:
self.pathnames = pathnames
self.flags = flags
self.queue_events(self.timeout)
#for pathname, flag in zip(pathnames, flags):
#if emitter.watch.is_recursive: # and pathname != emitter.watch.path:
# new_sub_snapshot = DirectorySnapshot(pathname, True)
# old_sub_snapshot = self.snapshot.copy(pathname)
# diff = new_sub_snapshot - old_sub_snapshot
# self.snapshot += new_subsnapshot
#else:
# new_snapshot = DirectorySnapshot(emitter.watch.path, False)
# diff = new_snapshot - emitter.snapshot
# emitter.snapshot = new_snapshot
# INFO: FSEvents reports directory notifications recursively
# by default, so we do not need to add subdirectory paths.
#pathnames = set([self.watch.path])
#if self.watch.is_recursive:
# for root, directory_names, _ in os.walk(self.watch.path):
# for directory_name in directory_names:
# full_path = absolute_path(
# os.path.join(root, directory_name))
# pathnames.add(full_path)
self.pathnames = [self.watch.path]
_fsevents.add_watch(self,
self.watch,
callback,
[self.watch.path])
_fsevents.read_events(self)
except Exception, e:
pass
finally:
__author__ = 'aum'
#get all content off given path
import time
import datetime
import os.path as p
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from watchdog.utils.dirsnapshot import DirectorySnapshot
if __name__ == "__main__":
snap = DirectorySnapshot(p.expanduser('~'))
for i in snap.paths:
print i,':', datetime.datetime.fromtimestamp(snap.stat_info(i).st_atime).ctime()
