def test_ignore_device(monkeypatch, p):
# Create a file and take a snapshot.
touch(p('file'))
ref = DirectorySnapshot(p(''))
wait()
def inode(self, path):
# This function will always return a different device_id,
# even for the same file.
result = inode_orig(self, path)
inode.times += 1
return result[0], result[1] + inode.times
inode.times = 0
# Set the custom inode function.
inode_orig = DirectorySnapshot.inode
monkeypatch.setattr(DirectorySnapshot, 'inode', inode)
# If we make the diff of the same directory, since by default the
# DirectorySnapshotDiff compares the snapshots using the device_id (and it will
# be different), it thinks that the same file has been deleted and created again.
snapshot = DirectorySnapshot(p(''))
diff_with_device = DirectorySnapshotDiff(ref, snapshot)
assert diff_with_device.files_deleted == [(p('file'))]
assert diff_with_device.files_created == [(p('file'))]
# Otherwise, if we choose to ignore the device, the file will not be detected as
# deleted and re-created.
snapshot = DirectorySnapshot(p(''))
diff_without_device = DirectorySnapshotDiff(ref,
snapshot,
ignore_device=True)
assert diff_without_device.files_deleted == []
assert diff_without_device.files_created == []
def _start(self):
# Local aliases to avoid namespace lookups in self
timeout = self.timeout
handler_func = self.event_handler.on_created
path = self.path
is_recursive = self.is_recursive
# Take an initial snapshot
snapshot = DirectorySnapshot(path, recursive=is_recursive)
while True:
# The latest snapshot ..
new_snapshot = DirectorySnapshot(path, recursive=is_recursive)
# .. difference between the old and new will return, in particular, new or modified files ..
diff = DirectorySnapshotDiff(snapshot, new_snapshot)
for path_created in diff.files_created:
handler_func(FileCreatedEvent(path_created))
for path_modified in diff.files_modified:
handler_func(FileModifiedEvent(path_modified))
# .. a new snapshot which will be treated as the old one in the next iteration ..
snapshot = DirectorySnapshot(path, recursive=is_recursive)
sleep(timeout)
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(DirMovedEvent(src_path, dest_path))
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved.sort(key=lambda item: (item.count("/"), item)):
self.queue_event(FileMovedEvent(src_path, dest_path))
def on_created(self, event) -> None:
global last_snapshot
self.logger.info("Detected new files!")
if os.path.isdir(self.path):
curr_snapshot = DirectorySnapshot(self.path)
files = [
file for file in DirectorySnapshotDiff(
last_snapshot, curr_snapshot).files_created
]
last_snapshot = curr_snapshot
for file_path in files:
upload_file(
api=self.api,
file_path=file_path,
logger=self.logger,
remove=self.remove,
deduplicate_api=self.deduplicate_api,
)
else:
upload_file(
api=self.api,
file_path=event.src_path,
logger=self.logger,
remove=self.remove,
deduplicate_api=self.deduplicate_api,
)
def checkSnapshot(self):
snapshot = DirectorySnapshot(self.aim_path)
diff = DirectorySnapshotDiff(self.snapshot, snapshot)
self.snapshot = snapshot
self.timer = None
# diff.files_created
# diff.files_deleted
# diff.files_modified
# diff.files_moved
# diff.dirs_modified
# diff.dirs_moved
# diff.dirs_deleted
# diff.dirs_created
if diff.files_modified:
print(diff.files_modified[0])
path_file = diff.files_modified[0]
last_line = self.get_last_line(path_file).decode('utf-8').rstrip()
print(last_line)
text_list = last_line.split(',')
result = {
'test_date': text_list[0],
'serial_no': text_list[4],
'test_attribute': text_list[5],
'rotate': text_list[6],
'dynamic_balance1': self.handler_dynamic_balance(text_list[7]),
'angle1': int(float(text_list[8]) + 0.5),
'dynamic_balance2': self.handler_dynamic_balance(text_list[9]),
'angle2': int(float(text_list[10]) + 0.5)
}
print(text_list)
print(result)
def queue_events(self, timeout):
time.sleep(timeout)
with self._lock:
if not self._snapshot:
return
new_snapshot = DirectorySnapshot(self.watch.path,
self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_permission_error(monkeypatch, p):
# Test that unreadable folders are not raising exceptions
mkdir(p('a', 'b', 'c'), parents=True)
ref = DirectorySnapshot(p(''))
def walk(self, root):
"""Generate a permission error on folder "a/b"."""
# Generate the permission error
if root.startswith(p('a', 'b')):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
# Mimic the original method
for entry in walk_orig(self, root):
yield entry
walk_orig = DirectorySnapshot.walk
monkeypatch.setattr(DirectorySnapshot, "walk", walk)
# Should NOT raise an OSError (EACCES)
new_snapshot = DirectorySnapshot(p(''))
monkeypatch.undo()
diff = DirectorySnapshotDiff(ref, new_snapshot)
assert repr(diff)
# Children of a/b/ are no more accessible and so removed in the new snapshot
assert diff.dirs_deleted == [(p('a', 'b', 'c'))]
def handle(self, *args, **options):
snapshots = {}
roots = self.find_static_roots()
if not roots:
self.write('No static roots has been found in your application')
return
for path in roots:
snapshots[path] = DirectorySnapshot(path)
start_message = 'Directories to watch:\n{}'.format('\n'.join(roots))
self.write(start_message)
events = ('files_created', 'files_deleted', 'files_modified',
'files_moved')
try:
while True:
for path in roots:
snapshot = DirectorySnapshot(path)
diff = DirectorySnapshotDiff(snapshot, snapshots[path])
if any(getattr(diff, event) for event in events):
snapshots[path] = snapshot
self.write('Changes detected')
call_command('collectstatic', interactive=False)
except KeyboardInterrupt:
pass
def queue_events(self, timeout):
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def push(self):
diff = DirectorySnapshotDiff(self.pushed_snapshot,
self.commited_snapshot)
for dir in diff.dirs_modified:
if dir in self.parts():
self.get_part(dir).push()
self.push_self()
def queue_events(self, timeout):
with self._lock:
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
time.sleep(timeout)
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive)
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def start(self, *args, **kwargs):
previous_snapshots = dict()
if os.path.exists(self._filename):
with open(self._filename) as f:
previous_snapshots = pickle.load(f)
for watcher, handlers in self._handlers.iteritems():
path = watcher.path
curr_snap = DirectorySnapshot(path)
pre_snap = previous_snapshots.get(path, _EmptySnapshot())
diff = DirectorySnapshotDiff(pre_snap, curr_snap)
for handler in handlers:
# Dispatch files modifications
for new_path in diff.files_created:
handler.dispatch(FileCreatedEvent(new_path))
for del_path in diff.files_deleted:
handler.dispatch(FileDeletedEvent(del_path))
for mod_path in diff.files_modified:
handler.dispatch(FileModifiedEvent(mod_path))
for mov_path in diff.files_moved:
handler.dispatch(FileMovedEvent(mov_path))
# Dispatch directories modifications
for new_dir in diff.dirs_created:
handler.dispatch(DirCreatedEvent(new_dir))
for del_dir in diff.dirs_deleted:
handler.dispatch(DirDeletedEvent(del_dir))
for mod_dir in diff.dirs_modified:
handler.dispatch(DirModifiedEvent(mod_dir))
for mov_dir in diff.dirs_moved:
handler.dispatch(DirMovedEvent(mov_dir))
Observer.start(self, *args, **kwargs)
def queue_events(self, timeout):
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
new_snapshot = self._take_snapshot()
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def test_move_from(p):
mkdir(p("dir1"))
mkdir(p("dir2"))
touch(p("dir1", "a"))
ref = DirectorySnapshot(p("dir1"))
mv(p("dir1", "a"), p("dir2", "b"))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p("dir1")))
assert diff.files_deleted == [p("dir1", "a")]
def getDifferencesSinceLastSync(self):
"""
Get differences between the current directory and backup directory since last sync.
"""
oldSnapshot = self.getSnapshotOld()
currentSnapshot = self.getSnapshotCurrent()
difference = DirectorySnapshotDiff(oldSnapshot, currentSnapshot)
return self.getDiffDict(difference)
def test_move_from(p):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
ref = DirectorySnapshot(p('dir1'))
mv(p('dir1', 'a'), p('dir2', 'b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('dir1')))
assert diff.files_deleted == [p('dir1', 'a')]
def __sub__(self, previous_dirsnap):
"""Allow subtracting a DirectorySnapshot object instance from
another.
:returns:
A :class:`DirectorySnapshotDiff` object.
"""
return DirectorySnapshotDiff(previous_dirsnap, self)
def test_detect_modify_for_moved_files(p):
touch(p('a'))
ref = DirectorySnapshot(p(''))
wait()
touch(p('a'))
mv(p('a'), p('b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('')))
assert diff.files_moved == [(p('a'), p('b'))]
assert diff.files_modified == [p('a')]
def test_dir_modify_on_move(p):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
ref = DirectorySnapshot(p(''))
wait()
mv(p('dir1', 'a'), p('dir2', 'b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('')))
assert set(diff.dirs_modified) == set([p('dir1'), p('dir2')])
def test_dir_modify_on_move(p):
mkdir(p("dir1"))
mkdir(p("dir2"))
touch(p("dir1", "a"))
ref = DirectorySnapshot(p(""))
wait()
mv(p("dir1", "a"), p("dir2", "b"))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p("")))
assert set(diff.dirs_modified) == set([p("dir1"), p("dir2")])
def test_move_internal(p):
mkdir(p('dir1'))
mkdir(p('dir2'))
touch(p('dir1', 'a'))
ref = DirectorySnapshot(p(''))
mv(p('dir1', 'a'), p('dir2', 'b'))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p('')))
assert diff.files_moved == [(p('dir1', 'a'), p('dir2', 'b'))]
assert diff.files_created == []
assert diff.files_deleted == []
def test_move_internal(p):
mkdir(p("dir1"))
mkdir(p("dir2"))
touch(p("dir1", "a"))
ref = DirectorySnapshot(p(""))
mv(p("dir1", "a"), p("dir2", "b"))
diff = DirectorySnapshotDiff(ref, DirectorySnapshot(p("")))
assert diff.files_moved == [(p("dir1", "a"), p("dir2", "b"))]
assert diff.files_created == []
assert diff.files_deleted == []
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except OSError as e:
if e.errno in DEFAULT_RESUMABLE_ERRNO:
return
elif e.errno == errno.ENOENT: # The monitor directory has been removed.
if not SUPPORT_ENOENT_RECOVERY:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
else:
raise
if self._snapshot.mount != new_snapshot.mount:
# The mount state of monitor directory has changed, such as a local directory is
# mounted or a NAS directory is unmounted.
if not SUPPORT_UNMOUNT_RECOVERY:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def get_diff(self):
current_snap = DirectorySnapshot(self.path)
diff = DirectorySnapshotDiff(self.snap, current_snap)
self.snap = current_snap
if len(diff.files_modified) != 0:
self.logger.modify(str(diff.files_modified))
if len(diff.files_created) != 0:
self.logger.create(str(diff.files_created))
changes = diff.files_modified + diff.files_created
if len(changes) != 0:
self.queue.put(changes)
def on_any_event(self, event):
LOG.debug('%s %s', event.event_type, event.src_path)
new_ref = Dir(self.path).hash()
snapshot = DirectorySnapshot(self.path, recursive=True)
diff = DirectorySnapshotDiff(self.ref_snapshot, snapshot)
# compare directory hashes to determine if recompile needed
if self.ref != new_ref:
self.print_changes(diff)
self.runtime.compile()
self.ref = new_ref
self.ref_snapshot = snapshot
def test_empty_snapshot(p):
# Create a file and declare a DirectorySnapshot and a DirectorySnapshotEmpty.
# When we make the diff, although both objects were declared with the same items on
# the directory, the file and directories created BEFORE the DirectorySnapshot will
# be detected as newly created.
touch(p('a'))
mkdir(p('b', 'c'), parents=True)
ref = DirectorySnapshot(p(''))
empty = EmptyDirectorySnapshot()
diff = DirectorySnapshotDiff(empty, ref)
assert diff.files_created == [p('a')]
assert sorted(diff.dirs_created) == sorted([p(''), p('b'), p('b', 'c')])
def main():
args = parser.parse_args()
root_path = args.root_path
course = Course(root_path)
snap1 = DirectorySnapshot(".")
snap2 = DirectorySnapshot(".")
diff = DirectorySnapshotDiff(snap1, snap2)
diff.
setup_watchers(course)
asyncore.loop()
def checkSnapshot(self):
snapshot = DirectorySnapshot(self.aim_path)
diff = DirectorySnapshotDiff(self.snapshot, snapshot)
self.snapshot = snapshot
self.timer = None
print("files_created:", diff.files_created)
print("files_deleted:", diff.files_deleted)
print("files_modified:", diff.files_modified)
print("files_moved:", diff.files_moved)
print("dirs_modified:", diff.dirs_modified)
print("dirs_moved:", diff.dirs_moved)
print("dirs_deleted:", diff.dirs_deleted)
print("dirs_created:", diff.dirs_created)
# 接下来就是你想干的啥就干点啥，或者该干点啥就干点啥
pass
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except Exception as e: # OSError:
logging.debug('queue_events _take_snapshot: %s' % e)
# stop commented to do not stop the thread if an
# error happened (i.e. watch folder not accessible temporarily)
self.queue_event(FileSystemErrorEvent(self.watch.path, e))
#self.stop()
return
events = DirectorySnapshotDiff(self._snapshot if self._snapshot else new_snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
def queue_events(self, timeout):
# We don't want to hit the disk continuously.
# timeout behaves like an interval for polling emitters.
if self.stopped_event.wait(timeout):
return
with self._lock:
if not self.should_keep_running():
return
# Get event diff between fresh snapshot and previous snapshot.
# Update snapshot.
try:
new_snapshot = self._take_snapshot()
except OSError as e:
self.queue_event(DirDeletedEvent(self.watch.path))
self.stop()
return
except Exception as e:
raise e
events = DirectorySnapshotDiff(self._snapshot, new_snapshot)
self._snapshot = new_snapshot
# Files.
for src_path in events.files_deleted:
self.queue_event(FileDeletedEvent(src_path))
for src_path in events.files_modified:
self.queue_event(FileModifiedEvent(src_path))
for src_path in events.files_created:
self.queue_event(FileCreatedEvent(src_path))
for src_path, dest_path in events.files_moved:
self.queue_event(FileMovedEvent(src_path, dest_path))
# Directories.
for src_path in events.dirs_deleted:
self.queue_event(DirDeletedEvent(src_path))
for src_path in events.dirs_modified:
self.queue_event(DirModifiedEvent(src_path))
for src_path in events.dirs_created:
self.queue_event(DirCreatedEvent(src_path))
for src_path, dest_path in events.dirs_moved:
self.queue_event(DirMovedEvent(src_path, dest_path))
