def _update_file_index(self, file_path, changes):
relative_path = pathext.normalize(
os.path.relpath(file_path, self.dir_path)
)
file_data = self._index.setdefault(relative_path, dict())
if not file_data:
# New file
file_hash = bintools.hash_file(file_path)
file_data['last_update'] = int(os.path.getmtime(file_path))
file_data['hash'] = file_hash
file_data['last_update_location'] = self.uuid
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = file_data['last_update']
changes.add(relative_path)
elif int(os.path.getmtime(file_path)) > file_data['last_update']:
# Check if file is actually changed or the system time is off
file_hash = bintools.hash_file(file_path)
if file_data['hash'] != file_hash:
# File modified locally (since last sync)
file_data['last_update'] = int(os.path.getmtime(file_path))
file_data['hash'] = file_hash
file_data['last_update_location'] = self.uuid
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = file_data['last_update']
changes.add(relative_path)
if 'deleted' in file_data:
file_data['last_update'] = datetime.now().timestamp()
file_data['hash'] = bintools.hash_file(file_path)
file_data['last_update_location'] = self.uuid
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = file_data['last_update']
changes.add(relative_path)
del file_data['deleted']
if 'not_found' in file_data:
del file_data['not_found']
def _update_file_index(self, file_path, changes):
relative_path = pathext.normalize(
os.path.relpath(file_path, self.dir_path))
file_data = self._index.setdefault(relative_path, dict())
if not file_data:
# New file
file_hash = bintools.hash_file(file_path)
file_data['last_update'] = int(os.path.getmtime(file_path))
file_data['hash'] = file_hash
file_data['last_update_location'] = self.uuid
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = file_data['last_update']
changes.add(relative_path)
elif int(os.path.getmtime(file_path)) > file_data['last_update']:
# Check if file is actually changed or the system time is off
file_hash = bintools.hash_file(file_path)
if file_data['hash'] != file_hash:
# File modified locally (since last sync)
file_data['last_update'] = int(os.path.getmtime(file_path))
file_data['hash'] = file_hash
file_data['last_update_location'] = self.uuid
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = file_data['last_update']
changes.add(relative_path)
if 'deleted' in file_data:
file_data['last_update'] = datetime.now().timestamp()
file_data['hash'] = bintools.hash_file(file_path)
file_data['last_update_location'] = self.uuid
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = file_data['last_update']
changes.add(relative_path)
del file_data['deleted']
if 'not_found' in file_data:
del file_data['not_found']
def update_index(self, save_index=True, force=False):
"""
Update self._index (use the get_index() method to get it).
Return True if index changed, False otherwise.
The index structure is:
<index> ::= {
<file_name>: <file_info>,
...
}
<file_name> ::= file path relative to directory top
<file_info> ::= {
'sync_log': {
<remote_uuid (as string)>: <timestamp>,
...
},
'last_update_location': <remote_uuid (or the local UUID) (str)>
'last_update': <timestamp>,
'hash': <md5 byte-string>,
[optional 'deleted': (True|False)]
}
<timestamp> ::= Datetime in unix timestamp (seconds).
Depends on the os time on the system on which
the change happened.
"""
changes = set()
with self.fs_access_lock:
for file_data in self._index.values():
file_data['not_found'] = True
for dirpath, dirnames, filenames in os.walk(self.dir_path):
for name in filenames:
file_path = pathext.normalize(os.path.join(dirpath, name))
if not re.search(self.IGNORE_PATTERNS, file_path):
self._update_file_index(file_path, changes)
# Mark each deleted file with the current timestamp
# and UUID to avoid conflicts and to propagate properly
timestamp = datetime.now().timestamp()
for file_name, file_data in self._index.items():
if 'not_found' in file_data:
del file_data['not_found']
if 'deleted' in file_data and file_data['deleted']:
# File has been deleted some time ago...
continue
# File has been deleted now
file_data['deleted'] = True
file_data['last_update'] = timestamp
file_data['last_update_location'] = self.uuid
file_data['hash'] = b''
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = timestamp
changes.add(file_name)
if changes:
self.last_update = datetime.now().timestamp()
if save_index and changes:
self.save_index()
if force:
self.index_updated.notify(None)
elif changes:
self.index_updated.notify(changes)
def update_index(self, save_index=True, force=False):
"""
Update self._index (use the get_index() method to get it).
Return True if index changed, False otherwise.
The index structure is:
<index> ::= {
<file_name>: <file_info>,
...
}
<file_name> ::= file path relative to directory top
<file_info> ::= {
'sync_log': {
<remote_uuid (as string)>: <timestamp>,
...
},
'last_update_location': <remote_uuid (or the local UUID) (str)>
'last_update': <timestamp>,
'hash': <md5 byte-string>,
[optional 'deleted': (True|False)]
}
<timestamp> ::= Datetime in unix timestamp (seconds).
Depends on the os time on the system on which
the change happened.
"""
changes = set()
with self.fs_access_lock:
for file_data in self._index.values():
file_data['not_found'] = True
for dirpath, dirnames, filenames in os.walk(self.dir_path):
for name in filenames:
file_path = pathext.normalize(os.path.join(dirpath, name))
if not re.search(self.IGNORE_PATTERNS, file_path):
self._update_file_index(file_path, changes)
# Mark each deleted file with the current timestamp
# and UUID to avoid conflicts and to propagate properly
timestamp = datetime.now().timestamp()
for file_name, file_data in self._index.items():
if 'not_found' in file_data:
del file_data['not_found']
if 'deleted' in file_data and file_data['deleted']:
# File has been deleted some time ago...
continue
# File has been deleted now
file_data['deleted'] = True
file_data['last_update'] = timestamp
file_data['last_update_location'] = self.uuid
file_data['hash'] = b''
sync_log = file_data.setdefault('sync_log', dict())
sync_log[self.uuid] = timestamp
changes.add(file_name)
if changes:
self.last_update = datetime.now().timestamp()
if save_index and changes:
self.save_index()
if force:
self.index_updated.notify(None)
elif changes:
self.index_updated.notify(changes)
