def __init__(self, path_converter):
super(NotifyIfModifiedAction, self).__init__()
self.file_modified = Signal(
str, # relative_path
float) # modified time
self._patch_converter = path_converter
def __init__(self, storage, get_sync_dir_size, tracker=None, parent=None):
"""
Constructor
@param storage Storage class instance [Storage]
@param tracker
Statictics event tracker instance [stat_tracking.Tracker] or None
"""
QObject.__init__(self, parent=parent)
self._storage = storage
self._observer = None
self._active = True
self._root_handlers = {}
self._tracker = tracker
self._reset_stats()
self._started = False
self.is_processing_offline = False
self._processed_offline_changes = False
self._get_sync_dir_size = get_sync_dir_size
self._special_dirs = dict()
self._special_files = set()
self._lock = RLock()
self.start.connect(self._start)
# Signal to be emitted when detecting offline changes
self.offline_event_occured = Signal(FsEvent, bool)
def __init__(self, path_converter, max_relpath_len, long_paths):
super(CheckLongPathAction, self).__init__()
self.long_path_added = Signal(FilePath)
self.long_path_removed = Signal(FilePath)
self._path_converter = path_converter
self._max_relpath_len = max_relpath_len
self._long_paths = long_paths
def __init__(self, storage, root):
super(FilesList, self).__init__()
self._storage = storage
self._pc = PathConverter(root)
self._files_dict = dict()
self._store_limit = FILE_LIST_COUNT_LIMIT * 10
self.file_list_changed = Signal()
self._lock = RLock()
self._last_sent = None
def __init__(self,
root,
db_file_created_cb=None,
extended_logging=True,
to_upgrade=True):
self.possibly_sync_folder_is_removed = Signal()
self.delete_copy = Signal(
str, # copy hash
bool) # with signature
self.db_or_disk_full = Signal()
self._db_file = join(get_copies_dir(root), 'copies.db')
new_db_file = not exists(self._db_file)
if new_db_file and callable(db_file_created_cb):
db_file_created_cb()
if to_upgrade and not new_db_file:
# Database migration. It can be executed before opening db
try:
upgrade_db("copies_db", db_filename=self._db_file)
except Exception as e:
remove_file(self._db_file)
new_db_file = True
logger.warning(
"Can't upgrade copies db. "
"Reason: (%s) Creating...", e)
if callable(db_file_created_cb):
db_file_created_cb()
self._engine = create_engine('sqlite:///{}'.format(
FilePath(self._db_file)))
self._Session = sessionmaker(bind=self._engine)
Base.metadata.create_all(self._engine, checkfirst=True)
if new_db_file:
try:
stamp_db("copies_db", db_filename=self._db_file)
except Exception as e:
logger.error("Error stamping copies db: %s", e)
self._lock = RLock()
self._root = root
self._extended_logging = extended_logging
if not self._extended_logging:
self._logger = None
else:
self._logger = logging.getLogger('copies_logger')
self._logger.debug("Copies init")
self._last_changes = defaultdict(int)
class NotifyIfDeletedAction(ActionBase):
def __init__(self, path_converter):
super(NotifyIfDeletedAction, self).__init__()
self.file_deleted = Signal(str) # relative_path
self._patch_converter = path_converter
def add_new_event(self, fs_event):
if fs_event.event_type == DELETE:
src_path = fs_event.src[: -len(FILE_LINK_SUFFIX)] \
if fs_event.is_link \
else fs_event.src
self.file_deleted.emit(
self._patch_converter.create_relpath(src_path))
def __init__(self, storage, path_converter, tracker=None):
super(UpdateStorageAction, self).__init__()
self._storage = storage
self._path_converter = path_converter
self.event_processed = Signal(FsEvent)
self._tracker = tracker
self._waiting = False
class NotifyIfModifiedAction(ActionBase):
def __init__(self, path_converter):
super(NotifyIfModifiedAction, self).__init__()
self.file_modified = Signal(
str, # relative_path
float) # modified time
self._patch_converter = path_converter
def add_new_event(self, fs_event):
if fs_event.event_type == MODIFY:
src_path = fs_event.src[: -len(FILE_LINK_SUFFIX)] \
if fs_event.is_link \
else fs_event.src
self.file_modified.emit(
self._patch_converter.create_relpath(src_path), fs_event.mtime)
def __init__(self):
super(ActionBase, self).__init__()
self._active = True
self.event_passed = Signal(FsEvent)
self.event_returned = Signal(FsEvent)
self.event_spawned = Signal(FsEvent)
self.event_suppressed = Signal(FsEvent)
self.event_is_processing = Signal(FsEvent)
self.event_passed.connect(self._on_event_passed)
self.event_returned.connect(self._on_event_returned)
self.event_suppressed.connect(self._on_event_suppressed)
self.event_spawned.connect(self._on_event_spawned)
def notify(self, signal_type, source_name, signal_data):
if signal_data is None:
return
signal = Signal(Signal.SOURCE_VIEW, source_name, signal_type, signal_data)
with self._mutex:
self._signals_handler.notify(signal)
class CheckLongPathAction(ActionBase):
def __init__(self, path_converter, max_relpath_len, long_paths):
super(CheckLongPathAction, self).__init__()
self.long_path_added = Signal(FilePath)
self.long_path_removed = Signal(FilePath)
self._path_converter = path_converter
self._max_relpath_len = max_relpath_len
self._long_paths = long_paths
def _on_new_event(self, fs_event):
if fs_event.event_type in (CREATE, MODIFY) and \
self._is_path_long(fs_event.src):
self._long_paths.add(fs_event.src)
self.long_path_added.emit(fs_event.src)
self.event_suppressed(fs_event)
elif fs_event.event_type == MOVE and \
self._is_path_long(fs_event.dst):
if self._is_path_long(fs_event.src):
self._long_paths.discard(fs_event.src)
self._long_paths.add(fs_event.dst)
self.long_path_added.emit(fs_event.dst)
self.long_path_removed.emit(fs_event.src)
self.event_suppressed(fs_event)
else:
self._long_paths.add(fs_event.dst)
self.long_path_added.emit(fs_event.dst)
fs_event.event_type = DELETE
self.event_passed(fs_event)
elif fs_event.event_type == MOVE and \
self._is_path_long(fs_event.src):
fs_event.event_type = CREATE
self.event_passed(fs_event)
elif fs_event.event_type == DELETE and \
self._is_path_long(fs_event.src):
self._long_paths.discard(fs_event.src)
self.long_path_removed.emit(fs_event.src)
self.event_suppressed(fs_event)
else:
self.event_passed(fs_event)
def _is_path_long(self, path):
rel_path = self._path_converter.create_relpath(path)
file_name = rel_path.split('/')[-1]
return len(bytes(rel_path.encode('utf-8'))) > \
self._max_relpath_len or \
len(bytes(file_name.encode('utf-8'))) > MAX_FILE_NAME_LEN
def __init__(self, path_converter, db_file_created_cb=None):
self._pc = path_converter
self.possibly_sync_folder_is_removed = Signal()
self.db_or_disk_full = Signal()
self._db_file = self._pc.create_abspath('.pvtbox/storage.db')
logger.debug("DB file: %s", self._db_file)
new_db_file = not exists(self._db_file)
if new_db_file and callable(db_file_created_cb):
db_file_created_cb()
make_dirs(self._db_file)
if not new_db_file:
# Database migration. It can be executed before opening db
try:
upgrade_db("storage_db", db_filename=self._db_file)
except Exception as e:
remove_file(self._db_file)
new_db_file = True
logger.warning(
"Can't upgrade storage db. "
"Reason: (%s) Creating...", e)
if callable(db_file_created_cb):
db_file_created_cb()
self._engine = create_engine('sqlite:///{}'.format(
FilePath(self._db_file)),
connect_args={
'timeout': 60 * 1000,
'check_same_thread': False,
})
self._engine.pool_timeout = 60 * 60 * 1000
self._Session = sessionmaker(bind=self._engine)
Base.metadata.create_all(self._engine, checkfirst=True)
if new_db_file:
try:
stamp_db("storage_db", db_filename=self._db_file)
except Exception as e:
logger.error("Error stamping storage db: %s", e)
self._lock = threading.RLock()
class NotifyIfMovedAction(ActionBase):
def __init__(self, path_converter):
super(NotifyIfMovedAction, self).__init__()
self.file_moved = Signal(str, str) # old, new file path
self._patch_converter = path_converter
def add_new_event(self, fs_event):
if fs_event.event_type == MOVE:
src_path = fs_event.src[: -len(FILE_LINK_SUFFIX)] \
if fs_event.is_link \
else fs_event.src
dst_path = fs_event.src[: -len(FILE_LINK_SUFFIX)] \
if fs_event.is_link \
else fs_event.dst
self.file_moved.emit(
self._patch_converter.create_relpath(src_path),
self._patch_converter.create_relpath(dst_path))
def __init__(self, config_file, check=True):
# Emitted when some parameter values has changed
# dict: name -> namedtuple(old_value, new_value)
self.settings_changed = Signal(dict)
self.encrypt = True
self.config_file_name = config_file
self.config = None
self._defaults = set()
self.refresh(check=check)
def __init__(self,
db,
event,
get_download_backups_mode,
is_smart_sync=False):
super(EventStrategy, self).__init__()
self.event = event
self.event_id = event.id if event else 0
self.file_id = event.file.id if event and event.file else 0
self.db = db
self._is_smart_sync = is_smart_sync
self._cached_file_path = None
self._events_queue = None
self._download_backups = get_download_backups_mode()
self._force_move = False
self.change_processing_events_counts = Signal(int,
int) # (local, remote)
self.append_local_event = Signal(Event, str, str, int, bool)
self.rename_or_delete_dst_path = Signal(str, int, Session, bool)
class MoveFileRecentCopyAction(ActionBase):
def __init__(self, root, copies_storage):
super(MoveFileRecentCopyAction, self).__init__()
self._root = root
self._copies_storage = copies_storage
self.copy_added = Signal(str)
def _on_new_event(self, fs_event):
file_synced_copy_name = FilePath(
join(get_copies_dir(self._root), fs_event.new_hash)).longpath
file_recent_copy_name = FilePath(fs_event.file_recent_copy).longpath
self._copies_storage.add_copy_reference(
fs_event.new_hash,
reason="MoveFileRecentCopyAction {}".format(fs_event.src))
if exists(file_recent_copy_name):
if not exists(file_synced_copy_name):
try:
shutil.move(file_recent_copy_name, file_synced_copy_name)
self.copy_added.emit(fs_event.new_hash)
except (OSError, IOError):
self._copies_storage.remove_copy_reference(
fs_event.new_hash,
reason="MoveFileRecentCopyAction {}".format(
fs_event.src))
self.event_returned(fs_event)
return
if stat(file_synced_copy_name).st_size != fs_event.file_size:
self.event_returned(fs_event)
return
fs_event.file_synced_copy = FilePath(file_synced_copy_name)
self.event_passed(fs_event)
else:
self.event_returned(fs_event)
def _is_sutable(self, fs_event):
return (not fs_event.is_dir and fs_event.event_type in (CREATE, MODIFY)
and fs_event.file_recent_copy)
class LocalProcessor(object):
def __init__(self, root, storage, path_converter, tracker):
self._root = root
self._path_converter = path_converter
self._tracker = tracker
self._storage = storage
self.event_is_arrived = Signal(dict, object)
def process(self, fs_event):
""" crates message about event """
template_message = dict(event=fs_event.event_type,
time=time.time(),
type=DIRECTORY if fs_event.is_dir else FILE)
src = fs_event.src if not fs_event.is_link \
else fs_event.src[: -len(FILE_LINK_SUFFIX)]
dst = fs_event.dst if not fs_event.is_link or \
fs_event.event_type != MOVE \
else fs_event.dst[: -len(FILE_LINK_SUFFIX)]
if fs_event.event_type == MOVE:
template_message['src'] = self._path_converter.create_relpath(src)
template_message['dst'] = self._path_converter.create_relpath(dst)
template_message['hash'] = fs_event.old_hash
else:
template_message['path'] = self._path_converter.create_relpath(src)
template_message['hash'] = fs_event.new_hash
template_message['old_hash'] = fs_event.old_hash
template_message['file_size'] = fs_event.file_size
self.event_is_arrived.check_connected()
self.event_is_arrived(template_message, fs_event)
if fs_event.event_type != MOVE:
path = template_message['path']
else:
path = template_message['src']
logger.info("Event type %s for path '%s'",
event_names[template_message['event']], path)
class MakeFileRecentCopyAction(ActionBase):
def __init__(self, root):
super(MakeFileRecentCopyAction, self).__init__()
self._root = root
self.no_disk_space = Signal(object, str, bool)
def _on_new_event(self, fs_event):
if fs_event.file_size + get_signature_file_size(fs_event.file_size) > \
get_free_space_by_filepath(fs_event.src):
self.no_disk_space.emit(fs_event, fs_event.src, False)
self.event_suppressed(fs_event)
return
file_recent_copy_name = FilePath(
join(get_copies_dir(self._root),
'recent_copy_' + str(fs_event.id)))
fs_event.file_recent_copy = file_recent_copy_name
recent_copy_longpath = FilePath(file_recent_copy_name).longpath
try:
copy_file(FilePath(fs_event.src).longpath, recent_copy_longpath)
except (OSError, IOError) as e:
if e.errno == errno.ENOSPC:
self.no_disk_space.emit(fs_event, fs_event.src, True)
self.event_suppressed(fs_event)
return
self.event_returned(fs_event)
return
recent_copy_size = stat(recent_copy_longpath).st_size
if recent_copy_size != fs_event.file_size:
self.event_returned(fs_event)
return
self.event_passed(fs_event)
def _is_sutable(self, fs_event):
return (not fs_event.is_dir and fs_event.event_type in (CREATE, MODIFY)
and fs_event.file_size and not fs_event.is_link)
class ActionBase(object):
def __init__(self):
super(ActionBase, self).__init__()
self._active = True
self.event_passed = Signal(FsEvent)
self.event_returned = Signal(FsEvent)
self.event_spawned = Signal(FsEvent)
self.event_suppressed = Signal(FsEvent)
self.event_is_processing = Signal(FsEvent)
self.event_passed.connect(self._on_event_passed)
self.event_returned.connect(self._on_event_returned)
self.event_suppressed.connect(self._on_event_suppressed)
self.event_spawned.connect(self._on_event_spawned)
def set_active(self, active=True):
self._active = active
def _on_event_passed(self, fs_event):
pass
def _on_event_returned(self, fs_event):
logger.debug('%s RETURNED the event %s for additional processing',
self.__class__.__name__, fs_event)
def _on_event_spawned(self, fs_event):
logger.debug('%s SPAWNED new event %s for processing',
self.__class__.__name__, fs_event)
def _on_event_suppressed(self, fs_event):
logger.debug('%s SUPRESSED the event %s', self.__class__.__name__,
fs_event)
def add_new_event(self, fs_event):
if self._is_sutable(fs_event):
logger.debug('%s received the event %s', self.__class__.__name__,
fs_event)
self.event_is_processing(fs_event)
self._on_new_event(fs_event)
else:
self.event_passed(fs_event)
def _is_sutable(self, fs_event):
return self._active
def _on_new_event(self, fs_event):
self.event_passed(fs_event)
def __init__(self, root, storage, path_converter, exceptions):
self._root = root
self._storage = storage
self._path_converter = path_converter
self._exceptions = exceptions
self._tmp_id = 0
self._tmp_id_lock = RLock()
self._init_temp_dir()
self.file_moved = Signal(str, str)
self.file_deleted = Signal(str)
self.file_modified = Signal(str, float)
self.access_denied = Signal(str)
def __init__(self,
root,
hidden_dirs,
hidden_files,
patterns=None,
is_special=False):
self._hidden_files = hidden_files
self._hidden_dirs = hidden_dirs
self._is_special = is_special
ignore_patterns = []
for hidden_file in hidden_files:
ignore_patterns.append('*' + hidden_file)
for hidden_dir in hidden_dirs:
ignore_patterns.append(
FilePath(op.join(root, hidden_dir)).longpath)
ignore_patterns.append(
FilePath(op.join(root, hidden_dir, '*')).longpath)
super(WatchdogHandler, self).__init__(ignore_patterns=ignore_patterns,
patterns=patterns)
self.event_is_arrived = Signal(FsEvent, bool)
def __init__(self, events_processing_delay, tracker):
super(DelayAction, self).__init__()
self.idle = Signal()
self.working = Signal()
self.file_added_to_indexing = Signal(FilePath)
self.file_removed_from_indexing = Signal(FilePath, bool)
self._started = False
self._delay = events_processing_delay
self._tracker = tracker
self._offline_delay = 0.5
self._delayed_paths = {}
self._delayed_move_dst_paths = set()
self._lock = threading.RLock()
self._time_provider = time.time
self._executor = ThreadPoolExecutor(max_workers=self.workers_count)
self._processing_events = set()
self._batch_checking_paths = set()
self._loud_events = set()
self._is_idle = True
self._sorted_offline_events = None
def __init__(self,
root,
events_processing_delay,
copies_storage,
get_sync_dir_size,
conflict_file_suffix='',
tracker=None,
storage=None,
excluded_dirs=(),
parent=None,
max_relpath_len=3096,
db_file_created_cb=None):
QObject.__init__(self, parent=parent)
freeze_support()
self._tracker = tracker
self._root = root
self._path_converter = PathConverter(self._root)
self._storage = storage if storage else Storage(
self._path_converter, db_file_created_cb)
self._copies_storage = copies_storage
self._copies_storage.delete_copy.connect(self.on_delete_copy)
self.possibly_sync_folder_is_removed = \
self._storage.possibly_sync_folder_is_removed
self.db_or_disk_full = self._storage.db_or_disk_full
self._get_sync_dir_size = get_sync_dir_size
self._conflict_file_suffix = conflict_file_suffix
self._rsync = Rsync
_hide_files = HIDDEN_FILES
_hide_dirs = HIDDEN_DIRS
self._clean_recent_copies()
self._actions = FsEventActions(
self._root,
events_processing_delay=events_processing_delay,
path_converter=self._path_converter,
storage=self._storage,
copies_storage=self._copies_storage,
rsync=self._rsync,
tracker=self._tracker,
parent=None,
max_relpath_len=max_relpath_len,
)
self._watch = WatchdogHandler(root=FilePath(self._root).longpath,
hidden_files=_hide_files,
hidden_dirs=_hide_dirs)
self._download_watch = WatchdogHandler(root=FilePath(
self._root).longpath,
hidden_files=_hide_files,
hidden_dirs=_hide_dirs,
patterns=['*.download'],
is_special=True)
self._observer = ObserverWrapper(self._storage,
self._get_sync_dir_size,
self._tracker,
parent=None)
self._observer.event_handled.connect(
self._observer.on_event_is_handled_slot)
self._actions.event_passed.connect(
lambda ev: self._observer.event_handled.emit(ev, False))
self._actions.event_suppressed.connect(
lambda ev: self._observer.event_handled.emit(ev, True))
# Add FS root for events tracking
self._observer.schedule(self._watch, root)
self._local_processor = LocalProcessor(self._root, self._storage,
self._path_converter,
self._tracker)
self.event_is_arrived = self._local_processor.event_is_arrived
self._quiet_processor = QuietProcessor(self._root, self._storage,
self._path_converter,
self.Exceptions)
self._files_list = FilesList(self._storage, self._root)
self._thread = QThread()
self._thread.started.connect(self._on_thread_started)
self._actions.moveToThread(self._thread)
self._observer.moveToThread(self._thread)
self._watch.event_is_arrived.connect(self._on_event_arrived)
self._download_watch.event_is_arrived.connect(self._on_event_arrived)
self._actions.event_passed.connect(self._local_processor.process)
self._local_events_flag = False
self._actions.event_passed.connect(self._set_local_events_flag)
self.error_happens = self._actions.error_happens
self.no_disk_space = self._actions.no_disk_space
self.idle = self._actions.idle
self.working = self._actions.working
self.file_added_to_ignore = self._actions.file_added_to_ignore
self.file_removed_from_ignore = self._actions.file_removed_from_ignore
self.file_added_to_indexing = self._actions.file_added_to_indexing
self.file_removed_from_indexing = self._actions.file_removed_from_indexing
self.file_added = self._actions.file_added
self.file_modified = self._actions.file_modified
self.file_deleted = Signal(str)
self._actions.file_deleted.connect(self.file_deleted)
self._quiet_processor.file_deleted.connect(self.file_deleted)
self._quiet_processor.file_modified.connect(self.file_modified)
self.file_moved = self._quiet_processor.file_moved
self._actions.file_moved.connect(lambda o, n: self.file_moved(o, n))
self.access_denied = self._quiet_processor.access_denied
self.file_list_changed = self._files_list.file_list_changed
self.file_added.connect(self._files_list.on_file_added)
self.file_deleted.connect(self._files_list.on_file_deleted)
self.file_moved.connect(self._files_list.on_file_moved)
self.file_modified.connect(self._files_list.on_file_modified)
self.idle.connect(self._files_list.on_idle)
self.process_offline.connect(self._observer.process_offline_changes)
self.copy_added = Signal(str)
self._actions.copy_added.connect(self.copy_added)
self._actions.rename_file.connect(self._rename_file)
self.special_file_event = Signal(
str, # path
int, # event type
str) # new path
self._special_files = list()
self._excluded_dirs = list(map(FilePath, excluded_dirs))
self._online_processing_allowed = False
self._online_modifies_processing_allowed = False
self._paths_with_modify_quiet = set()
class FilesystemMonitor(QObject):
"""
Class provides all functions needed to work with filesystem
in scope of project
"""
max_file_name_length = MAX_FILE_NAME_LEN - 5
selective_sync_conflict_suffix = "selective sync conflict"
started = pyqtSignal()
stopped = pyqtSignal()
process_offline = pyqtSignal(bool)
def __init__(self,
root,
events_processing_delay,
copies_storage,
get_sync_dir_size,
conflict_file_suffix='',
tracker=None,
storage=None,
excluded_dirs=(),
parent=None,
max_relpath_len=3096,
db_file_created_cb=None):
QObject.__init__(self, parent=parent)
freeze_support()
self._tracker = tracker
self._root = root
self._path_converter = PathConverter(self._root)
self._storage = storage if storage else Storage(
self._path_converter, db_file_created_cb)
self._copies_storage = copies_storage
self._copies_storage.delete_copy.connect(self.on_delete_copy)
self.possibly_sync_folder_is_removed = \
self._storage.possibly_sync_folder_is_removed
self.db_or_disk_full = self._storage.db_or_disk_full
self._get_sync_dir_size = get_sync_dir_size
self._conflict_file_suffix = conflict_file_suffix
self._rsync = Rsync
_hide_files = HIDDEN_FILES
_hide_dirs = HIDDEN_DIRS
self._clean_recent_copies()
self._actions = FsEventActions(
self._root,
events_processing_delay=events_processing_delay,
path_converter=self._path_converter,
storage=self._storage,
copies_storage=self._copies_storage,
rsync=self._rsync,
tracker=self._tracker,
parent=None,
max_relpath_len=max_relpath_len,
)
self._watch = WatchdogHandler(root=FilePath(self._root).longpath,
hidden_files=_hide_files,
hidden_dirs=_hide_dirs)
self._download_watch = WatchdogHandler(root=FilePath(
self._root).longpath,
hidden_files=_hide_files,
hidden_dirs=_hide_dirs,
patterns=['*.download'],
is_special=True)
self._observer = ObserverWrapper(self._storage,
self._get_sync_dir_size,
self._tracker,
parent=None)
self._observer.event_handled.connect(
self._observer.on_event_is_handled_slot)
self._actions.event_passed.connect(
lambda ev: self._observer.event_handled.emit(ev, False))
self._actions.event_suppressed.connect(
lambda ev: self._observer.event_handled.emit(ev, True))
# Add FS root for events tracking
self._observer.schedule(self._watch, root)
self._local_processor = LocalProcessor(self._root, self._storage,
self._path_converter,
self._tracker)
self.event_is_arrived = self._local_processor.event_is_arrived
self._quiet_processor = QuietProcessor(self._root, self._storage,
self._path_converter,
self.Exceptions)
self._files_list = FilesList(self._storage, self._root)
self._thread = QThread()
self._thread.started.connect(self._on_thread_started)
self._actions.moveToThread(self._thread)
self._observer.moveToThread(self._thread)
self._watch.event_is_arrived.connect(self._on_event_arrived)
self._download_watch.event_is_arrived.connect(self._on_event_arrived)
self._actions.event_passed.connect(self._local_processor.process)
self._local_events_flag = False
self._actions.event_passed.connect(self._set_local_events_flag)
self.error_happens = self._actions.error_happens
self.no_disk_space = self._actions.no_disk_space
self.idle = self._actions.idle
self.working = self._actions.working
self.file_added_to_ignore = self._actions.file_added_to_ignore
self.file_removed_from_ignore = self._actions.file_removed_from_ignore
self.file_added_to_indexing = self._actions.file_added_to_indexing
self.file_removed_from_indexing = self._actions.file_removed_from_indexing
self.file_added = self._actions.file_added
self.file_modified = self._actions.file_modified
self.file_deleted = Signal(str)
self._actions.file_deleted.connect(self.file_deleted)
self._quiet_processor.file_deleted.connect(self.file_deleted)
self._quiet_processor.file_modified.connect(self.file_modified)
self.file_moved = self._quiet_processor.file_moved
self._actions.file_moved.connect(lambda o, n: self.file_moved(o, n))
self.access_denied = self._quiet_processor.access_denied
self.file_list_changed = self._files_list.file_list_changed
self.file_added.connect(self._files_list.on_file_added)
self.file_deleted.connect(self._files_list.on_file_deleted)
self.file_moved.connect(self._files_list.on_file_moved)
self.file_modified.connect(self._files_list.on_file_modified)
self.idle.connect(self._files_list.on_idle)
self.process_offline.connect(self._observer.process_offline_changes)
self.copy_added = Signal(str)
self._actions.copy_added.connect(self.copy_added)
self._actions.rename_file.connect(self._rename_file)
self.special_file_event = Signal(
str, # path
int, # event type
str) # new path
self._special_files = list()
self._excluded_dirs = list(map(FilePath, excluded_dirs))
self._online_processing_allowed = False
self._online_modifies_processing_allowed = False
self._paths_with_modify_quiet = set()
def on_initial_sync_finished(self):
logger.debug("on_initial_sync_finished")
self._actions.on_initial_sync_finished()
if not self._actions.get_fs_events_count() \
and not self._observer.is_processing_offline:
self.idle.emit()
def _on_processed_offline_changes(self):
logger.debug("_on_processed_offline_changes")
if not self._actions.get_fs_events_count():
self.idle.emit()
def on_initial_sync_started(self):
logger.debug("on_initial_sync_started")
self._actions.on_initial_sync_started()
self._online_processing_allowed = False
self._online_modifies_processing_allowed = False
def start_online_processing(self):
logger.debug("start_online_processing")
if not self._online_processing_allowed:
logger.debug("start_online_processing, emit process_offline")
self.process_offline.emit(self._online_modifies_processing_allowed)
self._online_processing_allowed = True
def start_online_modifies_processing(self):
logger.debug("start_online_modifies_processing")
if not self._online_modifies_processing_allowed:
logger.debug(
"start_online_modifies_processing, emit process_offline")
self.process_offline.emit(True)
self._online_modifies_processing_allowed = True
def get_root(self):
return self._root
def root_exists(self):
return op.isdir(self._root)
def _on_thread_started(self):
logger.info("Start monitoring of '%s'", self._root)
self._observer.offline_event_occured.connect(self._on_event_arrived)
self._observer.processed_offline_changes.connect(
self._on_processed_offline_changes)
self.started.emit()
self._actions.start.emit()
self._observer.start.emit()
self._local_events_flag = False
@benchmark
def start(self):
logger.debug("start")
self._observer.set_active()
if self._thread.isRunning():
self._on_thread_started()
else:
self._thread.start()
self._files_list.start()
def stop(self):
logger.info("stopped monitoring")
try:
self._observer.offline_event_occured.disconnect(
self._on_event_arrived)
except RuntimeError:
logger.warning("Can't disconnect offline_event_occured")
try:
self._observer.processed_offline_changes.disconnect(
self._on_processed_offline_changes)
except RuntimeError:
logger.warning("Can't disconnect processed_offline_changes")
self._actions.stop()
self._observer.stop()
self._files_list.stop()
self.stopped.emit()
def quit(self):
self.stop()
self._thread.quit()
self._thread.wait()
def is_processing(self, file_path):
return self._actions.is_processing(
self._path_converter.create_abspath(file_path))
def is_known(self, file_path):
if file_path.endswith(FILE_LINK_SUFFIX):
file_path = file_path[:-len(FILE_LINK_SUFFIX)]
return self._storage.get_known_file(file_path) is not None
def process_offline_changes(self):
if self._local_events_flag:
self.process_offline.emit(self._online_modifies_processing_allowed)
self._local_events_flag = False
def _set_local_events_flag(self, fs_event):
if not fs_event.is_offline:
self._local_events_flag = True
def clean_storage(self):
self._storage.clean()
delete_file_links(self._root)
def clean_copies(self, with_files=True):
self._copies_storage.clean(with_files=with_files)
def move_files_to_copies(self):
with self._storage.create_session(read_only=False,
locked=True) as session:
files_with_hashes = session\
.query(File.relative_path, File.file_hash) \
.filter(File.is_folder == 0) \
.all()
copies_dir = get_copies_dir(self._root)
for (file, hashsum) in files_with_hashes:
hash_path = op.join(copies_dir, hashsum)
file_path = self._path_converter.create_abspath(file)
if not op.exists(hash_path):
try:
os.rename(file_path, hash_path)
except Exception as e:
logger.error("Error moving file to copy: %s", e)
remove_file(file_path)
abs_path = FilePath(self._root).longpath
folders_plus_hidden = [
self._path_converter.create_abspath(f)
for f in os.listdir(abs_path) if f not in HIDDEN_DIRS
]
for folder in folders_plus_hidden:
if not op.isdir(folder):
continue
try:
remove_dir(folder)
except Exception as e:
logger.error("Error removing dir '%s' (%s)", folder, e)
logger.info("Removed all files and folders")
self._storage.clean()
def clean(self):
files = self._storage.get_known_files()
for file in files:
try:
remove_file(file)
except Exception as e:
logger.error("Error removing file '%s' (%s)", file, e)
folders = self._storage.get_known_folders()
for folder in sorted(folders, key=len):
try:
remove_dir(folder)
except Exception as e:
logger.error("Error removing dir '%s' (%s)", folder, e)
logger.info("Removed all files and folders")
self._storage.clean()
def accept_delete(self,
path,
is_directory=False,
events_file_id=None,
is_offline=True):
'''
Processes file deletion
@param path Name of file relative to sync directory [unicode]
'''
full_path = self._path_converter.create_abspath(path)
object_type = 'directory' if is_directory else 'file'
logger.debug("Deleting '%s' %s...", path, object_type)
if is_directory:
self._quiet_processor.delete_directory(full_path, events_file_id)
else:
self._quiet_processor.delete_file(full_path, events_file_id,
is_offline)
self.file_removed_from_indexing.emit(FilePath(full_path), True)
logger.info("'%s' %s is deleted", path, object_type)
def set_patch_uuid(self, patch_path, diff_file_uuid):
shutil.move(patch_path, self.get_patch_path(diff_file_uuid))
def get_patch_path(self, diff_file_uuid):
return os.path.join(get_patches_dir(self._root), diff_file_uuid)
def create_directory(self, path, events_file_id):
full_path = self._path_converter.create_abspath(path)
try:
self._quiet_processor.create_directory(
full_path,
events_file_id=events_file_id,
wrong_file_id=self.Exceptions.WrongFileId)
except AssertionError:
self._on_event_arrived(
FsEvent(DELETE,
op.dirname(full_path),
True,
is_offline=True,
quiet=True))
raise
def apply_patch(self, filename, patch, new_hash, old_hash, events_file_id):
'''
Applies given patch for the file specified
@param filename Name of file relative to sync directory [unicode]
@param patch Patch data [dict]
'''
full_fn = self._path_converter.create_abspath(filename)
try:
self._apply_patch(full_fn,
patch,
new_hash,
old_hash,
events_file_id=events_file_id)
except AssertionError:
self._on_event_arrived(
FsEvent(DELETE,
op.dirname(full_fn),
True,
is_offline=True,
quiet=True))
raise
def accept_move(self,
src,
dst,
is_directory=False,
events_file_id=None,
is_offline=True):
src_full_path = self._path_converter.create_abspath(src)
dst_full_path = self._path_converter.create_abspath(dst)
try:
object_type = 'directory' if is_directory else 'file'
logger.debug("Moving '%s' %s to '%s'...", src, object_type, dst)
if is_directory:
self._quiet_processor.move_directory(
src_full_path,
dst_full_path,
events_file_id,
self.Exceptions.FileAlreadyExists,
self.Exceptions.FileNotFound,
wrong_file_id=self.Exceptions.WrongFileId)
else:
self._quiet_processor.move_file(
src_full_path,
dst_full_path,
events_file_id,
self.Exceptions.FileAlreadyExists,
self.Exceptions.FileNotFound,
wrong_file_id=self.Exceptions.WrongFileId,
is_offline=is_offline)
logger.info("'%s' %s is moved to '%s'", src, object_type, dst)
self.file_removed_from_indexing.emit(FilePath(src_full_path), True)
except AssertionError:
self._on_event_arrived(
FsEvent(DELETE,
op.dirname(dst_full_path),
True,
is_offline=True,
quiet=True))
raise
def change_events_file_id(self, old_id, new_id):
self._storage.change_events_file_id(old_id, new_id)
class Exceptions(object):
""" User-defined exceptions are stored here """
class FileNotFound(Exception):
""" File doesn't exist exception"""
def __init__(self, file):
self.file = file
def __str__(self):
return repr(self.file)
class FileAlreadyExists(Exception):
""" File already exists exception (for move) """
def __init__(self, path):
self.path = path
def __str__(self):
return "File already exists {}".format(self.path)
class AccessDenied(Exception):
""" Access denied exception (for move or delete) """
def __init__(self, path):
self.path = path
def __str__(self):
return "Access denied for {}".format(self.path)
class WrongFileId(Exception):
""" Wrong file if exception """
def __init__(self, path, file_id_expected=None, file_id_got=None):
self.path = path
self.file_id_expected = file_id_expected
self.file_id_got = file_id_got
def __str__(self):
return "Wrong file id for {}. Expected id {}. Got id {}".format(
self.path, self.file_id_expected, self.file_id_got)
class CopyDoesNotExists(Exception):
def __init__(self, hash):
self.hash = hash
def __str__(self):
return "Copy with hash {} does not exists".format(self.hash)
def _apply_patch(self,
filename,
patch,
new_hash,
old_hash,
silent=True,
events_file_id=None):
start_time = time.time()
patch_size = os.stat(patch).st_size
success = False
try:
patched_new_hash, old_hash = self._quiet_processor.patch_file(
filename,
patch,
silent=silent,
events_file_id=events_file_id,
wrong_file_id=self.Exceptions.WrongFileId)
assert patched_new_hash == new_hash
success = True
self.copy_added.emit(new_hash)
except Rsync.AlreadyPatched:
success = True
except:
raise
finally:
if self._tracker:
try:
file_size = os.stat(filename).st_size
except OSError:
file_size = 0
duration = time.time() - start_time
self._tracker.monitor_patch_accept(file_size, patch_size,
duration, success)
def generate_conflict_file_name(self,
filename,
is_folder=False,
name_suffix=None,
with_time=True):
orig_filename = filename
directory, filename = op.split(filename)
original_ext = ''
if is_folder:
original_name = filename
else:
# consider ext as 2 '.'-delimited last filename substrings
# if they don't contain spaces
dots_list = filename.split('.')
name_parts_len = len(dots_list)
for k in range(1, min(name_parts_len, 3)):
if ' ' in dots_list[-k]:
break
original_ext = '.{}{}'.format(dots_list[k], original_ext)
name_parts_len -= 1
original_name = '.'.join(dots_list[:name_parts_len])
index = 0
if name_suffix is None:
name_suffix = self._conflict_file_suffix
date_today = date.today().strftime('%d-%m-%y') if with_time else ''
suffix = '({} {})'.format(name_suffix, date_today)
while len(bytes(suffix.encode('utf-8'))) > \
int(self.max_file_name_length / 3):
suffix = suffix[int(len(suffix) / 2):]
name = '{}{}{}'.format(original_name, suffix, original_ext)
while True:
to_cut = len(bytes(name.encode('utf-8'))) - \
self.max_file_name_length
if to_cut <= 0:
break
if len(original_name) > to_cut:
original_name = original_name[:-to_cut]
else:
remained = to_cut - len(original_name) + 1
original_name = original_name[:1]
if remained < len(original_ext):
original_ext = original_ext[remained:]
else:
original_ext = original_ext[int(len(original_ext) / 2):]
name = '{}{}{}'.format(original_name, suffix, original_ext)
while op.exists(
self._path_converter.create_abspath(
FilePath(op.join(directory, name)))):
index += 1
name = '{}{} {}{}'.format(original_name, suffix, index,
original_ext)
conflict_file_name = FilePath(op.join(directory, name))
logger.info(
"Generated conflict file name: %s, original name: %s, "
"is_folder: %s, name_suffix: %s, with_time: %s",
conflict_file_name, orig_filename, is_folder, name_suffix,
with_time)
return conflict_file_name
def move_file(self, src, dst, is_offline=True):
src_full_path = self._path_converter.create_abspath(src)
dst_full_path = self._path_converter.create_abspath(dst)
is_offline = True if op.isdir(src_full_path) else is_offline
src_hard_path = self._quiet_processor.get_hard_path(
src_full_path, is_offline)
dst_hard_path = self._quiet_processor.get_hard_path(
dst_full_path, is_offline)
if not op.exists(src_hard_path):
raise self.Exceptions.FileNotFound(src_full_path)
elif op.exists(dst_hard_path):
raise self.Exceptions.FileAlreadyExists(dst_full_path)
dst_parent_folder_path = op.dirname(dst_full_path)
if not op.exists(dst_parent_folder_path):
self._on_event_arrived(
FsEvent(DELETE,
dst_parent_folder_path,
True,
is_offline=True,
quiet=True))
try:
os.rename(src_hard_path, dst_hard_path)
except OSError as e:
logger.warning("Can't move file (dir) %s. Reason: %s",
src_full_path, e)
if e.errno == errno.EACCES:
self._quiet_processor.access_denied()
raise self.Exceptions.AccessDenied(src_full_path)
else:
raise e
def copy_file(self, src, dst, is_directory=False, is_offline=True):
is_offline = True if is_directory else is_offline
src_full_path = self._path_converter.create_abspath(src)
dst_full_path = self._path_converter.create_abspath(dst)
src_hard_path = self._quiet_processor.get_hard_path(
src_full_path, is_offline)
dst_hard_path = self._quiet_processor.get_hard_path(
dst_full_path, is_offline)
if not op.exists(src_hard_path):
raise self.Exceptions.FileNotFound(src_full_path)
if is_directory:
shutil.copytree(src_full_path, dst_full_path)
else:
common.utils.copy_file(src_hard_path, dst_hard_path)
def restore_file_from_copy(self,
file_name,
copy_hash,
events_file_id,
search_by_id=False):
try:
old_hash = self._quiet_processor.create_file_from_copy(
file_name,
copy_hash,
silent=True,
events_file_id=events_file_id,
search_by_id=search_by_id,
wrong_file_id=self.Exceptions.WrongFileId,
copy_does_not_exists=self.Exceptions.CopyDoesNotExists)
except AssertionError:
self._on_event_arrived(
FsEvent(DELETE,
op.dirname(
self._path_converter.create_abspath(file_name)),
True,
is_offline=True,
quiet=True))
raise
return old_hash
def create_file_from_copy(self,
file_name,
copy_hash,
events_file_id,
search_by_id=False):
self.restore_file_from_copy(file_name,
copy_hash,
events_file_id=events_file_id,
search_by_id=search_by_id)
@benchmark
def make_copy_from_existing_files(self, copy_hash):
self._quiet_processor.make_copy_from_existing_files(copy_hash)
def create_empty_file(self,
file_name,
file_hash,
events_file_id,
search_by_id=False,
is_offline=True):
try:
self._quiet_processor.create_empty_file(
file_name,
file_hash,
silent=True,
events_file_id=events_file_id,
search_by_id=search_by_id,
wrong_file_id=self.Exceptions.WrongFileId,
is_offline=is_offline)
except AssertionError:
self._on_event_arrived(
FsEvent(DELETE,
op.dirname(
self._path_converter.create_abspath(file_name)),
True,
is_offline=True,
quiet=True))
raise
def on_delete_copy(self, hash, with_signature=True):
if not hash:
logger.error("Invalid hash '%s'", hash)
return
copy = op.join(get_copies_dir(self._root), hash)
try:
remove_file(copy)
logger.info("File copy deleted %s", copy)
if not with_signature:
return
signature = op.join(get_signatures_dir(self._root), hash)
remove_file(signature)
logger.info("File copy signature deleted %s", signature)
except Exception as e:
logger.error(
"Can't delete copy. "
"Possibly sync folder is removed %s", e)
self.possibly_sync_folder_is_removed()
def delete_old_signatures(self, delete_all=False):
logger.debug("Deleting old signatures...")
self._quiet_processor.delete_old_signatures(
get_signatures_dir(self._root), delete_all)
def path_exists(self, path, is_offline=True):
full_path = self._path_converter.create_abspath(path)
hard_path = self._quiet_processor.get_hard_path(full_path, is_offline)
return op.exists(hard_path)
def rename_excluded(self, rel_path):
logger.debug("Renaming excluded dir %s", rel_path)
new_path = self.generate_conflict_file_name(
rel_path,
name_suffix=self.selective_sync_conflict_suffix,
with_time=False)
self.move_file(rel_path, new_path)
def _rename_file(self, abs_path):
rel_path = self._path_converter.create_relpath(abs_path)
new_path = self.generate_conflict_file_name(rel_path,
is_folder=False,
name_suffix="",
with_time=True)
self.move_file(rel_path, new_path)
def db_file_exists(self):
return self._storage.db_file_exists()
def _clean_recent_copies(self):
mask = op.join(get_copies_dir(self._root), "*.recent_copy_[0-9]*")
recent_copies = glob.glob(mask)
list(map(os.remove, recent_copies))
def add_special_file(self, path):
self._special_files.append(path)
watch = None
if not (path in FilePath(self._root)):
watch = self._download_watch
self._observer.add_special_file(path, watch)
def remove_special_file(self, path):
logger.debug("Removing special file %s...", path)
if not (path in FilePath(self._root)):
self._observer.remove_special_file(path)
try:
self._special_files.remove(path)
except ValueError:
logger.warning("Can't remove special file %s from list %s", path,
self._special_files)
def change_special_file(self, old_file, new_file):
self.add_special_file(new_file)
self.remove_special_file(old_file)
def _on_event_arrived(self, fs_event, is_special=False):
logger.debug(
"Event arrived %s, special %s, online_processing_allowed: %s, "
"online_modifies_processing_allowed: %s", fs_event, is_special,
self._online_processing_allowed,
self._online_modifies_processing_allowed)
if is_special or fs_event.src in self._special_files:
self.special_file_event.emit(fs_event.src, fs_event.event_type,
fs_event.dst)
elif fs_event.is_offline or self._online_processing_allowed:
if not self._online_modifies_processing_allowed and \
not fs_event.is_offline and fs_event.event_type == MODIFY:
return
elif fs_event.src in self._paths_with_modify_quiet \
and fs_event.event_type in (CREATE, MODIFY):
fs_event.is_offline = True
fs_event.quiet = True
path = fs_event.src if fs_event.event_type == CREATE \
else fs_event.dst if fs_event.event_type == MOVE else ""
name = op.basename(path)
parent_path = op.dirname(path)
stripped_name = name.strip()
if stripped_name != name:
new_path = op.join(parent_path, stripped_name)
if op.exists(new_path):
new_path = self.generate_conflict_file_name(
new_path,
is_folder=fs_event.is_dir,
name_suffix="",
with_time=True)
logger.debug("Renaming '%s' to '%s'...", path, new_path)
os.rename(FilePath(path).longpath, FilePath(new_path).longpath)
path = new_path
if fs_event.event_type == CREATE:
fs_event.src = new_path
elif fs_event.event_type == MOVE:
fs_event.dst = new_path
hidden_dir = FilePath(
self._path_converter.create_abspath(HIDDEN_DIRS[0]))
if fs_event.event_type == MOVE:
if FilePath(fs_event.src) in hidden_dir or \
op.basename(fs_event.src).startswith('._'):
fs_event.event_type = CREATE
fs_event.src = fs_event.dst
fs_event.dst = None
elif FilePath(fs_event.dst) in hidden_dir or \
op.basename(fs_event.dst).startswith('._'):
fs_event.event_type = DELETE
fs_event.dst = None
if FilePath(fs_event.src) in hidden_dir or \
op.basename(fs_event.src).startswith('._'):
return
if FilePath(path) in self._excluded_dirs:
self.rename_excluded(self._path_converter.create_relpath(path))
else:
self._actions.add_new_event(fs_event)
def get_long_paths(self):
return self._actions.get_long_paths()
def set_excluded_dirs(self, excluded_dirs):
self._excluded_dirs = list(map(FilePath, excluded_dirs))
def remove_dir_from_excluded(self, directory):
try:
self._excluded_dirs.remove(directory)
except Exception as e:
logger.warning("Can't remove excluded dir %s from %s. Reason: %s",
directory, self._excluded_dirs, e)
def sync_events_file_id(self, file_path, events_file_id, is_folder):
self._quiet_processor.sync_events_file_id(file_path, events_file_id,
is_folder)
def sync_events_file_id_by_old_id(self, events_file_id,
old_events_file_id):
self._quiet_processor.sync_events_file_id_by_old_id(
events_file_id, old_events_file_id)
def set_collaboration_folder_icon(self, folder_name):
set_custom_folder_icon('collaboration', self._root, folder_name)
def reset_collaboration_folder_icon(self, folder_name):
reset_custom_folder_icon(self._root,
folder_name,
resource_name='collaboration')
def reset_all_collaboration_folder_icons(self):
root_folders = [
f for f in os.listdir(self._root)
if op.isdir(self._path_converter.create_abspath(f))
]
logger.debug("root_folders %s", root_folders)
list(map(self.reset_collaboration_folder_icon, root_folders))
def get_excluded_dirs_to_change(self,
excluded_dirs,
src_path,
dst_path=None):
src_path = FilePath(src_path)
if dst_path:
dst_path = FilePath(dst_path)
excluded_dirs = list(map(FilePath, excluded_dirs))
dirs_to_add = []
dirs_to_delete = list(filter(lambda ed: ed in src_path, excluded_dirs))
if dst_path is not None and \
not is_contained_in_dirs(dst_path, excluded_dirs):
# we have to add new excluded dirs only if folder is not moved
# to excluded dir
l = len(src_path)
dirs_to_add = [dst_path + d[l:] for d in dirs_to_delete]
logger.debug(
"get_excluded_dirs_to_change. "
"excluded_dirs %s, src_path %s, dst_path %s, "
"dirs_to_delete %s, dirs_to_add %s", excluded_dirs, src_path,
dst_path, dirs_to_delete, dirs_to_add)
return dirs_to_delete, dirs_to_add
def change_excluded_dirs(self, dirs_to_delete, dirs_to_add):
for directory in dirs_to_delete:
self.remove_dir_from_excluded(directory)
for directory in dirs_to_add:
self._excluded_dirs.append(directory)
def clear_excluded_dirs(self):
self._excluded_dirs = []
def get_fs_events_count(self):
return self._actions.get_fs_events_count()
def force_create_copies(self):
self._storage.clear_files_hash_mtime()
self.delete_old_signatures(delete_all=True)
self._local_events_flag = True
self.process_offline_changes()
def get_file_list(self):
return self._files_list.get()
def get_actual_events_file_id(self, path, is_folder=None):
abs_path = self._path_converter.create_abspath(path)
file = self._storage.get_known_file(abs_path, is_folder=is_folder)
return file.events_file_id if file else None
def is_directory(self, path):
abs_path = self._path_converter.create_abspath(path)
return op.isdir(abs_path)
def set_waiting(self, to_wait):
self._actions.set_waiting(to_wait)
def set_path_quiet(self, path):
logger.debug("Setting path %s quiet...", path)
self._paths_with_modify_quiet.add(FilePath(path))
def clear_paths_quiet(self):
logger.debug("Clearing quiet paths...")
self._paths_with_modify_quiet.clear()
def delete_files_with_empty_events_file_ids(self):
if self._storage.delete_files_with_empty_events_file_ids():
self.working.emit()
def is_file_in_storage(self, events_file_id):
return self._storage.get_known_file_by_id(events_file_id)
class Storage(object):
"""
Interface for requesting info on registered files and folders
"""
def __init__(self, path_converter, db_file_created_cb=None):
self._pc = path_converter
self.possibly_sync_folder_is_removed = Signal()
self.db_or_disk_full = Signal()
self._db_file = self._pc.create_abspath('.pvtbox/storage.db')
logger.debug("DB file: %s", self._db_file)
new_db_file = not exists(self._db_file)
if new_db_file and callable(db_file_created_cb):
db_file_created_cb()
make_dirs(self._db_file)
if not new_db_file:
# Database migration. It can be executed before opening db
try:
upgrade_db("storage_db", db_filename=self._db_file)
except Exception as e:
remove_file(self._db_file)
new_db_file = True
logger.warning(
"Can't upgrade storage db. "
"Reason: (%s) Creating...", e)
if callable(db_file_created_cb):
db_file_created_cb()
self._engine = create_engine('sqlite:///{}'.format(
FilePath(self._db_file)),
connect_args={
'timeout': 60 * 1000,
'check_same_thread': False,
})
self._engine.pool_timeout = 60 * 60 * 1000
self._Session = sessionmaker(bind=self._engine)
Base.metadata.create_all(self._engine, checkfirst=True)
if new_db_file:
try:
stamp_db("storage_db", db_filename=self._db_file)
except Exception as e:
logger.error("Error stamping storage db: %s", e)
self._lock = threading.RLock()
@contextmanager
def create_session(self, read_only=True, locked=False):
session = self._Session()
session.expire_on_commit = False
session.autoflush = False
if read_only:
session.flush = lambda: None
if not read_only and locked:
logger.debug("session %s acquiring lock...", hex(id(session)))
self._lock.acquire()
logger.debug("session %s acquired lock.", hex(id(session)))
try:
yield session
session.commit()
except OperationalError as e:
logger.warning("OperationalError: %s", e)
try:
session.rollback()
except Exception as e:
logger.warning(
"OperationalError, exception while trying to rollback session: %s",
e)
pass
if is_db_or_disk_full(e):
self.db_or_disk_full.emit()
else:
raise
except Exception as e:
logger.warning("Exception: %s", e)
session.rollback()
raise
finally:
if not read_only and locked:
self._lock.release()
logger.debug("session %s released lock.", hex(id(session)))
session.close()
@with_session(True)
def _get_known_paths(self,
is_folder,
parent_dir=None,
exclude_dirs=None,
session=None):
query = session.query(File.relative_path)
query = query.filter(File.is_folder == is_folder)
paths = query.all()
if parent_dir:
parent_dir = self._pc.create_relpath(parent_dir)
result = []
for path in paths:
if is_contained_in(path[0], parent_dir):
result.append(path)
paths = result
if exclude_dirs:
result = []
# Optimize perfomance using iterator based solution
for ed in exclude_dirs:
for pp in paths:
if not is_contained_in(pp[0], ed):
result.append(pp)
paths = result
return [FilePath(self._pc.create_abspath(x[0])) for x in paths]
@benchmark
def get_known_files(self,
parent_dir=None,
exclude_dirs=None,
session=None):
"""
Returns absolute paths of files known at the moment.
@param parent_dir Name of parent dir to limit results to [unicode]
@return Known files paths (absolute) [(unicode, )]
"""
return self._get_known_paths(is_folder=False,
parent_dir=parent_dir,
exclude_dirs=exclude_dirs,
session=session)
@benchmark
def get_known_folders(self,
parent_dir=None,
exclude_dirs=None,
session=None):
"""
Returns absolute paths of folders known at the moment
@param parent_dir Name of parent dir to limit results to [unicode]
@return Known folders paths (absolute) [(unicode, )]
"""
return self._get_known_paths(is_folder=True,
parent_dir=parent_dir,
exclude_dirs=exclude_dirs,
session=session)
@with_session(True)
def get_known_file(self, abs_path, is_folder=None, session=None):
rel_path = self._pc.create_relpath(abs_path)
query = session.query(File).filter(File.relative_path == rel_path)
if is_folder is not None:
query.filter(File.is_folder == is_folder)
return query.one_or_none()
@with_session(True)
def get_known_file_by_id(self, file_id, session=None):
return session.query(File)\
.filter(File.events_file_id == file_id)\
.one_or_none()
@with_session(False)
def get_new_file(self, abs_path, is_folder, session=None):
rel_path = self._pc.create_relpath(abs_path)
file = File(relative_path=rel_path, is_folder=is_folder)
return file
def update_file_signature(self, file, signature):
signature_path = self._pc.create_abspath(file.signature_rel_path)
make_dirs(signature_path)
with open(signature_path, 'wb') as f:
dump(signature, f, protocol=2)
def get_file_signature(self, file):
abs_path = self._pc.create_abspath(file.signature_rel_path)
try:
with open(abs_path, 'rb') as f:
return load(f)
except (IOError, OSError, EOFError):
return None
@with_session(False)
def save_file(self, file, session=None):
return session.merge(file)
@with_session(False)
def delete_file(self, file, session=None):
session.delete(file)
def clean(self):
try:
self._engine.execute("delete from files")
logger.info("Cleaned storage data base")
except Exception as e:
logger.error("Failed to clean DB (%s)", e)
if not self.db_file_exists():
raise e
@with_session(False)
def delete_directories(self, dirs=[], session=None):
paths_deleted = []
if not dirs:
return paths_deleted
files = session.query(File).all()
dirs_rel = [self._pc.create_relpath(p) for p in dirs]
for file in files:
if is_contained_in_dirs(file.relative_path, dirs_rel):
if not file.is_folder:
paths_deleted.append(file.relative_path)
session.delete(file)
return paths_deleted
def db_file_exists(self):
return exists(self._db_file) and getsize(self._db_file) > 0
@with_session(False, True)
def change_events_file_id(self, old_id, new_id, session=None):
file = self.get_known_file_by_id(old_id, session=session)
if file:
file.events_file_id = new_id
logger.debug("Changed events_file_id for %s from %s to %s",
file.relative_path, old_id, new_id)
else:
logger.warning("Could not find file with events_file_id = %s",
old_id)
@with_session(True)
def get_known_folder_children(self, parent_dir_rel_path, session=None):
path_like = parent_dir_rel_path + '/%'
children = session.query(File)\
.filter(
or_(File.relative_path == parent_dir_rel_path,
File.relative_path.like(path_like)))\
.all()
return children
@with_session(False, True)
def delete_known_folder_children(self, parent_dir_rel_path, session=None):
path_like = parent_dir_rel_path + '/%'
session.query(File)\
.filter(
or_(File.relative_path == parent_dir_rel_path,
File.relative_path.like(path_like)))\
.delete(synchronize_session=False)
@with_session(False, True)
def move_known_folder_children(self,
old_dir_rel_path,
new_dir_rel_path,
session=None):
path_like = old_dir_rel_path + '/%'
files = session.query(File) \
.filter(
or_(File.relative_path == old_dir_rel_path,
File.relative_path.like(path_like))) \
.all()
mappings = [{
'id':
f.id,
'relative_path':
FilePath(
join(new_dir_rel_path,
relpath(f.relative_path, old_dir_rel_path)))
} for f in files]
session.bulk_update_mappings(File, mappings)
@with_session(True)
def hash_in_storage(self, file_hash, session=None):
if not file_hash:
return None
files_count = session.query(func.count())\
.select_from(File)\
.filter(File.file_hash == file_hash)\
.scalar()
return files_count > 0
@with_session(False, True)
def clear_files_hash_mtime(self, session=None):
session.execute(
update(File).where(File.is_folder == 0).values(file_hash=None,
mtime=0))
@with_session(True)
def get_last_files(self, limit, offset=0, session=None):
files = session.query(File) \
.filter(File.is_folder == 0) \
.order_by(File.mtime.desc()) \
.offset(offset).limit(limit) \
.all()
return files
def get_file_by_hash(self, hash, exclude, session):
return session.query(File) \
.filter(File.file_hash == hash) \
.filter(File.id.notin_(exclude)) \
.first()
@with_session(False, True)
def delete_files_with_empty_events_file_ids(self, session=None):
files_with_empty_ids = session.query(File) \
.filter(File.events_file_id.is_(None)) \
.all()
for file in files_with_empty_ids:
if file.is_folder:
self.delete_known_folder_children(file.relative_path,
session=session)
type_str = "folder"
else:
session.delete(file)
type_str = "file"
logger.debug("Deleted %s %s with empty events_file_id", type_str,
file.relative_path)
return bool(files_with_empty_ids)
def __init__(self,
cfg,
main_cfg,
start_service,
exit_service,
parent=None,
size=None,
migrate=False,
dp=1,
get_offline_dirs=lambda: None,
set_offline_dirs=lambda o, no: None):
super(Settings, self).__init__()
self._cfg = cfg
self._main_cfg = main_cfg
self._start_service = start_service
self._exit_service = exit_service
self._parent = parent
self._size = size
self._dp = dp
self._get_offline_dirs = get_offline_dirs
self._set_offline_dirs = set_offline_dirs
self._dialog = QDialog(parent)
self._dialog.setWindowIcon(QIcon(':/images/icon.png'))
self._dialog.setAttribute(Qt.WA_MacFrameworkScaled)
self._ui = settings.Ui_Dialog()
self._ui.setupUi(self._dialog)
self._max_root_len = get_max_root_len(self._cfg)
self._migrate = migrate
self._migration = None
self._migration_cancelled = False
try:
self._ui.account_type.setText(
license_display_name_from_constant(self._cfg.license_type))
self._ui.account_type.setVisible(True)
self._ui.account_type_header.setVisible(True)
self._ui.account_upgrade.setVisible(True)
except KeyError:
pass
upgrade_license_types = (FREE_LICENSE, FREE_TRIAL_LICENSE)
if self._cfg.license_type in upgrade_license_types:
self._ui.account_upgrade.setText('<a href="{}">{}</a>'.format(
GET_PRO_URI.format(self._cfg.host), tr('Upgrade')))
self._ui.account_upgrade.setTextFormat(Qt.RichText)
self._ui.account_upgrade.setTextInteractionFlags(
Qt.TextBrowserInteraction)
self._ui.account_upgrade.setOpenExternalLinks(True)
self._ui.account_upgrade.setAlignment(Qt.AlignLeft)
else:
self._ui.account_upgrade.setText("")
self._ui.centralWidget.setFrameShape(QFrame.NoFrame)
self._ui.centralWidget.setLineWidth(1)
self._ui.language_comboBox.addItem(tr('English'))
self._ui.language_comboBox.setEnabled(False)
self._connect_slots()
self._set_fonts()
self._ui.tabWidget.setCurrentIndex(0)
self._smart_sync_dialog = None
self.logged_out = Signal(bool)
self.logging_disabled_changed = Signal(bool)
# FIXMe: without line below app crashes on exit after settings opened
self._dialog.mousePressEvent = self.on_mouse_press_event
class EventStrategy(object):
"""
Desribe the handling stratrgy for file events
"""
DUMMY_PAGE_SIZE = 500
def __init__(self,
db,
event,
get_download_backups_mode,
is_smart_sync=False):
super(EventStrategy, self).__init__()
self.event = event
self.event_id = event.id if event else 0
self.file_id = event.file.id if event and event.file else 0
self.db = db
self._is_smart_sync = is_smart_sync
self._cached_file_path = None
self._events_queue = None
self._download_backups = get_download_backups_mode()
self._force_move = False
self.change_processing_events_counts = Signal(int,
int) # (local, remote)
self.append_local_event = Signal(Event, str, str, int, bool)
self.rename_or_delete_dst_path = Signal(str, int, Session, bool)
''' Public methods templates ==============================================
'''
@atomic
def apply(self,
session=None,
fs=None,
excluded_dirs=None,
patches_storage=None,
collaborated_folders=(),
events_queue=None):
event = self.event
assert event.file_id
logger.debug('applying %s', self)
change_name = True
parent_found = True
if event.type != 'delete':
self._events_queue = events_queue
change_name, parent_found = self._apply_move_if_needed(
session, fs, excluded_dirs, patches_storage, events_queue)
if parent_found:
self._apply_event(session, fs, excluded_dirs, patches_storage)
if event.state == 'received' and not event.file.excluded:
# update file strategy cannot apply patch
return
logger.debug('after _apply_event %s', self)
self._set_actual_file_state_to_this(
session,
update_file_event=not event.file.excluded,
change_name=change_name)
if event.is_folder and event.type != 'delete':
self.set_collaborated_folder_icon(session, fs,
collaborated_folders)
self.db.expunge_parents(event.file, session)
logger.debug('applied %s', self)
def _apply_event(self, session, fs, excluded_dirs, patches_storage):
pass
def _create_file_from_copy(self, path, fs, search_by_id=False):
pass
def _apply_move_if_needed(self, session, fs, excluded_dirs,
patches_storage, events_queue):
event = self.event
assert event.file_id
parent_found = True
folder = self.find_folder_by_uuid(session, event.folder_uuid)
if folder == event.file.folder and event.file_name == event.file.name:
return True, parent_found
move_events = list(
filter(
lambda e: e.server_event_id and e.type == 'move' and
(not event.server_event_id or e.server_event_id > event.
server_event_id), event.file.events))
if move_events and not self._force_move and event.is_folder:
# skip this if we have subsequent moves
return False, parent_found
# Calculate object path for further use
event_path = event.file.path
if folder and not folder.is_existing and not folder.excluded:
logger.debug("Parent folder does not exist for %s", event_path)
parent_found = False
if self._process_parent_not_found(session):
fs.accept_delete(event_path,
is_directory=event.is_folder,
events_file_id=event.file_id,
is_offline=event.file.is_offline)
return True, parent_found
logger.debug('moving %s', event.file)
new_path = ('/'.join([folder.path, event.file_name])
if folder else event.file_name)
# Check whether event paths are excluded from sync
is_path_excluded = is_contained_in_dirs(event_path, excluded_dirs)
is_new_path_excluded = is_contained_in_dirs(new_path, excluded_dirs)
# Both source and destination paths are excluded
if is_path_excluded and is_new_path_excluded:
assert False, 'Excluded-excluded must never occur'
# None of source and destination paths are excluded
elif not is_path_excluded and not is_new_path_excluded:
# Regular move event processing
try:
fs.accept_move(event_path,
new_path,
is_directory=event.is_folder,
events_file_id=event.file_id,
is_offline=event.file.is_offline)
except fs.Exceptions.FileAlreadyExists:
if event.file.event_id and not event.file.is_deleted:
if not self._rename_or_delete_dst_path(
new_path, session, event.file.is_offline):
raise SkipEventForNow()
else:
# retry move after renaming new path
return self._apply_move_if_needed(
session, fs, excluded_dirs, patches_storage,
events_queue)
except fs.Exceptions.FileNotFound:
subsequent_local_moves_deletes = list(
filter(
lambda ev: ev.id > event.id and ev.type in
('delete', 'move') and ev.state in
('occured', 'conflicted', 'sent'), event.file.events))
if not subsequent_local_moves_deletes and \
not self.check_previous_delete(
session, events_queue, fs):
# file/folder moved or deleted locally and
# no events in db for now
# so wait
logger.warning("Source file (folder) %s not found.",
event_path)
raise SkipEventForNow()
except fs.Exceptions.WrongFileId:
if not self.event.is_folder or \
not self._apply_folder_delete_if_any(session, fs):
raise SkipEventForNow()
# retry move after deleting folder
return self._apply_move_if_needed(session, fs, excluded_dirs,
patches_storage,
events_queue)
except Exception as e:
# ignore move if file is unavailable
logger.warning("Can't move file (folder) %s. Reason %s",
event_path, e)
raise SkipEventForNow()
event.file.name = event.file_name
event.file.folder = folder
if folder:
event.file.folder_id = folder.id
# Source path is excluded
elif is_path_excluded and not is_new_path_excluded:
self.event.file.excluded = False
self.event.file.folder = folder
if event.is_folder:
# Create directory at destination path
fs.create_directory(new_path, self.event.file_id)
else:
# Create file at destination path
if self.event.file_size:
self._create_file_from_copy(new_path, fs)
else:
fs.create_empty_file(new_path,
self.event.file_hash,
self.event.file_id,
is_offline=self.event.file.is_offline)
# Destination path is excluded
elif not is_path_excluded and is_new_path_excluded:
if not hasattr(self, '_excluded_ready') or \
not self._excluded_ready:
self._excluded_ready = False
raise SkipExcludedMove
self.event.file.excluded = True
self.event.file.event_id = None
if not self.event.is_folder:
self.event.state = 'received'
else:
self.db.mark_child_excluded(self.event.file_id, session)
# Delete object at source path
fs.accept_delete(event_path,
is_directory=event.is_folder,
is_offline=event.file.is_offline)
return True, parent_found
def _rename_or_delete_dst_path(self, path, session, is_offline=True):
try:
self.rename_or_delete_dst_path.emit(path, self.event.file_id,
session, is_offline)
except RenameDstPathFailed:
return False
return True
@db_read
def ready_to_apply(self, session=None, is_deleted=False):
return self._ready_to_apply(session, is_deleted=is_deleted)
@benchmark
def _ready_to_apply(self, session, is_deleted=False, files=None):
ready = (not self.event.file.is_locally_modified and
(self.event.is_folder or self.ready_to_skip(session=session))
and not is_deleted
and self.event.state in ('registered', 'sent', 'downloaded'))
return ready
@atomic
def ready_to_register(self, session):
return self._ready_to_register(session)
def _ready_to_register(self, session):
return False
@benchmark
@db_read
def ready_to_skip(self, session=None):
event = self.event
file = event.file
file_event = file.event
if file.event_id and not file.event:
file_event = session.query(Event) \
.filter(Event.id == file.event_id) \
.one_or_none()
return (event.id and
(not file_event or file.event_id == event.last_event_id
or file_event.type == 'delete' or not event.last_event_id or
(file.last_skipped_event_id
and file.last_skipped_event_id == event.last_event_id)))
@db_read
def skip_if_file_will_be_deleted(self, session):
'''Should be overriden in concrete strategy, if it can be skipped'''
return not self.event.file.is_existing
@atomic
def skip(self, session, min_server_event_id=0, fs=None):
if self.event.type == 'delete':
self.event.file.event_id = self.event.id
if fs:
fs.sync_events_file_id_by_old_id(None, self.event.file_id)
else:
self.event.file.last_skipped_event_id = self.event.id
# add dummy delete if there is no delete event for file
file = session.query(File) \
.filter(File.id == self.event.file_id) \
.one_or_none()
if min_server_event_id and file:
delete_events = list(
filter(lambda e: e.type == 'delete', file.events))
if not delete_events:
self._add_dummy_delete(file, self.event,
min_server_event_id, session)
def postpone_after_save(self):
''' Should be overriden in concrete strategy
if it must be postponed after save in db'''
return False
@db_read
def file_will_be_deleted(self, session, file=None):
return self._file_will_be_deleted(
session=session, file=file if file else self.event.file)
@benchmark
def _file_will_be_deleted(self, session, file):
return file.events and file.events[-1].type == 'delete'
@db_read
def get_old_uuid(self, session):
if self.event.last_event:
return self.event.last_event.uuid
else:
return None
def set_collaborated_folder_icon(self, session, fs, collaborated_folders):
pass
def set_excluded_ready(self):
self._excluded_ready = True
''' Utility functions ===================================================
'''
def _apply_patch(self, fs, file_path, patch_uuid):
try:
fs.apply_patch(file_path, fs.get_patch_path(patch_uuid),
self.event.file_hash,
self.event.file_hash_before_event,
self.event.file_id)
except Exception as e:
logger.error("Can't apply patch %s for file %s. Error %s",
patch_uuid, file_path, e)
return False
return True
def _get_last_nonconflicted_state(self, session, fs):
''' Also make a conflicted copy of the current file state.'''
assert self.event.file_id
assert self.event.last_event_id, \
'Getting last nonconflicted state for create event'
event = session.query(Event) \
.filter(Event.server_event_id.isnot(None)) \
.filter(Event.file_id == self.event.file_id) \
.filter(Event.state.in_(['sent', 'downloaded'])) \
.filter(Event.type != 'delete') \
.filter(Event.id <= self.event.last_event_id) \
.order_by(Event.id.desc()).first()
assert event is not None, \
'Probably getting last nonconflicted state for create event'
return event
def _set_actual_file_state_to_this(self,
session,
update_file_event=True,
change_name=True):
file = self.event.file
if update_file_event:
file.event = self.event
file.event_id = self.event.id
file.last_skipped_event_id = None
if self.event.type in ('create', 'move') and change_name:
file.name = self.event.file_name
if not file.folder:
folder = self.find_folder_by_uuid(session,
self.event.folder_uuid)
file.folder_id = folder.id if folder else None
logger.debug(
"_set_actual_file_state_to_this. "
"event_id %s, file.event_id %s, file.name %s", self.event.id,
file.event_id, file.name)
def find_folder_by_uuid(self, session, uuid):
if not uuid:
return None
try:
folder = session.query(File) \
.filter(File.is_folder) \
.filter(File.uuid == uuid) \
.one()
except:
raise FolderUUIDNotFound(uuid)
return folder
def _get_target_path(self, session):
if self.event.state in ('occured', 'sent', 'conflicted'):
target_path = self.event.file.path
else:
target_path = self.db.get_path_from_event(self.event, session)
return target_path
def __str__(self):
return '{self.__class__.__name__}: {self.event}'.format(self=self)
def check_event_path_excluded(self, excluded_dirs):
return False
@db_read
def get_file_path(self, session):
if not self._cached_file_path:
self._cached_file_path = self.event.file.path
return self._cached_file_path
@atomic
def event_newer_than_applied(self, session):
return not self.event.file.event or \
self.event.server_event_id and \
(not self.event.file.event.server_event_id or
self.event.server_event_id >
self.event.file.event.server_event_id)
def make_conflicting_copy(self, fs):
raise NotImplemented()
@benchmark
def is_event_skipped(self, session):
event = self.event
try:
session.expire(event)
except Exception:
event = session.query(Event).filter(Event.id == event.id).one()
return event.file.last_skipped_event_id and \
event.id < event.file.last_skipped_event_id
def force_move(self):
self._force_move = True
def _add_dummy_delete_events(self, session=None):
assert self.event.file.is_folder
self._dummy_deletes = []
folder_uuid = self.event.file.uuid
self.db.get_files_by_folder_uuid(folder_uuid,
self._files_page_processor_cb,
include_folders=True,
include_deleted=False,
session=session)
self._save_dummy_delete_events(session)
def _files_page_processor_cb(self, files_page, folders_uuids, session):
file_ids = [f.id for f in files_page]
all_events = session.query(Event) \
.filter(Event.file_id.in_(tuple(file_ids))).all()
for file in files_page:
if self._events_queue.is_processing_stopped():
raise ProcessingAborted
events = filter(lambda e: e.file_id == file.id, all_events)
events = sorted(events,
key=lambda e: e.server_event_id
if e.server_event_id else 0,
reverse=True)
if events and not (
events[0].type == 'delete' or events[-1].type == 'delete'
and events[-1].server_event_id
and events[-1].server_event_id < 0
or events[0].type == 'move' and events[0].server_event_id
and events[0].folder_uuid not in folders_uuids):
min_server_event_id = self._events_queue\
.get_min_server_event_id()
session.expire(file)
if not file.uuid:
file.uuid = generate_uuid()
for one_event in events:
one_event.file_uuid = file.uuid
self._add_dummy_delete(file, events[0], min_server_event_id,
session)
self._events_queue.cancel_file_download(file.id, session)
self._events_queue.change_processing_events_counts(
remote_inc=1)
self._events_queue.events_added.set()
if len(self._dummy_deletes) >= self.DUMMY_PAGE_SIZE:
self._save_dummy_delete_events(session)
self._events_queue.allow_loading_remotes()
def _save_dummy_delete_events(self, session):
if self._dummy_deletes:
logger.debug("Saving %s dummy deletes in db",
len(self._dummy_deletes))
try:
session.bulk_insert_mappings(Event, self._dummy_deletes)
finally:
self._dummy_deletes = []
def _add_dummy_delete(self,
file,
event,
server_event_id,
session,
add_to_dummies=True):
msg = {
'event_id': server_event_id,
'event_type': 'delete',
'is_folder': file.is_folder,
'uuid': file.uuid,
'event_uuid': event.uuid,
'file_name': event.file_name,
'file_name_before_event': event.file_name,
'file_size': event.file_size,
'last_event_id': event.server_event_id,
'file_hash_before_event': event.file_hash,
'parent_folder_uuid': event.folder_uuid,
'timestamp': calendar.timegm(event.timestamp.utctimetuple()),
}
logger.debug("Formed dummy delete message '%s'...", msg)
new_event, _ = deserialize_event(msg)
new_event.last_event_id = event.id
new_event.file_id = file.id
new_event.state = 'downloaded'
if add_to_dummies:
if hasattr(self, "_dummy_deletes"):
# many dummy deletes
self._dummy_deletes.append(self.db.get_mapping(new_event))
else:
# one dummy delete
session.add(new_event)
return new_event
def _update_excluded_dirs(self,
fs,
excluded_dirs,
session=None,
signals=None,
change_in_db=True):
assert self.event.is_folder
if not excluded_dirs:
return
logger.debug("Updating excluded dirs")
if self.event.type == 'delete':
src_path = self.db.get_path_by_events(self.event, session)
dst_path = None
else: # self.event.type == 'move'
try:
prev_event = self.event.file.events[-2]
except IndexError:
logger.warning("No prev event for %s", self.event)
src_path = ""
else:
src_path = self.db.get_path_by_events(prev_event, session)
dst_path = self.db.get_path_by_events(self.event, session)
dirs_to_delete, dirs_to_add = fs.get_excluded_dirs_to_change(
excluded_dirs, src_path, dst_path)
if not dirs_to_delete:
return
if dirs_to_add:
change_in_db = False
signals.change_excluded_dirs.emit(dirs_to_delete, dirs_to_add)
if change_in_db:
for path in dirs_to_delete:
self._mark_dir_not_excluded(path, session)
def _mark_dir_not_excluded(self, path, session):
try:
folders = self.db.find_folders_by_future_path(path,
session=session,
include_deleted=True)
except Exception:
logger.error("Error finding folders %s by path", path)
return
assert folders, "Excluded dir has to be in db"
for folder in folders:
if folder.excluded:
folder.excluded = False
self.db.mark_child_excluded(folder.id,
session,
is_excluded=False)
def _apply_folder_delete_if_any(self, session, fs):
path = self.event.file.path
actual_file_id = fs.get_actual_events_file_id(path, is_folder=True)
logger.debug(
"Trying to delete folder %s "
"with actual events_file_id %s...", path, actual_file_id)
delete_events = session.query(Event) \
.filter(Event.file_id == actual_file_id) \
.filter(Event.type == 'delete') \
.filter(Event.state == 'downloaded') \
.all()
delete_events = sorted(filter(
lambda e: not e.file.event_id or e.id > e.file.event_id,
delete_events),
key=lambda e: e.server_event_id)
if not delete_events:
return False
delete_events[-1].file.event_id = delete_events[-1].id
fs.accept_delete(path,
is_directory=True,
events_file_id=actual_file_id)
self.change_processing_events_counts.emit(0, -1)
return True
def _generate_uuid(self, session):
while True:
uuid = generate_uuid()
events = session.query(Event).filter(Event.uuid == uuid).all()
if not events:
return uuid
logger.warning("Events with uuid '%s' exist %s", uuid, events)
def _process_parent_not_found(self, session):
with self.db.db_lock:
next_events = list(
filter(lambda e: e.id > self.event.id, self.event.file.events))
move_delete_exists = any(e.type in ('move', 'delete')
for e in next_events)
if move_delete_exists:
# don't do anything
return False
if self.event.is_folder:
self._add_dummy_delete_events(session)
delattr(self, "_dummy_deletes")
server_event_id = self._events_queue.get_min_server_event_id()
self._add_dummy_delete(self.event.file, self.event,
server_event_id, session)
return True
def _check_previous_delete(self, event, file_events, session, events_queue,
fs):
return False, False, None
@atomic
def check_previous_delete(self, session=None, events_queue=None, fs=None):
event = session.query(Event) \
.filter(Event.id == self.event.id) \
.one_or_none()
if not event:
return False
has_deletes, \
add_dummy, \
new_delete_event = self._check_previous_delete(
event, event.file.events, session, events_queue, fs)
if not has_deletes:
return False
if new_delete_event:
new_delete_event.file_id = event.file_id
session.add(new_delete_event)
if add_dummy:
min_server_event_id = events_queue.get_min_server_event_id()
self._add_dummy_delete(event.file, event, min_server_event_id,
session)
events_queue.change_processing_events_counts(remote_inc=1)
return True
@atomic
def add_dummy_if_parent_deleted(self, session=None, events_queue=None):
logger.debug("Adding dummy when parent is deleted...")
event = session.query(Event) \
.filter(Event.id == self.event.id) \
.one_or_none()
has_deletes = any(e.type == 'delete' for e in event.file.events)
if not event or has_deletes:
return False
folder = self.find_folder_by_uuid(session, event.folder_uuid)
if not folder or not folder.is_deleted_registered:
return False
if not self.event.file.folder_id:
self.event.file.folder_id = folder.id
min_server_event_id = events_queue.get_min_server_event_id()
self._add_dummy_delete(event.file, event, min_server_event_id, session)
events_queue.change_processing_events_counts(remote_inc=1)
return True
def _check_offline(self, session):
not_applied = 0
event = self.event
folder = self.find_folder_by_uuid(session, event.folder_uuid)
if folder and folder.is_offline and not event.file.is_offline:
not_applied = self.db.make_offline(event.file_uuid,
session=session,
is_offline=True)
return not_applied
class QuietProcessor(object):
def __init__(self, root, storage, path_converter, exceptions):
self._root = root
self._storage = storage
self._path_converter = path_converter
self._exceptions = exceptions
self._tmp_id = 0
self._tmp_id_lock = RLock()
self._init_temp_dir()
self.file_moved = Signal(str, str)
self.file_deleted = Signal(str)
self.file_modified = Signal(str, float)
self.access_denied = Signal(str)
def delete_file(self, full_path, events_file_id=None, is_offline=True):
full_path = unicodedata.normalize('NFC', full_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _full_path = self._get_file_by_id(events_file_id, session)
if not file:
if events_file_id is not None:
logger.warning("Skipping file deletion because "
"file with same events_file_id not found")
return
file = self._storage.get_known_file(full_path, session=session)
else:
full_path = _full_path
if file:
try:
remove_file(self.get_hard_path(full_path, is_offline))
except OSError as e:
logger.warning("Can't remove file. Reason: %s", e)
if e.errno == errno.EACCES:
self._raise_access_denied(full_path)
else:
raise e
self._storage.delete_file(file, session=session)
self.file_deleted.emit(self._path_converter.create_relpath(full_path))
def delete_directory(self, full_path, events_file_id=None):
full_path = unicodedata.normalize('NFC', full_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _full_path = self._get_file_by_id(events_file_id, session)
if file:
full_path = _full_path
elif events_file_id is not None:
logger.warning("Skipping directory deletion because "
"directory with same events_file_id not found")
return
rel_path = self._path_converter.create_relpath(full_path)
files = self._storage.get_known_folder_children(rel_path,
session=session)
try:
temp_path = join(self._temp_dir, basename(full_path))
if isdir(temp_path):
remove_dir(temp_path, suppress_not_exists_exception=True)
elif isfile(temp_path):
remove_file(temp_path)
if isdir(full_path):
os.rename(full_path, temp_path)
try:
remove_dir(temp_path,
suppress_not_exists_exception=True)
except Exception:
logger.debug("Dir %s delete failed", temp_path)
except OSError as e:
logger.warning("Can't remove dir %s. Reason: %s", full_path, e)
if e.errno == errno.EACCES:
self._raise_access_denied(full_path)
elif e.errno != errno.ENOENT: # directory does not exist
raise e
deleted_paths = [f.relative_path for f in files]
self._storage.delete_known_folder_children(rel_path,
session=session)
for path in deleted_paths:
self.file_deleted.emit(path)
def create_directory(self, full_path, events_file_id, wrong_file_id=None):
full_path = unicodedata.normalize('NFC', full_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
assert exists(dirname(full_path))
file = self._storage.get_known_file(full_path,
True,
session=session)
if file is None:
mkdir(full_path)
file = self._storage.get_new_file(full_path,
True,
session=session)
elif events_file_id and file.events_file_id and \
file.events_file_id != events_file_id and \
wrong_file_id:
logger.error("Wrong file id for %s. Expected %s. Got %s",
full_path, events_file_id,
file.events_file_id if file else None)
raise wrong_file_id(full_path, events_file_id,
file.events_file_id)
file.events_file_id = events_file_id
self._storage.save_file(file, session=session)
def patch_file(self,
full_fn,
patch_archive,
silent=True,
events_file_id=None,
wrong_file_id=None):
full_fn = unicodedata.normalize('NFC', full_fn)
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _full_path = self._get_file_by_id(events_file_id, session)
if file:
full_fn = _full_path
else:
file = self._storage.get_known_file(full_fn,
is_folder=False,
session=session)
if (file is None or file and events_file_id and
file.events_file_id
and file.events_file_id != events_file_id) and \
wrong_file_id:
logger.error("Wrong file id for %s. Expected %s. Got %s",
full_fn, events_file_id,
file.events_file_id if file else None)
raise wrong_file_id(full_fn, events_file_id,
file.events_file_id if file else None)
# file = self._storage.get_new_file(full_fn, False,
# session=session)
assert exists(dirname(full_fn))
hash, signature, old_hash = Rsync.accept_patch(
patch_archive=patch_archive,
unpatched_file=full_fn,
known_old_hash=file.file_hash if file else None,
root=self._root)
if silent:
file.mtime = os.stat(full_fn).st_mtime
file.size = os.stat(full_fn).st_size
file.file_hash = hash
file.events_file_id = events_file_id
file.was_updated = True
self._storage.save_file(file, session=session)
self._storage.update_file_signature(file, signature)
self.file_modified.emit(file.relative_path, file.mtime)
return hash, old_hash
def move_file(self,
src_full_path,
dst_full_path,
events_file_id=None,
already_exists=None,
file_not_found=None,
wrong_file_id=None,
is_offline=True):
dst_full_path = unicodedata.normalize('NFC', dst_full_path)
dst_rel_path = self._path_converter.create_relpath(dst_full_path)
src_full_path = unicodedata.normalize('NFC', src_full_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _full_path = self._get_file_by_id(events_file_id, session)
if not file:
file = self._storage.get_known_file(src_full_path,
False,
session=session)
else:
src_full_path = _full_path
src_rel_path = self._path_converter.create_relpath(src_full_path)
if src_rel_path == dst_rel_path or not self._check_paths_exist(
src_full_path, dst_full_path, already_exists,
file_not_found):
return
assert exists(dirname(dst_full_path))
if file:
if events_file_id and file.events_file_id and \
file.events_file_id != events_file_id and \
wrong_file_id:
logger.error("Wrong file id for %s. Expected %s. Got %s",
dst_full_path, events_file_id,
file.events_file_id)
raise wrong_file_id(src_full_path, events_file_id,
file.events_file_id)
file.relative_path = self._path_converter.create_relpath(
dst_full_path)
try:
shutil.move(src=self.get_hard_path(src_full_path,
is_offline),
dst=self.get_hard_path(dst_full_path,
is_offline))
except OSError as e:
logger.warning("Can't move file. Reason: %s", e)
if e.errno == errno.EACCES:
self._raise_access_denied(src_full_path)
else:
raise e
self._storage.save_file(file, session=session)
self.file_moved(src_rel_path, dst_rel_path)
def move_directory(self,
src_full_path,
dst_full_path,
events_file_id=None,
already_exists=None,
file_not_found=None,
wrong_file_id=None):
dst_full_path = unicodedata.normalize('NFC', dst_full_path)
dst_rel_path = self._path_converter.create_relpath(dst_full_path)
src_full_path = unicodedata.normalize('NFC', src_full_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _full_path = self._get_file_by_id(events_file_id, session)
if not file:
file = self._storage.get_known_file(src_full_path,
True,
session=session)
else:
src_full_path = _full_path
src_rel_path = self._path_converter.create_relpath(src_full_path)
if src_rel_path == dst_rel_path or not self._check_paths_exist(
src_full_path, dst_full_path, already_exists,
file_not_found):
return
assert exists(dirname(dst_full_path))
if file:
if events_file_id and file.events_file_id and \
file.events_file_id != events_file_id and \
wrong_file_id:
logger.error("Wrong file id for %s. Expected %s. Got %s",
src_full_path, events_file_id,
file.events_file_id if file else None)
raise wrong_file_id(src_full_path, events_file_id,
file.events_file_id)
try:
os.rename(src_full_path, dst_full_path)
except OSError as e:
logger.warning("Can't move dir %s. Reason: %s", src_full_path,
e)
if e.errno == errno.EACCES:
self._raise_access_denied(src_full_path)
else:
raise e
self._storage.move_known_folder_children(src_rel_path,
dst_rel_path,
session=session)
self.file_moved(src_rel_path, str(dst_rel_path))
def create_file_from_copy(self,
file_rel_path,
copy_hash,
silent,
events_file_id,
search_by_id=False,
wrong_file_id=None,
copy_does_not_exists=None):
dst_full_path = self._path_converter.create_abspath(file_rel_path)
copy_full_path = join(get_copies_dir(self._root), copy_hash)
if copy_does_not_exists is not None and not exists(copy_full_path):
if not self.make_copy_from_existing_files(copy_hash):
raise copy_does_not_exists(copy_hash)
return self._create_file(copy_full_path, dst_full_path, silent,
copy_hash, events_file_id, search_by_id,
wrong_file_id)
def make_copy_from_existing_files(self, copy_hash):
copy_full_path = join(get_copies_dir(self._root), copy_hash)
if exists(copy_full_path):
return True
tmp_full_path = self._get_temp_path(copy_full_path)
with self._storage.create_session(read_only=True,
locked=False) as session:
excludes = []
while True:
file = self._storage.get_file_by_hash(copy_hash,
exclude=excludes,
session=session)
if not file:
return False
file_path = self._path_converter.create_abspath(
file.relative_path)
if not exists(file_path):
excludes.append(file.id)
continue
try:
copy_file(file_path, tmp_full_path)
hash = Rsync.hash_from_block_checksum(
Rsync.block_checksum(tmp_full_path))
if hash == copy_hash:
os.rename(tmp_full_path, copy_full_path)
return True
else:
excludes.append(file.id)
remove_file(tmp_full_path)
except Exception as e:
logger.warning("Can't operate tmp file %s. Reason: (%s)",
tmp_full_path, e)
if file.id not in excludes:
excludes.append(file.id)
try:
remove_file(tmp_full_path)
except Exception:
tmp_full_path = self._get_temp_path(copy_full_path)
def _get_temp_path(self, copy_full_path):
while True:
with self._tmp_id_lock:
self._tmp_id += 1
tmp_full_path = "{}_{}.tmp".format(copy_full_path, self._tmp_id)
if not exists(tmp_full_path):
return tmp_full_path
def create_empty_file(self,
file_rel_path,
file_hash,
silent,
events_file_id,
search_by_id=False,
wrong_file_id=None,
is_offline=True):
dst_full_path = self._path_converter.create_abspath(file_rel_path)
self._create_file(None, dst_full_path, silent, file_hash,
events_file_id, search_by_id, wrong_file_id,
is_offline)
def _create_file(self,
src_full_path,
dst_full_path,
silent,
file_hash,
events_file_id,
search_by_id,
wrong_file_id,
is_offline=True):
with self._storage.create_session(read_only=False,
locked=True) as session:
file = None
file_exists = False
was_updated = True
if search_by_id:
file, _full_path = self._get_file_by_id(
events_file_id, session)
if file:
dst_full_path = _full_path
assert exists(dirname(dst_full_path))
hard_path = self.get_hard_path(dst_full_path, is_offline)
if not file:
file = self._storage.get_known_file(dst_full_path,
is_folder=False,
session=session)
if file and events_file_id and file.events_file_id and \
file.events_file_id != events_file_id and \
wrong_file_id:
logger.error("Wrong file id for %s. Expected %s. Got %s",
dst_full_path, events_file_id,
file.events_file_id)
raise wrong_file_id(dst_full_path, events_file_id,
file.events_file_id)
if file:
file_exists = file.file_hash == file_hash and \
(exists(dst_full_path) and is_offline or
exists(hard_path) and not is_offline)
logger.debug(
"The fact that file %s with same hash "
"already exists in storage and filesystem is %s",
dst_full_path, file_exists)
if file is None:
# if search_by_id and wrong_file_id:
# logger.error("Wrong file id for %s. Expected %s. Got None",
# dst_full_path, events_file_id)
# raise wrong_file_id(dst_full_path,
# events_file_id,
# None)
file = self._storage.get_new_file(dst_full_path,
False,
session=session)
was_updated = False
old_hash = file.file_hash
signature = None
if not file_exists:
if src_full_path:
# create file from copy
if not exists(get_signature_path(file_hash)):
signature = Rsync.block_checksum(src_full_path)
tmp_full_path = self._get_temp_path(src_full_path)
copy_file(src_full_path, tmp_full_path)
try:
remove_file(dst_full_path)
os.rename(tmp_full_path, dst_full_path)
copy_time(dst_full_path + FILE_LINK_SUFFIX,
dst_full_path)
remove_file(dst_full_path + FILE_LINK_SUFFIX)
except Exception as e:
logger.warning(
"Can't rename to dst file %s. "
"Reason: %s", dst_full_path, e)
try:
remove_file(tmp_full_path)
except Exception:
pass
raise e
else:
create_empty_file(hard_path)
if not is_offline:
self.write_events_file_id(hard_path, events_file_id)
set_ext_invisible(hard_path)
if hard_path.endswith(FILE_LINK_SUFFIX):
copy_time(dst_full_path, hard_path)
remove_file(dst_full_path)
else:
copy_time(hard_path, dst_full_path)
remove_file(dst_full_path + FILE_LINK_SUFFIX)
if silent:
file.mtime = os.stat(hard_path).st_mtime
file.size = os.stat(hard_path).st_size
file.file_hash = file_hash
file.events_file_id = events_file_id
file.was_updated = was_updated
logger.debug("Saving file. id=%s", file.events_file_id)
self._storage.save_file(file, session=session)
if src_full_path and signature:
# create file from copy
self._storage.update_file_signature(file, signature)
if was_updated:
self.file_modified.emit(file.relative_path, file.mtime)
return old_hash
def sync_events_file_id(self, file_path, events_file_id, is_folder):
full_path = self._path_converter.create_abspath(file_path)
with self._storage.create_session(read_only=False,
locked=True) as session:
file = self._storage.get_known_file(full_path,
is_folder=is_folder,
session=session)
if file:
file.events_file_id = events_file_id
self._storage.save_file(file, session=session)
else:
logger.warning("Can't sync events_file_id for path %s",
file_path)
def sync_events_file_id_by_old_id(self, events_file_id,
old_events_file_id):
with self._storage.create_session(read_only=False,
locked=True) as session:
file, _ = self._get_file_by_id(old_events_file_id, session)
if file:
file.events_file_id = events_file_id
self._storage.save_file(file, session=session)
else:
logger.debug("Can't sync events_file_id for old_id %s",
old_events_file_id)
def _get_file_by_id(self, events_file_id, session):
file = full_path = None
if events_file_id:
file = self._storage.get_known_file_by_id(events_file_id, session)
if file:
full_path = self._path_converter.create_abspath(
file.relative_path)
else:
logger.warning("Can't find file by id %s", events_file_id)
return file, full_path
def _check_paths_exist(self, src_full_path, dst_full_path, already_exists,
file_not_found):
if exists(dst_full_path):
if exists(src_full_path):
if already_exists:
raise already_exists(dst_full_path)
else:
return False
else:
logger.debug(
"Destination exists %s, source does not exist %s."
" Moving accepted", dst_full_path, src_full_path)
return False
if not exists(src_full_path):
if file_not_found:
raise file_not_found(src_full_path)
else:
return False
return True
def delete_old_signatures(self, signatures_dir, delete_all=False):
# we believe that signatires dir contains only signature files
# and no subdirs
try:
signatures_to_delete = os.listdir(signatures_dir)
except Exception as e:
logger.warning("Can't delete old signatures. Reason: %s", e)
return
if not delete_all:
# taking storage lock to prevent adding new signatures
# during deletion
with self._storage.create_session(read_only=False,
locked=True) as session:
signatures_to_delete = filter(
lambda h: not self._storage.hash_in_storage(
h, session=session), signatures_to_delete)
try:
list(
map(lambda s: remove_file(join(signatures_dir, s)),
signatures_to_delete))
except Exception as e:
logger.warning("Can't delete old signatures. Reason: %s", e)
def _init_temp_dir(self):
self._temp_dir = get_temp_dir(self._root)
if exists(self._temp_dir):
try:
remove_dir(self._temp_dir)
except Exception as e:
logger.warning("Can't remove temp dir. Reason: %s", e)
self._temp_dir = get_temp_dir(self._root, create=True)
def _raise_access_denied(self, full_path):
self.access_denied(full_path)
raise self._exceptions.AccessDenied(full_path)
def get_hard_path(self, full_path, is_offline=True):
suffix = "" if is_offline else FILE_LINK_SUFFIX
return full_path + suffix
def write_events_file_id(self, hard_path, events_file_id):
with open(hard_path, 'wb') as f:
pickle.dump(events_file_id, f)
def __init__(self, path_converter):
super(NotifyIfMovedAction, self).__init__()
self.file_moved = Signal(str, str) # old, new file path
self._patch_converter = path_converter
def __init__(self, path_converter):
super(NotifyIfDeletedAction, self).__init__()
self.file_deleted = Signal(str) # relative_path
self._patch_converter = path_converter
