def _find_remote_child_match_or_create(self, parent_pair, child_info): local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref # Try to get the local definition if not linked child_pair = self._dao.get_state_from_local(local_path) # Case of duplication (the file can exists in with a __x) or local rename if child_pair is None and parent_pair is not None and self._local_client.exists(parent_pair.local_path): for child in self._local_client.get_children_info(parent_pair.local_path): if self._local_client.get_remote_id(child.path) == child_info.uid: if '__' in child.name: log.debug('Found a deduplication case: %r on %r', child_info, child.path) else: log.debug('Found a local rename case: %r on %r', child_info, child.path) child_pair = self._dao.get_state_from_local(child.path) break if child_pair is not None: if child_pair.remote_ref is not None and child_pair.remote_ref != child_info.uid: log.debug("Got an existing pair with different id: %r | %r", child_pair, child_info) else: if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): # Local rename if child_pair.local_path != local_path: child_pair.local_state = 'moved' child_pair.remote_state = 'unknown' local_info = self._local_client.get_info(child_pair.local_path) self._dao.update_local_state(child_pair, local_info) self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) else: self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) # Use version+1 as we just update the remote info synced = self._dao.synchronize_state(child_pair, version=child_pair.version + 1) if not synced: # Try again, might happen that it has been modified locally and remotely child_pair = self._dao.get_state_from_id(child_pair.id) if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests( child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): self._dao.synchronize_state(child_pair) child_pair = self._dao.get_state_from_id(child_pair.id) synced = child_pair.pair_state == 'synchronized' # Can be updated in previous call if synced: self._engine.stop_processor_on(child_pair.local_path) # Push the remote_Id log.debug('Set remote ID on %r / %r == %r', child_pair, child_pair.local_path, child_pair.local_path) self._local_client.set_remote_id(child_pair.local_path, child_info.uid) if child_pair.folderish: self._dao.queue_children(child_pair) else: child_pair.remote_state = 'modified' self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True) return child_pair, False row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) child_pair = self._dao.get_state_from_id(row_id, from_write=True) return child_pair, True
def _find_remote_child_match_or_create(self, parent_pair, child_info): local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref # Try to get the local definition if not linked child_pair = self._dao.get_state_from_local(local_path) if child_pair is not None: # Should compare to xattr remote uid if child_pair.remote_ref is not None: child_pair = None else: self._dao.update_remote_state(child_pair, child_info, remote_parent_path) if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): # Use version+1 as we just update the remote info if not self._dao.synchronize_state(child_pair, version=child_pair.version + 1): # Try again, might happens that it has been modified locally and remotelly child_pair = self._dao.get_state_from_id(child_pair.id) if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests( child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): self._dao.synchronize_state(child_pair) # Push the remote_Id self._local_client.set_remote_id(local_path, child_info.uid) if child_pair.folderish: self._dao.queue_children(child_pair) child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True) return child_pair, False row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) child_pair = self._dao.get_state_from_id(row_id, from_write=True) return child_pair, True
def _find_remote_child_match_or_create(self, parent_pair, child_info): local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref # Try to get the local definition if not linked child_pair = self._dao.get_state_from_local(local_path) # Case of duplication: the file can exists in with a __x if child_pair is None and parent_pair is not None and self._local_client.exists(parent_pair.local_path): for child in self._local_client.get_children_info(parent_pair.local_path): # Skip any file without __ as it cannot be a deduplicate if '__' not in child.name: continue if self._local_client.get_remote_id(child.path) == child_info.uid: log.debug("Found a deduplication case: %r on %s", child_info, child.path) child_pair = self._dao.get_state_from_local(child.path) break if child_pair is not None: if child_pair.remote_ref is not None and child_pair.remote_ref != child_info.uid: log.debug("Got an existing pair with different id: %r | %r", child_pair, child_info) child_pair = None else: if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) # Use version+1 as we just update the remote info synced = self._dao.synchronize_state(child_pair, version=child_pair.version + 1) if not synced: # Try again, might happen that it has been modified locally and remotely child_pair = self._dao.get_state_from_id(child_pair.id) if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests( child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): self._dao.synchronize_state(child_pair) child_pair = self._dao.get_state_from_id(child_pair.id) synced = child_pair.pair_state == 'synchronized' # Can be updated in previous call if synced: self._engine.stop_processor_on(child_pair.local_path) # Push the remote_Id log.debug("set remote id on: %r / %s == %s", child_pair, child_pair.local_path, local_path) self._local_client.set_remote_id(local_path, child_info.uid) if child_pair.folderish: self._dao.queue_children(child_pair) else: child_pair.remote_state = 'modified' self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True) return child_pair, False row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) child_pair = self._dao.get_state_from_id(row_id, from_write=True) return child_pair, True
def _find_remote_child_match_or_create(self, parent_pair, child_info): local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref # Try to get the local definition if not linked child_pair = self._dao.get_state_from_local(local_path) # Case of duplication: the file can exists in with a __x if child_pair is None and parent_pair is not None and self._local_client.exists(parent_pair.local_path): for child in self._local_client.get_children_info(parent_pair.local_path): # Skip any file without __ as it cannot be a deduplicate if '__' not in child.name: continue if self._local_client.get_remote_id(child.path) == child_info.uid: log.debug("Found a deduplication case: %r on %s", child_info, child.path) child_pair = self._dao.get_state_from_local(child.path) break if child_pair is not None: if child_pair.remote_ref is not None: child_pair = None else: if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) # Use version+1 as we just update the remote info synced = self._dao.synchronize_state(child_pair, version=child_pair.version + 1) if not synced: # Try again, might happen that it has been modified locally and remotely child_pair = self._dao.get_state_from_id(child_pair.id) if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests( child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)): self._dao.synchronize_state(child_pair) child_pair = self._dao.get_state_from_id(child_pair.id) synced = child_pair.pair_state == 'synchronized' # Can be updated in previous call if synced: self._engine.stop_processor_on(child_pair.local_path) # Push the remote_Id self._local_client.set_remote_id(local_path, child_info.uid) if child_pair.folderish: self._dao.queue_children(child_pair) else: child_pair.remote_state = 'modified' self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path) child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True) return child_pair, False row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) child_pair = self._dao.get_state_from_id(row_id, from_write=True) return child_pair, True
def _scan_pair(self, remote_path): if remote_path is None: return remote_path = str(remote_path) if self._dao.is_filter(remote_path): # Skip if filter return if remote_path[-1:] == '/': remote_path = remote_path[0:-1] remote_ref = os.path.basename(remote_path) parent_path = os.path.dirname(remote_path) if parent_path == '/': parent_path = '' # If pair is present already try: child_info = self._client.get_info(remote_ref) except NotFound: # The folder has been deleted return doc_pair = self._dao.get_state_from_remote_with_path( remote_ref, parent_path) if doc_pair is not None: self._do_scan_remote(doc_pair, child_info) return log.debug('parent_path: %r\t%r\t%r', parent_path, os.path.basename(parent_path), os.path.dirname(parent_path)) parent_pair = self._dao.get_state_from_remote_with_path( os.path.basename(parent_path), os.path.dirname(parent_path)) log.debug("scan_pair: parent_pair: %r", parent_pair) if parent_pair is None: return local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref if os.path.dirname(child_info.path) == remote_parent_path: row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) doc_pair = self._dao.get_state_from_id(row_id, from_write=True) if child_info.folderish: self._do_scan_remote(doc_pair, child_info) else: log.debug('Remote scan_pair: %r is not available', remote_path) self._scan_remote()
def _scan_pair(self, remote_path): if remote_path is None: return remote_path = str(remote_path) if self._dao.is_filter(remote_path): # Skip if filter return if remote_path[-1:] == '/': remote_path = remote_path[0:-1] remote_ref = os.path.basename(remote_path) parent_path = os.path.dirname(remote_path) if parent_path == '/': parent_path = '' # If pair is present already try: child_info = self._client.get_info(remote_ref) except NotFound: # The folder has been deleted return doc_pair = self._dao.get_state_from_remote_with_path(remote_ref, parent_path) if doc_pair is not None: log.debug("Remote scan_pair: %s", doc_pair.local_path) self._do_scan_remote(doc_pair, child_info) log.debug("Remote scan_pair ended: %s", doc_pair.local_path) return log.debug("parent_path: '%s'\t'%s'\t'%s'", parent_path, os.path.basename(parent_path), os.path.dirname(parent_path)) parent_pair = self._dao.get_state_from_remote_with_path(os.path.basename(parent_path), os.path.dirname(parent_path)) log.debug("scan_pair: parent_pair: %r", parent_pair) if parent_pair is None: return local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref if os.path.dirname(child_info.path) == remote_parent_path: row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) doc_pair = self._dao.get_state_from_id(row_id, from_write=True) if child_info.folderish: log.debug("Remote scan_pair: %s", doc_pair.local_path) self._do_scan_remote(doc_pair, child_info) log.debug("Remote scan_pair ended: %s", doc_pair.local_path) else: log.debug("Remote scan_pair: %s is not available, Do full scan", remote_path) self._scan_remote()
def _find_remote_child_match_or_create(self, parent_pair, child_info): if not parent_pair.local_path: # The parent folder has an empty local_path, # it probably means that it has been put in error as a duplicate # by a processor => ignoring this child. log.debug('Ignoring child %r of a duplicate folder in error %r', child_info, parent_pair) return None, None local_path = path_join(parent_pair.local_path, safe_filename(child_info.name)) remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref # Try to get the local definition if not linked child_pair = self._dao.get_state_from_local(local_path) # Case of duplication (the file can exists in with a __x) or local rename if child_pair is None and parent_pair is not None and self._local_client.exists( parent_pair.local_path): for child in self._local_client.get_children_info( parent_pair.local_path): if self._local_client.get_remote_id( child.path) == child_info.uid: if '__' in child.name: log.debug('Found a deduplication case: %r on %r', child_info, child.path) else: log.debug('Found a local rename case: %r on %r', child_info, child.path) child_pair = self._dao.get_state_from_local(child.path) break if child_pair is not None: if child_pair.remote_ref is not None and child_pair.remote_ref != child_info.uid: log.debug("Got an existing pair with different id: %r | %r", child_pair, child_info) else: if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests( child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm )): # Local rename if child_pair.local_path != local_path: child_pair.local_state = 'moved' child_pair.remote_state = 'unknown' local_info = self._local_client.get_info( child_pair.local_path) self._dao.update_local_state(child_pair, local_info) self._dao.update_remote_state( child_pair, child_info, remote_parent_path=remote_parent_path) else: self._dao.update_remote_state( child_pair, child_info, remote_parent_path=remote_parent_path) # Use version+1 as we just update the remote info synced = self._dao.synchronize_state( child_pair, version=child_pair.version + 1) if not synced: # Try again, might happen that it has been modified locally and remotely child_pair = self._dao.get_state_from_id( child_pair.id) if (child_pair.folderish == child_info.folderish and self._local_client.is_equal_digests( child_pair.local_digest, child_info.digest, child_pair.local_path, remote_digest_algorithm=child_info. digest_algorithm)): self._dao.synchronize_state(child_pair) child_pair = self._dao.get_state_from_id( child_pair.id) synced = child_pair.pair_state == 'synchronized' # Can be updated in previous call if synced: self._engine.stop_processor_on( child_pair.local_path) # Push the remote_Id log.debug('Set remote ID on %r / %r == %r', child_pair, child_pair.local_path, child_pair.local_path) self._local_client.set_remote_id( child_pair.local_path, child_info.uid) if child_pair.folderish: self._dao.queue_children(child_pair) else: child_pair.remote_state = 'modified' self._dao.update_remote_state( child_pair, child_info, remote_parent_path=remote_parent_path) child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True) return child_pair, False row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path) child_pair = self._dao.get_state_from_id(row_id, from_write=True) return child_pair, True