def remove_copy_reference(self, hash, reason="", postponed=False): if postponed: self._last_changes[hash] -= 1 copy_count = self._last_changes[hash] else: with self.create_session() as session: copy = session.query(Copy)\ .filter(Copy.hash == hash)\ .one_or_none() if not copy: logger.warning( "Trying to remove copy reference " "for non-existant copy %s", hash) return else: copy.count -= 1 copy_count = copy.count session.merge(copy) logger.debug( "File copy reference removed, %s, " "count: %s, postponed is %s", hash, copy_count, postponed) if not self._extended_logging: return copies_dir = get_copies_dir(self._root) self._logger.debug( "File copy reference removed, %s, count: %s, " "postponed is %s. File exists: %s. Reason: %s", hash, copy_count, postponed, exists(join(copies_dir, hash)), reason)
def move_files_to_copies(self): with self._storage.create_session(read_only=False, locked=True) as session: files_with_hashes = session\ .query(File.relative_path, File.file_hash) \ .filter(File.is_folder == 0) \ .all() copies_dir = get_copies_dir(self._root) for (file, hashsum) in files_with_hashes: hash_path = op.join(copies_dir, hashsum) file_path = self._path_converter.create_abspath(file) if not op.exists(hash_path): try: os.rename(file_path, hash_path) except Exception as e: logger.error("Error moving file to copy: %s", e) remove_file(file_path) abs_path = FilePath(self._root).longpath folders_plus_hidden = [ self._path_converter.create_abspath(f) for f in os.listdir(abs_path) if f not in HIDDEN_DIRS ] for folder in folders_plus_hidden: if not op.isdir(folder): continue try: remove_dir(folder) except Exception as e: logger.error("Error removing dir '%s' (%s)", folder, e) logger.info("Removed all files and folders") self._storage.clean()
def _on_new_event(self, fs_event): if fs_event.file_size + get_signature_file_size(fs_event.file_size) > \ get_free_space_by_filepath(fs_event.src): self.no_disk_space.emit(fs_event, fs_event.src, False) self.event_suppressed(fs_event) return file_recent_copy_name = FilePath( join(get_copies_dir(self._root), 'recent_copy_' + str(fs_event.id))) fs_event.file_recent_copy = file_recent_copy_name recent_copy_longpath = FilePath(file_recent_copy_name).longpath try: copy_file(FilePath(fs_event.src).longpath, recent_copy_longpath) except (OSError, IOError) as e: if e.errno == errno.ENOSPC: self.no_disk_space.emit(fs_event, fs_event.src, True) self.event_suppressed(fs_event) return self.event_returned(fs_event) return recent_copy_size = stat(recent_copy_longpath).st_size if recent_copy_size != fs_event.file_size: self.event_returned(fs_event) return self.event_passed(fs_event)
def add_copy_reference(self, hash, reason="", postponed=False): if postponed: self._last_changes[hash] += 1 copy_count = self._last_changes[hash] else: with self.create_session() as session: copy = session.query(Copy)\ .filter(Copy.hash == hash)\ .one_or_none() if copy is None: copy = Copy(hash=hash, count=0) copy.count += 1 copy_count = copy.count session.merge(copy) logger.debug( "File copy reference added, %s, " "count: %s, postponed is %s", hash, copy_count, postponed) if not self._extended_logging: return copies_dir = get_copies_dir(self._root) self._logger.debug( "File copy reference added, %s, count: %s. " "postponed is %s. File exists: %s. " "Reason: %s", hash, copy_count, postponed, exists(join(copies_dir, hash)), reason)
def _on_new_event(self, fs_event): file_synced_copy_name = FilePath( join(get_copies_dir(self._root), fs_event.new_hash)).longpath file_recent_copy_name = FilePath(fs_event.file_recent_copy).longpath self._copies_storage.add_copy_reference( fs_event.new_hash, reason="MoveFileRecentCopyAction {}".format(fs_event.src)) if exists(file_recent_copy_name): if not exists(file_synced_copy_name): try: shutil.move(file_recent_copy_name, file_synced_copy_name) self.copy_added.emit(fs_event.new_hash) except (OSError, IOError): self._copies_storage.remove_copy_reference( fs_event.new_hash, reason="MoveFileRecentCopyAction {}".format( fs_event.src)) self.event_returned(fs_event) return if stat(file_synced_copy_name).st_size != fs_event.file_size: self.event_returned(fs_event) return fs_event.file_synced_copy = FilePath(file_synced_copy_name) self.event_passed(fs_event) else: self.event_returned(fs_event)
def get_url(): filename = config.get_main_option("filename") if filename is None: filename = ensure_unicode( join(get_copies_dir(get_data_dir(), create=True), 'copies.db')) url = config.get_main_option("sqlalchemy.url") url = ensure_unicode(url) url = url.format(filename=FilePath(filename)) return url
def _log_db(self, session): if not self._extended_logging: return copies_dir = get_copies_dir(self._root) copies = session.query(Copy).all() for i, copy in enumerate(copies): self._logger.debug("Copy %s: %s. File exists: %s", i, copy, exists(join(copies_dir, copy.hash)))
def _begin_file_download(self, download_manager, fs, signals, on_load_task_success, on_load_task_failure, session=None): from .update_file_strategy import RemoteUpdateFileStrategy from .delete_file_strategy import RemoteDeleteFileStrategy assert self.event.file_hash or self.event.file_hash_before_event size = self.event.file_size hash = self.event.file_hash if self.event.file_hash \ else self.event.file_hash_before_event path = join(get_copies_dir(fs.get_root()), hash) if exists(path): self.download_success = True return self.download_success else: self.download_success = False logger.debug( 'Starting downloading file: %s into: %s, size:%s (event uuid: %s)', self.event.file.name, path, size, self.event.uuid) signals.downloading_started.emit(self) is_silent = self.event.type == 'delete' or \ self.file_will_be_deleted() priority = DOWNLOAD_PRIORITY_REVERSED_PATCH if is_silent \ else DOWNLOAD_PRIORITY_FILE display_name = 'Syncing backup for file {}' if is_silent \ else 'Syncing file {}' display_name = display_name.format(self.event.file_name) target_file_path = self.db.get_path_from_event(self.event, session) files_info = [{ "target_file_path": target_file_path, "mtime": get_local_time_from_timestamp(self.event.timestamp), "is_created": not isinstance(self, RemoteUpdateFileStrategy), "is_deleted": isinstance(self, RemoteDeleteFileStrategy) }] download_manager.add_file_download(priority, self.event.uuid, self.event.file_size, hash, path, display_name, on_load_task_success, on_load_task_failure, files_info) return False
def __init__(self, root, db_file_created_cb=None, extended_logging=True, to_upgrade=True): self.possibly_sync_folder_is_removed = Signal() self.delete_copy = Signal( str, # copy hash bool) # with signature self.db_or_disk_full = Signal() self._db_file = join(get_copies_dir(root), 'copies.db') new_db_file = not exists(self._db_file) if new_db_file and callable(db_file_created_cb): db_file_created_cb() if to_upgrade and not new_db_file: # Database migration. It can be executed before opening db try: upgrade_db("copies_db", db_filename=self._db_file) except Exception as e: remove_file(self._db_file) new_db_file = True logger.warning( "Can't upgrade copies db. " "Reason: (%s) Creating...", e) if callable(db_file_created_cb): db_file_created_cb() self._engine = create_engine('sqlite:///{}'.format( FilePath(self._db_file))) self._Session = sessionmaker(bind=self._engine) Base.metadata.create_all(self._engine, checkfirst=True) if new_db_file: try: stamp_db("copies_db", db_filename=self._db_file) except Exception as e: logger.error("Error stamping copies db: %s", e) self._lock = RLock() self._root = root self._extended_logging = extended_logging if not self._extended_logging: self._logger = None else: self._logger = logging.getLogger('copies_logger') self._logger.debug("Copies init") self._last_changes = defaultdict(int)
def create_file_from_copy(self, file_rel_path, copy_hash, silent, events_file_id, search_by_id=False, wrong_file_id=None, copy_does_not_exists=None): dst_full_path = self._path_converter.create_abspath(file_rel_path) copy_full_path = join(get_copies_dir(self._root), copy_hash) if copy_does_not_exists is not None and not exists(copy_full_path): if not self.make_copy_from_existing_files(copy_hash): raise copy_does_not_exists(copy_hash) return self._create_file(copy_full_path, dst_full_path, silent, copy_hash, events_file_id, search_by_id, wrong_file_id)
def start_file_download(self, file_info): data_path = self._cfg.sync_directory if self._cfg else get_data_dir() download_path = op.join(get_copies_dir(data_path), file_info.file_hash) logger.info("Initiating downloading of file '%s' to '%s'...", file_info.fullname, download_path) def on_success(task): self.signals.download_success.emit(task.id, file_info, download_path) def on_failure(task): self.signals.download_failure.emit(task.id, file_info) if not file_info.size: create_empty_file(file_info.fullname) self._tasks[self._current_share_hash].remove(file_info.event_uuid) self._finish_task_download() return elif not self._cfg.download_backups: self._sync.make_copy_from_existing_files(file_info.file_hash) files_info = [{ "target_file_path": self._get_path_relative_to_share(file_info), "mtime": 0, # mtime == 0 => shared file "is_created": None, "is_deleted": None }] self._download_manager.add_file_download( DOWNLOAD_PRIORITY_FILE, file_info.event_uuid, file_info.size, file_info.file_hash, download_path, 'Downloading shared file {}'.format(file_info.name), on_success, on_failure, files_info=files_info, )
def remove_copies_not_in_db(self): with self.create_session() as session: copies = session.query(Copy).all() exclude_files = {copy.hash for copy in copies} exclude_files.add('copies.db') copies_dir = get_copies_dir(self._root) try: files_to_delete = set(listdir(copies_dir)) - exclude_files files_to_delete = map(lambda f: join(copies_dir, f), files_to_delete) list( map( remove_file, filter( lambda f: isfile(f) and not f.endswith('.download') and not f.endswith('.info'), files_to_delete))) except Exception as e: self.possibly_sync_folder_is_removed() logger.warning("Can't remove copies files. Reason: %s", e)
def on_delete_copy(self, hash, with_signature=True): if not hash: logger.error("Invalid hash '%s'", hash) return copy = op.join(get_copies_dir(self._root), hash) try: remove_file(copy) logger.info("File copy deleted %s", copy) if not with_signature: return signature = op.join(get_signatures_dir(self._root), hash) remove_file(signature) logger.info("File copy signature deleted %s", signature) except Exception as e: logger.error( "Can't delete copy. " "Possibly sync folder is removed %s", e) self.possibly_sync_folder_is_removed()
def make_copy_from_existing_files(self, copy_hash): copy_full_path = join(get_copies_dir(self._root), copy_hash) if exists(copy_full_path): return True tmp_full_path = self._get_temp_path(copy_full_path) with self._storage.create_session(read_only=True, locked=False) as session: excludes = [] while True: file = self._storage.get_file_by_hash(copy_hash, exclude=excludes, session=session) if not file: return False file_path = self._path_converter.create_abspath( file.relative_path) if not exists(file_path): excludes.append(file.id) continue try: copy_file(file_path, tmp_full_path) hash = Rsync.hash_from_block_checksum( Rsync.block_checksum(tmp_full_path)) if hash == copy_hash: os.rename(tmp_full_path, copy_full_path) return True else: excludes.append(file.id) remove_file(tmp_full_path) except Exception as e: logger.warning("Can't operate tmp file %s. Reason: (%s)", tmp_full_path, e) if file.id not in excludes: excludes.append(file.id) try: remove_file(tmp_full_path) except Exception: tmp_full_path = self._get_temp_path(copy_full_path)
def get_copy_file_path(self, hash): return join(get_copies_dir(self._root), hash)
def _clean_recent_copies(self): mask = op.join(get_copies_dir(self._root), "*.recent_copy_[0-9]*") recent_copies = glob.glob(mask) list(map(os.remove, recent_copies))
def _accept_patch(patch_info, patch_data, unpatched_file, root): file_blocks_hashes = SortedDict() blocksize = patch_info['blocksize'] temp_name = os.path.join(get_patches_dir(root), '.patching_' + generate_uuid()) blocks = SortedDict( (int(k), v) for k, v in patch_info['blocks'].items()) source_file = None if op.exists(unpatched_file): source_file = open(unpatched_file, "rb") with open(temp_name, "wb") as temp_file: # count = 0 # min = 999999999.0 # max = 0.0 # avg = 0.0 # sum = 0.0 for offset, block in blocks.items(): # count += 1 # start_time = time.time() block_offset = int(block['offset']) if block['new']: patch_data.seek(block_offset) data_size = block['data_size'] data = patch_data.read(data_size) else: if block['from_patch']: patch_offset = blocks[block_offset]['offset'] data_size = blocks[block_offset].get( 'data_size', blocksize) patch_data.seek(patch_offset) data = patch_data.read(data_size) else: if source_file is None: raise IOError("Source file not found") source_file.seek(block_offset) data = source_file.read(blocksize) temp_file.seek(offset) temp_file.write(data) file_blocks_hashes[offset] = block['hash'] # diff = time.time() - start_time # min = diff if diff < min else min # max = diff if diff > max else max # avg = diff if avg == 0 else (avg + diff) / 2 # sum += diff # logger.debug( # 'processed block %s:%s in %s', count, len(blocks), diff) # logger.debug( # 'processing blocks time:%s, min:%s, max:%s, avg:%s', # sum, min, max, avg) if source_file: source_file.close() logger.debug('calculating patched file signature') file_signature = Rsync.block_checksum(temp_name, blocksize=blocksize) logger.debug('calculated patched file signature') if file_signature != file_blocks_hashes: remove_file(temp_name) raise IOError( "Invalid patch result, expected signature: {}, actual: {}". format(file_blocks_hashes, file_signature)) new_hash = patch_info['new_hash'] logger.debug('moving patched file') copy = join(get_copies_dir(root), new_hash) if not exists(copy): copy_file(temp_name, copy) shutil.move(temp_name, unpatched_file) logger.debug('moved patched file') return new_hash, file_blocks_hashes, patch_info['old_hash']