def remove_copy_reference(self, hash, reason="", postponed=False):
if postponed:
self._last_changes[hash] -= 1
copy_count = self._last_changes[hash]
else:
with self.create_session() as session:
copy = session.query(Copy)\
.filter(Copy.hash == hash)\
.one_or_none()
if not copy:
logger.warning(
"Trying to remove copy reference "
"for non-existant copy %s", hash)
return
else:
copy.count -= 1
copy_count = copy.count
session.merge(copy)
logger.debug(
"File copy reference removed, %s, "
"count: %s, postponed is %s", hash, copy_count, postponed)
if not self._extended_logging:
return
copies_dir = get_copies_dir(self._root)
self._logger.debug(
"File copy reference removed, %s, count: %s, "
"postponed is %s. File exists: %s. Reason: %s", hash, copy_count,
postponed, exists(join(copies_dir, hash)), reason)
def move_files_to_copies(self):
with self._storage.create_session(read_only=False,
locked=True) as session:
files_with_hashes = session\
.query(File.relative_path, File.file_hash) \
.filter(File.is_folder == 0) \
.all()
copies_dir = get_copies_dir(self._root)
for (file, hashsum) in files_with_hashes:
hash_path = op.join(copies_dir, hashsum)
file_path = self._path_converter.create_abspath(file)
if not op.exists(hash_path):
try:
os.rename(file_path, hash_path)
except Exception as e:
logger.error("Error moving file to copy: %s", e)
remove_file(file_path)
abs_path = FilePath(self._root).longpath
folders_plus_hidden = [
self._path_converter.create_abspath(f)
for f in os.listdir(abs_path) if f not in HIDDEN_DIRS
]
for folder in folders_plus_hidden:
if not op.isdir(folder):
continue
try:
remove_dir(folder)
except Exception as e:
logger.error("Error removing dir '%s' (%s)", folder, e)
logger.info("Removed all files and folders")
self._storage.clean()
def _on_new_event(self, fs_event):
if fs_event.file_size + get_signature_file_size(fs_event.file_size) > \
get_free_space_by_filepath(fs_event.src):
self.no_disk_space.emit(fs_event, fs_event.src, False)
self.event_suppressed(fs_event)
return
file_recent_copy_name = FilePath(
join(get_copies_dir(self._root),
'recent_copy_' + str(fs_event.id)))
fs_event.file_recent_copy = file_recent_copy_name
recent_copy_longpath = FilePath(file_recent_copy_name).longpath
try:
copy_file(FilePath(fs_event.src).longpath, recent_copy_longpath)
except (OSError, IOError) as e:
if e.errno == errno.ENOSPC:
self.no_disk_space.emit(fs_event, fs_event.src, True)
self.event_suppressed(fs_event)
return
self.event_returned(fs_event)
return
recent_copy_size = stat(recent_copy_longpath).st_size
if recent_copy_size != fs_event.file_size:
self.event_returned(fs_event)
return
self.event_passed(fs_event)
def add_copy_reference(self, hash, reason="", postponed=False):
if postponed:
self._last_changes[hash] += 1
copy_count = self._last_changes[hash]
else:
with self.create_session() as session:
copy = session.query(Copy)\
.filter(Copy.hash == hash)\
.one_or_none()
if copy is None:
copy = Copy(hash=hash, count=0)
copy.count += 1
copy_count = copy.count
session.merge(copy)
logger.debug(
"File copy reference added, %s, "
"count: %s, postponed is %s", hash, copy_count, postponed)
if not self._extended_logging:
return
copies_dir = get_copies_dir(self._root)
self._logger.debug(
"File copy reference added, %s, count: %s. "
"postponed is %s. File exists: %s. "
"Reason: %s", hash, copy_count, postponed,
exists(join(copies_dir, hash)), reason)
def _on_new_event(self, fs_event):
file_synced_copy_name = FilePath(
join(get_copies_dir(self._root), fs_event.new_hash)).longpath
file_recent_copy_name = FilePath(fs_event.file_recent_copy).longpath
self._copies_storage.add_copy_reference(
fs_event.new_hash,
reason="MoveFileRecentCopyAction {}".format(fs_event.src))
if exists(file_recent_copy_name):
if not exists(file_synced_copy_name):
try:
shutil.move(file_recent_copy_name, file_synced_copy_name)
self.copy_added.emit(fs_event.new_hash)
except (OSError, IOError):
self._copies_storage.remove_copy_reference(
fs_event.new_hash,
reason="MoveFileRecentCopyAction {}".format(
fs_event.src))
self.event_returned(fs_event)
return
if stat(file_synced_copy_name).st_size != fs_event.file_size:
self.event_returned(fs_event)
return
fs_event.file_synced_copy = FilePath(file_synced_copy_name)
self.event_passed(fs_event)
else:
self.event_returned(fs_event)
def get_url():
filename = config.get_main_option("filename")
if filename is None:
filename = ensure_unicode(
join(get_copies_dir(get_data_dir(), create=True), 'copies.db'))
url = config.get_main_option("sqlalchemy.url")
url = ensure_unicode(url)
url = url.format(filename=FilePath(filename))
return url
def _log_db(self, session):
if not self._extended_logging:
return
copies_dir = get_copies_dir(self._root)
copies = session.query(Copy).all()
for i, copy in enumerate(copies):
self._logger.debug("Copy %s: %s. File exists: %s", i, copy,
exists(join(copies_dir, copy.hash)))
def _begin_file_download(self,
download_manager,
fs,
signals,
on_load_task_success,
on_load_task_failure,
session=None):
from .update_file_strategy import RemoteUpdateFileStrategy
from .delete_file_strategy import RemoteDeleteFileStrategy
assert self.event.file_hash or self.event.file_hash_before_event
size = self.event.file_size
hash = self.event.file_hash if self.event.file_hash \
else self.event.file_hash_before_event
path = join(get_copies_dir(fs.get_root()), hash)
if exists(path):
self.download_success = True
return self.download_success
else:
self.download_success = False
logger.debug(
'Starting downloading file: %s into: %s, size:%s (event uuid: %s)',
self.event.file.name, path, size, self.event.uuid)
signals.downloading_started.emit(self)
is_silent = self.event.type == 'delete' or \
self.file_will_be_deleted()
priority = DOWNLOAD_PRIORITY_REVERSED_PATCH if is_silent \
else DOWNLOAD_PRIORITY_FILE
display_name = 'Syncing backup for file {}' if is_silent \
else 'Syncing file {}'
display_name = display_name.format(self.event.file_name)
target_file_path = self.db.get_path_from_event(self.event, session)
files_info = [{
"target_file_path":
target_file_path,
"mtime":
get_local_time_from_timestamp(self.event.timestamp),
"is_created":
not isinstance(self, RemoteUpdateFileStrategy),
"is_deleted":
isinstance(self, RemoteDeleteFileStrategy)
}]
download_manager.add_file_download(priority, self.event.uuid,
self.event.file_size, hash, path,
display_name, on_load_task_success,
on_load_task_failure, files_info)
return False
def __init__(self,
root,
db_file_created_cb=None,
extended_logging=True,
to_upgrade=True):
self.possibly_sync_folder_is_removed = Signal()
self.delete_copy = Signal(
str, # copy hash
bool) # with signature
self.db_or_disk_full = Signal()
self._db_file = join(get_copies_dir(root), 'copies.db')
new_db_file = not exists(self._db_file)
if new_db_file and callable(db_file_created_cb):
db_file_created_cb()
if to_upgrade and not new_db_file:
# Database migration. It can be executed before opening db
try:
upgrade_db("copies_db", db_filename=self._db_file)
except Exception as e:
remove_file(self._db_file)
new_db_file = True
logger.warning(
"Can't upgrade copies db. "
"Reason: (%s) Creating...", e)
if callable(db_file_created_cb):
db_file_created_cb()
self._engine = create_engine('sqlite:///{}'.format(
FilePath(self._db_file)))
self._Session = sessionmaker(bind=self._engine)
Base.metadata.create_all(self._engine, checkfirst=True)
if new_db_file:
try:
stamp_db("copies_db", db_filename=self._db_file)
except Exception as e:
logger.error("Error stamping copies db: %s", e)
self._lock = RLock()
self._root = root
self._extended_logging = extended_logging
if not self._extended_logging:
self._logger = None
else:
self._logger = logging.getLogger('copies_logger')
self._logger.debug("Copies init")
self._last_changes = defaultdict(int)
def create_file_from_copy(self,
file_rel_path,
copy_hash,
silent,
events_file_id,
search_by_id=False,
wrong_file_id=None,
copy_does_not_exists=None):
dst_full_path = self._path_converter.create_abspath(file_rel_path)
copy_full_path = join(get_copies_dir(self._root), copy_hash)
if copy_does_not_exists is not None and not exists(copy_full_path):
if not self.make_copy_from_existing_files(copy_hash):
raise copy_does_not_exists(copy_hash)
return self._create_file(copy_full_path, dst_full_path, silent,
copy_hash, events_file_id, search_by_id,
wrong_file_id)
def start_file_download(self, file_info):
data_path = self._cfg.sync_directory if self._cfg else get_data_dir()
download_path = op.join(get_copies_dir(data_path), file_info.file_hash)
logger.info("Initiating downloading of file '%s' to '%s'...",
file_info.fullname, download_path)
def on_success(task):
self.signals.download_success.emit(task.id, file_info,
download_path)
def on_failure(task):
self.signals.download_failure.emit(task.id, file_info)
if not file_info.size:
create_empty_file(file_info.fullname)
self._tasks[self._current_share_hash].remove(file_info.event_uuid)
self._finish_task_download()
return
elif not self._cfg.download_backups:
self._sync.make_copy_from_existing_files(file_info.file_hash)
files_info = [{
"target_file_path":
self._get_path_relative_to_share(file_info),
"mtime":
0, # mtime == 0 => shared file
"is_created":
None,
"is_deleted":
None
}]
self._download_manager.add_file_download(
DOWNLOAD_PRIORITY_FILE,
file_info.event_uuid,
file_info.size,
file_info.file_hash,
download_path,
'Downloading shared file {}'.format(file_info.name),
on_success,
on_failure,
files_info=files_info,
)
def remove_copies_not_in_db(self):
with self.create_session() as session:
copies = session.query(Copy).all()
exclude_files = {copy.hash for copy in copies}
exclude_files.add('copies.db')
copies_dir = get_copies_dir(self._root)
try:
files_to_delete = set(listdir(copies_dir)) - exclude_files
files_to_delete = map(lambda f: join(copies_dir, f),
files_to_delete)
list(
map(
remove_file,
filter(
lambda f: isfile(f) and not f.endswith('.download') and
not f.endswith('.info'), files_to_delete)))
except Exception as e:
self.possibly_sync_folder_is_removed()
logger.warning("Can't remove copies files. Reason: %s", e)
def on_delete_copy(self, hash, with_signature=True):
if not hash:
logger.error("Invalid hash '%s'", hash)
return
copy = op.join(get_copies_dir(self._root), hash)
try:
remove_file(copy)
logger.info("File copy deleted %s", copy)
if not with_signature:
return
signature = op.join(get_signatures_dir(self._root), hash)
remove_file(signature)
logger.info("File copy signature deleted %s", signature)
except Exception as e:
logger.error(
"Can't delete copy. "
"Possibly sync folder is removed %s", e)
self.possibly_sync_folder_is_removed()
def make_copy_from_existing_files(self, copy_hash):
copy_full_path = join(get_copies_dir(self._root), copy_hash)
if exists(copy_full_path):
return True
tmp_full_path = self._get_temp_path(copy_full_path)
with self._storage.create_session(read_only=True,
locked=False) as session:
excludes = []
while True:
file = self._storage.get_file_by_hash(copy_hash,
exclude=excludes,
session=session)
if not file:
return False
file_path = self._path_converter.create_abspath(
file.relative_path)
if not exists(file_path):
excludes.append(file.id)
continue
try:
copy_file(file_path, tmp_full_path)
hash = Rsync.hash_from_block_checksum(
Rsync.block_checksum(tmp_full_path))
if hash == copy_hash:
os.rename(tmp_full_path, copy_full_path)
return True
else:
excludes.append(file.id)
remove_file(tmp_full_path)
except Exception as e:
logger.warning("Can't operate tmp file %s. Reason: (%s)",
tmp_full_path, e)
if file.id not in excludes:
excludes.append(file.id)
try:
remove_file(tmp_full_path)
except Exception:
tmp_full_path = self._get_temp_path(copy_full_path)
def get_copy_file_path(self, hash):
return join(get_copies_dir(self._root), hash)
def _clean_recent_copies(self):
mask = op.join(get_copies_dir(self._root), "*.recent_copy_[0-9]*")
recent_copies = glob.glob(mask)
list(map(os.remove, recent_copies))
def _accept_patch(patch_info, patch_data, unpatched_file, root):
file_blocks_hashes = SortedDict()
blocksize = patch_info['blocksize']
temp_name = os.path.join(get_patches_dir(root),
'.patching_' + generate_uuid())
blocks = SortedDict(
(int(k), v) for k, v in patch_info['blocks'].items())
source_file = None
if op.exists(unpatched_file):
source_file = open(unpatched_file, "rb")
with open(temp_name, "wb") as temp_file:
# count = 0
# min = 999999999.0
# max = 0.0
# avg = 0.0
# sum = 0.0
for offset, block in blocks.items():
# count += 1
# start_time = time.time()
block_offset = int(block['offset'])
if block['new']:
patch_data.seek(block_offset)
data_size = block['data_size']
data = patch_data.read(data_size)
else:
if block['from_patch']:
patch_offset = blocks[block_offset]['offset']
data_size = blocks[block_offset].get(
'data_size', blocksize)
patch_data.seek(patch_offset)
data = patch_data.read(data_size)
else:
if source_file is None:
raise IOError("Source file not found")
source_file.seek(block_offset)
data = source_file.read(blocksize)
temp_file.seek(offset)
temp_file.write(data)
file_blocks_hashes[offset] = block['hash']
# diff = time.time() - start_time
# min = diff if diff < min else min
# max = diff if diff > max else max
# avg = diff if avg == 0 else (avg + diff) / 2
# sum += diff
# logger.debug(
# 'processed block %s:%s in %s', count, len(blocks), diff)
# logger.debug(
# 'processing blocks time:%s, min:%s, max:%s, avg:%s',
# sum, min, max, avg)
if source_file:
source_file.close()
logger.debug('calculating patched file signature')
file_signature = Rsync.block_checksum(temp_name, blocksize=blocksize)
logger.debug('calculated patched file signature')
if file_signature != file_blocks_hashes:
remove_file(temp_name)
raise IOError(
"Invalid patch result, expected signature: {}, actual: {}".
format(file_blocks_hashes, file_signature))
new_hash = patch_info['new_hash']
logger.debug('moving patched file')
copy = join(get_copies_dir(root), new_hash)
if not exists(copy):
copy_file(temp_name, copy)
shutil.move(temp_name, unpatched_file)
logger.debug('moved patched file')
return new_hash, file_blocks_hashes, patch_info['old_hash']
