def _find_remote_child_match_or_create(self, parent_pair, child_info):
local_path = path_join(parent_pair.local_path, safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
# Try to get the local definition if not linked
child_pair = self._dao.get_state_from_local(local_path)
# Case of duplication (the file can exists in with a __x) or local rename
if child_pair is None and parent_pair is not None and self._local_client.exists(parent_pair.local_path):
for child in self._local_client.get_children_info(parent_pair.local_path):
if self._local_client.get_remote_id(child.path) == child_info.uid:
if '__' in child.name:
log.debug('Found a deduplication case: %r on %r', child_info, child.path)
else:
log.debug('Found a local rename case: %r on %r', child_info, child.path)
child_pair = self._dao.get_state_from_local(child.path)
break
if child_pair is not None:
if child_pair.remote_ref is not None and child_pair.remote_ref != child_info.uid:
log.debug("Got an existing pair with different id: %r | %r", child_pair, child_info)
else:
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest,
child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)):
# Local rename
if child_pair.local_path != local_path:
child_pair.local_state = 'moved'
child_pair.remote_state = 'unknown'
local_info = self._local_client.get_info(child_pair.local_path)
self._dao.update_local_state(child_pair, local_info)
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
else:
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
# Use version+1 as we just update the remote info
synced = self._dao.synchronize_state(child_pair, version=child_pair.version + 1)
if not synced:
# Try again, might happen that it has been modified locally and remotely
child_pair = self._dao.get_state_from_id(child_pair.id)
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(
child_pair.local_digest, child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm)):
self._dao.synchronize_state(child_pair)
child_pair = self._dao.get_state_from_id(child_pair.id)
synced = child_pair.pair_state == 'synchronized'
# Can be updated in previous call
if synced:
self._engine.stop_processor_on(child_pair.local_path)
# Push the remote_Id
log.debug('Set remote ID on %r / %r == %r', child_pair, child_pair.local_path, child_pair.local_path)
self._local_client.set_remote_id(child_pair.local_path, child_info.uid)
if child_pair.folderish:
self._dao.queue_children(child_pair)
else:
child_pair.remote_state = 'modified'
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True)
return child_pair, False
row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path)
child_pair = self._dao.get_state_from_id(row_id, from_write=True)
return child_pair, True
def _find_remote_child_match_or_create(self, parent_pair, child_info):
local_path = path_join(parent_pair.local_path, safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
# Try to get the local definition if not linked
child_pair = self._dao.get_state_from_local(local_path)
if child_pair is not None:
# Should compare to xattr remote uid
if child_pair.remote_ref is not None:
child_pair = None
else:
self._dao.update_remote_state(child_pair, child_info, remote_parent_path)
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm)):
# Use version+1 as we just update the remote info
if not self._dao.synchronize_state(child_pair, version=child_pair.version + 1):
# Try again, might happens that it has been modified locally and remotelly
child_pair = self._dao.get_state_from_id(child_pair.id)
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(
child_pair.local_digest, child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm)):
self._dao.synchronize_state(child_pair)
# Push the remote_Id
self._local_client.set_remote_id(local_path, child_info.uid)
if child_pair.folderish:
self._dao.queue_children(child_pair)
child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True)
return child_pair, False
row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path)
child_pair = self._dao.get_state_from_id(row_id, from_write=True)
return child_pair, True
def _find_remote_child_match_or_create(self, parent_pair, child_info):
local_path = path_join(parent_pair.local_path, safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
# Try to get the local definition if not linked
child_pair = self._dao.get_state_from_local(local_path)
# Case of duplication: the file can exists in with a __x
if child_pair is None and parent_pair is not None and self._local_client.exists(parent_pair.local_path):
for child in self._local_client.get_children_info(parent_pair.local_path):
# Skip any file without __ as it cannot be a deduplicate
if '__' not in child.name:
continue
if self._local_client.get_remote_id(child.path) == child_info.uid:
log.debug("Found a deduplication case: %r on %s", child_info, child.path)
child_pair = self._dao.get_state_from_local(child.path)
break
if child_pair is not None:
if child_pair.remote_ref is not None and child_pair.remote_ref != child_info.uid:
log.debug("Got an existing pair with different id: %r | %r", child_pair, child_info)
child_pair = None
else:
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest,
child_pair.local_path, remote_digest_algorithm=child_info.digest_algorithm)):
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
# Use version+1 as we just update the remote info
synced = self._dao.synchronize_state(child_pair, version=child_pair.version + 1)
if not synced:
# Try again, might happen that it has been modified locally and remotely
child_pair = self._dao.get_state_from_id(child_pair.id)
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(
child_pair.local_digest, child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm)):
self._dao.synchronize_state(child_pair)
child_pair = self._dao.get_state_from_id(child_pair.id)
synced = child_pair.pair_state == 'synchronized'
# Can be updated in previous call
if synced:
self._engine.stop_processor_on(child_pair.local_path)
# Push the remote_Id
log.debug("set remote id on: %r / %s == %s", child_pair, child_pair.local_path, local_path)
self._local_client.set_remote_id(local_path, child_info.uid)
if child_pair.folderish:
self._dao.queue_children(child_pair)
else:
child_pair.remote_state = 'modified'
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True)
return child_pair, False
row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path)
child_pair = self._dao.get_state_from_id(row_id, from_write=True)
return child_pair, True
def _find_remote_child_match_or_create(self, parent_pair, child_info):
local_path = path_join(parent_pair.local_path, safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
# Try to get the local definition if not linked
child_pair = self._dao.get_state_from_local(local_path)
# Case of duplication: the file can exists in with a __x
if child_pair is None and parent_pair is not None and self._local_client.exists(parent_pair.local_path):
for child in self._local_client.get_children_info(parent_pair.local_path):
# Skip any file without __ as it cannot be a deduplicate
if '__' not in child.name:
continue
if self._local_client.get_remote_id(child.path) == child_info.uid:
log.debug("Found a deduplication case: %r on %s", child_info, child.path)
child_pair = self._dao.get_state_from_local(child.path)
break
if child_pair is not None:
if child_pair.remote_ref is not None:
child_pair = None
else:
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(child_pair.local_digest, child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm)):
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
# Use version+1 as we just update the remote info
synced = self._dao.synchronize_state(child_pair, version=child_pair.version + 1)
if not synced:
# Try again, might happen that it has been modified locally and remotely
child_pair = self._dao.get_state_from_id(child_pair.id)
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(
child_pair.local_digest, child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm)):
self._dao.synchronize_state(child_pair)
child_pair = self._dao.get_state_from_id(child_pair.id)
synced = child_pair.pair_state == 'synchronized'
# Can be updated in previous call
if synced:
self._engine.stop_processor_on(child_pair.local_path)
# Push the remote_Id
self._local_client.set_remote_id(local_path, child_info.uid)
if child_pair.folderish:
self._dao.queue_children(child_pair)
else:
child_pair.remote_state = 'modified'
self._dao.update_remote_state(child_pair, child_info, remote_parent_path=remote_parent_path)
child_pair = self._dao.get_state_from_id(child_pair.id, from_write=True)
return child_pair, False
row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path)
child_pair = self._dao.get_state_from_id(row_id, from_write=True)
return child_pair, True
def _scan_pair(self, remote_path):
if remote_path is None:
return
remote_path = str(remote_path)
if self._dao.is_filter(remote_path):
# Skip if filter
return
if remote_path[-1:] == '/':
remote_path = remote_path[0:-1]
remote_ref = os.path.basename(remote_path)
parent_path = os.path.dirname(remote_path)
if parent_path == '/':
parent_path = ''
# If pair is present already
try:
child_info = self._client.get_info(remote_ref)
except NotFound:
# The folder has been deleted
return
doc_pair = self._dao.get_state_from_remote_with_path(
remote_ref, parent_path)
if doc_pair is not None:
self._do_scan_remote(doc_pair, child_info)
return
log.debug('parent_path: %r\t%r\t%r', parent_path,
os.path.basename(parent_path), os.path.dirname(parent_path))
parent_pair = self._dao.get_state_from_remote_with_path(
os.path.basename(parent_path), os.path.dirname(parent_path))
log.debug("scan_pair: parent_pair: %r", parent_pair)
if parent_pair is None:
return
local_path = path_join(parent_pair.local_path,
safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
if os.path.dirname(child_info.path) == remote_parent_path:
row_id = self._dao.insert_remote_state(child_info,
remote_parent_path,
local_path,
parent_pair.local_path)
doc_pair = self._dao.get_state_from_id(row_id, from_write=True)
if child_info.folderish:
self._do_scan_remote(doc_pair, child_info)
else:
log.debug('Remote scan_pair: %r is not available', remote_path)
self._scan_remote()
def _scan_pair(self, remote_path):
if remote_path is None:
return
remote_path = str(remote_path)
if self._dao.is_filter(remote_path):
# Skip if filter
return
if remote_path[-1:] == '/':
remote_path = remote_path[0:-1]
remote_ref = os.path.basename(remote_path)
parent_path = os.path.dirname(remote_path)
if parent_path == '/':
parent_path = ''
# If pair is present already
try:
child_info = self._client.get_info(remote_ref)
except NotFound:
# The folder has been deleted
return
doc_pair = self._dao.get_state_from_remote_with_path(remote_ref, parent_path)
if doc_pair is not None:
log.debug("Remote scan_pair: %s", doc_pair.local_path)
self._do_scan_remote(doc_pair, child_info)
log.debug("Remote scan_pair ended: %s", doc_pair.local_path)
return
log.debug("parent_path: '%s'\t'%s'\t'%s'", parent_path, os.path.basename(parent_path),
os.path.dirname(parent_path))
parent_pair = self._dao.get_state_from_remote_with_path(os.path.basename(parent_path),
os.path.dirname(parent_path))
log.debug("scan_pair: parent_pair: %r", parent_pair)
if parent_pair is None:
return
local_path = path_join(parent_pair.local_path, safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
if os.path.dirname(child_info.path) == remote_parent_path:
row_id = self._dao.insert_remote_state(child_info, remote_parent_path, local_path, parent_pair.local_path)
doc_pair = self._dao.get_state_from_id(row_id, from_write=True)
if child_info.folderish:
log.debug("Remote scan_pair: %s", doc_pair.local_path)
self._do_scan_remote(doc_pair, child_info)
log.debug("Remote scan_pair ended: %s", doc_pair.local_path)
else:
log.debug("Remote scan_pair: %s is not available, Do full scan", remote_path)
self._scan_remote()
def _find_remote_child_match_or_create(self, parent_pair, child_info):
if not parent_pair.local_path:
# The parent folder has an empty local_path,
# it probably means that it has been put in error as a duplicate
# by a processor => ignoring this child.
log.debug('Ignoring child %r of a duplicate folder in error %r',
child_info, parent_pair)
return None, None
local_path = path_join(parent_pair.local_path,
safe_filename(child_info.name))
remote_parent_path = parent_pair.remote_parent_path + '/' + parent_pair.remote_ref
# Try to get the local definition if not linked
child_pair = self._dao.get_state_from_local(local_path)
# Case of duplication (the file can exists in with a __x) or local rename
if child_pair is None and parent_pair is not None and self._local_client.exists(
parent_pair.local_path):
for child in self._local_client.get_children_info(
parent_pair.local_path):
if self._local_client.get_remote_id(
child.path) == child_info.uid:
if '__' in child.name:
log.debug('Found a deduplication case: %r on %r',
child_info, child.path)
else:
log.debug('Found a local rename case: %r on %r',
child_info, child.path)
child_pair = self._dao.get_state_from_local(child.path)
break
if child_pair is not None:
if child_pair.remote_ref is not None and child_pair.remote_ref != child_info.uid:
log.debug("Got an existing pair with different id: %r | %r",
child_pair, child_info)
else:
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(
child_pair.local_digest,
child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.digest_algorithm
)):
# Local rename
if child_pair.local_path != local_path:
child_pair.local_state = 'moved'
child_pair.remote_state = 'unknown'
local_info = self._local_client.get_info(
child_pair.local_path)
self._dao.update_local_state(child_pair, local_info)
self._dao.update_remote_state(
child_pair,
child_info,
remote_parent_path=remote_parent_path)
else:
self._dao.update_remote_state(
child_pair,
child_info,
remote_parent_path=remote_parent_path)
# Use version+1 as we just update the remote info
synced = self._dao.synchronize_state(
child_pair, version=child_pair.version + 1)
if not synced:
# Try again, might happen that it has been modified locally and remotely
child_pair = self._dao.get_state_from_id(
child_pair.id)
if (child_pair.folderish == child_info.folderish
and self._local_client.is_equal_digests(
child_pair.local_digest,
child_info.digest,
child_pair.local_path,
remote_digest_algorithm=child_info.
digest_algorithm)):
self._dao.synchronize_state(child_pair)
child_pair = self._dao.get_state_from_id(
child_pair.id)
synced = child_pair.pair_state == 'synchronized'
# Can be updated in previous call
if synced:
self._engine.stop_processor_on(
child_pair.local_path)
# Push the remote_Id
log.debug('Set remote ID on %r / %r == %r', child_pair,
child_pair.local_path, child_pair.local_path)
self._local_client.set_remote_id(
child_pair.local_path, child_info.uid)
if child_pair.folderish:
self._dao.queue_children(child_pair)
else:
child_pair.remote_state = 'modified'
self._dao.update_remote_state(
child_pair,
child_info,
remote_parent_path=remote_parent_path)
child_pair = self._dao.get_state_from_id(child_pair.id,
from_write=True)
return child_pair, False
row_id = self._dao.insert_remote_state(child_info, remote_parent_path,
local_path,
parent_pair.local_path)
child_pair = self._dao.get_state_from_id(row_id, from_write=True)
return child_pair, True
