def handle(self):
logging.info('Downloading file "%s" to "%s".', self.remote_item.id,
self.local_abspath)
try:
tmp_name = self.repo.path_filter.get_temp_name(
self.remote_item.name)
tmp_path = self.repo.local_root + self.parent_relpath + '/' + tmp_name
item_request = self.repo.authenticator.client.item(
drive=self.repo.drive.id, id=self.remote_item.id)
item_mtime, item_mtime_editable = get_item_modified_datetime(
self.remote_item)
item_request_call(self.repo, item_request.download, tmp_path)
hashes = self.remote_item.file.hashes
if hashes is None or hashes.sha1_hash is None or hashes.sha1_hash == sha1_value(
tmp_path):
item_size_local = os.path.getsize(tmp_path)
os.rename(tmp_path, self.local_abspath)
fix_owner_and_timestamp(self.local_abspath,
self.repo.context.user_uid,
datetime_to_timestamp(item_mtime))
self.repo.update_item(self.remote_item, self.parent_relpath,
item_size_local)
logging.info('Finished downloading item "%s".',
self.remote_item.id)
return True
else:
# We assumed server's SHA-1 value is always correct -- might not be true.
logging.error('Hash mismatch for downloaded file "%s".',
self.local_abspath)
os.remove(tmp_path)
except (onedrivesdk.error.OneDriveError, OSError) as e:
logging.error('Error when downloading file "%s": %s.',
self.remote_item.id, e)
return False
def test_datetime_to_timestamp_implicit_utc(self):
"""
onedrivesdk-python returns datetime objects that do not have tzinfo but onedrive_client wants it to be explicit.
This test case check if the machine and program can handle implicit UTC.
"""
self.assertEqual(self.DT_UTC_OBJ.float_timestamp,
od_dateutils.datetime_to_timestamp(self.DT_OBJ))
def _handle_local_file(self, item_name, item_record, item_stat, item_local_abspath):
"""
:param str item_name:
:param onedrive_client.od_repo.ItemRecord | None item_record:
:param posix.stat_result | None item_stat:
:param str item_local_abspath:
"""
if self.repo.path_filter.should_ignore(self.rel_path + '/' + item_name, False):
logging.debug('Ignored local file "%s/%s".', self.rel_path, item_name)
return
if item_stat is None:
logging.info('Local-only file "%s" existed when scanning but is now gone. Skip it.', item_local_abspath)
if item_record is not None:
self.repo.delete_item(item_record.item_name, item_record.parent_path, False)
if self.assume_remote_unchanged:
self.task_pool.add_task(delete_item.DeleteRemoteItemTask(
repo=self.repo, task_pool=self.task_pool, parent_relpath=self.rel_path,
item_name=item_name, item_id=item_record.item_id, is_folder=False))
return
if item_record is not None and item_record.type == ItemRecordType.FILE:
record_ts = datetime_to_timestamp(item_record.modified_time)
equal_ts = diff_timestamps(item_stat.st_mtime, record_ts) == 0
if item_stat.st_size == item_record.size_local and \
(equal_ts or item_record.sha1_hash and item_record.sha1_hash == sha1_value(item_local_abspath)):
# Local file matches record.
if self.assume_remote_unchanged:
if not equal_ts:
fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, record_ts)
else:
logging.debug('Local file "%s" used to exist remotely but not found. Delete it.',
item_local_abspath)
send2trash(item_local_abspath)
self.repo.delete_item(item_record.item_name, item_record.parent_path, False)
return
logging.debug('Local file "%s" is different from when it was last synced. Upload it.', item_local_abspath)
elif item_record is not None:
# Record is a dir but local entry is a file.
if self.assume_remote_unchanged:
logging.info('Remote item for local file "%s" is a directory that has been deleted locally. '
'Delete the remote item and upload the file.', item_local_abspath)
if not delete_item.DeleteRemoteItemTask(
repo=self.repo, task_pool=self.task_pool, parent_relpath=self.rel_path,
item_name=item_name, item_id=item_record.item_id, is_folder=True).handle():
logging.error('Failed to delete outdated remote directory "%s/%s" of Drive %s.',
self.rel_path, item_name, self.repo.drive.id)
# Keep the record so that the branch can be revisited next time.
return
logging.debug('Local file "%s" is new to OneDrive. Upload it.', item_local_abspath)
self.task_pool.add_task(upload_file.UploadFileTask(
self.repo, self.task_pool, self.item_request, self.rel_path, item_name))
def update_timestamp_and_record(self, new_item, item_local_stat):
remote_mtime, remote_mtime_w = get_item_modified_datetime(new_item)
if not remote_mtime_w:
# last_modified_datetime attribute is not modifiable in OneDrive server. Update local mtime.
fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid,
datetime_to_timestamp(remote_mtime))
else:
file_system_info = FileSystemInfo()
file_system_info.last_modified_date_time = datetime.utcfromtimestamp(item_local_stat.st_mtime)
updated_item = Item()
updated_item.file_system_info = file_system_info
item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, id=new_item.id)
new_item = item_request_call(self.repo, item_request.update, updated_item)
self.repo.update_item(new_item, self.parent_relpath, item_local_stat.st_size)
def _handle_remote_file_without_record(self, remote_item, item_stat, item_local_abspath, all_local_items):
"""
Handle the case in which a remote item is not found in the database. The local item may or may not exist.
:param onedrivesdk.model.item.Item remote_item:
:param posix.stat_result | None item_stat:
:param str item_local_abspath:
:param [str] all_local_items:
"""
if item_stat is None:
# The file does not exist locally, and there is no record in database. The safest approach is probably
# download the file and update record.
self.task_pool.add_task(
download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path))
elif os.path.isdir(item_local_abspath):
# Remote path is file yet local path is a dir.
logging.info('Path "%s" is a folder yet the remote item is a file. Keep both.', item_local_abspath)
self._rename_local_and_download_remote(remote_item, all_local_items)
else:
# We first compare timestamp and size -- if both properties match then we think the items are identical
# and just update the database record. Otherwise if sizes are equal, we calculate hash of local item to
# determine if they are the same. If so we update timestamp of local item and update database record.
# If the remote item has different hash, then we rename the local one and download the remote one so that no
# information is lost.
remote_mtime, remote_mtime_w = get_item_modified_datetime(remote_item)
remote_mtime_ts = datetime_to_timestamp(remote_mtime)
equal_ts = diff_timestamps(remote_mtime_ts, item_stat.st_mtime) == 0
equal_attr = remote_item.size == item_stat.st_size and equal_ts
# Because of the size mismatch issue, we can't use size not being equal as a shortcut for hash not being
# equal. When the bug is fixed we can do it.
if equal_attr or hash_match(item_local_abspath, remote_item):
if not equal_ts:
logging.info('Local file "%s" has same content but wrong timestamp. '
'Remote: mtime=%s, w=%s, ts=%s, size=%d. '
'Local: ts=%s, size=%d. Fix it.',
item_local_abspath,
remote_mtime, remote_mtime_w, remote_mtime_ts, remote_item.size,
item_stat.st_mtime, item_stat.st_size)
fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts)
self.repo.update_item(remote_item, self.rel_path, item_stat.st_size)
else:
self._rename_local_and_download_remote(remote_item, all_local_items)
def test_datetime_to_timestamp_non_utc(self):
ts = od_dateutils.datetime_to_timestamp(self.DT_NONUTC_OBJ)
self.assertEqual(
self.DT_UTC_OBJ.float_timestamp - self.DT_OFFSET.total_seconds(),
ts)
self.assertEqual(self.DT_NONUTC_OBJ.float_timestamp, ts)
def test_datetime_to_timestamp_explicit_utc(self):
ts = od_dateutils.datetime_to_timestamp(self.DT_UTC_OBJ)
self.assertEqual(self.DT_UTC_OBJ.float_timestamp, ts)
self.assertEqual(self.DT_TS, ts)
def _handle_remote_file_with_record(self, remote_item, item_record, item_stat, item_local_abspath, all_local_items):
"""
:param onedrivesdk.model.item.Item remote_item:
:param onedrive_client.od_repo.ItemRecord item_record:
:param posix.stat_result | None item_stat:
:param str item_local_abspath:
:param [str] all_local_items:
"""
# In this case we have all three pieces of information -- remote item metadata, database record, and local inode
# stats. The best case is that all of them agree, and the worst case is that they all disagree.
if os.path.isdir(item_local_abspath):
# Remote item is a file yet the local item is a folder.
if item_record and item_record.type == ItemRecordType.FOLDER:
# TODO: Use the logic in handle_local_folder to solve this.
send2trash(item_local_abspath)
self.repo.delete_item(remote_item.name, self.rel_path, True)
else:
# When db record does not exist or says the path is a file, then it does not agree with local inode
# and the information is useless. We delete it and sync both remote and local items.
if item_record:
self.repo.delete_item(remote_item.name, self.rel_path, False)
return self._handle_remote_file_without_record(remote_item, None, item_local_abspath, all_local_items)
remote_mtime, _ = get_item_modified_datetime(remote_item)
local_mtime_ts = item_stat.st_mtime if item_stat else None
remote_mtime_ts = datetime_to_timestamp(remote_mtime)
record_mtime_ts = datetime_to_timestamp(item_record.modified_time)
try:
remote_sha1_hash = remote_item.file.hashes.sha1_hash
except AttributeError:
remote_sha1_hash = None
if (remote_item.id == item_record.item_id and remote_item.c_tag == item_record.c_tag or
remote_item.size == item_record.size and
diff_timestamps(remote_mtime_ts, record_mtime_ts) == 0):
# The remote item metadata matches the database record. So this item has been synced before.
if item_stat is None:
# The local file was synced but now is gone. Delete remote one as well.
logging.debug('Local file "%s" is gone but remote item matches db record. Delete remote item.',
item_local_abspath)
self.task_pool.add_task(delete_item.DeleteRemoteItemTask(
self.repo, self.task_pool, self.rel_path, remote_item.name, remote_item.id, False))
elif (item_stat.st_size == item_record.size_local and
(diff_timestamps(local_mtime_ts, record_mtime_ts) == 0 or
remote_sha1_hash and remote_sha1_hash == sha1_value(item_local_abspath))):
# If the local file matches the database record (i.e., same mtime timestamp or same content),
# simply return. This is the best case.
if diff_timestamps(local_mtime_ts, remote_mtime_ts) != 0:
logging.info('File "%s" seems to have same content but different timestamp (%f, %f). Fix it.',
item_local_abspath, local_mtime_ts, remote_mtime_ts)
fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts)
self.repo.update_item(remote_item, self.rel_path, item_stat.st_size)
else:
# Content of local file has changed. Because we assume the remote item was synced before, we overwrite
# the remote item with local one.
# API Issue: size field may not match file size.
# Refer to https://github.com/OneDrive/onedrive-sdk-python/issues/88
# Workaround -- storing both remote and local sizes.
logging.debug('File "%s" was changed locally and the remote version is known old. Upload it.',
item_local_abspath)
self.task_pool.add_task(upload_file.UploadFileTask(
self.repo, self.task_pool, self.item_request, self.rel_path, remote_item.name))
else:
# The remote file metadata and database record disagree.
if item_stat is None:
# If the remote file is the one on record, then the remote one is newer than the deleted local file
# so it should be downloaded. If they are not the same, then the remote one should definitely
# be kept. So the remote file needs to be kept and downloaded anyway.
logging.debug('Local file "%s" is gone but remote item disagrees with db record. Download it.',
item_local_abspath)
self.task_pool.add_task(
download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path))
elif item_stat.st_size == item_record.size_local and \
(diff_timestamps(local_mtime_ts, record_mtime_ts) == 0 or
item_record.sha1_hash and item_record.sha1_hash == sha1_value(item_local_abspath)):
# Local file agrees with database record. This means that the remote file is strictly newer.
# The local file can be safely overwritten.
logging.debug('Local file "%s" agrees with db record but remote item is different. Overwrite local.',
item_local_abspath)
self.task_pool.add_task(
download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path))
else:
# So both the local file and remote file have been changed after the record was created.
equal_ts = diff_timestamps(local_mtime_ts, remote_mtime_ts) == 0
if (item_stat.st_size == remote_item.size and (
(equal_ts or remote_sha1_hash and remote_sha1_hash == sha1_value(item_local_abspath)))):
# Fortunately the two files seem to be the same.
# Here the logic is written as if there is no size mismatch issue.
logging.debug(
'Local file "%s" seems to have same content with remote but record disagrees. Fix db record.',
item_local_abspath)
if not equal_ts:
fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts)
self.repo.update_item(remote_item, self.rel_path, item_stat.st_size)
else:
# Worst case we keep both files.
logging.debug('Local file "%s" differs from db record and remote item. Keep both versions.',
item_local_abspath)
self._rename_local_and_download_remote(remote_item, all_local_items)