def handle(self): logging.info('Downloading file "%s" to "%s".', self.remote_item.id, self.local_abspath) try: tmp_name = self.repo.path_filter.get_temp_name( self.remote_item.name) tmp_path = self.repo.local_root + self.parent_relpath + '/' + tmp_name item_request = self.repo.authenticator.client.item( drive=self.repo.drive.id, id=self.remote_item.id) item_mtime, item_mtime_editable = get_item_modified_datetime( self.remote_item) item_request_call(self.repo, item_request.download, tmp_path) hashes = self.remote_item.file.hashes if hashes is None or hashes.sha1_hash is None or hashes.sha1_hash == sha1_value( tmp_path): item_size_local = os.path.getsize(tmp_path) os.rename(tmp_path, self.local_abspath) fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid, datetime_to_timestamp(item_mtime)) self.repo.update_item(self.remote_item, self.parent_relpath, item_size_local) logging.info('Finished downloading item "%s".', self.remote_item.id) return True else: # We assumed server's SHA-1 value is always correct -- might not be true. logging.error('Hash mismatch for downloaded file "%s".', self.local_abspath) os.remove(tmp_path) except (onedrivesdk.error.OneDriveError, OSError) as e: logging.error('Error when downloading file "%s": %s.', self.remote_item.id, e) return False
def update_item(self, item, parent_relpath, size_local=0, status=ItemRecordStatus.OK): """ :param onedrivesdk.model.item.Item item: :param str parent_relpath: :param int size_local: :param int status: """ sha1_hash = None file_facet = item.file if file_facet: item_type = ItemRecordType.FILE hash_facet = file_facet.hashes if hash_facet: sha1_hash = hash_facet.sha1_hash elif item.folder: item_type = ItemRecordType.FOLDER else: raise ValueError('Unknown type of item "%s (%s)".' % (item.name, item.id)) parent_reference = item.parent_reference modified_time, _ = get_item_modified_datetime(item) modified_time_str = datetime_to_str(modified_time) created_time_str = datetime_to_str(get_item_created_datetime(item)) with self._lock, self._conn: self._conn.execute( 'INSERT OR REPLACE INTO items (id, type, name, parent_id, parent_path, etag, ' 'ctag, size, size_local, created_time, modified_time, status, sha1_hash, record_time)' ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (item.id, item_type, item.name, parent_reference.id, parent_relpath, item.e_tag, item.c_tag, item.size, size_local, created_time_str, modified_time_str, status, sha1_hash, str(datetime.utcnow().isoformat()) + 'Z'))
def test_get_item_modified_datetime_modifiable(self): fs = FileSystemInfo() fs.last_modified_date_time = self.SAMPLE_ARROW_OBJ.datetime self.item.file_system_info = fs t, w = od_api_helper.get_item_modified_datetime(self.item) self.assertTrue( w, 'fileSystemInfo.lastModifiedDateTime should be modifiable.') self.assertEqual(self.SAMPLE_ARROW_OBJ, t)
def update_timestamp_and_record(self, new_item, item_local_stat): remote_mtime, remote_mtime_w = get_item_modified_datetime(new_item) if not remote_mtime_w: # last_modified_datetime attribute is not modifiable in OneDrive server. Update local mtime. fix_owner_and_timestamp(self.local_abspath, self.repo.context.user_uid, datetime_to_timestamp(remote_mtime)) else: file_system_info = FileSystemInfo() file_system_info.last_modified_date_time = datetime.utcfromtimestamp(item_local_stat.st_mtime) updated_item = Item() updated_item.file_system_info = file_system_info item_request = self.repo.authenticator.client.item(drive=self.repo.drive.id, id=new_item.id) new_item = item_request_call(self.repo, item_request.update, updated_item) self.repo.update_item(new_item, self.parent_relpath, item_local_stat.st_size)
def _handle_remote_file_without_record(self, remote_item, item_stat, item_local_abspath, all_local_items): """ Handle the case in which a remote item is not found in the database. The local item may or may not exist. :param onedrivesdk.model.item.Item remote_item: :param posix.stat_result | None item_stat: :param str item_local_abspath: :param [str] all_local_items: """ if item_stat is None: # The file does not exist locally, and there is no record in database. The safest approach is probably # download the file and update record. self.task_pool.add_task( download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path)) elif os.path.isdir(item_local_abspath): # Remote path is file yet local path is a dir. logging.info('Path "%s" is a folder yet the remote item is a file. Keep both.', item_local_abspath) self._rename_local_and_download_remote(remote_item, all_local_items) else: # We first compare timestamp and size -- if both properties match then we think the items are identical # and just update the database record. Otherwise if sizes are equal, we calculate hash of local item to # determine if they are the same. If so we update timestamp of local item and update database record. # If the remote item has different hash, then we rename the local one and download the remote one so that no # information is lost. remote_mtime, remote_mtime_w = get_item_modified_datetime(remote_item) remote_mtime_ts = datetime_to_timestamp(remote_mtime) equal_ts = diff_timestamps(remote_mtime_ts, item_stat.st_mtime) == 0 equal_attr = remote_item.size == item_stat.st_size and equal_ts # Because of the size mismatch issue, we can't use size not being equal as a shortcut for hash not being # equal. When the bug is fixed we can do it. if equal_attr or hash_match(item_local_abspath, remote_item): if not equal_ts: logging.info('Local file "%s" has same content but wrong timestamp. ' 'Remote: mtime=%s, w=%s, ts=%s, size=%d. ' 'Local: ts=%s, size=%d. Fix it.', item_local_abspath, remote_mtime, remote_mtime_w, remote_mtime_ts, remote_item.size, item_stat.st_mtime, item_stat.st_size) fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts) self.repo.update_item(remote_item, self.rel_path, item_stat.st_size) else: self._rename_local_and_download_remote(remote_item, all_local_items)
def _check_item_props(self, item, record, expected_type=od_repo.ItemRecordType.FILE): """ :param onedrivesdk.Item item: :param onedrive_client.od_repo.ItemRecord record: """ self.assertIsInstance(record, od_repo.ItemRecord) self.assertEqual(item.id, record.item_id) self.assertEqual(item.name, record.item_name) self.assertEqual(item.c_tag, record.c_tag) self.assertEqual(item.parent_reference.id, record.parent_id) self.assertEqual(od_api_helper.get_item_created_datetime(item), record.created_time) mtime, w = od_api_helper.get_item_modified_datetime(item) self.assertEqual(mtime, record.modified_time) self.assertEqual(expected_type, record.type) if expected_type == od_repo.ItemRecordType.FILE: self.assertEqual(item.size, record.size) self.assertEqual(item.file.hashes.sha1_hash, record.sha1_hash)
def test_get_item_modified_datetime_unmodifiable(self): self.item.last_modified_date_time = self.SAMPLE_ARROW_OBJ.datetime t, w = od_api_helper.get_item_modified_datetime(self.item) self.assertFalse(w, 'lastModifiedDateTime should be immutable.') self.assertEqual(self.SAMPLE_ARROW_OBJ, t)
def _handle_remote_file_with_record(self, remote_item, item_record, item_stat, item_local_abspath, all_local_items): """ :param onedrivesdk.model.item.Item remote_item: :param onedrive_client.od_repo.ItemRecord item_record: :param posix.stat_result | None item_stat: :param str item_local_abspath: :param [str] all_local_items: """ # In this case we have all three pieces of information -- remote item metadata, database record, and local inode # stats. The best case is that all of them agree, and the worst case is that they all disagree. if os.path.isdir(item_local_abspath): # Remote item is a file yet the local item is a folder. if item_record and item_record.type == ItemRecordType.FOLDER: # TODO: Use the logic in handle_local_folder to solve this. send2trash(item_local_abspath) self.repo.delete_item(remote_item.name, self.rel_path, True) else: # When db record does not exist or says the path is a file, then it does not agree with local inode # and the information is useless. We delete it and sync both remote and local items. if item_record: self.repo.delete_item(remote_item.name, self.rel_path, False) return self._handle_remote_file_without_record(remote_item, None, item_local_abspath, all_local_items) remote_mtime, _ = get_item_modified_datetime(remote_item) local_mtime_ts = item_stat.st_mtime if item_stat else None remote_mtime_ts = datetime_to_timestamp(remote_mtime) record_mtime_ts = datetime_to_timestamp(item_record.modified_time) try: remote_sha1_hash = remote_item.file.hashes.sha1_hash except AttributeError: remote_sha1_hash = None if (remote_item.id == item_record.item_id and remote_item.c_tag == item_record.c_tag or remote_item.size == item_record.size and diff_timestamps(remote_mtime_ts, record_mtime_ts) == 0): # The remote item metadata matches the database record. So this item has been synced before. if item_stat is None: # The local file was synced but now is gone. Delete remote one as well. logging.debug('Local file "%s" is gone but remote item matches db record. Delete remote item.', item_local_abspath) self.task_pool.add_task(delete_item.DeleteRemoteItemTask( self.repo, self.task_pool, self.rel_path, remote_item.name, remote_item.id, False)) elif (item_stat.st_size == item_record.size_local and (diff_timestamps(local_mtime_ts, record_mtime_ts) == 0 or remote_sha1_hash and remote_sha1_hash == sha1_value(item_local_abspath))): # If the local file matches the database record (i.e., same mtime timestamp or same content), # simply return. This is the best case. if diff_timestamps(local_mtime_ts, remote_mtime_ts) != 0: logging.info('File "%s" seems to have same content but different timestamp (%f, %f). Fix it.', item_local_abspath, local_mtime_ts, remote_mtime_ts) fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts) self.repo.update_item(remote_item, self.rel_path, item_stat.st_size) else: # Content of local file has changed. Because we assume the remote item was synced before, we overwrite # the remote item with local one. # API Issue: size field may not match file size. # Refer to https://github.com/OneDrive/onedrive-sdk-python/issues/88 # Workaround -- storing both remote and local sizes. logging.debug('File "%s" was changed locally and the remote version is known old. Upload it.', item_local_abspath) self.task_pool.add_task(upload_file.UploadFileTask( self.repo, self.task_pool, self.item_request, self.rel_path, remote_item.name)) else: # The remote file metadata and database record disagree. if item_stat is None: # If the remote file is the one on record, then the remote one is newer than the deleted local file # so it should be downloaded. If they are not the same, then the remote one should definitely # be kept. So the remote file needs to be kept and downloaded anyway. logging.debug('Local file "%s" is gone but remote item disagrees with db record. Download it.', item_local_abspath) self.task_pool.add_task( download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path)) elif item_stat.st_size == item_record.size_local and \ (diff_timestamps(local_mtime_ts, record_mtime_ts) == 0 or item_record.sha1_hash and item_record.sha1_hash == sha1_value(item_local_abspath)): # Local file agrees with database record. This means that the remote file is strictly newer. # The local file can be safely overwritten. logging.debug('Local file "%s" agrees with db record but remote item is different. Overwrite local.', item_local_abspath) self.task_pool.add_task( download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path)) else: # So both the local file and remote file have been changed after the record was created. equal_ts = diff_timestamps(local_mtime_ts, remote_mtime_ts) == 0 if (item_stat.st_size == remote_item.size and ( (equal_ts or remote_sha1_hash and remote_sha1_hash == sha1_value(item_local_abspath)))): # Fortunately the two files seem to be the same. # Here the logic is written as if there is no size mismatch issue. logging.debug( 'Local file "%s" seems to have same content with remote but record disagrees. Fix db record.', item_local_abspath) if not equal_ts: fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts) self.repo.update_item(remote_item, self.rel_path, item_stat.st_size) else: # Worst case we keep both files. logging.debug('Local file "%s" differs from db record and remote item. Keep both versions.', item_local_abspath) self._rename_local_and_download_remote(remote_item, all_local_items)