예제 #1
0
    def _handle_local_file(self, item_name, item_record, item_stat, item_local_abspath):
        """
        :param str item_name:
        :param onedrive_client.od_repo.ItemRecord | None item_record:
        :param posix.stat_result | None item_stat:
        :param str item_local_abspath:
        """
        if self.repo.path_filter.should_ignore(self.rel_path + '/' + item_name, False):
            logging.debug('Ignored local file "%s/%s".', self.rel_path, item_name)
            return

        if item_stat is None:
            logging.info('Local-only file "%s" existed when scanning but is now gone. Skip it.', item_local_abspath)
            if item_record is not None:
                self.repo.delete_item(item_record.item_name, item_record.parent_path, False)
                if self.assume_remote_unchanged:
                    self.task_pool.add_task(delete_item.DeleteRemoteItemTask(
                        repo=self.repo, task_pool=self.task_pool, parent_relpath=self.rel_path,
                        item_name=item_name, item_id=item_record.item_id, is_folder=False))
            return

        if item_record is not None and item_record.type == ItemRecordType.FILE:
            record_ts = datetime_to_timestamp(item_record.modified_time)
            equal_ts = diff_timestamps(item_stat.st_mtime, record_ts) == 0
            if item_stat.st_size == item_record.size_local and \
                    (equal_ts or item_record.sha1_hash and item_record.sha1_hash == sha1_value(item_local_abspath)):
                # Local file matches record.
                if self.assume_remote_unchanged:
                    if not equal_ts:
                        fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, record_ts)
                else:
                    logging.debug('Local file "%s" used to exist remotely but not found. Delete it.',
                                  item_local_abspath)
                    send2trash(item_local_abspath)
                    self.repo.delete_item(item_record.item_name, item_record.parent_path, False)
                return
            logging.debug('Local file "%s" is different from when it was last synced. Upload it.', item_local_abspath)
        elif item_record is not None:
            # Record is a dir but local entry is a file.
            if self.assume_remote_unchanged:
                logging.info('Remote item for local file "%s" is a directory that has been deleted locally. '
                             'Delete the remote item and upload the file.', item_local_abspath)
                if not delete_item.DeleteRemoteItemTask(
                        repo=self.repo, task_pool=self.task_pool, parent_relpath=self.rel_path,
                        item_name=item_name, item_id=item_record.item_id, is_folder=True).handle():
                    logging.error('Failed to delete outdated remote directory "%s/%s" of Drive %s.',
                                  self.rel_path, item_name, self.repo.drive.id)
                    # Keep the record so that the branch can be revisited next time.
                    return
            logging.debug('Local file "%s" is new to OneDrive. Upload it.', item_local_abspath)

        self.task_pool.add_task(upload_file.UploadFileTask(
            self.repo, self.task_pool, self.item_request, self.rel_path, item_name))
예제 #2
0
    def _handle_remote_folder(self, remote_item, item_local_abspath, record, all_local_items):
        if not self.deep_merge:
            return
        try:
            remote_dir_matches_record = self._remote_dir_matches_record(remote_item, record)
            if os.path.isfile(item_local_abspath):
                # Remote item is a directory but local item is a file.
                if remote_dir_matches_record:
                    # The remote item is very LIKELY to be outdated.
                    logging.warning('Local path "%s" is a file but its remote counterpart is a folder which seems to '
                                    'be synced before. Will delete the remote folder. To restore it, go to '
                                    'OneDrive.com and move the folder out of Recycle Bin.', item_local_abspath)
                    delete_item.DeleteRemoteItemTask(
                        self.repo, self.task_pool, self.rel_path, remote_item.name, remote_item.id, True).handle()
                    self.task_pool.add_task(upload_file.UploadFileTask(
                        self.repo, self.task_pool, self.item_request, self.rel_path, remote_item.name))
                    return
                # If the remote metadata doesn't agree with record, keep both by renaming the local file.
                all_local_items.add(
                    rename_with_suffix(self.local_abspath, remote_item.name, self.repo.context.host_name))

            if not os.path.exists(item_local_abspath):
                if remote_dir_matches_record:
                    logging.debug('Local dir "%s" is gone but db record matches remote metadata. Delete remote dir.',
                                  item_local_abspath)
                    self.task_pool.add_task(delete_item.DeleteRemoteItemTask(
                        self.repo, self.task_pool, self.rel_path, remote_item.name, remote_item.id, True))
                    return
                # Local directory does not exist. Create it.
                logging.debug('Create missing directory "%s".', item_local_abspath)
                mkdir(item_local_abspath, uid=self.repo.context.user_uid, exist_ok=True)

            # The database is temporarily corrupted until the whole dir is merged. But unfortunately we returned early.
            self.repo.update_item(remote_item, self.rel_path, 0)
            self.task_pool.add_task(MergeDirectoryTask(
                repo=self.repo, task_pool=self.task_pool, rel_path=self.rel_path + '/' + remote_item.name,
                item_request=self.repo.authenticator.client.item(drive=self.repo.drive.id, id=remote_item.id),
                assume_remote_unchanged=remote_dir_matches_record,
                parent_remote_unchanged=self.assume_remote_unchanged))
        except OSError as e:
            logging.error('Error occurred when merging directory "%s": %s', item_local_abspath, e)
예제 #3
0
    def ensure_remote_path_is_dir(self, repo, rel_path):
        """
        Make sure the path is a folder in remote repository. If the path does not exist, create it. If the path is a
        file, rename the file and create the dir. Return False if the remote path can't be made a dir.
        :param onedrive_client.od_repo.OneDriveLocalRepository repo:
        :param str rel_path:
        :return True | False:
        """
        if rel_path == '':
            # Drive root is guaranteed a directory.
            return True
        item_request = repo.authenticator.client.item(drive=repo.drive.id, path=rel_path)
        parent_relpath, item_name = os.path.split(rel_path)
        if parent_relpath == '/':
            parent_relpath = ''

        try:
            item = item_request_call(repo, item_request.get)

            # Return True if the remote path exists and is a directory.
            if item.folder is not None:
                return item_name == item.name

            # Remote path is not a directory. Try renaming it and if renaming fails, deleting it.
            new_name = get_filename_with_incremented_count(item_name)
            logging.info('Remote item "%s" in Drive %s is not a directory. Try renaming it to "%s".',
                         rel_path, repo.drive.id, new_name)
            if not move_item.MoveItemTask(repo=repo, task_pool=self.task_pool,
                                          parent_relpath=parent_relpath, item_name=item_name,
                                          new_name=new_name, is_folder=False).handle():
                if not delete_item.DeleteRemoteItemTask(repo=repo, task_pool=self.task_pool,
                                                        parent_relpath=parent_relpath,
                                                        item_name=item_name, is_folder=False).handle():
                    logging.warning('Failed to rename or delete remote item "%s" in Drive %s.',
                                    rel_path, repo.drive.id)
                    return False
        except onedrivesdk.error.OneDriveError as e:
            if e.code != onedrivesdk.error.ErrorCode.ItemNotFound:
                return False

        if not merge_dir.CreateFolderTask(repo=repo, task_pool=self.task_pool,
                                          item_name=item_name, parent_relpath=parent_relpath,
                                          upload_if_success=False, abort_if_local_gone=True).handle():
            logging.critical('Failed to create remote directory "%s" on Drive %s.', rel_path, repo.drive.id)
            return False
        return True
예제 #4
0
    def _handle_unpaired_move_from(self, from_ev, from_flags, from_parent_dir=None, from_parent_relpath=None,
                                   from_repo=None, from_item_record=None):
        """
        :param inotify_simple.Event from_ev:
        :param [inotify_simple.flags] from_flags:
        :param str | None from_parent_dir:
        :param str | None from_parent_relpath:
        :param onedrive_client.od_repo.OneDriveLocalRepository | None from_repo:
        :param onedrive_client.od_repo.ItemRecord | None from_item_record:
        """

        if from_parent_dir is None or from_repo is None:
            from_repo, from_parent_dir = self.watch_descriptors[from_ev.wd]

        if from_parent_relpath is None:
            from_parent_relpath = self._local_abspath_to_relpath(from_repo, from_parent_dir)

        if from_item_record is None:
            from_item_record = from_repo.get_item_by_path(item_name=from_ev.name, parent_relpath=from_parent_relpath)

        item_relpath = from_parent_relpath + '/' + from_ev.name

        item_request, item = self._get_remote_item(from_repo, item_relpath)

        if item and from_item_record and item.id == from_item_record.item_id and item.e_tag == from_item_record.e_tag:
            logging.info('Will remove item "%s/%s" in Drive %s.', from_parent_relpath, from_ev.name, from_repo.drive.id)
            try:
                self._squash_tasks(from_repo, item_relpath)
                self.task_queue.append(delete_item.DeleteRemoteItemTask(
                    repo=from_repo, task_pool=self.task_pool, parent_relpath=from_parent_relpath,
                    item_name=from_ev.name,
                    item_id=from_item_record.item_id, is_folder=from_item_record.type == ItemRecordType.FOLDER))
            except ParentTaskExistsException as e:
                logging.info('Task on path "%s" will be covered by %s. Skip adding.', item_relpath, e.task)
        else:
            logging.info('Uncertain status of item "%s" in Drive %s for %s. Fallback to dir merge.',
                         item_relpath, from_repo.drive.id, str(from_ev))
            self._add_merge_dir_task(from_repo, from_parent_relpath)
예제 #5
0
    def _handle_local_folder(self, item_name, item_record, item_local_abspath):
        """
        :param str item_name:
        :param onedrive_client.od_repo.ItemRecord | None item_record:
        :param str item_local_abspath:
        """
        if not self.deep_merge:
            return

        if self.repo.path_filter.should_ignore(self.rel_path + '/' + item_name, True):
            logging.debug('Ignored local directory "%s/%s".', self.rel_path, item_name)
            return

        if item_record is not None and item_record.type == ItemRecordType.FOLDER:
            if self.assume_remote_unchanged:
                rel_path = self.rel_path + '/' + item_name
                self.task_pool.add_task(MergeDirectoryTask(
                    repo=self.repo, task_pool=self.task_pool, rel_path=rel_path,
                    item_request=self.repo.authenticator.client.item(drive=self.repo.drive.id, path=rel_path),
                    assume_remote_unchanged=True, parent_remote_unchanged=self.assume_remote_unchanged))
            else:
                send2trash(item_local_abspath)
                self.repo.delete_item(item_name, self.rel_path, True)
            return
            # try:
            #     # If there is any file accessed after the time when the record was created, do not delete the dir.
            #     # Instead, upload it back.
            #     # As a note, the API will return HTTP 404 Not Found after the item was deleted. So we cannot know from
            #     # API when the item was deleted. Otherwise this deletion time should be the timestamp to use.
            #     # TODO: A second best timestamp is the latest timestamp of any children item under this dir.
            #     visited_files = subprocess.check_output(
            #         ['find', item_local_abspath, '-type', 'f',
            #         '(', '-newermt', item_record.record_time_str, '-o',
            #          '-newerat', item_record.record_time_str, ')', '-print'], universal_newlines=True)
            #     if visited_files == '':
            #         logging.info('Local item "%s" was deleted remotely and not used since %s. Delete it locally.',
            #                      item_local_abspath, item_record.record_time_str)
            #         send2trash(item_local_abspath)
            #         self.repo.delete_item(item_name, self.rel_path, True)
            #         return
            #     logging.info('Local directory "%s" was deleted remotely but locally used. Upload it back.')
            # except subprocess.CalledProcessError as e:
            #     logging.error('Error enumerating files in "%s" accessed after "%s": %s.',
            #                   item_local_abspath, item_record.record_time_str, e)
            # except OSError as e:
            #     logging.error('Error checking local folder "%s": %s.', item_local_abspath, e)
        elif item_record is not None:
            if self.assume_remote_unchanged:
                logging.info('Remote item for local dir "%s" is a file that has been deleted locally. '
                             'Delete the remote item and upload the file.', item_local_abspath)
                if not delete_item.DeleteRemoteItemTask(
                        repo=self.repo, task_pool=self.task_pool, parent_relpath=self.rel_path,
                        item_name=item_name, item_id=item_record.item_id, is_folder=False).handle():
                    logging.error('Failed to delete outdated remote directory "%s/%s" of Drive %s.',
                                  self.rel_path, item_name, self.repo.drive.id)
                    # Keep the record so that the branch can be revisited next time.
                    return

        # Either we decide to upload the item above, or the folder does not exist remotely and we have no reference
        # whether it existed remotely or not in the past. Better upload it back.
        logging.info('Local directory "%s" seems new. Upload it.', item_local_abspath)
        self.task_pool.add_task(CreateFolderTask(
            self.repo, self.task_pool, item_name, self.rel_path, True, True))
예제 #6
0
    def _handle_remote_file_with_record(self, remote_item, item_record, item_stat, item_local_abspath, all_local_items):
        """
        :param onedrivesdk.model.item.Item remote_item:
        :param onedrive_client.od_repo.ItemRecord item_record:
        :param posix.stat_result | None item_stat:
        :param str item_local_abspath:
        :param [str] all_local_items:
        """
        # In this case we have all three pieces of information -- remote item metadata, database record, and local inode
        # stats. The best case is that all of them agree, and the worst case is that they all disagree.

        if os.path.isdir(item_local_abspath):
            # Remote item is a file yet the local item is a folder.
            if item_record and item_record.type == ItemRecordType.FOLDER:
                # TODO: Use the logic in handle_local_folder to solve this.
                send2trash(item_local_abspath)
                self.repo.delete_item(remote_item.name, self.rel_path, True)
            else:
                # When db record does not exist or says the path is a file, then it does not agree with local inode
                # and the information is useless. We delete it and sync both remote and local items.
                if item_record:
                    self.repo.delete_item(remote_item.name, self.rel_path, False)
            return self._handle_remote_file_without_record(remote_item, None, item_local_abspath, all_local_items)

        remote_mtime, _ = get_item_modified_datetime(remote_item)
        local_mtime_ts = item_stat.st_mtime if item_stat else None
        remote_mtime_ts = datetime_to_timestamp(remote_mtime)
        record_mtime_ts = datetime_to_timestamp(item_record.modified_time)
        try:
            remote_sha1_hash = remote_item.file.hashes.sha1_hash
        except AttributeError:
            remote_sha1_hash = None

        if (remote_item.id == item_record.item_id and remote_item.c_tag == item_record.c_tag or
            remote_item.size == item_record.size and
                diff_timestamps(remote_mtime_ts, record_mtime_ts) == 0):
            # The remote item metadata matches the database record. So this item has been synced before.
            if item_stat is None:
                # The local file was synced but now is gone. Delete remote one as well.
                logging.debug('Local file "%s" is gone but remote item matches db record. Delete remote item.',
                              item_local_abspath)
                self.task_pool.add_task(delete_item.DeleteRemoteItemTask(
                    self.repo, self.task_pool, self.rel_path, remote_item.name, remote_item.id, False))
            elif (item_stat.st_size == item_record.size_local and
                  (diff_timestamps(local_mtime_ts, record_mtime_ts) == 0 or
                   remote_sha1_hash and remote_sha1_hash == sha1_value(item_local_abspath))):
                # If the local file matches the database record (i.e., same mtime timestamp or same content),
                # simply return. This is the best case.
                if diff_timestamps(local_mtime_ts, remote_mtime_ts) != 0:
                    logging.info('File "%s" seems to have same content but different timestamp (%f, %f). Fix it.',
                                 item_local_abspath, local_mtime_ts, remote_mtime_ts)
                    fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts)
                    self.repo.update_item(remote_item, self.rel_path, item_stat.st_size)
            else:
                # Content of local file has changed. Because we assume the remote item was synced before, we overwrite
                # the remote item with local one.
                # API Issue: size field may not match file size.
                # Refer to https://github.com/OneDrive/onedrive-sdk-python/issues/88
                # Workaround -- storing both remote and local sizes.
                logging.debug('File "%s" was changed locally and the remote version is known old. Upload it.',
                              item_local_abspath)
                self.task_pool.add_task(upload_file.UploadFileTask(
                    self.repo, self.task_pool, self.item_request, self.rel_path, remote_item.name))
        else:
            # The remote file metadata and database record disagree.
            if item_stat is None:
                # If the remote file is the one on record, then the remote one is newer than the deleted local file
                # so it should be downloaded. If they are not the same, then the remote one should definitely
                # be kept. So the remote file needs to be kept and downloaded anyway.
                logging.debug('Local file "%s" is gone but remote item disagrees with db record. Download it.',
                              item_local_abspath)
                self.task_pool.add_task(
                    download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path))
            elif item_stat.st_size == item_record.size_local and \
                    (diff_timestamps(local_mtime_ts, record_mtime_ts) == 0 or
                     item_record.sha1_hash and item_record.sha1_hash == sha1_value(item_local_abspath)):
                # Local file agrees with database record. This means that the remote file is strictly newer.
                # The local file can be safely overwritten.
                logging.debug('Local file "%s" agrees with db record but remote item is different. Overwrite local.',
                              item_local_abspath)
                self.task_pool.add_task(
                    download_file.DownloadFileTask(self.repo, self.task_pool, remote_item, self.rel_path))
            else:
                # So both the local file and remote file have been changed after the record was created.
                equal_ts = diff_timestamps(local_mtime_ts, remote_mtime_ts) == 0
                if (item_stat.st_size == remote_item.size and (
                        (equal_ts or remote_sha1_hash and remote_sha1_hash == sha1_value(item_local_abspath)))):
                    # Fortunately the two files seem to be the same.
                    # Here the logic is written as if there is no size mismatch issue.
                    logging.debug(
                        'Local file "%s" seems to have same content with remote but record disagrees. Fix db record.',
                        item_local_abspath)
                    if not equal_ts:
                        fix_owner_and_timestamp(item_local_abspath, self.repo.context.user_uid, remote_mtime_ts)
                    self.repo.update_item(remote_item, self.rel_path, item_stat.st_size)
                else:
                    # Worst case we keep both files.
                    logging.debug('Local file "%s" differs from db record and remote item. Keep both versions.',
                                  item_local_abspath)
                    self._rename_local_and_download_remote(remote_item, all_local_items)