Example #1
0
def _upload_large_file(dbx: dropbox.Dropbox, path: Path,
                       chunk_size: int) -> Iterator[Tuple[int, int]]:
    file_size = path.stat().st_size
    num_chunks = math.ceil(file_size / chunk_size)
    curr_chunk = 0
    yield curr_chunk, num_chunks
    with path.open(mode="rb") as f:
        upload_session_start_result = dbx.files_upload_session_start(
            f.read(chunk_size))
        curr_chunk += 1
        yield curr_chunk, num_chunks
        logger.debug(f"Uploaded chunk {curr_chunk}/{num_chunks} of {path}")
        cursor = dropbox.files.UploadSessionCursor(
            session_id=upload_session_start_result.session_id, offset=f.tell())
        commit = dropbox.files.CommitInfo(
            path=_upload_path(path), mode=dropbox.files.WriteMode("overwrite"))

        while f.tell() < file_size:
            if (file_size - f.tell()) <= chunk_size:
                dbx.files_upload_session_finish(f.read(chunk_size), cursor,
                                                commit)
            else:
                dbx.files_upload_session_append_v2(f.read(chunk_size), cursor)
                cursor.offset = f.tell()
            curr_chunk += 1
            logger.debug(f"Uploaded chunk {curr_chunk}/{num_chunks} of {path}")
            yield curr_chunk, num_chunks
Example #2
0
def upload_chunked(dropbox_client: dropbox.Dropbox, local_file_path: str,
                   remote_file_path: str) -> None:
    """ Uploads a file in chucks to Dropbox, allowing it to resume on (connection) failure. """
    logger.info('Dropbox: Syncing file %s', remote_file_path)

    write_mode = dropbox.files.WriteMode.overwrite

    file_handle = open(local_file_path, 'rb')
    file_size = os.path.getsize(local_file_path)

    # Many thanks to https://stackoverflow.com/documentation/dropbox-api/409/uploading-a-file/1927/uploading-a-file-usin
    # g-the-dropbox-python-sdk#t=201610181733061624381
    CHUNK_SIZE = 2 * 1024 * 1024

    # Small uploads should be transfers at one go.
    if file_size <= CHUNK_SIZE:
        dropbox_client.files_upload(file_handle.read(),
                                    remote_file_path,
                                    mode=write_mode)

    # Large uploads can be sent in chunks, by creating a session allowing multiple separate uploads.
    else:
        upload_session_start_result = dropbox_client.files_upload_session_start(
            file_handle.read(CHUNK_SIZE))

        cursor = dropbox.files.UploadSessionCursor(
            session_id=upload_session_start_result.session_id,
            offset=file_handle.tell())
        commit = dropbox.files.CommitInfo(path=remote_file_path,
                                          mode=write_mode)

        # We keep sending the data in chunks, until we reach the last one, then we instruct Dropbox to finish the upload
        # by combining all the chunks sent previously.
        while file_handle.tell() < file_size:
            if (file_size - file_handle.tell()) <= CHUNK_SIZE:
                dropbox_client.files_upload_session_finish(
                    file_handle.read(CHUNK_SIZE), cursor, commit)
            else:
                dropbox_client.files_upload_session_append_v2(
                    file_handle.read(CHUNK_SIZE), cursor)
                cursor.offset = file_handle.tell()

    file_handle.close()
Example #3
0
def upload_to_dropbox(access_token,
                      dropbox_path,
                      file_path,
                      progress_callback=None):
    dbx = Dropbox(access_token)
    with open(file_path, 'rb') as file:
        chunk = file.read(CHUNK_SIZE)
        offset = len(chunk)

        upload_session = dbx.files_upload_session_start(chunk)
        progress_callback and progress_callback(offset)

        while True:
            chunk = file.read(CHUNK_SIZE)
            if not chunk:
                break
            dbx.files_upload_session_append_v2(
                chunk,
                UploadSessionCursor(
                    upload_session.session_id,
                    offset,
                ),
            )
            offset += len(chunk)
            progress_callback and progress_callback(offset)

        file_metadata = dbx.files_upload_session_finish(
            b'',
            UploadSessionCursor(
                upload_session.session_id,
                offset=offset,
            ),
            CommitInfo(
                dropbox_path,
                # When writing the file it won't overwrite an existing file, just add
                # another file like "filename (2).txt"
                WriteMode('add'),
            ),
        )
        progress_callback and progress_callback(offset)
        return file_metadata.path_display
Example #4
0
def upload_to_dropbox(access_token, dropbox_path, file_path, progress_callback=None):
    dbx = Dropbox(access_token)
    with open(file_path, 'rb') as file:
        chunk = file.read(CHUNK_SIZE)
        offset = len(chunk)

        upload_session = dbx.files_upload_session_start(chunk)
        progress_callback and progress_callback(offset)

        while True:
            chunk = file.read(CHUNK_SIZE)
            if not chunk:
                break
            dbx.files_upload_session_append_v2(
                chunk,
                UploadSessionCursor(
                    upload_session.session_id,
                    offset,
                ),
            )
            offset += len(chunk)
            progress_callback and progress_callback(offset)

        file_metadata = dbx.files_upload_session_finish(
            b'',
            UploadSessionCursor(
                upload_session.session_id,
                offset=offset,
            ),
            CommitInfo(
                dropbox_path,
                # When writing the file it won't overwrite an existing file, just add
                # another file like "filename (2).txt"
                WriteMode('add'),
            ),
        )
        progress_callback and progress_callback(offset)
        return file_metadata.path_display
Example #5
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=None, root_path=None):
        oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN')
        self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure a token auth at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")
        self.client = Dropbox(oauth2_access_token)

    def _full_path(self, name):
        if name == '/':
            name = ''
        return safe_join(self.root_path, name).replace('\\', '/')

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)
        metadata = self.client.files_get_metadata(full_path)
        for entry in metadata['contents']:
            entry['path'] = entry['path'].replace(full_path, '', 1)
            entry['path'] = entry['path'].replace('/', '', 1)
            if entry['is_dir']:
                directories.append(entry['path'])
            else:
                files.append(entry['path'])
        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata['bytes']

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
        return mod_time

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
        return acc_time

    def url(self, name):
        media = self.client.files_get_temporary_link(self._full_path(name))
        return media.link

    def _open(self, name, mode='rb'):
        remote_file = DropBoxFile(self._full_path(name), self)
        return remote_file

    def _save(self, name, content):
        content.open()
        if content.size <= self.CHUNK_SIZE:
            self.client.files_upload(content.read(), self._full_path(name))
        else:
            self._chunked_upload(content, self._full_path(name))
        content.close()
        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE)
        )
        cursor = UploadSessionCursor(
            session_id=upload_session.session_id,
            offset=content.tell()
        )
        commit = CommitInfo(path=dest_path)

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit
                )
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor
                )
                cursor.offset = content.tell()
Example #6
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""
    location = setting('DROPBOX_ROOT_PATH', '/')
    oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN')
    timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT)
    write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE)

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout,
                 write_mode=write_mode):
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure an auth token at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")

        self.root_path = root_path
        self.write_mode = write_mode
        self.client = Dropbox(oauth2_access_token, timeout=timeout)

    def _full_path(self, name):
        if name == '/':
            name = ''
        
        # If the machine is windows do not append the drive letter to file path
        if os.name == 'nt':
            final_path = os.path.join(self.root_path, name).replace('\\', '/')
            
            # Separator on linux system
            sep = '//'
            base_path = self.root_path

            if (not os.path.normcase(final_path).startswith(os.path.normcase(base_path + sep)) and
                    os.path.normcase(final_path) != os.path.normcase(base_path) and
                    os.path.dirname(os.path.normcase(base_path)) != os.path.normcase(base_path)):
                raise SuspiciousFileOperation(
                    'The joined path ({}) is located outside of the base path '
                    'component ({})'.format(final_path, base_path))
            
            return final_path
        
        else:
            return safe_join(self.root_path, name).replace('\\', '/')

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)

        if full_path == '/':
            full_path = ''

        metadata = self.client.files_list_folder(full_path)
        for entry in metadata.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(entry.name)
            else:
                files.append(entry.name)
        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.size

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.server_modified

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.client_modified

    def url(self, name):
        media = self.client.files_get_temporary_link(self._full_path(name))
        return media.link

    def _open(self, name, mode='rb'):
        remote_file = DropBoxFile(self._full_path(name), self)
        return remote_file

    def _save(self, name, content):
        content.open()
        if content.size <= self.CHUNK_SIZE:
            self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode))
        else:
            self._chunked_upload(content, self._full_path(name))
        content.close()
        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE)
        )
        cursor = UploadSessionCursor(
            session_id=upload_session.session_id,
            offset=content.tell()
        )
        commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode))

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit
                )
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor
                )
                cursor.offset = content.tell()

    def get_available_name(self, name, max_length=None):
        """Overwrite existing file with the same name."""
        name = self._full_path(name)
        if self.write_mode == 'overwrite':
            return get_available_overwrite_name(name, max_length)
        return super().get_available_name(name, max_length)
Example #7
0
class DropboxConnection(object):
    class Parameters(object):
        def __init__(self, auth_token, remote_dir_path):
            self._auth_token = auth_token
            self._remote_dir_path = remote_dir_path

        @property
        def auth_token(self):
            return self._auth_token

        @property
        def remote_dir_path(self):
            return constants.DROPBOX_APP_PATH_PREFIX / self._remote_dir_path

        def __str__(self):
            return """
                        remote_dir_path: {}
                   """.format(self.remote_dir_path)

    @classmethod
    def get_client_from_params(cls, dropbox_parameters):
        #Dropbox connection placeholder
        dropbox = None

        if dropbox_parameters:
            dropbox_params = DropboxConnection.Parameters(
                dropbox_parameters[0], dropbox_parameters[1])
            dropbox = DropboxConnection(dropbox_params)

        return dropbox

    @classmethod
    def get_client(cls, auth_token, remote_dir_path):
        #Initialize the paramters
        params = DropboxConnection.Parameters(auth_token, remote_dir_path)

        #Create dropbox client
        client = DropboxConnection(params)

        return client

    def __init__(self, params):
        #Required parameters
        self._params = params

        #Derived parameters
        self._client = Dropbox(self._params.auth_token)

        #Logging
        self._logger = logging.get_logger(__name__)

    def upload(self, source_file_path):
        """It upload the source files to the dropbox.

        Arguments:
            source_file_path {string} -- The source file path.

        Raises:
            ValueError -- It raise value error for invalid files.
        """

        #Validate parameters
        if not source_file_path:
            raise ValueError("Invalid source file path")

        with open(source_file_path, 'rb') as handle:
            #Source file name
            source_file_name = Path(source_file_path).name

            #Remote path
            remote_file_path = (self._params.remote_dir_path /
                                source_file_name).as_posix()

            #File size
            upload_size = path.getsize(source_file_path)

            #Upload the files based on the upload size
            if upload_size <= constants.DROPBOX_CHUNK_SIZE:
                self._logger.info(
                    'Preparing to upload small file: %s with size: %d to: %s',
                    source_file_path, upload_size, remote_file_path)

                self._upload_small_file(handle, remote_file_path)
            else:
                self._logger.info(
                    'Preparing to upload large file: %s with size: %d to: %s',
                    source_file_path, upload_size, remote_file_path)

                self._upload_large_file(handle, upload_size, remote_file_path)

            self._logger.info('Uploaded: %s', source_file_path)

    def _upload_small_file(self, handle, remote_file_path):
        """It uploads a small source files to the dropbox.

        Arguments:
            handle {A File handle} -- The source file handle.
            remote_file_path {string} -- The destination path of the file.
        """
        self._client.files_upload(handle.read(),
                                  remote_file_path,
                                  mode=Dropbox_WriteMode.overwrite)

    def _upload_large_file(self, handle, upload_size, remote_file_path):
        """It uploads a large source files to the dropbox.

        Arguments:
            handle {A File handle} -- The source file handle.
            upload_size {int} -- The number of bytes to be uploaded.
            remote_file_path {string} -- The destination path of the file.
        """
        #Upload session
        session = self._client.files_upload_session_start(
            handle.read(constants.DROPBOX_CHUNK_SIZE))
        cursor = Dropbox_UploadSessionCursor(session_id=session.session_id,
                                             offset=handle.tell())

        #Upload look
        with tqdm(desc='Uploading: {}'.format(remote_file_path),
                  total=upload_size) as pbar:
            #Update the progress bar for the session start reads
            pbar.update(handle.tell())

            while handle.tell() < upload_size:
                #Calculate remaining bytes
                remaining_bytes = upload_size - handle.tell()

                #If it is the last chunk, finalize the upload
                if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE:
                    #Commit info
                    commit = Dropbox_CommitInfo(
                        path=remote_file_path,
                        mode=Dropbox_WriteMode.overwrite)

                    #Finish upload
                    self._client.files_upload_session_finish(
                        handle.read(remaining_bytes), cursor, commit)

                    #Update progress
                    pbar.update(remaining_bytes)
                #More than chunk size remaining to upload
                else:
                    self._client.files_upload_session_append_v2(
                        handle.read(constants.DROPBOX_CHUNK_SIZE), cursor)

                    #Update the cursor
                    cursor.offset = handle.tell()

                    #Update the progress
                    pbar.update(constants.DROPBOX_CHUNK_SIZE)

                #Refresh the progress bar
                pbar.refresh()

    def download(self, remote_file_path):
        """It downloads the remote files from the dropbox.

        Arguments:
            remote_file_path {Path} -- The path to the remote file.

        Raises:
            ValueError -- It raise value error for invalid file name.
        """
        #Validate parameters
        if not remote_file_path:
            raise ValueError("Invalid remote file path")

        #Destination file path
        dest_file_path = remote_file_path

        #Full remote file path
        remote_file_path = self._params.remote_dir_path / remote_file_path

        #Download file size placeholder
        download_size = 0

        try:
            download_size = self._client.files_get_metadata(
                remote_file_path.as_posix()).size
        except ApiError as e:
            raise FileNotFoundError(
                'File: {} is not found'.format(remote_file_path.as_posix()), e)

        self._logger.info('Preparing file download: %s with size: %d to: %s',
                          remote_file_path, download_size, dest_file_path)

        #Download the file
        self._download_file(dest_file_path, remote_file_path, download_size)

        self._logger.info('Completed the file download: %s to: %s',
                          remote_file_path, dest_file_path)

    def _download_file(self, dest_file_path, remote_file_path, download_size):
        """It downloads the remote files from the dropbox.

        Arguments:
            remote_file_path {A Path object} -- The path of the remote file.
            dest_file_path {string} -- The destination file path.
            download_size {int} -- The number of bytes to be downloaded.
        """
        #Download
        _, result = self._client.files_download(remote_file_path.as_posix())

        #Temporary dest_file_name
        tmp_dest_file_path = "{}.tmp".format(dest_file_path)

        with open(tmp_dest_file_path, 'wb') as handle:
            with tqdm(desc='Downloading: {}'.format(
                    remote_file_path.as_posix()),
                      total=download_size) as pbar:
                for bytes_read in result.iter_content(
                        constants.DROPBOX_CHUNK_SIZE):
                    handle.write(bytes_read)

                    #Update the progress
                    pbar.update(len(bytes_read))

        if Path(tmp_dest_file_path).exists():
            rename(tmp_dest_file_path, dest_file_path)

    def list(self, dir_path=Path(), file_name_prefix=''):
        """It lists the files in the dropbox folder that starts with the given prefix.

        Arguments:
            file_name_prefix {string} -- The prefix to filter the results.
        """
        #Candidate directory whose contents are to be listed
        candidate_dir_path = self._params.remote_dir_path / dir_path
        self._logger.info('Enumerating: %s with file_name_prefix: %s',
                          candidate_dir_path, file_name_prefix)

        #Call the downstream API
        response = self._client.files_list_folder(
            candidate_dir_path.as_posix())

        #Output list placeholder
        files = []
        sizes = []

        if response.entries:
            #Log the response summary
            self._logger.info('Got %d files in: %s', len(response.entries),
                              candidate_dir_path)

            #Extract the name of files satisfying the input criteria from the response entries.
            file_infos = [(dir_path / entry.name, entry.size)
                          for entry in response.entries
                          if entry.name.startswith(file_name_prefix)]

            files, sizes = zip(*file_infos)

        return files, sizes
Example #8
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=None, root_path=None):
        oauth2_access_token = oauth2_access_token or setting(
            'DROPBOX_OAUTH2_TOKEN')
        self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure a token auth at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")
        self.client = Dropbox(oauth2_access_token)

    def _full_path(self, path):
        path = PurePosixPath(self.root_path) / path
        path = str(path)

        if path == '/':
            path = ''

        return path

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)
        result = self.client.files_list_folder(full_path)

        for entry in result.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(entry.name)
            else:
                files.append(entry.name)

        assert not result.has_more, "FIXME: Not implemented!"

        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.size

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.server_modified

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        # Note to the unwary, this is actually an mtime
        return metadata.client_modified

    def url(self, name):
        try:
            media = self.client.files_get_temporary_link(self._full_path(name))
            return media.link
        except ApiError:
            raise ValueError("This file is not accessible via a URL.")

    def _open(self, name, mode='rb'):
        return DropBoxFile(self._full_path(name), self)

    def _save(self, name, content):
        try:
            content.open()

            if content.size <= self.CHUNK_SIZE:
                self.client.files_upload(content.read(), self._full_path(name))
            else:
                self._chunked_upload(content, self._full_path(name))

        finally:
            content.close()

        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE))
        cursor = UploadSessionCursor(session_id=upload_session.session_id,
                                     offset=content.tell())
        commit = CommitInfo(path=dest_path)

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit)
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor)
                cursor.offset = content.tell()
Example #9
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=None, root_path=None, timeout=None):
        oauth2_access_token = oauth2_access_token or setting(
            'DROPBOX_OAUTH2_TOKEN')
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure an auth token at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")

        self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
        timeout = timeout or setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT)
        self.client = Dropbox(oauth2_access_token, timeout=timeout)

    def _full_path(self, name):
        if name == '/':
            name = ''
        return safe_join(self.root_path, name).replace('\\', '/')

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)

        if full_path == '/':
            full_path = ''

        metadata = self.client.files_list_folder(full_path)
        for entry in metadata.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(entry.name)
            else:
                files.append(entry.name)
        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.size

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.server_modified

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.client_modified

    def url(self, name):
        media = self.client.files_get_temporary_link(self._full_path(name))
        return media.link

    def _open(self, name, mode='rb'):
        remote_file = DropBoxFile(self._full_path(name), self)
        return remote_file

    def _save(self, name, content):
        content.open()
        if content.size <= self.CHUNK_SIZE:
            self.client.files_upload(content.read(), self._full_path(name))
        else:
            self._chunked_upload(content, self._full_path(name))
        content.close()
        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE))
        cursor = UploadSessionCursor(session_id=upload_session.session_id,
                                     offset=content.tell())
        commit = CommitInfo(path=dest_path)

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit)
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor)
                cursor.offset = content.tell()
Example #10
0
def upload(dropbox_helper_id, access_token, size, max_retries):
    from .models import DropboxUploadHelper
    helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id)
    dbx = Dropbox(access_token)

    try:
        with open(helper.src, 'rb') as f:
            chunk = f.read(CHUNK_SIZE)
            offset = len(chunk)

            upload_session = dbx.files_upload_session_start(chunk)

            while True:
                chunk = f.read(CHUNK_SIZE)
                if not chunk:
                    break
                helper.progress = offset / size
                helper.save()
                dbx.files_upload_session_append_v2(
                    chunk,
                    UploadSessionCursor(
                        upload_session.session_id,
                        offset,
                    ),
                )
                offset += len(chunk)

            file_metadata = dbx.files_upload_session_finish(
                b'',
                UploadSessionCursor(
                    upload_session.session_id,
                    offset=offset,
                ),
                CommitInfo(
                    '/{}'.format(os.path.basename(helper.src)),
                    # When writing the file it won't overwrite an existing file, just add
                    # another file like "filename (2).txt"
                    WriteMode('add'),
                ),
            )
    except Exception as e:
        helper.failure_reason = str(e)
        helper.save()

    couch_user = CouchUser.get_by_username(helper.user.username)
    if helper.failure_reason is None:
        path_link_metadata = dbx.sharing_create_shared_link_with_settings(
            file_metadata.path_display,
            SharedLinkSettings(
                requested_visibility=RequestedVisibility.team_only, ),
        )
        context = {
            'share_url':
            path_link_metadata.url,
            'path':
            os.path.join(
                u'Apps',
                settings.DROPBOX_APP_NAME,
                path_link_metadata.name,
            )
        }
        with localize(couch_user.get_language_code()):
            subject = _(u'{} has been uploaded to dropbox!'.format(
                helper.dest))
            html_content = render_to_string(
                'dropbox/emails/upload_success.html', context)
            text_content = render_to_string(
                'dropbox/emails/upload_success.txt', context)
    else:
        context = {'reason': helper.failure_reason, 'path': helper.dest}
        with localize(couch_user.get_language_code()):
            subject = _(u'{} has failed to upload to dropbox'.format(
                helper.dest))
            html_content = render_to_string('dropbox/emails/upload_error.html',
                                            context)
            text_content = render_to_string('dropbox/emails/upload_error.txt',
                                            context)

    send_HTML_email(
        subject,
        helper.user.email,
        html_content,
        text_content=text_content,
    )