def _upload_large_file(dbx: dropbox.Dropbox, path: Path, chunk_size: int) -> Iterator[Tuple[int, int]]: file_size = path.stat().st_size num_chunks = math.ceil(file_size / chunk_size) curr_chunk = 0 yield curr_chunk, num_chunks with path.open(mode="rb") as f: upload_session_start_result = dbx.files_upload_session_start( f.read(chunk_size)) curr_chunk += 1 yield curr_chunk, num_chunks logger.debug(f"Uploaded chunk {curr_chunk}/{num_chunks} of {path}") cursor = dropbox.files.UploadSessionCursor( session_id=upload_session_start_result.session_id, offset=f.tell()) commit = dropbox.files.CommitInfo( path=_upload_path(path), mode=dropbox.files.WriteMode("overwrite")) while f.tell() < file_size: if (file_size - f.tell()) <= chunk_size: dbx.files_upload_session_finish(f.read(chunk_size), cursor, commit) else: dbx.files_upload_session_append_v2(f.read(chunk_size), cursor) cursor.offset = f.tell() curr_chunk += 1 logger.debug(f"Uploaded chunk {curr_chunk}/{num_chunks} of {path}") yield curr_chunk, num_chunks
def upload_chunked(dropbox_client: dropbox.Dropbox, local_file_path: str, remote_file_path: str) -> None: """ Uploads a file in chucks to Dropbox, allowing it to resume on (connection) failure. """ logger.info('Dropbox: Syncing file %s', remote_file_path) write_mode = dropbox.files.WriteMode.overwrite file_handle = open(local_file_path, 'rb') file_size = os.path.getsize(local_file_path) # Many thanks to https://stackoverflow.com/documentation/dropbox-api/409/uploading-a-file/1927/uploading-a-file-usin # g-the-dropbox-python-sdk#t=201610181733061624381 CHUNK_SIZE = 2 * 1024 * 1024 # Small uploads should be transfers at one go. if file_size <= CHUNK_SIZE: dropbox_client.files_upload(file_handle.read(), remote_file_path, mode=write_mode) # Large uploads can be sent in chunks, by creating a session allowing multiple separate uploads. else: upload_session_start_result = dropbox_client.files_upload_session_start( file_handle.read(CHUNK_SIZE)) cursor = dropbox.files.UploadSessionCursor( session_id=upload_session_start_result.session_id, offset=file_handle.tell()) commit = dropbox.files.CommitInfo(path=remote_file_path, mode=write_mode) # We keep sending the data in chunks, until we reach the last one, then we instruct Dropbox to finish the upload # by combining all the chunks sent previously. while file_handle.tell() < file_size: if (file_size - file_handle.tell()) <= CHUNK_SIZE: dropbox_client.files_upload_session_finish( file_handle.read(CHUNK_SIZE), cursor, commit) else: dropbox_client.files_upload_session_append_v2( file_handle.read(CHUNK_SIZE), cursor) cursor.offset = file_handle.tell() file_handle.close()
def upload_to_dropbox(access_token, dropbox_path, file_path, progress_callback=None): dbx = Dropbox(access_token) with open(file_path, 'rb') as file: chunk = file.read(CHUNK_SIZE) offset = len(chunk) upload_session = dbx.files_upload_session_start(chunk) progress_callback and progress_callback(offset) while True: chunk = file.read(CHUNK_SIZE) if not chunk: break dbx.files_upload_session_append_v2( chunk, UploadSessionCursor( upload_session.session_id, offset, ), ) offset += len(chunk) progress_callback and progress_callback(offset) file_metadata = dbx.files_upload_session_finish( b'', UploadSessionCursor( upload_session.session_id, offset=offset, ), CommitInfo( dropbox_path, # When writing the file it won't overwrite an existing file, just add # another file like "filename (2).txt" WriteMode('add'), ), ) progress_callback and progress_callback(offset) return file_metadata.path_display
def upload_to_dropbox(access_token, dropbox_path, file_path, progress_callback=None): dbx = Dropbox(access_token) with open(file_path, 'rb') as file: chunk = file.read(CHUNK_SIZE) offset = len(chunk) upload_session = dbx.files_upload_session_start(chunk) progress_callback and progress_callback(offset) while True: chunk = file.read(CHUNK_SIZE) if not chunk: break dbx.files_upload_session_append_v2( chunk, UploadSessionCursor( upload_session.session_id, offset, ), ) offset += len(chunk) progress_callback and progress_callback(offset) file_metadata = dbx.files_upload_session_finish( b'', UploadSessionCursor( upload_session.session_id, offset=offset, ), CommitInfo( dropbox_path, # When writing the file it won't overwrite an existing file, just add # another file like "filename (2).txt" WriteMode('add'), ), ) progress_callback and progress_callback(offset) return file_metadata.path_display
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def _full_path(self, name): if name == '/': name = '' return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) metadata = self.client.files_get_metadata(full_path) for entry in metadata['contents']: entry['path'] = entry['path'].replace(full_path, '', 1) entry['path'] = entry['path'].replace('/', '', 1) if entry['is_dir']: directories.append(entry['path']) else: files.append(entry['path']) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata['bytes'] def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT) return mod_time def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT) return acc_time def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name)) else: self._chunked_upload(content, self._full_path(name)) content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE) ) cursor = UploadSessionCursor( session_id=upload_session.session_id, offset=content.tell() ) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit ) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor ) cursor.offset = content.tell()
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" location = setting('DROPBOX_ROOT_PATH', '/') oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN') timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE) CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout, write_mode=write_mode): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path self.write_mode = write_mode self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): if name == '/': name = '' # If the machine is windows do not append the drive letter to file path if os.name == 'nt': final_path = os.path.join(self.root_path, name).replace('\\', '/') # Separator on linux system sep = '//' base_path = self.root_path if (not os.path.normcase(final_path).startswith(os.path.normcase(base_path + sep)) and os.path.normcase(final_path) != os.path.normcase(base_path) and os.path.dirname(os.path.normcase(base_path)) != os.path.normcase(base_path)): raise SuspiciousFileOperation( 'The joined path ({}) is located outside of the base path ' 'component ({})'.format(final_path, base_path)) return final_path else: return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) if full_path == '/': full_path = '' metadata = self.client.files_list_folder(full_path) for entry in metadata.entries: if isinstance(entry, FolderMetadata): directories.append(entry.name) else: files.append(entry.name) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.size def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.server_modified def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.client_modified def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode)) else: self._chunked_upload(content, self._full_path(name)) content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE) ) cursor = UploadSessionCursor( session_id=upload_session.session_id, offset=content.tell() ) commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode)) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit ) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor ) cursor.offset = content.tell() def get_available_name(self, name, max_length=None): """Overwrite existing file with the same name.""" name = self._full_path(name) if self.write_mode == 'overwrite': return get_available_overwrite_name(name, max_length) return super().get_available_name(name, max_length)
class DropboxConnection(object): class Parameters(object): def __init__(self, auth_token, remote_dir_path): self._auth_token = auth_token self._remote_dir_path = remote_dir_path @property def auth_token(self): return self._auth_token @property def remote_dir_path(self): return constants.DROPBOX_APP_PATH_PREFIX / self._remote_dir_path def __str__(self): return """ remote_dir_path: {} """.format(self.remote_dir_path) @classmethod def get_client_from_params(cls, dropbox_parameters): #Dropbox connection placeholder dropbox = None if dropbox_parameters: dropbox_params = DropboxConnection.Parameters( dropbox_parameters[0], dropbox_parameters[1]) dropbox = DropboxConnection(dropbox_params) return dropbox @classmethod def get_client(cls, auth_token, remote_dir_path): #Initialize the paramters params = DropboxConnection.Parameters(auth_token, remote_dir_path) #Create dropbox client client = DropboxConnection(params) return client def __init__(self, params): #Required parameters self._params = params #Derived parameters self._client = Dropbox(self._params.auth_token) #Logging self._logger = logging.get_logger(__name__) def upload(self, source_file_path): """It upload the source files to the dropbox. Arguments: source_file_path {string} -- The source file path. Raises: ValueError -- It raise value error for invalid files. """ #Validate parameters if not source_file_path: raise ValueError("Invalid source file path") with open(source_file_path, 'rb') as handle: #Source file name source_file_name = Path(source_file_path).name #Remote path remote_file_path = (self._params.remote_dir_path / source_file_name).as_posix() #File size upload_size = path.getsize(source_file_path) #Upload the files based on the upload size if upload_size <= constants.DROPBOX_CHUNK_SIZE: self._logger.info( 'Preparing to upload small file: %s with size: %d to: %s', source_file_path, upload_size, remote_file_path) self._upload_small_file(handle, remote_file_path) else: self._logger.info( 'Preparing to upload large file: %s with size: %d to: %s', source_file_path, upload_size, remote_file_path) self._upload_large_file(handle, upload_size, remote_file_path) self._logger.info('Uploaded: %s', source_file_path) def _upload_small_file(self, handle, remote_file_path): """It uploads a small source files to the dropbox. Arguments: handle {A File handle} -- The source file handle. remote_file_path {string} -- The destination path of the file. """ self._client.files_upload(handle.read(), remote_file_path, mode=Dropbox_WriteMode.overwrite) def _upload_large_file(self, handle, upload_size, remote_file_path): """It uploads a large source files to the dropbox. Arguments: handle {A File handle} -- The source file handle. upload_size {int} -- The number of bytes to be uploaded. remote_file_path {string} -- The destination path of the file. """ #Upload session session = self._client.files_upload_session_start( handle.read(constants.DROPBOX_CHUNK_SIZE)) cursor = Dropbox_UploadSessionCursor(session_id=session.session_id, offset=handle.tell()) #Upload look with tqdm(desc='Uploading: {}'.format(remote_file_path), total=upload_size) as pbar: #Update the progress bar for the session start reads pbar.update(handle.tell()) while handle.tell() < upload_size: #Calculate remaining bytes remaining_bytes = upload_size - handle.tell() #If it is the last chunk, finalize the upload if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE: #Commit info commit = Dropbox_CommitInfo( path=remote_file_path, mode=Dropbox_WriteMode.overwrite) #Finish upload self._client.files_upload_session_finish( handle.read(remaining_bytes), cursor, commit) #Update progress pbar.update(remaining_bytes) #More than chunk size remaining to upload else: self._client.files_upload_session_append_v2( handle.read(constants.DROPBOX_CHUNK_SIZE), cursor) #Update the cursor cursor.offset = handle.tell() #Update the progress pbar.update(constants.DROPBOX_CHUNK_SIZE) #Refresh the progress bar pbar.refresh() def download(self, remote_file_path): """It downloads the remote files from the dropbox. Arguments: remote_file_path {Path} -- The path to the remote file. Raises: ValueError -- It raise value error for invalid file name. """ #Validate parameters if not remote_file_path: raise ValueError("Invalid remote file path") #Destination file path dest_file_path = remote_file_path #Full remote file path remote_file_path = self._params.remote_dir_path / remote_file_path #Download file size placeholder download_size = 0 try: download_size = self._client.files_get_metadata( remote_file_path.as_posix()).size except ApiError as e: raise FileNotFoundError( 'File: {} is not found'.format(remote_file_path.as_posix()), e) self._logger.info('Preparing file download: %s with size: %d to: %s', remote_file_path, download_size, dest_file_path) #Download the file self._download_file(dest_file_path, remote_file_path, download_size) self._logger.info('Completed the file download: %s to: %s', remote_file_path, dest_file_path) def _download_file(self, dest_file_path, remote_file_path, download_size): """It downloads the remote files from the dropbox. Arguments: remote_file_path {A Path object} -- The path of the remote file. dest_file_path {string} -- The destination file path. download_size {int} -- The number of bytes to be downloaded. """ #Download _, result = self._client.files_download(remote_file_path.as_posix()) #Temporary dest_file_name tmp_dest_file_path = "{}.tmp".format(dest_file_path) with open(tmp_dest_file_path, 'wb') as handle: with tqdm(desc='Downloading: {}'.format( remote_file_path.as_posix()), total=download_size) as pbar: for bytes_read in result.iter_content( constants.DROPBOX_CHUNK_SIZE): handle.write(bytes_read) #Update the progress pbar.update(len(bytes_read)) if Path(tmp_dest_file_path).exists(): rename(tmp_dest_file_path, dest_file_path) def list(self, dir_path=Path(), file_name_prefix=''): """It lists the files in the dropbox folder that starts with the given prefix. Arguments: file_name_prefix {string} -- The prefix to filter the results. """ #Candidate directory whose contents are to be listed candidate_dir_path = self._params.remote_dir_path / dir_path self._logger.info('Enumerating: %s with file_name_prefix: %s', candidate_dir_path, file_name_prefix) #Call the downstream API response = self._client.files_list_folder( candidate_dir_path.as_posix()) #Output list placeholder files = [] sizes = [] if response.entries: #Log the response summary self._logger.info('Got %d files in: %s', len(response.entries), candidate_dir_path) #Extract the name of files satisfying the input criteria from the response entries. file_infos = [(dir_path / entry.name, entry.size) for entry in response.entries if entry.name.startswith(file_name_prefix)] files, sizes = zip(*file_infos) return files, sizes
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting( 'DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = Dropbox(oauth2_access_token) def _full_path(self, path): path = PurePosixPath(self.root_path) / path path = str(path) if path == '/': path = '' return path def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) result = self.client.files_list_folder(full_path) for entry in result.entries: if isinstance(entry, FolderMetadata): directories.append(entry.name) else: files.append(entry.name) assert not result.has_more, "FIXME: Not implemented!" return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.size def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.server_modified def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) # Note to the unwary, this is actually an mtime return metadata.client_modified def url(self, name): try: media = self.client.files_get_temporary_link(self._full_path(name)) return media.link except ApiError: raise ValueError("This file is not accessible via a URL.") def _open(self, name, mode='rb'): return DropBoxFile(self._full_path(name), self) def _save(self, name, content): try: content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name)) else: self._chunked_upload(content, self._full_path(name)) finally: content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session.session_id, offset=content.tell()) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor) cursor.offset = content.tell()
class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" CHUNK_SIZE = 4 * 1024 * 1024 def __init__(self, oauth2_access_token=None, root_path=None, timeout=None): oauth2_access_token = oauth2_access_token or setting( 'DROPBOX_OAUTH2_TOKEN') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure an auth token at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') timeout = timeout or setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT) self.client = Dropbox(oauth2_access_token, timeout=timeout) def _full_path(self, name): if name == '/': name = '' return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): self.client.files_delete(self._full_path(name)) def exists(self, name): try: return bool(self.client.files_get_metadata(self._full_path(name))) except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) if full_path == '/': full_path = '' metadata = self.client.files_list_folder(full_path) for entry in metadata.entries: if isinstance(entry, FolderMetadata): directories.append(entry.name) else: files.append(entry.name) return directories, files def size(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.size def modified_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.server_modified def accessed_time(self, name): metadata = self.client.files_get_metadata(self._full_path(name)) return metadata.client_modified def url(self, name): media = self.client.files_get_temporary_link(self._full_path(name)) return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): content.open() if content.size <= self.CHUNK_SIZE: self.client.files_upload(content.read(), self._full_path(name)) else: self._chunked_upload(content, self._full_path(name)) content.close() return name def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session.session_id, offset=content.tell()) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor) cursor.offset = content.tell()
def upload(dropbox_helper_id, access_token, size, max_retries): from .models import DropboxUploadHelper helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id) dbx = Dropbox(access_token) try: with open(helper.src, 'rb') as f: chunk = f.read(CHUNK_SIZE) offset = len(chunk) upload_session = dbx.files_upload_session_start(chunk) while True: chunk = f.read(CHUNK_SIZE) if not chunk: break helper.progress = offset / size helper.save() dbx.files_upload_session_append_v2( chunk, UploadSessionCursor( upload_session.session_id, offset, ), ) offset += len(chunk) file_metadata = dbx.files_upload_session_finish( b'', UploadSessionCursor( upload_session.session_id, offset=offset, ), CommitInfo( '/{}'.format(os.path.basename(helper.src)), # When writing the file it won't overwrite an existing file, just add # another file like "filename (2).txt" WriteMode('add'), ), ) except Exception as e: helper.failure_reason = str(e) helper.save() couch_user = CouchUser.get_by_username(helper.user.username) if helper.failure_reason is None: path_link_metadata = dbx.sharing_create_shared_link_with_settings( file_metadata.path_display, SharedLinkSettings( requested_visibility=RequestedVisibility.team_only, ), ) context = { 'share_url': path_link_metadata.url, 'path': os.path.join( u'Apps', settings.DROPBOX_APP_NAME, path_link_metadata.name, ) } with localize(couch_user.get_language_code()): subject = _(u'{} has been uploaded to dropbox!'.format( helper.dest)) html_content = render_to_string( 'dropbox/emails/upload_success.html', context) text_content = render_to_string( 'dropbox/emails/upload_success.txt', context) else: context = {'reason': helper.failure_reason, 'path': helper.dest} with localize(couch_user.get_language_code()): subject = _(u'{} has failed to upload to dropbox'.format( helper.dest)) html_content = render_to_string('dropbox/emails/upload_error.html', context) text_content = render_to_string('dropbox/emails/upload_error.txt', context) send_HTML_email( subject, helper.user.email, html_content, text_content=text_content, )