def upload_file(self, file_to_upload, path): file_size = self._get_file_size(file_to_upload) file_to_upload.seek(0) if file_size <= DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT: logging.debug('Using single request to upload file') self.__dropbox_client.files_upload(file_to_upload.read(), path) else: logging.debug('Using multi-request upload session for this file') session = None offset = 0 chunk = file_to_upload.read( DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT) while len(chunk) == DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT: if session is None: logging.debug('Initializing upload session') session = self.__dropbox_client.files_upload_session_start( chunk) else: logging.debug('Appending to session %s at offset %d', session.session_id, offset) self.__dropbox_client.files_upload_session_append_v2( chunk, UploadSessionCursor(session.session_id, offset)) offset += len(chunk) chunk = file_to_upload.read( DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT) logging.debug('Finishing session %s', session.session_id) commit_info = CommitInfo(path=path, mode=WriteMode('add'), autorename=False) self.__dropbox_client.files_upload_session_finish( chunk, UploadSessionCursor(session.session_id, offset), commit_info)
def upload_large_dropbox_file(client, source_full_path, destination_full_path): """ Uploads a large (>CHUNK_SIZE) single file to Dropbox. """ file_size = os.path.getsize(source_full_path) with open(source_full_path, 'rb') as f: try: upload_session_start_result = client.files_upload_session_start( f.read(CHUNK_SIZE)) session_id = upload_session_start_result.session_id cursor = UploadSessionCursor(session_id=session_id, offset=f.tell()) commit = CommitInfo(path=destination_full_path) while f.tell() < file_size: if ((file_size - f.tell()) <= CHUNK_SIZE): print( client.files_upload_session_finish( f.read(CHUNK_SIZE), cursor, commit)) else: client.files_upload_session_append(f.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell() except ApiError as e: print(f'Failed to upload file {source_full_path}') print(f'{source_full_path} successfully uploaded to ' \ f'{destination_full_path}')
def make_backup_dropbox(self, ts, name, dump_stream, info_file, info_file_content, cloud_params): # Upload two backup objects to Dropbox DropboxService = self.env['ir.config_parameter'].get_dropbox_service() folder_path = self.env['ir.config_parameter'].get_param("odoo_backup_sh_dropbox.dropbox_folder_path") info_file_size = info_file.tell() dump_stream.seek(0) info_file.seek(0) for obj, obj_name, file_size in \ [[dump_stream, compute_backup_filename(name, ts, info_file_content.get('encrypted')), info_file_content.get("backup_size") * 1024 * 1024], [info_file, compute_backup_info_filename(name, ts), info_file_size]]: # The full path to upload the file to, including the file name full_path = "{folder_path}/{file_name}".format( folder_path=folder_path, file_name=obj_name, ) # from here: https://www.dropboxforum.com/t5/API-Support-Feedback/python-upload-big-file-example/m-p/166627/highlight/true#M6013 if file_size <= CHUNK_SIZE: DropboxService.files_upload(obj.read(), full_path) else: upload_session_start_result = DropboxService.files_upload_session_start(obj.read(CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=obj.tell()) commit = CommitInfo(path=full_path) while obj.tell() < file_size: if ((file_size - obj.tell()) <= CHUNK_SIZE): DropboxService.files_upload_session_finish(obj.read(CHUNK_SIZE), cursor, commit) else: DropboxService.files_upload_session_append(obj.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = obj.tell()
def _upload_session(self, bytes, finish): if self._session is None: start_result = self._dbx.files_upload_session_start(bytes) self._session = start_result.session_id elif finish: cursor = UploadSessionCursor(self._session, self._total_bytes) info = CommitInfo(path=self._upload_path, mode=WriteMode('overwrite'), client_modified=self._file_retriever.modified) self._dbx.files_upload_session_finish(bytes, cursor, info) else: cursor = UploadSessionCursor(self._session, self._total_bytes) self._dbx.files_upload_session_append_v2(bytes, cursor)
def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session.session_id, offset=content.tell()) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor) cursor.offset = content.tell()
def dropbox_upload_export(access_token, local_path, remote_fname): client = Dropbox(access_token) with open(local_path, 'rb') as inf: chunk_size = int(1.5e8 - 1) next_bytes = inf.read(chunk_size) upload_session_result = client.files_upload_session_start(next_bytes) uploaded_so_far = len(next_bytes) next_bytes = inf.read(chunk_size) cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far) while next_bytes: client.files_upload_session_append_v2(next_bytes, cursor) next_bytes = inf.read(chunk_size) uploaded_so_far += chunk_size cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far) client.files_upload_session_finish('', cursor, CommitInfo(remote_fname))
def put(self, source, dest, retry=True): dest = self._fix_slashes(dest) if (self.client is not None): # open the file and get its size f = open(source, 'rb') f_size = os.path.getsize(source) try: if (f_size < self.MAX_CHUNK): # use the regular upload self.client.files_upload(f.read(), dest, mode=WriteMode('overwrite')) else: # start the upload session upload_session = self.client.files_upload_session_start( f.read(self.MAX_CHUNK)) upload_cursor = UploadSessionCursor( upload_session.session_id, f.tell()) while (f.tell() < f_size): # check if we should finish the upload if ((f_size - f.tell()) <= self.MAX_CHUNK): # upload and close self.client.files_upload_session_finish( f.read(self.MAX_CHUNK), upload_cursor, CommitInfo(dest, mode=WriteMode('overwrite'))) else: # upload a part and store the offset self.client.files_upload_session_append_v2( f.read(self.MAX_CHUNK), upload_cursor) upload_cursor.offset = f.tell() # if no errors we're good! return True except Exception as anError: utils.log(str(anError)) # if we have an exception retry if (retry): return self.put(source, dest, False) else: # tried once already, just quit return False else: return False
def sync(self, file_path: Path) -> None: """Sync a file to Dropbox.""" # Check if Dropbox token is valid. if self.valid is False: error = "Dropbox token is invalid!" self.success = False self.output = error log.error(error) return None # This is size what can be uploaded as one chunk. # When file is bigger than that, this will be uploaded # in multiple parts. chunk_size = 4 * 1024 * 1024 temp_file, recompressed = self.compress(file_path) upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}" try: with temp_file as f: file_size = os.stat(f.name).st_size log.debug(file_size) if file_size <= chunk_size: self.client.files_upload(f.read(), upload_path, WriteMode.overwrite) else: session_start = self.client.files_upload_session_start( f.read(chunk_size)) cursor = UploadSessionCursor(session_start.session_id, offset=f.tell()) # Commit contains path in Dropbox and write mode about file commit = CommitInfo(upload_path, WriteMode.overwrite) while f.tell() < file_size: if (file_size - f.tell()) <= chunk_size: self.client.files_upload_session_finish( f.read(chunk_size), cursor, commit) else: self.client.files_upload_session_append( f.read(chunk_size), cursor.session_id, cursor.offset) cursor.offset = f.tell() self.success = True except (ApiError, HttpError) as e: log.error(e) self.success = False self.output = str(e)
def upload_file(dbx, LOCALFILE): """Uploads a local file to Dropbox in chunks <=4MiB each""" with open(LOCALFILE, 'rb') as f: BACKUPPATH = '/' + os.path.split(LOCALFILE)[1] file_size = os.path.getsize(LOCALFILE) if file_size <= CHUNK_SIZE: dbx.files_upload(f.read(), BACKUPPATH, mode=WriteMode('overwrite')) else: upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=f.tell()) commit = CommitInfo(path=BACKUPPATH) while f.tell() < file_size: if ((file_size - f.tell()) <= CHUNK_SIZE): dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit) else: dbx.files_upload_session_append(f.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell()
def upload_to_dropbox(access_token, dropbox_path, file_path, progress_callback=None): dbx = Dropbox(access_token) with open(file_path, 'rb') as file: chunk = file.read(CHUNK_SIZE) offset = len(chunk) upload_session = dbx.files_upload_session_start(chunk) progress_callback and progress_callback(offset) while True: chunk = file.read(CHUNK_SIZE) if not chunk: break dbx.files_upload_session_append_v2( chunk, UploadSessionCursor( upload_session.session_id, offset, ), ) offset += len(chunk) progress_callback and progress_callback(offset) file_metadata = dbx.files_upload_session_finish( b'', UploadSessionCursor( upload_session.session_id, offset=offset, ), CommitInfo( dropbox_path, # When writing the file it won't overwrite an existing file, just add # another file like "filename (2).txt" WriteMode('add'), ), ) progress_callback and progress_callback(offset) return file_metadata.path_display
def put_file_chunked(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug( 'dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries is_eof = False # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed # chunk while not is_eof or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) is_eof = f.tell() >= file_size if not is_eof and len(buf) == 0: continue # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if not is_eof: assert len(buf) != 0 log.Debug( 'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append( buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug( 'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish( buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError ) and error.is_incorrect_offset(): # Server reports that we should send another chunk. # Most likely this is caused by network error during # previous upload attempt. In such case we'll get # expected offset from server and it's enough to just # seek() and retry again new_offset = error.get_incorrect_offset( ).correct_offset log.Debug( 'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something # strange and no safe way to recover raise BackendException( "dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if not self.user_authenticated(): self.login() if retry_number == 0: raise # We don't know for sure, was partial upload successful or # not. So it's better to retry smaller amount to avoid extra # reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) return res_metadata finally: f.close()
def __exit__(self, exc_type, exc_val, exc_tb): commit = CommitInfo(path=path, mute=True) cur = UploadSessionCursor(self._sess_id, self._data_offset) self._dbx.files_upload_session_finish(b'', cur, commit)
def upload(dropbox_helper_id, access_token, size, max_retries): from .models import DropboxUploadHelper helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id) dbx = Dropbox(access_token) try: with open(helper.src, 'rb') as f: chunk = f.read(CHUNK_SIZE) offset = len(chunk) upload_session = dbx.files_upload_session_start(chunk) while True: chunk = f.read(CHUNK_SIZE) if not chunk: break helper.progress = offset / size helper.save() dbx.files_upload_session_append_v2( chunk, UploadSessionCursor( upload_session.session_id, offset, ), ) offset += len(chunk) file_metadata = dbx.files_upload_session_finish( b'', UploadSessionCursor( upload_session.session_id, offset=offset, ), CommitInfo( '/{}'.format(os.path.basename(helper.src)), # When writing the file it won't overwrite an existing file, just add # another file like "filename (2).txt" WriteMode('add'), ), ) except Exception as e: helper.failure_reason = str(e) helper.save() couch_user = CouchUser.get_by_username(helper.user.username) if helper.failure_reason is None: path_link_metadata = dbx.sharing_create_shared_link_with_settings( file_metadata.path_display, SharedLinkSettings( requested_visibility=RequestedVisibility.team_only, ), ) context = { 'share_url': path_link_metadata.url, 'path': os.path.join( u'Apps', settings.DROPBOX_APP_NAME, path_link_metadata.name, ) } with localize(couch_user.get_language_code()): subject = _(u'{} has been uploaded to dropbox!'.format( helper.dest)) html_content = render_to_string( 'dropbox/emails/upload_success.html', context) text_content = render_to_string( 'dropbox/emails/upload_success.txt', context) else: context = {'reason': helper.failure_reason, 'path': helper.dest} with localize(couch_user.get_language_code()): subject = _(u'{} has failed to upload to dropbox'.format( helper.dest)) html_content = render_to_string('dropbox/emails/upload_error.html', context) text_content = render_to_string('dropbox/emails/upload_error.txt', context) send_HTML_email( subject, helper.user.email, html_content, text_content=text_content, )