def upload_large_dropbox_file(client, source_full_path, destination_full_path): """ Uploads a large (>CHUNK_SIZE) single file to Dropbox. """ file_size = os.path.getsize(source_full_path) with open(source_full_path, 'rb') as f: try: upload_session_start_result = client.files_upload_session_start( f.read(CHUNK_SIZE)) session_id = upload_session_start_result.session_id cursor = UploadSessionCursor(session_id=session_id, offset=f.tell()) commit = CommitInfo(path=destination_full_path) while f.tell() < file_size: if ((file_size - f.tell()) <= CHUNK_SIZE): print( client.files_upload_session_finish( f.read(CHUNK_SIZE), cursor, commit)) else: client.files_upload_session_append(f.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell() except ApiError as e: print(f'Failed to upload file {source_full_path}') print(f'{source_full_path} successfully uploaded to ' \ f'{destination_full_path}')
def make_backup_dropbox(self, ts, name, dump_stream, info_file, info_file_content, cloud_params): # Upload two backup objects to Dropbox DropboxService = self.env['ir.config_parameter'].get_dropbox_service() folder_path = self.env['ir.config_parameter'].get_param("odoo_backup_sh_dropbox.dropbox_folder_path") info_file_size = info_file.tell() dump_stream.seek(0) info_file.seek(0) for obj, obj_name, file_size in \ [[dump_stream, compute_backup_filename(name, ts, info_file_content.get('encrypted')), info_file_content.get("backup_size") * 1024 * 1024], [info_file, compute_backup_info_filename(name, ts), info_file_size]]: # The full path to upload the file to, including the file name full_path = "{folder_path}/{file_name}".format( folder_path=folder_path, file_name=obj_name, ) # from here: https://www.dropboxforum.com/t5/API-Support-Feedback/python-upload-big-file-example/m-p/166627/highlight/true#M6013 if file_size <= CHUNK_SIZE: DropboxService.files_upload(obj.read(), full_path) else: upload_session_start_result = DropboxService.files_upload_session_start(obj.read(CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=obj.tell()) commit = CommitInfo(path=full_path) while obj.tell() < file_size: if ((file_size - obj.tell()) <= CHUNK_SIZE): DropboxService.files_upload_session_finish(obj.read(CHUNK_SIZE), cursor, commit) else: DropboxService.files_upload_session_append(obj.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = obj.tell()
def _upload_large_file(self, handle, upload_size, remote_file_path): """It uploads a large source files to the dropbox. Arguments: handle {A File handle} -- The source file handle. upload_size {int} -- The number of bytes to be uploaded. remote_file_path {string} -- The destination path of the file. """ #Upload session session = self._client.files_upload_session_start( handle.read(constants.DROPBOX_CHUNK_SIZE)) cursor = Dropbox_UploadSessionCursor(session_id=session.session_id, offset=handle.tell()) #Upload look with tqdm(desc='Uploading: {}'.format(remote_file_path), total=upload_size) as pbar: #Update the progress bar for the session start reads pbar.update(handle.tell()) while handle.tell() < upload_size: #Calculate remaining bytes remaining_bytes = upload_size - handle.tell() #If it is the last chunk, finalize the upload if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE: #Commit info commit = Dropbox_CommitInfo( path=remote_file_path, mode=Dropbox_WriteMode.overwrite) #Finish upload self._client.files_upload_session_finish( handle.read(remaining_bytes), cursor, commit) #Update progress pbar.update(remaining_bytes) #More than chunk size remaining to upload else: self._client.files_upload_session_append_v2( handle.read(constants.DROPBOX_CHUNK_SIZE), cursor) #Update the cursor cursor.offset = handle.tell() #Update the progress pbar.update(constants.DROPBOX_CHUNK_SIZE) #Refresh the progress bar pbar.refresh()
def _chunked_upload(self, content, dest_path): upload_session = self.client.files_upload_session_start( content.read(self.CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session.session_id, offset=content.tell()) commit = CommitInfo(path=dest_path) while content.tell() < content.size: if (content.size - content.tell()) <= self.CHUNK_SIZE: self.client.files_upload_session_finish( content.read(self.CHUNK_SIZE), cursor, commit) else: self.client.files_upload_session_append_v2( content.read(self.CHUNK_SIZE), cursor) cursor.offset = content.tell()
def put(self, source, dest, retry=True): dest = self._fix_slashes(dest) if (self.client is not None): # open the file and get its size f = open(source, 'rb') f_size = os.path.getsize(source) try: if (f_size < self.MAX_CHUNK): # use the regular upload self.client.files_upload(f.read(), dest, mode=WriteMode('overwrite')) else: # start the upload session upload_session = self.client.files_upload_session_start( f.read(self.MAX_CHUNK)) upload_cursor = UploadSessionCursor( upload_session.session_id, f.tell()) while (f.tell() < f_size): # check if we should finish the upload if ((f_size - f.tell()) <= self.MAX_CHUNK): # upload and close self.client.files_upload_session_finish( f.read(self.MAX_CHUNK), upload_cursor, CommitInfo(dest, mode=WriteMode('overwrite'))) else: # upload a part and store the offset self.client.files_upload_session_append_v2( f.read(self.MAX_CHUNK), upload_cursor) upload_cursor.offset = f.tell() # if no errors we're good! return True except Exception as anError: utils.log(str(anError)) # if we have an exception retry if (retry): return self.put(source, dest, False) else: # tried once already, just quit return False else: return False
def sync(self, file_path: Path) -> None: """Sync a file to Dropbox.""" # Check if Dropbox token is valid. if self.valid is False: error = "Dropbox token is invalid!" self.success = False self.output = error log.error(error) return None # This is size what can be uploaded as one chunk. # When file is bigger than that, this will be uploaded # in multiple parts. chunk_size = 4 * 1024 * 1024 temp_file, recompressed = self.compress(file_path) upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}" try: with temp_file as f: file_size = os.stat(f.name).st_size log.debug(file_size) if file_size <= chunk_size: self.client.files_upload(f.read(), upload_path, WriteMode.overwrite) else: session_start = self.client.files_upload_session_start( f.read(chunk_size)) cursor = UploadSessionCursor(session_start.session_id, offset=f.tell()) # Commit contains path in Dropbox and write mode about file commit = CommitInfo(upload_path, WriteMode.overwrite) while f.tell() < file_size: if (file_size - f.tell()) <= chunk_size: self.client.files_upload_session_finish( f.read(chunk_size), cursor, commit) else: self.client.files_upload_session_append( f.read(chunk_size), cursor.session_id, cursor.offset) cursor.offset = f.tell() self.success = True except (ApiError, HttpError) as e: log.error(e) self.success = False self.output = str(e)
def upload_file(dbx, LOCALFILE): """Uploads a local file to Dropbox in chunks <=4MiB each""" with open(LOCALFILE, 'rb') as f: BACKUPPATH = '/' + os.path.split(LOCALFILE)[1] file_size = os.path.getsize(LOCALFILE) if file_size <= CHUNK_SIZE: dbx.files_upload(f.read(), BACKUPPATH, mode=WriteMode('overwrite')) else: upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE)) cursor = UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=f.tell()) commit = CommitInfo(path=BACKUPPATH) while f.tell() < file_size: if ((file_size - f.tell()) <= CHUNK_SIZE): dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit) else: dbx.files_upload_session_append(f.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell()
def put(self,source,dest,retry=True): dest = self._fix_slashes(dest) if(self.client != None): #open the file and get its size f = open(source,'rb') f_size = os.path.getsize(source) try: if(f_size < self.MAX_CHUNK): #use the regular upload response = self.client.files_upload(f.read(),dest,mode=WriteMode('overwrite')) else: #start the upload session upload_session = self.client.files_upload_session_start(f.read(self.MAX_CHUNK)) upload_cursor = UploadSessionCursor(upload_session.session_id,f.tell()) while(f.tell() < f_size): #check if we should finish the upload if((f_size - f.tell()) <= self.MAX_CHUNK): #upload and close self.client.files_upload_session_finish(f.read(self.MAX_CHUNK),upload_cursor,CommitInfo(dest,mode=WriteMode('overwrite'))) else: #upload a part and store the offset self.client.files_upload_session_append_v2(f.read(self.MAX_CHUNK),upload_cursor) upload_cursor.offset = f.tell() #if no errors we're good! return True except Exception as anError: utils.log(str(anError)) #if we have an exception retry if(retry): return self.put(source,dest,False) else: #tried once already, just quit return False else: return False
def put_file_chunked(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug( 'dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries is_eof = False # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed # chunk while not is_eof or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) is_eof = f.tell() >= file_size if not is_eof and len(buf) == 0: continue # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if not is_eof: assert len(buf) != 0 log.Debug( 'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append( buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug( 'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish( buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError ) and error.is_incorrect_offset(): # Server reports that we should send another chunk. # Most likely this is caused by network error during # previous upload attempt. In such case we'll get # expected offset from server and it's enough to just # seek() and retry again new_offset = error.get_incorrect_offset( ).correct_offset log.Debug( 'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something # strange and no safe way to recover raise BackendException( "dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if not self.user_authenticated(): self.login() if retry_number == 0: raise # We don't know for sure, was partial upload successful or # not. So it's better to retry smaller amount to avoid extra # reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) return res_metadata finally: f.close()
def _put(self, source_path, remote_filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: progress.report_transfer(0, file_size) buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed chunk while (f.tell() < file_size) or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if len(buf) != 0: log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset(): # Server reports that we should send another chunk. Most likely this is caused by # network error during previous upload attempt. In such case we'll get expected offset # from server and it's enough to just seek() and retry again new_offset = error.get_incorrect_offset().correct_offset log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something strange and no safe way to recover raise BackendException("dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if retry_number == 0: raise # We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) # A few sanity checks if res_metadata.path_display != remote_path: raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path)) if res_metadata.size != file_size: raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size)) finally: f.close()