def put(self,source,dest,retry=True): dest = self._fix_slashes(dest) if(self.client != None): #open the file and get its size f = open(source,'rb') f_size = os.path.getsize(source) try: if(f_size < self.MAX_CHUNK): #use the regular upload response = self.client.files_upload(f.read(),dest,mode=WriteMode('overwrite')) else: #start the upload session upload_session = self.client.files_upload_session_start(f.read(self.MAX_CHUNK)) upload_cursor = UploadSessionCursor(upload_session.session_id,f.tell()) while(f.tell() < f_size): #check if we should finish the upload if((f_size - f.tell()) <= self.MAX_CHUNK): #upload and close self.client.files_upload_session_finish(f.read(self.MAX_CHUNK),upload_cursor,CommitInfo(dest,mode=WriteMode('overwrite'))) else: #upload a part and store the offset self.client.files_upload_session_append_v2(f.read(self.MAX_CHUNK),upload_cursor) upload_cursor.offset = f.tell() #if no errors we're good! return True except Exception as anError: utils.log(str(anError)) #if we have an exception retry if(retry): return self.put(source,dest,False) else: #tried once already, just quit return False else: return False
def put(self, source, dest, retry=True): dest = self._fix_slashes(dest) if(self.client is not None): # open the file and get its size f = open(source, 'rb') f_size = os.path.getsize(source) try: if(f_size < self.MAX_CHUNK): # use the regular upload self.client.files_upload(f.read(), dest, mode=WriteMode('overwrite')) else: # start the upload session upload_session = self.client.files_upload_session_start(f.read(self.MAX_CHUNK)) upload_cursor = UploadSessionCursor(upload_session.session_id, f.tell()) while(f.tell() < f_size): # check if we should finish the upload if((f_size - f.tell()) <= self.MAX_CHUNK): # upload and close self.client.files_upload_session_finish(f.read(self.MAX_CHUNK), upload_cursor, CommitInfo(dest, mode=WriteMode('overwrite'))) else: # upload a part and store the offset self.client.files_upload_session_append_v2(f.read(self.MAX_CHUNK), upload_cursor) upload_cursor.offset = f.tell() # if no errors we're good! return True except Exception as anError: utils.log(str(anError)) # if we have an exception retry if(retry): return self.put(source, dest, False) else: # tried once already, just quit return False else: return False
def put_file_chunked(self, source_path, remote_path): if not self.user_authenticated(): self.login() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug( 'dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries is_eof = False # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed # chunk while not is_eof or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) is_eof = f.tell() >= file_size if not is_eof and len(buf) == 0: continue # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if not is_eof: assert len(buf) != 0 log.Debug( 'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append( buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug( 'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish( buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError ) and error.is_incorrect_offset(): # Server reports that we should send another chunk. # Most likely this is caused by network error during # previous upload attempt. In such case we'll get # expected offset from server and it's enough to just # seek() and retry again new_offset = error.get_incorrect_offset( ).correct_offset log.Debug( 'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something # strange and no safe way to recover raise BackendException( "dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if not self.user_authenticated(): self.login() if retry_number == 0: raise # We don't know for sure, was partial upload successful or # not. So it's better to retry smaller amount to avoid extra # reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) return res_metadata finally: f.close()
def _put(self, source_path, remote_filename): remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')) remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip() file_size = os.path.getsize(source_path.name) f = source_path.open('rb') try: progress.report_transfer(0, file_size) buf = f.read(DPBX_UPLOAD_CHUNK_SIZE) log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size)) upload_sid = self.api_client.files_upload_session_start(buf) log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid) upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell()) commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True) res_metadata = None progress.report_transfer(f.tell(), file_size) requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries # We're doing our own error handling and retrying logic because # we can benefit from Dpbx chunked upload and retry only failed chunk while (f.tell() < file_size) or not res_metadata: try: if requested_offset is not None: upload_cursor.offset = requested_offset if f.tell() != upload_cursor.offset: f.seek(upload_cursor.offset) buf = f.read(current_chunk_size) # reset temporary status variables requested_offset = None current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE retry_number = globals.num_retries if len(buf) != 0: log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset) else: log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset)) res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info) upload_cursor.offset = f.tell() log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size)) progress.report_transfer(upload_cursor.offset, file_size) except ApiError as e: error = e.error if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset(): # Server reports that we should send another chunk. Most likely this is caused by # network error during previous upload attempt. In such case we'll get expected offset # from server and it's enough to just seek() and retry again new_offset = error.get_incorrect_offset().correct_offset log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset)) if requested_offset is not None: # chunk failed even after seek attempt. Something strange and no safe way to recover raise BackendException("dpbx: unable to chunk upload") else: # will seek and retry requested_offset = new_offset continue raise except ConnectionError as e: log.Debug('dpbx,files_upload_session_append: %s' % e) retry_number -= 1 if retry_number == 0: raise # We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload log.Info('dpbx: sleeping a bit before chunk retry') time.sleep(30) current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5 requested_offset = None continue if f.tell() != file_size: raise BackendException('dpbx: something wrong') log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata) progress.report_transfer(f.tell(), file_size) # A few sanity checks if res_metadata.path_display != remote_path: raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path)) if res_metadata.size != file_size: raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size)) finally: f.close()
def __exit__(self, exc_type, exc_val, exc_tb): commit = CommitInfo(path=path, mute=True) cur = UploadSessionCursor(self._sess_id, self._data_offset) self._dbx.files_upload_session_finish(b'', cur, commit)
def write(self, content): cur = UploadSessionCursor(self._sess_id, self._data_offset) self._dbx.files_upload_session_append_v2(content, cur) self._data_offset += len(content)
def upload(dropbox_helper_id, access_token, size, max_retries): from .models import DropboxUploadHelper helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id) dbx = Dropbox(access_token) try: with open(helper.src, 'rb') as f: chunk = f.read(CHUNK_SIZE) offset = len(chunk) upload_session = dbx.files_upload_session_start(chunk) while True: chunk = f.read(CHUNK_SIZE) if not chunk: break helper.progress = offset / size helper.save() dbx.files_upload_session_append_v2( chunk, UploadSessionCursor( upload_session.session_id, offset, ), ) offset += len(chunk) file_metadata = dbx.files_upload_session_finish( b'', UploadSessionCursor( upload_session.session_id, offset=offset, ), CommitInfo( '/{}'.format(os.path.basename(helper.src)), # When writing the file it won't overwrite an existing file, just add # another file like "filename (2).txt" WriteMode('add'), ), ) except Exception as e: helper.failure_reason = str(e) helper.save() couch_user = CouchUser.get_by_username(helper.user.username) if helper.failure_reason is None: path_link_metadata = dbx.sharing_create_shared_link_with_settings( file_metadata.path_display, SharedLinkSettings( requested_visibility=RequestedVisibility.team_only, ), ) context = { 'share_url': path_link_metadata.url, 'path': os.path.join( u'Apps', settings.DROPBOX_APP_NAME, path_link_metadata.name, ) } with localize(couch_user.get_language_code()): subject = _(u'{} has been uploaded to dropbox!'.format( helper.dest)) html_content = render_to_string( 'dropbox/emails/upload_success.html', context) text_content = render_to_string( 'dropbox/emails/upload_success.txt', context) else: context = {'reason': helper.failure_reason, 'path': helper.dest} with localize(couch_user.get_language_code()): subject = _(u'{} has failed to upload to dropbox'.format( helper.dest)) html_content = render_to_string('dropbox/emails/upload_error.html', context) text_content = render_to_string('dropbox/emails/upload_error.txt', context) send_HTML_email( subject, helper.user.email, html_content, text_content=text_content, )