def upload_file(self, file_to_upload, path):
file_size = self._get_file_size(file_to_upload)
file_to_upload.seek(0)
if file_size <= DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT:
logging.debug('Using single request to upload file')
self.__dropbox_client.files_upload(file_to_upload.read(), path)
else:
logging.debug('Using multi-request upload session for this file')
session = None
offset = 0
chunk = file_to_upload.read(
DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT)
while len(chunk) == DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT:
if session is None:
logging.debug('Initializing upload session')
session = self.__dropbox_client.files_upload_session_start(
chunk)
else:
logging.debug('Appending to session %s at offset %d',
session.session_id, offset)
self.__dropbox_client.files_upload_session_append_v2(
chunk, UploadSessionCursor(session.session_id, offset))
offset += len(chunk)
chunk = file_to_upload.read(
DropboxClient.SINGLE_REQ_UPLOAD_SIZE_LIMIT)
logging.debug('Finishing session %s', session.session_id)
commit_info = CommitInfo(path=path,
mode=WriteMode('add'),
autorename=False)
self.__dropbox_client.files_upload_session_finish(
chunk, UploadSessionCursor(session.session_id, offset),
commit_info)
def upload_large_dropbox_file(client, source_full_path, destination_full_path):
"""
Uploads a large (>CHUNK_SIZE) single file to Dropbox.
"""
file_size = os.path.getsize(source_full_path)
with open(source_full_path, 'rb') as f:
try:
upload_session_start_result = client.files_upload_session_start(
f.read(CHUNK_SIZE))
session_id = upload_session_start_result.session_id
cursor = UploadSessionCursor(session_id=session_id,
offset=f.tell())
commit = CommitInfo(path=destination_full_path)
while f.tell() < file_size:
if ((file_size - f.tell()) <= CHUNK_SIZE):
print(
client.files_upload_session_finish(
f.read(CHUNK_SIZE), cursor, commit))
else:
client.files_upload_session_append(f.read(CHUNK_SIZE),
cursor.session_id,
cursor.offset)
cursor.offset = f.tell()
except ApiError as e:
print(f'Failed to upload file {source_full_path}')
print(f'{source_full_path} successfully uploaded to ' \
f'{destination_full_path}')
def make_backup_dropbox(self, ts, name, dump_stream, info_file, info_file_content, cloud_params):
# Upload two backup objects to Dropbox
DropboxService = self.env['ir.config_parameter'].get_dropbox_service()
folder_path = self.env['ir.config_parameter'].get_param("odoo_backup_sh_dropbox.dropbox_folder_path")
info_file_size = info_file.tell()
dump_stream.seek(0)
info_file.seek(0)
for obj, obj_name, file_size in \
[[dump_stream, compute_backup_filename(name, ts, info_file_content.get('encrypted')), info_file_content.get("backup_size") * 1024 * 1024],
[info_file, compute_backup_info_filename(name, ts), info_file_size]]:
# The full path to upload the file to, including the file name
full_path = "{folder_path}/{file_name}".format(
folder_path=folder_path,
file_name=obj_name,
)
# from here: https://www.dropboxforum.com/t5/API-Support-Feedback/python-upload-big-file-example/m-p/166627/highlight/true#M6013
if file_size <= CHUNK_SIZE:
DropboxService.files_upload(obj.read(), full_path)
else:
upload_session_start_result = DropboxService.files_upload_session_start(obj.read(CHUNK_SIZE))
cursor = UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=obj.tell())
commit = CommitInfo(path=full_path)
while obj.tell() < file_size:
if ((file_size - obj.tell()) <= CHUNK_SIZE):
DropboxService.files_upload_session_finish(obj.read(CHUNK_SIZE), cursor, commit)
else:
DropboxService.files_upload_session_append(obj.read(CHUNK_SIZE), cursor.session_id,
cursor.offset)
cursor.offset = obj.tell()
def _upload_session(self, bytes, finish):
if self._session is None:
start_result = self._dbx.files_upload_session_start(bytes)
self._session = start_result.session_id
elif finish:
cursor = UploadSessionCursor(self._session, self._total_bytes)
info = CommitInfo(path=self._upload_path,
mode=WriteMode('overwrite'),
client_modified=self._file_retriever.modified)
self._dbx.files_upload_session_finish(bytes, cursor, info)
else:
cursor = UploadSessionCursor(self._session, self._total_bytes)
self._dbx.files_upload_session_append_v2(bytes, cursor)
def _chunked_upload(self, content, dest_path):
upload_session = self.client.files_upload_session_start(
content.read(self.CHUNK_SIZE))
cursor = UploadSessionCursor(session_id=upload_session.session_id,
offset=content.tell())
commit = CommitInfo(path=dest_path)
while content.tell() < content.size:
if (content.size - content.tell()) <= self.CHUNK_SIZE:
self.client.files_upload_session_finish(
content.read(self.CHUNK_SIZE), cursor, commit)
else:
self.client.files_upload_session_append_v2(
content.read(self.CHUNK_SIZE), cursor)
cursor.offset = content.tell()
def dropbox_upload_export(access_token, local_path, remote_fname):
client = Dropbox(access_token)
with open(local_path, 'rb') as inf:
chunk_size = int(1.5e8 - 1)
next_bytes = inf.read(chunk_size)
upload_session_result = client.files_upload_session_start(next_bytes)
uploaded_so_far = len(next_bytes)
next_bytes = inf.read(chunk_size)
cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far)
while next_bytes:
client.files_upload_session_append_v2(next_bytes, cursor)
next_bytes = inf.read(chunk_size)
uploaded_so_far += chunk_size
cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far)
client.files_upload_session_finish('', cursor, CommitInfo(remote_fname))
def put(self, source, dest, retry=True):
dest = self._fix_slashes(dest)
if (self.client is not None):
# open the file and get its size
f = open(source, 'rb')
f_size = os.path.getsize(source)
try:
if (f_size < self.MAX_CHUNK):
# use the regular upload
self.client.files_upload(f.read(),
dest,
mode=WriteMode('overwrite'))
else:
# start the upload session
upload_session = self.client.files_upload_session_start(
f.read(self.MAX_CHUNK))
upload_cursor = UploadSessionCursor(
upload_session.session_id, f.tell())
while (f.tell() < f_size):
# check if we should finish the upload
if ((f_size - f.tell()) <= self.MAX_CHUNK):
# upload and close
self.client.files_upload_session_finish(
f.read(self.MAX_CHUNK), upload_cursor,
CommitInfo(dest, mode=WriteMode('overwrite')))
else:
# upload a part and store the offset
self.client.files_upload_session_append_v2(
f.read(self.MAX_CHUNK), upload_cursor)
upload_cursor.offset = f.tell()
# if no errors we're good!
return True
except Exception as anError:
utils.log(str(anError))
# if we have an exception retry
if (retry):
return self.put(source, dest, False)
else:
# tried once already, just quit
return False
else:
return False
def sync(self, file_path: Path) -> None:
"""Sync a file to Dropbox."""
# Check if Dropbox token is valid.
if self.valid is False:
error = "Dropbox token is invalid!"
self.success = False
self.output = error
log.error(error)
return None
# This is size what can be uploaded as one chunk.
# When file is bigger than that, this will be uploaded
# in multiple parts.
chunk_size = 4 * 1024 * 1024
temp_file, recompressed = self.compress(file_path)
upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}"
try:
with temp_file as f:
file_size = os.stat(f.name).st_size
log.debug(file_size)
if file_size <= chunk_size:
self.client.files_upload(f.read(), upload_path,
WriteMode.overwrite)
else:
session_start = self.client.files_upload_session_start(
f.read(chunk_size))
cursor = UploadSessionCursor(session_start.session_id,
offset=f.tell())
# Commit contains path in Dropbox and write mode about file
commit = CommitInfo(upload_path, WriteMode.overwrite)
while f.tell() < file_size:
if (file_size - f.tell()) <= chunk_size:
self.client.files_upload_session_finish(
f.read(chunk_size), cursor, commit)
else:
self.client.files_upload_session_append(
f.read(chunk_size), cursor.session_id,
cursor.offset)
cursor.offset = f.tell()
self.success = True
except (ApiError, HttpError) as e:
log.error(e)
self.success = False
self.output = str(e)
def upload_file(dbx, LOCALFILE):
"""Uploads a local file to Dropbox in chunks <=4MiB each"""
with open(LOCALFILE, 'rb') as f:
BACKUPPATH = '/' + os.path.split(LOCALFILE)[1]
file_size = os.path.getsize(LOCALFILE)
if file_size <= CHUNK_SIZE:
dbx.files_upload(f.read(), BACKUPPATH, mode=WriteMode('overwrite'))
else:
upload_session_start_result = dbx.files_upload_session_start(f.read(CHUNK_SIZE))
cursor = UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=f.tell())
commit = CommitInfo(path=BACKUPPATH)
while f.tell() < file_size:
if ((file_size - f.tell()) <= CHUNK_SIZE):
dbx.files_upload_session_finish(f.read(CHUNK_SIZE), cursor, commit)
else:
dbx.files_upload_session_append(f.read(CHUNK_SIZE), cursor.session_id, cursor.offset)
cursor.offset = f.tell()
def upload_to_dropbox(access_token,
dropbox_path,
file_path,
progress_callback=None):
dbx = Dropbox(access_token)
with open(file_path, 'rb') as file:
chunk = file.read(CHUNK_SIZE)
offset = len(chunk)
upload_session = dbx.files_upload_session_start(chunk)
progress_callback and progress_callback(offset)
while True:
chunk = file.read(CHUNK_SIZE)
if not chunk:
break
dbx.files_upload_session_append_v2(
chunk,
UploadSessionCursor(
upload_session.session_id,
offset,
),
)
offset += len(chunk)
progress_callback and progress_callback(offset)
file_metadata = dbx.files_upload_session_finish(
b'',
UploadSessionCursor(
upload_session.session_id,
offset=offset,
),
CommitInfo(
dropbox_path,
# When writing the file it won't overwrite an existing file, just add
# another file like "filename (2).txt"
WriteMode('add'),
),
)
progress_callback and progress_callback(offset)
return file_metadata.path_display
def put_file_chunked(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug(
'dpbx,files_upload_session_start([%d bytes]), total: %d' %
(len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id,
f.tell())
commit_info = CommitInfo(remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
is_eof = False
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed
# chunk
while not is_eof or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
is_eof = f.tell() >= file_size
if not is_eof and len(buf) == 0:
continue
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if not is_eof:
assert len(buf) != 0
log.Debug(
'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(
buf, upload_cursor.session_id,
upload_cursor.offset)
else:
log.Debug(
'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(
buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' %
(upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError
) and error.is_incorrect_offset():
# Server reports that we should send another chunk.
# Most likely this is caused by network error during
# previous upload attempt. In such case we'll get
# expected offset from server and it's enough to just
# seek() and retry again
new_offset = error.get_incorrect_offset(
).correct_offset
log.Debug(
'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)'
% (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something
# strange and no safe way to recover
raise BackendException(
"dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if not self.user_authenticated():
self.login()
if retry_number == 0:
raise
# We don't know for sure, was partial upload successful or
# not. So it's better to retry smaller amount to avoid extra
# reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
return res_metadata
finally:
f.close()
def __exit__(self, exc_type, exc_val, exc_tb):
commit = CommitInfo(path=path, mute=True)
cur = UploadSessionCursor(self._sess_id, self._data_offset)
self._dbx.files_upload_session_finish(b'', cur, commit)
def upload(dropbox_helper_id, access_token, size, max_retries):
from .models import DropboxUploadHelper
helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id)
dbx = Dropbox(access_token)
try:
with open(helper.src, 'rb') as f:
chunk = f.read(CHUNK_SIZE)
offset = len(chunk)
upload_session = dbx.files_upload_session_start(chunk)
while True:
chunk = f.read(CHUNK_SIZE)
if not chunk:
break
helper.progress = offset / size
helper.save()
dbx.files_upload_session_append_v2(
chunk,
UploadSessionCursor(
upload_session.session_id,
offset,
),
)
offset += len(chunk)
file_metadata = dbx.files_upload_session_finish(
b'',
UploadSessionCursor(
upload_session.session_id,
offset=offset,
),
CommitInfo(
'/{}'.format(os.path.basename(helper.src)),
# When writing the file it won't overwrite an existing file, just add
# another file like "filename (2).txt"
WriteMode('add'),
),
)
except Exception as e:
helper.failure_reason = str(e)
helper.save()
couch_user = CouchUser.get_by_username(helper.user.username)
if helper.failure_reason is None:
path_link_metadata = dbx.sharing_create_shared_link_with_settings(
file_metadata.path_display,
SharedLinkSettings(
requested_visibility=RequestedVisibility.team_only, ),
)
context = {
'share_url':
path_link_metadata.url,
'path':
os.path.join(
u'Apps',
settings.DROPBOX_APP_NAME,
path_link_metadata.name,
)
}
with localize(couch_user.get_language_code()):
subject = _(u'{} has been uploaded to dropbox!'.format(
helper.dest))
html_content = render_to_string(
'dropbox/emails/upload_success.html', context)
text_content = render_to_string(
'dropbox/emails/upload_success.txt', context)
else:
context = {'reason': helper.failure_reason, 'path': helper.dest}
with localize(couch_user.get_language_code()):
subject = _(u'{} has failed to upload to dropbox'.format(
helper.dest))
html_content = render_to_string('dropbox/emails/upload_error.html',
context)
text_content = render_to_string('dropbox/emails/upload_error.txt',
context)
send_HTML_email(
subject,
helper.user.email,
html_content,
text_content=text_content,
)
