def _chunked_upload(self, fp, size, progress_callback):
url = self._client.get_url(self._url_template_content_chunked,
path=self.path)
chunks = list(
base.split_file_into_chunks(
fp, size, self._upload_chunk_size)) # need count of chunks
chunk_count = len(chunks)
headers = {}
for chunk_number, chunk in enumerate(chunks, 1): # count from 1 not 0
headers['x-egnyte-chunk-num'] = "%d" % chunk_number
headers['content-length'] = str(chunk.size)
if chunk_number == chunk_count: # last chunk
headers['x-egnyte-last-chunk'] = "true"
retries = max(self._upload_retries, 1)
while retries > 0:
r = self._client.POST(url, data=chunk, headers=headers)
server_sha = r.headers['x-egnyte-chunk-sha512-checksum']
our_sha = chunk.sha.hexdigest()
if server_sha == our_sha:
break
retries -= 1
# TODO: retry network errors too
# TODO: refactor common parts of chunked and standard upload
if retries == 0:
raise exc.ChecksumError("Failed to upload file chunk", {
"chunk_number": chunk_number,
"start_position": chunk.position
})
exc.default.check_response(r)
if chunk_number == 1:
headers['x-egnyte-upload-id'] = r.headers['x-egnyte-upload-id']
if progress_callback is not None:
progress_callback(self, size,
chunk_number * self._upload_chunk_size)
def _chunked_upload(self, fp, size, progress_callback):
url = self._client.get_url(self._url_template_content_chunked, path=self.path)
chunks = list(base.split_file_into_chunks(fp, size, self._upload_chunk_size)) # need count of chunks
chunk_count = len(chunks)
headers = {}
for chunk_number, chunk in enumerate(chunks, 1): # count from 1 not 0
headers['x-egnyte-chunk-num'] = "%d" % chunk_number
headers['content-length'] = chunk.size
if chunk_number == chunk_count: # last chunk
headers['x-egnyte-last-chunk'] = "true"
retries = max(self._upload_retries, 1)
while retries > 0:
r = self._client.POST(url, data=chunk, headers=headers)
server_sha = r.headers['x-egnyte-chunk-sha512-checksum']
our_sha = chunk.sha.hexdigest()
if server_sha == our_sha:
break
retries -= 1
# TODO: retry network errors too
# TODO: refactor common parts of chunked and standard upload
if retries == 0:
raise exc.ChecksumError("Failed to upload file chunk", {"chunk_number": chunk_number, "start_position": chunk.position})
exc.default.check_response(r)
if chunk_number == 1:
headers['x-egnyte-upload-id'] = r.headers['x-egnyte-upload-id']
if progress_callback is not None:
progress_callback(self, size, chunk_number * self._upload_chunk_size)
def _chunked_upload(self, fp, size, threads, chunksize_mb,
progress_callback):
url = self._client.get_url(self._url_template_content_chunked,
path=self.path)
chunks = list(
base.split_file_into_chunks(fp, size, chunksize_mb *
MEGABYTES)) # need count of chunks
chunk_count = len(chunks)
# upload first chunk
chunk = chunks.pop()
headers = self._upload_single_chunk(url,
1,
chunk,
size,
chunksize_mb,
progress_callback,
is_last_chunk=chunk_count == 1)
upload_id = headers['x-egnyte-upload-id']
# upload intermediate chunks in separate threads
if chunk_count > 2:
n = 1
for chunk_slice in six.moves.zip_longest(*[iter(chunks[:-1])] *
threads):
threads = []
for c in chunk_slice:
if c:
n += 1
threads.append(
threading.Thread(target=self._upload_single_chunk,
args=(url, n, c, size,
chunksize_mb,
progress_callback),
kwargs={'upload_id': upload_id}))
for t in threads:
t.start()
for t in threads:
t.join()
# upload last chunk
if chunk_count > 1:
self._upload_single_chunk(url,
chunk_count,
chunks[-1],
size,
chunksize_mb,
progress_callback,
upload_id=upload_id,
is_last_chunk=True)
def _chunked_upload(self, fp, size, threads, chunksize_mb, progress_callback):
url = self._client.get_url(self._url_template_content_chunked, path=self.path)
chunks = list(base.split_file_into_chunks(fp, size, chunksize_mb*MEGABYTES)) # need count of chunks
chunk_count = len(chunks)
# upload first chunk
chunk = chunks.pop()
headers = self._upload_single_chunk(url, 1, chunk, size, chunksize_mb, progress_callback, is_last_chunk=chunk_count==1)
upload_id = headers['x-egnyte-upload-id']
# upload intermediate chunks in separate threads
if chunk_count > 2:
n = 1
for chunk_slice in six.moves.zip_longest(*[iter(chunks[:-1])]*threads):
threads = []
for c in chunk_slice:
if c:
n += 1
threads.append(threading.Thread(target=self._upload_single_chunk, args=(url, n, c, size, chunksize_mb, progress_callback), kwargs={'upload_id': upload_id}))
for t in threads:
t.start()
for t in threads:
t.join()
# upload last chunk
if chunk_count > 1:
self._upload_single_chunk(url, chunk_count, chunks[-1], size, chunksize_mb, progress_callback, upload_id=upload_id, is_last_chunk=True)