def put(self, key, data=None, acl=None, metadata={}, mimetype=None,
transformer=None, headers={},args=None,subresource=None):
if isinstance(data, six.text_type):
data = data.encode(self.default_encoding)
headers = headers.copy()
if mimetype:
headers["Content-Type"] = str(mimetype)
elif "Content-Type" not in headers:
headers["Content-Type"] = guess_mimetype(key)
headers.update(metadata_headers(metadata))
if acl: headers["X-AMZ-ACL"] = acl
if transformer: data = transformer(headers, data)
if "Content-Length" not in headers:
# if isinstance(data, file) isinstance(data, FileChunkIO):
if hasattr(data,'fileno'):
headers["Content-Length"] = str(getSize(data.name))
elif hasattr(data,'__len__'):
headers["Content-Length"] = str(len(data))
else:
raise ValueError("Content-Length must be defined!!")
# if "s-sina-sha1" not in headers:
# headers["s-sina-sha1"] = aws_md5(data)
scsreq = self.request(method="PUT", key=key, data=data, headers=headers,
args=args, subresource=subresource)
scsResponse = self.send(scsreq)
return scsResponse
def put(self, key, data=None, acl=None, metadata={}, mimetype=None,
transformer=None, headers={},args=None,subresource=None):
if isinstance(data, six.text_type):
data = data.encode(self.default_encoding)
headers = headers.copy()
if mimetype:
headers["Content-Type"] = str(mimetype)
elif "Content-Type" not in headers:
headers["Content-Type"] = guess_mimetype(key)
headers.update(metadata_headers(metadata))
if acl: headers["X-AMZ-ACL"] = acl
if transformer: data = transformer(headers, data)
if "Content-Length" not in headers:
# if isinstance(data, file) isinstance(data, FileChunkIO):
if hasattr(data,'fileno'):
headers["Content-Length"] = str(getSize(data.name))
elif hasattr(data,'__len__'):
headers["Content-Length"] = str(len(data))
else:
raise ValueError("Content-Length must be defined!!")
# if "s-sina-sha1" not in headers:
# headers["s-sina-sha1"] = aws_md5(data)
scsreq = self.request(method="PUT", key=key, data=data, headers=headers,
args=args, subresource=subresource)
scsResponse = self.send(scsreq)
return scsResponse
def multipart_upload(self,
key_name,
source_path,
acl=None,
metadata={},
mimetype=None,
headers={},
cb=None,
num_cb=None,
part_failed_cb=None):
try:
# multipart portions copyright Fabian Topfstedt
# https://pypi.python.org/pypi/filechunkio/1.5
import math
import mimetypes
from multiprocessing import Pool
from sinastorage.vendored.filechunkio import FileChunkIO
multipart_capable = True
parallel_processes = 4
min_bytes_per_chunk = 5 * 1024 * 1024 #每片分片最大文件大小
usage_flag_multipart_capable = """ [--multipart]"""
usage_string_multipart_capable = """
multipart - Upload files as multiple parts. This needs filechunkio.
Requires ListBucket, ListMultipartUploadParts,
ListBucketMultipartUploads and PutObject permissions."""
except ImportError as err:
multipart_capable = False
usage_flag_multipart_capable = ""
usage_string_multipart_capable = '\n\n "' + \
err.message[len('No module named '):] + \
'" is missing for multipart support '
raise err
"""
Parallel multipart upload.
"""
multipart = self.initiate_multipart_upload(key_name, acl, metadata,
mimetype, headers)
source_size = getSize(source_path)
bytes_per_chunk = max(
int(math.sqrt(min_bytes_per_chunk) * math.sqrt(source_size)),
min_bytes_per_chunk)
chunk_amount = int(math.ceil(source_size / float(bytes_per_chunk)))
multipart.bytes_per_part = bytes_per_chunk
multipart.parts_amount = chunk_amount
pool = Pool(processes=parallel_processes)
i = 0
for part in multipart.get_next_part():
offset = i * bytes_per_chunk
remaining_bytes = source_size - offset
chunk_bytes = min([bytes_per_chunk, remaining_bytes])
pool.apply_async(func=_upload_part,
args=(
self.name,
key_name,
multipart.upload_id,
multipart.parts_amount,
part,
source_path,
offset,
chunk_bytes,
cb,
num_cb,
part_failed_cb,
),
callback=lambda part: multipart.parts.append(part)
if part is not None else None)
# partResult = _upload_part(bucketName, key_name, multipart.upload_id, multipart.parts_amount, part, source_path, offset, chunk_bytes,
# cb, num_cb)
# multipart.parts.append(partResult)
i = i + 1
pool.close()
pool.join()
if len(multipart.parts) == chunk_amount:
self.complete_multipart_upload(multipart)
# multipart.complete_upload()
# key = bucket.get_key(keyname)
# key.set_acl(acl)
else:
# mp.cancel_upload()
# print len(multipart.parts) , chunk_amount
six.print_(len(multipart.parts), chunk_amount)
raise RuntimeError("multipart upload is failed!!")
def multipart_upload(self, key_name, source_path, acl=None, metadata={}, mimetype=None,
headers={}, cb=None, num_cb=None):
try:
# multipart portions copyright Fabian Topfstedt
# https://pypi.python.org/pypi/filechunkio/1.5
import math
import mimetypes
from multiprocessing import Pool
from sinastorage.vendored.filechunkio import FileChunkIO
multipart_capable = True
parallel_processes = 4
min_bytes_per_chunk = 5 * 1024 * 1024 #每片分片最大文件大小
usage_flag_multipart_capable = """ [--multipart]"""
usage_string_multipart_capable = """
multipart - Upload files as multiple parts. This needs filechunkio.
Requires ListBucket, ListMultipartUploadParts,
ListBucketMultipartUploads and PutObject permissions."""
except ImportError as err:
multipart_capable = False
usage_flag_multipart_capable = ""
usage_string_multipart_capable = '\n\n "' + \
err.message[len('No module named '):] + \
'" is missing for multipart support '
raise err
"""
Parallel multipart upload.
"""
multipart = self.initiate_multipart_upload(key_name, acl, metadata, mimetype, headers)
source_size = getSize(source_path)
bytes_per_chunk = max(int(math.sqrt(min_bytes_per_chunk) * math.sqrt(source_size)),
min_bytes_per_chunk)
chunk_amount = int(math.ceil(source_size / float(bytes_per_chunk)))
multipart.bytes_per_part = bytes_per_chunk
multipart.parts_amount = chunk_amount
pool = Pool(processes=parallel_processes)
i = 0
for part in multipart.get_next_part():
offset = i * bytes_per_chunk
remaining_bytes = source_size - offset
chunk_bytes = min([bytes_per_chunk, remaining_bytes])
pool.apply_async(func = _upload_part,
args = (self.name, key_name, multipart.upload_id,
multipart.parts_amount, part, source_path,
offset, chunk_bytes,cb, num_cb,),
callback = lambda part : multipart.parts.append(part))
# partResult = _upload_part(bucketName, key_name, multipart.upload_id, multipart.parts_amount, part, source_path, offset, chunk_bytes,
# cb, num_cb)
# multipart.parts.append(partResult)
i = i + 1
pool.close()
pool.join()
if len(multipart.parts) == chunk_amount:
self.complete_multipart_upload(multipart)
# multipart.complete_upload()
# key = bucket.get_key(keyname)
# key.set_acl(acl)
else:
# mp.cancel_upload()
# print len(multipart.parts) , chunk_amount
six.print_(len(multipart.parts) , chunk_amount)
raise RuntimeError("multipart upload is failed!!")
