def put(self, key, data=None, acl=None, metadata={}, mimetype=None, transformer=None, headers={},args=None,subresource=None): if isinstance(data, unicode): data = data.encode(self.default_encoding) headers = headers.copy() if mimetype: headers["Content-Type"] = str(mimetype) elif "Content-Type" not in headers: headers["Content-Type"] = guess_mimetype(key) headers.update(metadata_headers(metadata)) if acl: headers["X-AMZ-ACL"] = acl if transformer: data = transformer(headers, data) if "Content-Length" not in headers: # if isinstance(data, file) isinstance(data, FileChunkIO): if hasattr(data,'fileno'): headers["Content-Length"] = str(getSize(data.name)) elif hasattr(data,'__len__'): headers["Content-Length"] = str(len(data)) else: raise ValueError("Content-Length must be defined!!") # if "s-sina-sha1" not in headers: # headers["s-sina-sha1"] = aws_md5(data) scsreq = self.request(method="PUT", key=key, data=data, headers=headers, args=args, subresource=subresource) scsResponse = self.send(scsreq) return scsResponse
def multipart_upload(self, key_name, source_path, acl=None, metadata={}, mimetype=None, headers={}, cb=None, num_cb=None): try: # multipart portions copyright Fabian Topfstedt # https://pypi.python.org/pypi/filechunkio/1.5 import math import mimetypes from multiprocessing import Pool from filechunkio import FileChunkIO multipart_capable = True parallel_processes = 4 min_bytes_per_chunk = 5 * 1024 * 1024 #每片分片最大文件大小 usage_flag_multipart_capable = """ [--multipart]""" usage_string_multipart_capable = """ multipart - Upload files as multiple parts. This needs filechunkio. Requires ListBucket, ListMultipartUploadParts, ListBucketMultipartUploads and PutObject permissions.""" except ImportError as err: multipart_capable = False usage_flag_multipart_capable = "" usage_string_multipart_capable = '\n\n "' + \ err.message[len('No module named '):] + \ '" is missing for multipart support ' raise err """ Parallel multipart upload. """ multipart = self.initiate_multipart_upload(key_name, acl, metadata, mimetype, headers) source_size = getSize(source_path) bytes_per_chunk = max(int(math.sqrt(min_bytes_per_chunk) * math.sqrt(source_size)), min_bytes_per_chunk) chunk_amount = int(math.ceil(source_size / float(bytes_per_chunk))) multipart.bytes_per_part = bytes_per_chunk multipart.parts_amount = chunk_amount pool = Pool(processes=parallel_processes) i = 0 for part in multipart.get_next_part(): offset = i * bytes_per_chunk remaining_bytes = source_size - offset chunk_bytes = min([bytes_per_chunk, remaining_bytes]) pool.apply_async(_upload_part, args = (self.name, key_name, multipart.upload_id, multipart.parts_amount, part, source_path, offset, chunk_bytes, cb, num_cb,), callback=lambda part : multipart.parts.append(part)) # partResult = _upload_part(bucketName, key_name, multipart.upload_id, multipart.parts_amount, part, source_path, offset, chunk_bytes, # cb, num_cb) # multipart.parts.append(partResult) i = i + 1 pool.close() pool.join() if len(multipart.parts) == chunk_amount: self.complete_multipart_upload(multipart) # multipart.complete_upload() # key = bucket.get_key(keyname) # key.set_acl(acl) else: # mp.cancel_upload() print len(multipart.parts) , chunk_amount raise RuntimeError("multipart upload is failed!!")