Ejemplo n.º 1
0
    def put(self, key, data=None, acl=None, metadata={}, mimetype=None,
            transformer=None, headers={},args=None,subresource=None):
        if isinstance(data, unicode):
            data = data.encode(self.default_encoding)
        headers = headers.copy()
        if mimetype:
            headers["Content-Type"] = str(mimetype)
        elif "Content-Type" not in headers:
            headers["Content-Type"] = guess_mimetype(key)
        headers.update(metadata_headers(metadata))
        if acl: headers["X-AMZ-ACL"] = acl
        if transformer: data = transformer(headers, data)
        if "Content-Length" not in headers:
#             if isinstance(data, file)  isinstance(data, FileChunkIO):
            if hasattr(data,'fileno'):
                headers["Content-Length"] = str(getSize(data.name))
            elif hasattr(data,'__len__'):
                headers["Content-Length"] = str(len(data))
            else:
                raise ValueError("Content-Length must be defined!!")
                
#         if "s-sina-sha1" not in headers:
#             headers["s-sina-sha1"] = aws_md5(data)
        scsreq = self.request(method="PUT", key=key, data=data, headers=headers, 
                              args=args, subresource=subresource)
        scsResponse = self.send(scsreq)
        return scsResponse
Ejemplo n.º 2
0
    def multipart_upload(self, key_name, source_path, acl=None, metadata={}, mimetype=None,
            headers={}, cb=None, num_cb=None):
        try:
            # multipart portions copyright Fabian Topfstedt
            # https://pypi.python.org/pypi/filechunkio/1.5
        
            import math
            import mimetypes
            from multiprocessing import Pool
            from filechunkio import FileChunkIO
            multipart_capable = True
            parallel_processes = 4
            min_bytes_per_chunk = 5 * 1024 * 1024                     #每片分片最大文件大小
            usage_flag_multipart_capable = """ [--multipart]"""
            usage_string_multipart_capable = """
                multipart - Upload files as multiple parts. This needs filechunkio.
                            Requires ListBucket, ListMultipartUploadParts,
                            ListBucketMultipartUploads and PutObject permissions."""
        except ImportError as err:
            multipart_capable = False
            usage_flag_multipart_capable = ""
            usage_string_multipart_capable = '\n\n     "' + \
                err.message[len('No module named '):] + \
                '" is missing for multipart support '
            
            raise err
            
        """
        Parallel multipart upload.
        """
        multipart = self.initiate_multipart_upload(key_name, acl, metadata, mimetype, headers)
        source_size = getSize(source_path)
        bytes_per_chunk = max(int(math.sqrt(min_bytes_per_chunk) * math.sqrt(source_size)),
                              min_bytes_per_chunk)
        chunk_amount = int(math.ceil(source_size / float(bytes_per_chunk)))
        multipart.bytes_per_part = bytes_per_chunk
        multipart.parts_amount = chunk_amount
        
        pool = Pool(processes=parallel_processes)
        i = 0
        for part in multipart.get_next_part():
            offset = i * bytes_per_chunk
            remaining_bytes = source_size - offset
            chunk_bytes = min([bytes_per_chunk, remaining_bytes])
            pool.apply_async(_upload_part, args = (self.name, key_name, multipart.upload_id, multipart.parts_amount, part, source_path, offset, chunk_bytes,
                                            cb, num_cb,), callback=lambda part : multipart.parts.append(part))
#             partResult = _upload_part(bucketName, key_name, multipart.upload_id, multipart.parts_amount, part, source_path, offset, chunk_bytes,
#                                             cb, num_cb)
            
#             multipart.parts.append(partResult)
            
            i = i + 1
            
        pool.close()
        pool.join()
    
        if len(multipart.parts) == chunk_amount:
            self.complete_multipart_upload(multipart)
#             multipart.complete_upload()
#             key = bucket.get_key(keyname)
#             key.set_acl(acl)
        else:
#             mp.cancel_upload()
            print  len(multipart.parts) , chunk_amount

            raise RuntimeError("multipart upload is failed!!")