Example #1
0
    def upload_part(self, seq, offset, chunk_size, labels, buffer = '', remote_status = None):
        """
        Upload a file chunk
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
        """
        # TODO implement Content-MD5
        debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size))

        if remote_status is not None:
            if int(remote_status['size']) == chunk_size:
                checksum = calculateChecksum(buffer, self.file, offset, chunk_size, self.s3.config.send_chunk)
                remote_checksum = remote_status['checksum'].strip('"\'')
                if remote_checksum == checksum:
                    warning("MultiPart: size and md5sum match for %s part %d, skipping." % (self.uri, seq))
                    self.parts[seq] = remote_status['checksum']
                    return
                else:
                    warning("MultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading."
                            % (remote_checksum, checksum, self.uri, seq))
            else:
                warning("MultiPart: size (%d vs %d) does not match for %s part %d, reuploading."
                        % (int(remote_status['size']), chunk_size, self.uri, seq))

        headers = { "content-length": str(chunk_size) }
        query_string = "?partNumber=%i&uploadId=%s" % (seq, encode_to_s3(self.upload_id))
        request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)
        response = self.s3.send_file(request, self.file, labels, buffer, offset = offset, chunk_size = chunk_size)
        self.parts[seq] = response["headers"]["etag"]
        return response
Example #2
0
def sign_string_v4(method='GET', host='', canonical_uri='/', params={}, region='us-east-1', cur_headers={}, body=''):
    service = 's3'

    cfg = Config.Config()
    access_key = cfg.access_key
    secret_key = cfg.secret_key

    t = datetime.datetime.utcnow()
    amzdate = t.strftime('%Y%m%dT%H%M%SZ')
    datestamp = t.strftime('%Y%m%d')

    canonical_querystring = '&'.join(['%s=%s' % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())])

    splits = canonical_uri.split('?')

    canonical_uri = quote_param(splits[0], quote_backslashes=False)
    canonical_querystring += '&'.join([('%s' if '=' in qs else '%s=') % qs for qs in splits[1:]])

    if type(body) == type(sha256('')):
        payload_hash = body.hexdigest()
    else:
        payload_hash = sha256(body).hexdigest()

    canonical_headers = {'host' : host,
                         'x-amz-content-sha256': payload_hash,
                         'x-amz-date' : amzdate
                         }
    signed_headers = 'host;x-amz-content-sha256;x-amz-date'

    for header in cur_headers.keys():
        # avoid duplicate headers and previous Authorization
        if header == 'Authorization' or header in signed_headers.split(';'):
            continue
        canonical_headers[header.strip()] = str(cur_headers[header]).strip()
        signed_headers += ';' + header.strip()

    # sort headers into a string
    canonical_headers_str = ''
    for k, v in sorted(canonical_headers.items()):
        canonical_headers_str += k + ":" + v + "\n"

    canonical_headers = canonical_headers_str
    debug(u"canonical_headers = %s" % canonical_headers)
    signed_headers = ';'.join(sorted(signed_headers.split(';')))

    canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
    debug('Canonical Request:\n%s\n----------------------' % canonical_request)

    algorithm = 'AWS4-HMAC-SHA256'
    credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
    string_to_sign = algorithm + '\n' +  amzdate + '\n' +  credential_scope + '\n' +  sha256(canonical_request).hexdigest()
    signing_key = getSignatureKey(secret_key, datestamp, region, service)
    signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
    authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ',' +  'SignedHeaders=' + signed_headers + ',' + 'Signature=' + signature
    headers = dict(cur_headers.items() + {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256': payload_hash}.items())
    debug("signature-v4 headers: %s" % headers)
    return headers
Example #3
0
def sign_string_v4(method='GET', host='', canonical_uri='/', params={}, region='us-east-1', cur_headers={}, body=''):
    service = 's3'

    cfg = Config.Config()
    access_key = cfg.access_key
    secret_key = cfg.secret_key

    t = datetime.datetime.utcnow()
    amzdate = t.strftime('%Y%m%dT%H%M%SZ')
    datestamp = t.strftime('%Y%m%d')

    canonical_querystring = '&'.join(['%s=%s' % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())])

    splits = canonical_uri.split('?')

    canonical_uri = quote_param(splits[0], quote_backslashes=False)
    canonical_querystring += '&'.join([('%s' if '=' in qs else '%s=') % qs for qs in splits[1:]])

    if type(body) == type(sha256('')):
        payload_hash = body.hexdigest()
    else:
        payload_hash = sha256(body).hexdigest()

    canonical_headers = {'host' : host,
                         'x-amz-content-sha256': payload_hash,
                         'x-amz-date' : amzdate
                         }
    signed_headers = 'host;x-amz-content-sha256;x-amz-date'

    for header in cur_headers.keys():
        # avoid duplicate headers and previous Authorization
        if header == 'Authorization' or header in signed_headers.split(';'):
            continue
        canonical_headers[header.strip()] = str(cur_headers[header]).strip()
        signed_headers += ';' + header.strip()

    # sort headers into a string
    canonical_headers_str = ''
    for k, v in sorted(canonical_headers.items()):
        canonical_headers_str += k + ":" + v + "\n"

    canonical_headers = canonical_headers_str
    debug(u"canonical_headers = %s" % canonical_headers)
    signed_headers = ';'.join(sorted(signed_headers.split(';')))

    canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
    debug('Canonical Request:\n%s\n----------------------' % canonical_request)

    algorithm = 'AWS4-HMAC-SHA256'
    credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
    string_to_sign = algorithm + '\n' +  amzdate + '\n' +  credential_scope + '\n' +  sha256(canonical_request).hexdigest()
    signing_key = getSignatureKey(secret_key, datestamp, region, service)
    signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
    authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ',' +  'SignedHeaders=' + signed_headers + ',' + 'Signature=' + signature
    headers = dict(cur_headers.items() + {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256': payload_hash}.items())
    debug("signature-v4 headers: %s" % headers)
    return headers
Example #4
0
    def upload_part(self,
                    seq,
                    offset,
                    chunk_size,
                    labels,
                    buffer='',
                    remote_status=None):
        """
        Upload a file chunk
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
        """
        # TODO implement Content-MD5
        debug("Uploading part %i of %r (%s bytes)" %
              (seq, self.upload_id, chunk_size))

        if remote_status is not None:
            if int(remote_status['size']) == chunk_size:
                checksum = calculateChecksum(buffer, self.file, offset,
                                             chunk_size,
                                             self.s3.config.send_chunk)
                remote_checksum = remote_status['checksum'].strip('"\'')
                if remote_checksum == checksum:
                    warning(
                        "MultiPart: size and md5sum match for %s part %d, skipping."
                        % (self.uri, seq))
                    self.parts[seq] = remote_status['checksum']
                    return
                else:
                    warning(
                        "MultiPart: checksum (%s vs %s) does not match for %s part %d, reuploading."
                        % (remote_checksum, checksum, self.uri, seq))
            else:
                warning(
                    "MultiPart: size (%d vs %d) does not match for %s part %d, reuploading."
                    % (int(remote_status['size']), chunk_size, self.uri, seq))

        headers = {"content-length": str(chunk_size)}
        query_string = "?partNumber=%i&uploadId=%s" % (
            seq, encode_to_s3(self.upload_id))
        request = self.s3.create_request("OBJECT_PUT",
                                         uri=self.uri,
                                         headers=headers,
                                         extra=query_string)
        response = self.s3.send_file(request,
                                     self.file,
                                     labels,
                                     buffer,
                                     offset=offset,
                                     chunk_size=chunk_size)
        self.parts[seq] = response["headers"]["etag"]
        return response
Example #5
0
    def complete_multipart_upload(self):
        """
        Finish a multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
        """
        debug("MultiPart: Completing upload: %s" % self.upload_id)

        parts_xml = []
        part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>"
        for seq, etag in self.parts.items():
            parts_xml.append(part_xml % (seq, etag))
        body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % (
            "".join(parts_xml))

        headers = {"content-length": str(len(body))}
        request = self.s3.create_request("OBJECT_POST",
                                         uri=self.uri,
                                         headers=headers,
                                         extra="?uploadId=%s" %
                                         encode_to_s3(self.upload_id),
                                         body=body)
        response = self.s3.send_request(request)

        return response
Example #6
0
def getSignatureKey(key, dateStamp, regionName, serviceName):
    kDate = sign(encode_to_s3('AWS4' + key), dateStamp)
    kRegion = sign(kDate, regionName)
    kService = sign(kRegion, serviceName)
    kSigning = sign(kService, 'aws4_request')
    return kSigning
Example #7
0
def sign(key, msg):
    return hmac.new(key, encode_to_s3(msg), sha256).digest()
Example #8
0
    def complete_multipart_upload(self):
        """
        Finish a multipart upload
        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
        """
        debug("MultiPart: Completing upload: %s" % self.upload_id)

        parts_xml = []
        part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>"
        for seq, etag in self.parts.items():
            parts_xml.append(part_xml % (seq, etag))
        body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml))

        headers = { "content-length": str(len(body)) }
        request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = headers, extra = "?uploadId=%s" % encode_to_s3(self.upload_id), body = body)
        response = self.s3.send_request(request)

        return response
Example #9
0
def sign_string_v4(method="GET", host="", canonical_uri="/", params={}, region="us-east-1", cur_headers={}, body=""):
    service = "s3"

    cfg = Config.Config()
    access_key = cfg.access_key
    secret_key = cfg.secret_key

    t = datetime.datetime.utcnow()
    amzdate = t.strftime("%Y%m%dT%H%M%SZ")
    datestamp = t.strftime("%Y%m%d")

    canonical_querystring = "&".join(
        ["%s=%s" % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())]
    )

    splits = canonical_uri.split("?")

    canonical_uri = quote_param(splits[0], quote_backslashes=False)
    canonical_querystring += "&".join([("%s" if "=" in qs else "%s=") % qs for qs in splits[1:]])

    if type(body) == type(sha256("")):
        payload_hash = body.hexdigest()
    else:
        payload_hash = sha256(body).hexdigest()

    canonical_headers = {"host": host, "x-amz-content-sha256": payload_hash, "x-amz-date": amzdate}
    signed_headers = "host;x-amz-content-sha256;x-amz-date"

    for header in cur_headers.keys():
        # avoid duplicate headers and previous Authorization
        if header == "Authorization" or header in signed_headers.split(";"):
            continue
        canonical_headers[header.strip()] = str(cur_headers[header]).strip()
        signed_headers += ";" + header.strip()

    # sort headers into a string
    canonical_headers_str = ""
    for k, v in sorted(canonical_headers.items()):
        canonical_headers_str += k + ":" + v + "\n"

    canonical_headers = canonical_headers_str
    debug(u"canonical_headers = %s" % canonical_headers)
    signed_headers = ";".join(sorted(signed_headers.split(";")))

    canonical_request = (
        method
        + "\n"
        + canonical_uri
        + "\n"
        + canonical_querystring
        + "\n"
        + canonical_headers
        + "\n"
        + signed_headers
        + "\n"
        + payload_hash
    )
    debug("Canonical Request:\n%s\n----------------------" % canonical_request)

    algorithm = "AWS4-HMAC-SHA256"
    credential_scope = datestamp + "/" + region + "/" + service + "/" + "aws4_request"
    string_to_sign = algorithm + "\n" + amzdate + "\n" + credential_scope + "\n" + sha256(canonical_request).hexdigest()
    signing_key = getSignatureKey(secret_key, datestamp, region, service)
    signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
    authorization_header = (
        algorithm
        + " "
        + "Credential="
        + access_key
        + "/"
        + credential_scope
        + ","
        + "SignedHeaders="
        + signed_headers
        + ","
        + "Signature="
        + signature
    )
    headers = dict(
        cur_headers.items()
        + {"x-amz-date": amzdate, "Authorization": authorization_header, "x-amz-content-sha256": payload_hash}.items()
    )
    debug("signature-v4 headers: %s" % headers)
    return headers
Example #10
0
def getSignatureKey(key, dateStamp, regionName, serviceName):
    kDate = sign(encode_to_s3("AWS4" + key), dateStamp)
    kRegion = sign(kDate, regionName)
    kService = sign(kRegion, serviceName)
    kSigning = sign(kService, "aws4_request")
    return kSigning
Example #11
0
def sign(key, msg):
    return hmac.new(key, encode_to_s3(msg), sha256).digest()