def _is_unmodified(self, s3_key):
local_path = self._get_local_path_from_key(s3_key.name)
with open(local_path, 'rb') as f:
if utils.f_sizeof(f) != s3_key.size:
return False
if not '-' in s3_key.md5:
local_md5 = hexlify(utils.f_md5(f))
else:
chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE)
m = hashlib.md5()
for chunk in chunks:
m.update(utils.f_md5(chunk))
local_md5 = "{}-{}".format(hexlify(m.digest()), len(chunks))
return local_md5 == s3_key.md5
def _multipart_upload(self, f):
upload_id = self._initiate_multipart_upload()
# Upload individual parts
upload_reqs = []
parts = []
chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE)
for r in range(len(chunks)):
upload_reqs.append([
self._multipart_upload_part,
chunks[r],
r+1,
upload_id
])
parts.append((r+1, hexlify(utils.f_md5(chunks[r]))))
try:
self.conn.join(upload_reqs)
except:
self._abort_multipart_upload(upload_id)
raise
self._complete_multipart_upload(upload_id, parts)
def _multipart_upload(self, file_like_object):
self.log_upload(file_like_object)
upload_id = self._initiate_multipart_upload()
# Upload individual parts
upload_reqs = []
parts = []
chunks = utils.f_chunk(file_like_object, constants.MULTIPART_PART_SIZE)
for r in range(len(chunks)):
upload_reqs.append([
self._multipart_upload_part,
chunks[r],
r+1,
upload_id
])
parts.append((r+1, hexlify(utils.f_md5(chunks[r]))))
try:
self.conn.join(upload_reqs)
except:
self._abort_multipart_upload(upload_id)
raise
self._complete_multipart_upload(upload_id, parts)
def test_f_md5():
s = StringIO('hello')
s.seek(2, 0)
expected = '5d41402abc4b2a76b9719d911017c592'
assert hexlify(utils.f_md5(s)) == expected
assert s.tell() == 2
def make_request(self, method, bucket, key=None, params=None, data=None,
headers=None):
# Remove params that are set to None
if isinstance(params, dict):
for k, v in params.copy().items():
if v is None:
params.pop(k)
# Construct target url
url = 'http://{}.{}'.format(bucket, self.hostname)
url += '/{}'.format(key) if key is not None else '/'
if isinstance(params, dict) and len(params) > 0:
url += '?{}'.format(urllib.urlencode(params))
elif isinstance(params, basestring):
url += '?{}'.format(params)
# Make headers case insensitive
if headers is None:
headers = {}
headers = CaseInsensitiveDict(headers)
headers['Host'] = '{}.{}'.format(bucket, self.hostname)
if data is not None:
try:
raw_md5 = utils.f_md5(data)
except:
m = hashlib.md5()
m.update(data)
raw_md5 = m.digest()
md5 = b64encode(raw_md5)
headers['Content-MD5'] = md5
else:
md5 = ''
try:
content_type = headers['Content-Type']
except KeyError:
content_type = ''
date = formatdate(timeval=None, localtime=False, usegmt=True)
headers['x-amz-date'] = date
# Construct canonicalized amz headers string
canonicalized_amz_headers = ''
amz_keys = [k for k in list(headers.keys()) if k.startswith('x-amz-')]
for k in sorted(amz_keys):
v = headers[k].strip()
canonicalized_amz_headers += '{}:{}\n'.format(k.lower(), v)
# Construct canonicalized resource string
canonicalized_resource = '/' + bucket
canonicalized_resource += '/' if key is None else '/{}'.format(key)
if isinstance(params, basestring):
canonicalized_resource += '?{}'.format(params)
elif isinstance(params, dict) and len(params) > 0:
canonicalized_resource += '?{}'.format(urllib.urlencode(params))
# Construct string to sign
string_to_sign = method.upper() + '\n'
string_to_sign += md5 + '\n'
string_to_sign += content_type + '\n'
string_to_sign += '\n' # date is always set through x-amz-date
string_to_sign += canonicalized_amz_headers + canonicalized_resource
# Create signature
h = hmac.new(self.secret_access_key, string_to_sign, hashlib.sha1)
signature = b64encode(h.digest())
# Set authorization header
auth_head = 'AWS {}:{}'.format(self.access_key_id, signature)
headers['Authorization'] = auth_head
# Prepare Request
req = Request(method, url, data=data, headers=headers).prepare()
# Log request data.
# Prepare request beforehand so requests-altered headers show.
# Combine into a single message so we don't have to bother with
# locking to make lines appear together.
log_message = '{} {}\n'.format(method, url)
log_message += 'headers:'
for k in sorted(req.headers.keys()):
log_message += '\n {}: {}'.format(k, req.headers[k])
log.debug(log_message)
# Send request
resp = Session().send(req)
# Update stats, log response data.
self.stats[method.upper()] += 1
log.debug('response: {} ({} {})'.format(resp.status_code, method, url))
# Handle errors
if resp.status_code/100 != 2:
soup = BeautifulSoup(resp.text)
error = soup.find('error')
log_message = "S3 replied with non 2xx response code!!!!\n"
log_message += ' request: {} {}\n'.format(method, url)
for c in error.children:
error_name = c.name
error_message = c.text.encode('unicode_escape')
log_message += ' {}: {}\n'.format(error_name, error_message)
log.debug(log_message)
code = error.find('code').text
message = error.find('message').text
raise S3ResponseError(code, message, resp)
return resp
def test_f_md5():
s = StringIO('hello')
s.seek(2, 0)
expected = '5d41402abc4b2a76b9719d911017c592'
assert hexlify(utils.f_md5(s)) == expected
assert s.tell() == 2
def make_request(self,
method,
bucket,
key=None,
subresource=None,
params=None,
data=None,
headers=None):
# Remove params that are set to None
if params is None:
params = {}
for k, v in params.copy().items():
if v is None:
params.pop(k)
# Construct target url
url = 'http://{}/{}'.format(self.hostname, bucket)
url += '/{}'.format(key) if key is not None else '/'
if subresource is not None:
url += '?{}'.format(subresource)
elif len(params) > 0:
url += '?{}'.format(urllib.urlencode(params))
# Make headers case insensitive
if headers is None:
headers = {}
headers = CaseInsensitiveDict(headers)
headers['Host'] = self.hostname
if self.temporary_security_token is not None:
headers['x-amz-security-token'] = self.temporary_security_token
if data is not None:
try:
raw_md5 = utils.f_md5(data)
except:
m = hashlib.md5()
m.update(data)
raw_md5 = m.digest()
md5 = b64encode(raw_md5)
headers['Content-MD5'] = md5
else:
md5 = ''
try:
content_type = headers['Content-Type']
except KeyError:
content_type = ''
date = formatdate(timeval=None, localtime=False, usegmt=True)
headers['x-amz-date'] = date
# Construct canonicalized amz headers string
canonicalized_amz_headers = ''
amz_keys = [k for k in list(headers.keys()) if k.startswith('x-amz-')]
for k in sorted(amz_keys):
v = headers[k].strip()
k = k.lower()
canonicalized_amz_headers += '{}:{}\n'.format(k, v)
# Construct canonicalized resource string
canonicalized_resource = '/' + bucket
canonicalized_resource += '/' if key is None else '/{}'.format(key)
if subresource is not None:
canonicalized_resource += '?{}'.format(subresource)
elif len(params) > 0:
canonicalized_resource += '?{}'.format(urllib.urlencode(params))
# Construct string to sign
string_to_sign = method.upper() + '\n'
string_to_sign += md5 + '\n'
string_to_sign += content_type + '\n'
string_to_sign += '\n' # date is always set through x-amz-date
string_to_sign += canonicalized_amz_headers + canonicalized_resource
# Create signature
h = hmac.new(self.secret_access_key, string_to_sign, hashlib.sha1)
signature = b64encode(h.digest())
# Set authorization header
auth_head = 'AWS {}:{}'.format(self.access_key_id, signature)
headers['Authorization'] = auth_head
# Prepare Request
req = Request(method, url, data=data, headers=headers).prepare()
# Log request data.
# Prepare request beforehand so requests-altered headers show.
# Combine into a single message so we don't have to bother with
# locking to make lines appear together.
log_message = '{} {}\n'.format(method, url)
log_message += 'string to sign: {}\n'.format(repr(string_to_sign))
log_message += 'headers:'
for k in sorted(req.headers.keys()):
log_message += '\n {}: {}'.format(k, req.headers[k])
log.debug(log_message)
# Send request
resp = Session().send(req)
# Update stats, log response data.
self.stats[method.upper()] += 1
log.debug('response: {} ({} {})'.format(resp.status_code, method, url))
# Handle errors
if resp.status_code / 100 != 2:
soup = BeautifulSoup(resp.text)
error = soup.find('error')
log_message = "S3 replied with non 2xx response code!!!!\n"
log_message += ' request: {} {}\n'.format(method, url)
for c in error.children:
error_name = c.name
error_message = c.text.encode('unicode_escape')
log_message += ' {}: {}\n'.format(error_name, error_message)
log.debug(log_message)
code = error.find('code').text
message = error.find('message').text
raise S3ResponseError(code, message, resp)
return resp
