def _is_unmodified(self, s3_key): local_path = self._get_local_path_from_key(s3_key.name) with open(local_path, 'rb') as f: if utils.f_sizeof(f) != s3_key.size: return False if not '-' in s3_key.md5: local_md5 = hexlify(utils.f_md5(f)) else: chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE) m = hashlib.md5() for chunk in chunks: m.update(utils.f_md5(chunk)) local_md5 = "{}-{}".format(hexlify(m.digest()), len(chunks)) return local_md5 == s3_key.md5
def _multipart_upload(self, f): upload_id = self._initiate_multipart_upload() # Upload individual parts upload_reqs = [] parts = [] chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE) for r in range(len(chunks)): upload_reqs.append([ self._multipart_upload_part, chunks[r], r+1, upload_id ]) parts.append((r+1, hexlify(utils.f_md5(chunks[r])))) try: self.conn.join(upload_reqs) except: self._abort_multipart_upload(upload_id) raise self._complete_multipart_upload(upload_id, parts)
def _multipart_upload(self, file_like_object): self.log_upload(file_like_object) upload_id = self._initiate_multipart_upload() # Upload individual parts upload_reqs = [] parts = [] chunks = utils.f_chunk(file_like_object, constants.MULTIPART_PART_SIZE) for r in range(len(chunks)): upload_reqs.append([ self._multipart_upload_part, chunks[r], r+1, upload_id ]) parts.append((r+1, hexlify(utils.f_md5(chunks[r])))) try: self.conn.join(upload_reqs) except: self._abort_multipart_upload(upload_id) raise self._complete_multipart_upload(upload_id, parts)
def test_f_md5(): s = StringIO('hello') s.seek(2, 0) expected = '5d41402abc4b2a76b9719d911017c592' assert hexlify(utils.f_md5(s)) == expected assert s.tell() == 2
def make_request(self, method, bucket, key=None, params=None, data=None, headers=None): # Remove params that are set to None if isinstance(params, dict): for k, v in params.copy().items(): if v is None: params.pop(k) # Construct target url url = 'http://{}.{}'.format(bucket, self.hostname) url += '/{}'.format(key) if key is not None else '/' if isinstance(params, dict) and len(params) > 0: url += '?{}'.format(urllib.urlencode(params)) elif isinstance(params, basestring): url += '?{}'.format(params) # Make headers case insensitive if headers is None: headers = {} headers = CaseInsensitiveDict(headers) headers['Host'] = '{}.{}'.format(bucket, self.hostname) if data is not None: try: raw_md5 = utils.f_md5(data) except: m = hashlib.md5() m.update(data) raw_md5 = m.digest() md5 = b64encode(raw_md5) headers['Content-MD5'] = md5 else: md5 = '' try: content_type = headers['Content-Type'] except KeyError: content_type = '' date = formatdate(timeval=None, localtime=False, usegmt=True) headers['x-amz-date'] = date # Construct canonicalized amz headers string canonicalized_amz_headers = '' amz_keys = [k for k in list(headers.keys()) if k.startswith('x-amz-')] for k in sorted(amz_keys): v = headers[k].strip() canonicalized_amz_headers += '{}:{}\n'.format(k.lower(), v) # Construct canonicalized resource string canonicalized_resource = '/' + bucket canonicalized_resource += '/' if key is None else '/{}'.format(key) if isinstance(params, basestring): canonicalized_resource += '?{}'.format(params) elif isinstance(params, dict) and len(params) > 0: canonicalized_resource += '?{}'.format(urllib.urlencode(params)) # Construct string to sign string_to_sign = method.upper() + '\n' string_to_sign += md5 + '\n' string_to_sign += content_type + '\n' string_to_sign += '\n' # date is always set through x-amz-date string_to_sign += canonicalized_amz_headers + canonicalized_resource # Create signature h = hmac.new(self.secret_access_key, string_to_sign, hashlib.sha1) signature = b64encode(h.digest()) # Set authorization header auth_head = 'AWS {}:{}'.format(self.access_key_id, signature) headers['Authorization'] = auth_head # Prepare Request req = Request(method, url, data=data, headers=headers).prepare() # Log request data. # Prepare request beforehand so requests-altered headers show. # Combine into a single message so we don't have to bother with # locking to make lines appear together. log_message = '{} {}\n'.format(method, url) log_message += 'headers:' for k in sorted(req.headers.keys()): log_message += '\n {}: {}'.format(k, req.headers[k]) log.debug(log_message) # Send request resp = Session().send(req) # Update stats, log response data. self.stats[method.upper()] += 1 log.debug('response: {} ({} {})'.format(resp.status_code, method, url)) # Handle errors if resp.status_code/100 != 2: soup = BeautifulSoup(resp.text) error = soup.find('error') log_message = "S3 replied with non 2xx response code!!!!\n" log_message += ' request: {} {}\n'.format(method, url) for c in error.children: error_name = c.name error_message = c.text.encode('unicode_escape') log_message += ' {}: {}\n'.format(error_name, error_message) log.debug(log_message) code = error.find('code').text message = error.find('message').text raise S3ResponseError(code, message, resp) return resp
def make_request(self, method, bucket, key=None, subresource=None, params=None, data=None, headers=None): # Remove params that are set to None if params is None: params = {} for k, v in params.copy().items(): if v is None: params.pop(k) # Construct target url url = 'http://{}/{}'.format(self.hostname, bucket) url += '/{}'.format(key) if key is not None else '/' if subresource is not None: url += '?{}'.format(subresource) elif len(params) > 0: url += '?{}'.format(urllib.urlencode(params)) # Make headers case insensitive if headers is None: headers = {} headers = CaseInsensitiveDict(headers) headers['Host'] = self.hostname if self.temporary_security_token is not None: headers['x-amz-security-token'] = self.temporary_security_token if data is not None: try: raw_md5 = utils.f_md5(data) except: m = hashlib.md5() m.update(data) raw_md5 = m.digest() md5 = b64encode(raw_md5) headers['Content-MD5'] = md5 else: md5 = '' try: content_type = headers['Content-Type'] except KeyError: content_type = '' date = formatdate(timeval=None, localtime=False, usegmt=True) headers['x-amz-date'] = date # Construct canonicalized amz headers string canonicalized_amz_headers = '' amz_keys = [k for k in list(headers.keys()) if k.startswith('x-amz-')] for k in sorted(amz_keys): v = headers[k].strip() k = k.lower() canonicalized_amz_headers += '{}:{}\n'.format(k, v) # Construct canonicalized resource string canonicalized_resource = '/' + bucket canonicalized_resource += '/' if key is None else '/{}'.format(key) if subresource is not None: canonicalized_resource += '?{}'.format(subresource) elif len(params) > 0: canonicalized_resource += '?{}'.format(urllib.urlencode(params)) # Construct string to sign string_to_sign = method.upper() + '\n' string_to_sign += md5 + '\n' string_to_sign += content_type + '\n' string_to_sign += '\n' # date is always set through x-amz-date string_to_sign += canonicalized_amz_headers + canonicalized_resource # Create signature h = hmac.new(self.secret_access_key, string_to_sign, hashlib.sha1) signature = b64encode(h.digest()) # Set authorization header auth_head = 'AWS {}:{}'.format(self.access_key_id, signature) headers['Authorization'] = auth_head # Prepare Request req = Request(method, url, data=data, headers=headers).prepare() # Log request data. # Prepare request beforehand so requests-altered headers show. # Combine into a single message so we don't have to bother with # locking to make lines appear together. log_message = '{} {}\n'.format(method, url) log_message += 'string to sign: {}\n'.format(repr(string_to_sign)) log_message += 'headers:' for k in sorted(req.headers.keys()): log_message += '\n {}: {}'.format(k, req.headers[k]) log.debug(log_message) # Send request resp = Session().send(req) # Update stats, log response data. self.stats[method.upper()] += 1 log.debug('response: {} ({} {})'.format(resp.status_code, method, url)) # Handle errors if resp.status_code / 100 != 2: soup = BeautifulSoup(resp.text) error = soup.find('error') log_message = "S3 replied with non 2xx response code!!!!\n" log_message += ' request: {} {}\n'.format(method, url) for c in error.children: error_name = c.name error_message = c.text.encode('unicode_escape') log_message += ' {}: {}\n'.format(error_name, error_message) log.debug(log_message) code = error.find('code').text message = error.find('message').text raise S3ResponseError(code, message, resp) return resp