def test_proxy_ssl_with_verification(self, ssl_mock, http_response_mock,
create_connection_mock):
type(http_response_mock.return_value).status = mock.PropertyMock(
return_value=200)
conn = AWSAuthConnection('mockservice.s3.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
proxy_port=80)
conn.https_validate_certificates = True
dummy_cert = {
'subjectAltName':
(('DNS', 's3.amazonaws.com'), ('DNS', '*.s3.amazonaws.com')),
}
mock_sock = mock.Mock()
create_connection_mock.return_value = mock_sock
mock_sslSock = mock.Mock()
mock_sslSock.getpeercert.return_value = dummy_cert
mock_context = mock.Mock()
mock_context.wrap_socket.return_value = mock_sslSock
ssl_mock.create_default_context.return_value = mock_context
# Attempt to call proxy_ssl and make sure it works
conn.proxy_ssl('mockservice.s3.amazonaws.com', 80)
mock_sslSock.getpeercert.assert_called_once_with()
mock_context.wrap_socket.assert_called_once_with(
mock_sock, server_hostname='mockservice.s3.amazonaws.com')
def test_connection_behind_proxy_without_explicit_port(self):
os.environ['http_proxy'] = "http://127.0.0.1"
conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
port=8180)
self.assertEqual(conn.proxy, '127.0.0.1')
self.assertEqual(conn.proxy_port, 8180)
del os.environ['http_proxy']
def test_get_proxy_url_with_auth(self):
conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
proxy="127.0.0.1",
proxy_user="******",
proxy_pass="******",
proxy_port="8180")
self.assertEqual(conn.get_proxy_url_with_auth(),
'http://john.doe:[email protected]:8180')
def test_connection_behind_proxy(self):
os.environ['http_proxy'] = "http://john.doe:[email protected]:8180"
conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False)
self.assertEqual(conn.proxy, '127.0.0.1')
self.assertEqual(conn.proxy_user, 'john.doe')
self.assertEqual(conn.proxy_pass, 'p4ssw0rd')
self.assertEqual(conn.proxy_port, '8180')
del os.environ['http_proxy']
def setUp(self):
self.connection = AWSAuthConnection('s3.amazonaws.com')
self.non_retriable_code = 404
self.retry_status_codes = [301, 400]
self.success_response = self.create_response(200)
self.default_host = 'bucket.s3.amazonaws.com'
self.retry_region = RETRY_REGION_BYTES.decode('utf-8')
self.default_retried_host = ('bucket.s3.%s.amazonaws.com' %
self.retry_region)
self.test_headers = [('x-amz-bucket-region', self.retry_region)]
def get_euca_authenticator(self):
"""
This method centralizes configuration of the EucaAuthenticator.
"""
host = self._get_ufs_host_setting_()
port = self._get_ufs_port_setting_()
validate_certs = asbool(self.request.registry.settings.get('connection.ssl.validation', False))
conn = AWSAuthConnection(None, aws_access_key_id='', aws_secret_access_key='')
ca_certs_file = conn.ca_certificates_file
conn = None
ca_certs_file = self.request.registry.settings.get('connection.ssl.certfile', ca_certs_file)
auth = EucaAuthenticator(host, port, True, validate_certs=validate_certs, ca_certs=ca_certs_file)
return auth
def get_cert_verification_info(self):
validate_certs = asbool(
self.request.registry.settings.get('connection.ssl.validation',
False))
conn = AWSAuthConnection(None,
aws_access_key_id='',
aws_secret_access_key='')
ca_certs_file = conn.ca_certificates_file
conn = None
ca_certs_file = self.request.registry.settings.get(
'connection.ssl.certfile', ca_certs_file)
ret = namedtuple('ret', 'validate_certs ca_certs_file')
return ret(validate_certs=validate_certs, ca_certs_file=ca_certs_file)
def test_proxy_ssl(self, ssl_mock, http_response_mock,
create_connection_mock):
type(http_response_mock.return_value).status = mock.PropertyMock(
return_value=200)
conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
proxy_port=80)
conn.https_validate_certificates = False
# Attempt to call proxy_ssl and make sure it works
conn.proxy_ssl('mockservice.cc-zone-1.amazonaws.com', 80)
def test_build_base_http_request_noproxy(self):
os.environ['no_proxy'] = 'mockservice.cc-zone-1.amazonaws.com'
conn = AWSAuthConnection('mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False,
proxy="127.0.0.1",
proxy_user="******",
proxy_pass="******",
proxy_port="8180")
request = conn.build_base_http_request('GET', '/', None)
del os.environ['no_proxy']
self.assertEqual(request.path, '/')
def test_get_path(self):
conn = AWSAuthConnection(
'mockservice.cc-zone-1.amazonaws.com',
aws_access_key_id='access_key',
aws_secret_access_key='secret',
suppress_consec_slashes=False
)
# Test some sample paths for mangling.
self.assertEqual(conn.get_path('/'), '/')
self.assertEqual(conn.get_path('image.jpg'), '/image.jpg')
self.assertEqual(conn.get_path('folder/image.jpg'), '/folder/image.jpg')
self.assertEqual(conn.get_path('folder//image.jpg'), '/folder//image.jpg')
# Ensure leading slashes aren't removed.
# See https://github.com/boto/boto/issues/1387
self.assertEqual(conn.get_path('/folder//image.jpg'), '/folder//image.jpg')
self.assertEqual(conn.get_path('/folder////image.jpg'), '/folder////image.jpg')
self.assertEqual(conn.get_path('///folder////image.jpg'), '///folder////image.jpg')
def handle_aws_login(self):
session = self.request.session
if self.aws_login_form.validate():
package = self.request.params.get('package')
package = base64.decodestring(package)
aws_region = self.request.params.get('aws-region')
validate_certs = asbool(self.request.registry.settings.get('connection.ssl.validation', False))
conn = AWSAuthConnection(None, aws_access_key_id='', aws_secret_access_key='')
ca_certs_file = conn.ca_certificates_file
conn = None
ca_certs_file = self.request.registry.settings.get('connection.ssl.certfile', ca_certs_file)
auth = AWSAuthenticator(package=package, validate_certs=validate_certs, ca_certs=ca_certs_file)
try:
creds = auth.authenticate(timeout=10)
logging.info(u"Authenticated AWS user from {ip}".format(ip=BaseView.get_remote_addr(self.request)))
default_region = self.request.registry.settings.get('aws.default.region', 'us-east-1')
session.invalidate() # Refresh session
session['cloud_type'] = 'aws'
session['auth_type'] = 'keys'
self._assign_session_creds(session, creds)
session['region'] = aws_region if aws_region else default_region
session['username_label'] = u'{user}...@AWS'.format(user=creds.access_key[:8])
session['supported_platforms'] = self.get_account_attributes(['supported-platforms'])
session['default_vpc'] = self.get_account_attributes(['default-vpc'])
conn = ConnectionManager.aws_connection(
session['region'], creds.access_key, creds.secret_key, creds.session_token, 'vpc')
vpcs = conn.get_all_vpcs()
if not vpcs or len(vpcs) == 0:
# remove vpc from supported-platforms
if 'VPC' in session.get('supported_platforms', []):
session.get('supported_platforms').remove('VPC')
headers = remember(self.request, creds.access_key[:8])
return HTTPFound(location=self.came_from, headers=headers)
except HTTPError as err:
if err.msg == 'Forbidden':
msg = _(u'Invalid access key and/or secret key.')
self.login_form_errors.append(msg)
except URLError as err:
if err.reason.find('ssl') > -1:
msg = INVALID_SSL_CERT_MSG
else:
msg = _(u'No response from host')
self.login_form_errors.append(msg)
return self.render_dict
def put_stream(self, bucket, label, fp, metadata={}, cb=None, num_cb=None):
if metadata is None:
metadata = {"_owner": getpass.getuser()}
path = "/" + bucket + "/" + label
content_type = metadata.get("_format", "application/octet-stream")
metadata = self.ckan.storage_metadata_set(path, metadata)
BufferSize = 65536 ## set to something very small to make sure
## chunking is working properly
headers = {'Content-Type': content_type}
#if content_type is None:
# content_type = mimetypes.guess_type(filename)[0] or "text/plain"
#headers['Content-Type'] = content_type
#if content_encoding is not None:
# headers['Content-Encoding'] = content_encoding
m = md5()
fp.seek(0)
s = fp.read(BufferSize)
while s:
m.update(s)
s = fp.read(BufferSize)
self.size = fp.tell()
fp.seek(0)
self.md5 = m.hexdigest()
headers['Content-MD5'] = base64.encodestring(m.digest()).rstrip('\n')
headers['Content-Length'] = str(self.size)
headers['Expect'] = '100-Continue'
host, headers = self.ckan.storage_auth_get(path, headers)
def sender(http_conn, method, path, data, headers):
http_conn.putrequest(method, path)
for key in headers:
http_conn.putheader(key, headers[key])
http_conn.endheaders()
fp.seek(0)
http_conn.set_debuglevel(
0) ### XXX set to e.g. 4 to see what going on
if cb:
if num_cb > 2:
cb_count = self.size / BufferSize / (num_cb - 2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = total_bytes = 0
cb(total_bytes, self.size)
l = fp.read(BufferSize)
while len(l) > 0:
http_conn.send(l)
if cb:
total_bytes += len(l)
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes, self.size)
i = 0
l = fp.read(BufferSize)
if cb:
cb(total_bytes, self.size)
response = http_conn.getresponse()
body = response.read()
fp.seek(0)
if response.status == 500 or response.status == 503 or \
response.getheader('location'):
# we'll try again
return response
elif response.status >= 200 and response.status <= 299:
self.etag = response.getheader('etag')
if self.etag != '"%s"' % self.md5:
raise Exception('ETag from S3 did not match computed MD5')
return response
else:
#raise provider.storage_response_error(
# response.status, response.reason, body)
raise Exception(response.status, response.reason, body)
awsc = AWSAuthConnection(host,
aws_access_key_id="key_id",
aws_secret_access_key="secret")
awsc._mexe('PUT', path, None, headers, sender=sender)
metadata = self.ckan.storage_metadata_update(path, {})
from pprint import pprint
pprint(metadata)
def proxy_upload(self,
path,
filename,
content_type=None,
content_encoding=None,
cb=None,
num_cb=None):
"""
This is the main function that uploads. We assume the bucket
and key (== path) exists. What we do here is simple. Calculate
the headers we will need, (e.g. md5, content-type, etc). Then
we ask the self.get_proxy_config method to fill in the authentication
information and tell us which remote host we should talk to
for the upload. From there, the rest is ripped from
boto.key.Key.send_file
"""
from boto.connection import AWSAuthConnection
import mimetypes
from hashlib import md5
import base64
BufferSize = 65536 ## set to something very small to make sure
## chunking is working properly
fp = open(filename)
headers = {'Content-Type': content_type}
if content_type is None:
content_type = mimetypes.guess_type(filename)[0] or "text/plain"
headers['Content-Type'] = content_type
if content_encoding is not None:
headers['Content-Encoding'] = content_encoding
m = md5()
fp.seek(0)
s = fp.read(BufferSize)
while s:
m.update(s)
s = fp.read(BufferSize)
self.size = fp.tell()
fp.seek(0)
self.md5 = m.hexdigest()
headers['Content-MD5'] = base64.encodestring(m.digest()).rstrip('\n')
headers['Content-Length'] = str(self.size)
headers['Expect'] = '100-Continue'
host, headers = self.get_proxy_config(headers, path)
### how to do this same thing with curl instead...
print("curl -i --trace-ascii foo.log -T %s -H %s https://%s%s" %
(filename, " -H ".join("'%s: %s'" % (k, v)
for k, v in headers.items()), host, path))
def sender(http_conn, method, path, data, headers):
http_conn.putrequest(method, path)
for key in headers:
http_conn.putheader(key, headers[key])
http_conn.endheaders()
fp.seek(0)
http_conn.set_debuglevel(
0) ### XXX set to e.g. 4 to see what going on
if cb:
if num_cb > 2:
cb_count = self.size / BufferSize / (num_cb - 2)
elif num_cb < 0:
cb_count = -1
else:
cb_count = 0
i = total_bytes = 0
cb(total_bytes, self.size)
l = fp.read(BufferSize)
while len(l) > 0:
http_conn.send(l)
if cb:
total_bytes += len(l)
i += 1
if i == cb_count or cb_count == -1:
cb(total_bytes, self.size)
i = 0
l = fp.read(BufferSize)
if cb:
cb(total_bytes, self.size)
response = http_conn.getresponse()
body = response.read()
fp.seek(0)
if response.status == 500 or response.status == 503 or \
response.getheader('location'):
# we'll try again
return response
elif response.status >= 200 and response.status <= 299:
self.etag = response.getheader('etag')
if self.etag != '"%s"' % self.md5:
raise Exception('ETag from S3 did not match computed MD5')
return response
else:
#raise provider.storage_response_error(
# response.status, response.reason, body)
raise Exception(response.status, response.reason, body)
awsc = AWSAuthConnection(host,
aws_access_key_id="key_id",
aws_secret_access_key="secret")
awsc._mexe('PUT', path, None, headers, sender=sender)