def _password_client(self, body=None, headers=None, decode=True):
"""Client for the Password Server."""
port = CONF.cloudstack.password_server_port
with contextlib.closing(
http_client.HTTPConnection(self._metadata_host,
port,
timeout=TIMEOUT)) as connection:
try:
connection.request("GET", "/", body=body, headers=headers)
response = connection.getresponse()
except http_client.HTTPException as exc:
LOG.error("Request failed: %s", exc)
raise
content = response.read()
if decode:
content = encoding.get_as_string(content)
if response.status != 200:
raise http_client.HTTPException(
"%(status)s %(reason)s - %(message)r", {
"status": response.status,
"reason": response.reason,
"message": content
})
return content
def get(http, path, root=METADATA_ROOT, recursive=None):
"""Fetch a resource from the metadata server.
Args:
http: an object to be used to make HTTP requests.
path: A string indicating the resource to retrieve. For example,
'instance/service-accounts/default'
root: A string indicating the full path to the metadata server root.
recursive: A boolean indicating whether to do a recursive query of
metadata. See
https://cloud.google.com/compute/docs/metadata#aggcontents
Returns:
A dictionary if the metadata server returns JSON, otherwise a string.
Raises:
http_client.HTTPException if an error corrured while
retrieving metadata.
"""
url = urlparse.urljoin(root, path)
url = _helpers._add_query_parameter(url, 'recursive', recursive)
response, content = transport.request(http, url, headers=METADATA_HEADERS)
if response.status == http_client.OK:
decoded = _helpers._from_bytes(content)
if response['content-type'] == 'application/json':
return json.loads(decoded)
else:
return decoded
else:
raise http_client.HTTPException(
'Failed to retrieve {0} from the Google Compute Engine'
'metadata service. Response:\n{1}'.format(url, response))
def test_bad_http_request(self):
"""Test response to raising an exception during a post"""
# Get a mock version of urlopen(), but with the side effect of having an exception of
# type http_client.HTTPException raised when it's called
mock_urlopen = self.get_openurl_patcher(side_effect=http_client.HTTPException("oops"))
q = queue.Queue()
obj = weewx.restx.AmbientThread(q,
manager_dict=None,
station=TestAmbient.station,
password=TestAmbient.password,
server_url=TestAmbient.server_url,
protocol_name=TestAmbient.protocol_name,
max_tries=1,
log_success=True,
log_failure=True,
)
record = get_record()
q.put(record)
q.put(None)
# Set up mocks of log.debug() and log.error(). Then we'll check that they got called as we expected
with mock.patch('weewx.restx.log.debug') as mock_logdbg:
with mock.patch('weewx.restx.log.error') as mock_logerr:
obj.run()
# log.debug() should have been called twice...
mock_logdbg.assert_called_once_with('Test-Ambient: Failed upload attempt 1: oops')
# ... and log.error() once with the failed post.
mock_logerr.assert_called_once_with('Test-Ambient: Failed to publish record '
'2018-03-22 00:00:00 PDT (1521702000): '
'Failed upload after 1 tries')
# Now check that our mock urlopen() was called with the parameters we expected.
matcher = TestAmbient.get_matcher(TestAmbient.server_url, TestAmbient.station, TestAmbient.password)
mock_urlopen.assert_called_once_with(matcher, data=None, timeout=10)
def test_ChunkedTransferEncodingStreamWriter_write_on_exception_still_closes():
data = b'BOGUS DATA'
with mock.patch('girder_worker.docker.io.httplib.HTTPConnection', autospec=True):
s = ChunkedTransferEncodingStreamWriter('http://bogus.url.com/')
s.conn.send.side_effect = httplib.HTTPException('Bogus Exception')
with pytest.raises(httplib.HTTPException):
s.write(data)
s.conn.close.assert_called_once()
assert s._closed is True
def _do_request(self, method, action, body=None):
conn = http_client.HTTPConnection(self.host, self.port)
url = self.url_prefix + action
headers = {}
if body is not None:
body = json.dumps(body)
headers['Content-Type'] = 'application/json'
conn.request(method, url, body, headers)
res = conn.getresponse()
if res.status in (http_client.OK, http_client.CREATED,
http_client.ACCEPTED, http_client.NO_CONTENT):
return res
raise http_client.HTTPException(
res, 'code %d reason %s' % (res.status, res.reason),
res.getheaders(), res.read())
def do_request(self, http_request):
"""
Send an HTTP request
:type http_request: :class:`vmware.vapi.protocol.client.http_lib.HTTPRequest` # pylint: disable=line-too-long
:param http_request: The http request to be sent
:rtype: :class:`vmware.vapi.protocol.client.http_lib.HTTPResponse`
:return: The http response received
"""
# pylint can't detect request, getresponse and close methods from
# Http(s)Connection/UnixSocketConnection
# pylint: disable=E1103
request_ctx = http_request.headers
request = http_request.body
content_type = request_ctx.get('Content-Type')
if not content_type:
# For http, content-type must be set
raise Exception('do_request: request_ctx content-type not set')
response_ctx, response = {'Content-Type': content_type}, None
if request:
request_length = len(request)
# Send request
headers = {'Cookie': self.cookie, 'Content-Type': content_type}
if self.accept_compress_response:
headers['Accept-Encoding'] = 'gzip, deflate'
try:
conn = self._get_connection()
logger.debug('do_request: request_len %d', request_length)
if use_connection_pool:
resp = conn.request(method=http_request.method,
url=self.path,
body=request,
headers=headers,
preload_content=False)
else:
conn.request(method=http_request.method,
url=self.path,
body=request,
headers=headers)
resp = conn.getresponse()
except:
logger.exception('do_request() failed')
raise
# Debug
# logger.debug('do_request: response headers', resp.getheaders())
cookie = resp.getheader('Set-Cookie')
if cookie:
self.cookie = cookie
status = resp.status
if status in [200, 500]:
try:
encoding = resp.getheader('Content-Encoding', 'identity').lower() # pylint: disable=line-too-long
if encoding in ['gzip', 'deflate']:
response = resp.read(decode_content=True)
else:
response = resp.read()
logger.debug('do_request: response len %d', len(response))
except:
conn.close()
raise
else:
if resp:
resp.read()
content_type = resp.getheader('Content-Type')
if content_type:
response_ctx['Content-Type'] = content_type
else:
raise http_client.HTTPException('%d %s' % (resp.status, resp.reason)) # pylint: disable=line-too-long
if self.cookie:
response_ctx['Cookie'] = self.cookie
return HTTPResponse(status=status, headers=response_ctx, body=response)
def do_request(self, http_request):
"""
Send an HTTP request
:type http_request: :class:`vmware.vapi.protocol.client.http_lib.HTTPRequest`
:param http_request: The http request to be sent
:rtype: :class:`vmware.vapi.protocol.client.http_lib.HTTPResponse`
:return: The http response received
"""
# pylint can't detect request, getresponse and close methods from
# Http(s)Connection/UnixSocketConnection
# pylint: disable=E1103
request_ctx = http_request.headers
request = http_request.body
content_type = request_ctx.get('Content-Type')
if not content_type:
# For http, content-type must be set
raise Exception('do_request: request_ctx content-type not set')
response_ctx, response = {'Content-Type': content_type}, None
if request:
request_length = len(request)
# Send request
headers = {'Cookie': self.cookie, 'Content-Type': content_type}
if self.accept_compress_response:
headers['Accept-Encoding'] = 'gzip, deflate'
try:
conn = self._get_connection()
logger.debug('do_request: request_len %d', request_length)
conn.request(http_request.method, self.path, request, headers)
resp = conn.getresponse()
except (socket.error, http_client.HTTPException):
raise
# Debug
# logger.debug('do_request: response headers', resp.getheaders())
cookie = resp.getheader('Set-Cookie')
if cookie:
self.cookie = cookie
status = resp.status
if status == 200 or status == 500:
try:
encoding = resp.getheader('Content-Encoding',
'identity').lower()
if encoding == 'gzip':
resp_reader = GzipReader(resp,
encoding=GzipReader.GZIP)
elif encoding == 'deflate':
resp_reader = GzipReader(resp,
encoding=GzipReader.DEFLATE)
else:
resp_reader = resp
response = resp_reader.read()
logger.debug('do_request: response len %d', len(response))
except:
conn.close()
raise
else:
if resp_reader:
resp_reader.read()
self._return_connection(conn)
content_type = resp.getheader('Content-Type')
if content_type:
response_ctx['Content-Type'] = content_type
else:
conn.close()
raise http_client.HTTPException('%d %s' %
(resp.status, resp.reason))
if self.cookie:
response_ctx['Cookie'] = self.cookie
return HTTPResponse(status=status, headers=response_ctx, body=response)
class AppAssertionCredentialsTests(unittest2.TestCase):
def test_constructor(self):
credentials = gce.AppAssertionCredentials()
self.assertIsNone(credentials.assertion_type, None)
self.assertIsNone(credentials.service_account_email)
self.assertIsNone(credentials.scopes)
self.assertTrue(credentials.invalid)
@mock.patch('warnings.warn')
def test_constructor_with_scopes(self, warn_mock):
scope = 'http://example.com/a http://example.com/b'
scopes = scope.split()
credentials = gce.AppAssertionCredentials(scopes=scopes)
self.assertEqual(credentials.scopes, None)
self.assertEqual(credentials.assertion_type, None)
warn_mock.assert_called_once_with(gce._SCOPES_WARNING)
def test_to_json(self):
credentials = gce.AppAssertionCredentials()
with self.assertRaises(NotImplementedError):
credentials.to_json()
def test_from_json(self):
with self.assertRaises(NotImplementedError):
gce.AppAssertionCredentials.from_json({})
@mock.patch('oauth2client.contrib._metadata.get_token',
side_effect=[('A', datetime.datetime.min),
('B', datetime.datetime.max)])
@mock.patch('oauth2client.contrib._metadata.get_service_account_info',
return_value=SERVICE_ACCOUNT_INFO)
def test_refresh_token(self, get_info, get_token):
http_request = mock.MagicMock()
http_mock = mock.MagicMock(request=http_request)
credentials = gce.AppAssertionCredentials()
credentials.invalid = False
credentials.service_account_email = '*****@*****.**'
self.assertIsNone(credentials.access_token)
credentials.get_access_token(http=http_mock)
self.assertEqual(credentials.access_token, 'A')
self.assertTrue(credentials.access_token_expired)
get_token.assert_called_with(http_request,
service_account='*****@*****.**')
credentials.get_access_token(http=http_mock)
self.assertEqual(credentials.access_token, 'B')
self.assertFalse(credentials.access_token_expired)
get_token.assert_called_with(http_request,
service_account='*****@*****.**')
get_info.assert_not_called()
def test_refresh_token_failed_fetch(self):
http_request = request_mock(
http_client.NOT_FOUND, 'application/json',
json.dumps({
'access_token': 'a',
'expires_in': 100
}))
credentials = gce.AppAssertionCredentials()
credentials.invalid = False
credentials.service_account_email = '*****@*****.**'
with self.assertRaises(client.HttpAccessTokenRefreshError):
credentials._refresh(http_request)
def test_serialization_data(self):
credentials = gce.AppAssertionCredentials()
with self.assertRaises(NotImplementedError):
getattr(credentials, 'serialization_data')
def test_create_scoped_required(self):
credentials = gce.AppAssertionCredentials()
self.assertFalse(credentials.create_scoped_required())
def test_sign_blob_not_implemented(self):
credentials = gce.AppAssertionCredentials([])
with self.assertRaises(NotImplementedError):
credentials.sign_blob(b'blob')
@mock.patch('oauth2client.contrib._metadata.get_service_account_info',
return_value=SERVICE_ACCOUNT_INFO)
def test_retrieve_scopes(self, metadata):
http_request = mock.MagicMock()
http_mock = mock.MagicMock(request=http_request)
credentials = gce.AppAssertionCredentials()
self.assertTrue(credentials.invalid)
self.assertIsNone(credentials.scopes)
scopes = credentials.retrieve_scopes(http_mock)
self.assertEqual(scopes, SERVICE_ACCOUNT_INFO['scopes'])
self.assertFalse(credentials.invalid)
credentials.retrieve_scopes(http_mock)
# Assert scopes weren't refetched
metadata.assert_called_once_with(http_request,
service_account='default')
@mock.patch('oauth2client.contrib._metadata.get_service_account_info',
side_effect=http_client.HTTPException('No Such Email'))
def test_retrieve_scopes_bad_email(self, metadata):
http_request = mock.MagicMock()
http_mock = mock.MagicMock(request=http_request)
credentials = gce.AppAssertionCredentials(email='*****@*****.**')
with self.assertRaises(http_client.HTTPException):
credentials.retrieve_scopes(http_mock)
metadata.assert_called_once_with(http_request,
service_account='*****@*****.**')
def test_save_to_well_known_file(self):
import os
ORIGINAL_ISDIR = os.path.isdir
try:
os.path.isdir = lambda path: True
credentials = gce.AppAssertionCredentials()
with self.assertRaises(NotImplementedError):
client.save_to_well_known_file(credentials)
finally:
os.path.isdir = ORIGINAL_ISDIR
class AppAssertionCredentialsTests(unittest.TestCase):
def test_constructor(self):
credentials = gce.AppAssertionCredentials()
self.assertIsNone(credentials.assertion_type, None)
self.assertIsNone(credentials.service_account_email)
self.assertIsNone(credentials.scopes)
self.assertTrue(credentials.invalid)
@mock.patch('warnings.warn')
def test_constructor_with_scopes(self, warn_mock):
scope = 'http://example.com/a http://example.com/b'
scopes = scope.split()
credentials = gce.AppAssertionCredentials(scopes=scopes)
self.assertEqual(credentials.scopes, None)
self.assertEqual(credentials.assertion_type, None)
warn_mock.assert_called_once_with(gce._SCOPES_WARNING)
def test_to_json(self):
credentials = gce.AppAssertionCredentials()
with self.assertRaises(NotImplementedError):
credentials.to_json()
def test_from_json(self):
with self.assertRaises(NotImplementedError):
gce.AppAssertionCredentials.from_json({})
@mock.patch('oauth2client.contrib._metadata.get_token',
side_effect=[('A', datetime.datetime.min),
('B', datetime.datetime.max)])
@mock.patch('oauth2client.contrib._metadata.get_service_account_info',
return_value=SERVICE_ACCOUNT_INFO)
def test_refresh_token(self, get_info, get_token):
http_mock = object()
credentials = gce.AppAssertionCredentials()
credentials.invalid = False
credentials.service_account_email = '*****@*****.**'
self.assertIsNone(credentials.access_token)
credentials.get_access_token(http=http_mock)
self.assertEqual(credentials.access_token, 'A')
self.assertTrue(credentials.access_token_expired)
get_token.assert_called_with(http_mock,
service_account='*****@*****.**')
credentials.get_access_token(http=http_mock)
self.assertEqual(credentials.access_token, 'B')
self.assertFalse(credentials.access_token_expired)
get_token.assert_called_with(http_mock,
service_account='*****@*****.**')
get_info.assert_not_called()
def test_refresh_token_failed_fetch(self):
headers = {
'status': http_client.NOT_FOUND,
'content-type': 'application/json',
}
response = json.dumps({'access_token': 'a', 'expires_in': 100})
http = http_mock.HttpMock(headers=headers, data=response)
credentials = gce.AppAssertionCredentials()
credentials.invalid = False
credentials.service_account_email = '*****@*****.**'
with self.assertRaises(client.HttpAccessTokenRefreshError):
credentials._refresh(http)
# Verify mock.
self.assertEqual(http.requests, 1)
expected_uri = _metadata.METADATA_ROOT + METADATA_PATH
self.assertEqual(http.uri, expected_uri)
self.assertEqual(http.method, 'GET')
self.assertIsNone(http.body)
self.assertEqual(http.headers, _metadata.METADATA_HEADERS)
def test_serialization_data(self):
credentials = gce.AppAssertionCredentials()
with self.assertRaises(NotImplementedError):
getattr(credentials, 'serialization_data')
def test_create_scoped_required(self):
credentials = gce.AppAssertionCredentials()
self.assertFalse(credentials.create_scoped_required())
def test_sign_blob_not_implemented(self):
credentials = gce.AppAssertionCredentials([])
with self.assertRaises(NotImplementedError):
credentials.sign_blob(b'blob')
@mock.patch('oauth2client.contrib._metadata.get_service_account_info',
return_value=SERVICE_ACCOUNT_INFO)
def test_retrieve_scopes(self, metadata):
http_mock = object()
credentials = gce.AppAssertionCredentials()
self.assertTrue(credentials.invalid)
self.assertIsNone(credentials.scopes)
scopes = credentials.retrieve_scopes(http_mock)
self.assertEqual(scopes, SERVICE_ACCOUNT_INFO['scopes'])
self.assertFalse(credentials.invalid)
credentials.retrieve_scopes(http_mock)
# Assert scopes weren't refetched
metadata.assert_called_once_with(http_mock,
service_account='default')
@mock.patch('oauth2client.contrib._metadata.get_service_account_info',
side_effect=http_client.HTTPException('No Such Email'))
def test_retrieve_scopes_bad_email(self, metadata):
http_mock = object()
credentials = gce.AppAssertionCredentials(email='*****@*****.**')
with self.assertRaises(http_client.HTTPException):
credentials.retrieve_scopes(http_mock)
metadata.assert_called_once_with(http_mock,
service_account='*****@*****.**')
def test_save_to_well_known_file(self):
import os
ORIGINAL_ISDIR = os.path.isdir
try:
os.path.isdir = lambda path: True
credentials = gce.AppAssertionCredentials()
with self.assertRaises(NotImplementedError):
client.save_to_well_known_file(credentials)
finally:
os.path.isdir = ORIGINAL_ISDIR
def test_custom_metadata_root_from_env(self):
headers = {'content-type': 'application/json'}
http = http_mock.HttpMock(headers=headers, data='{}')
fake_metadata_root = 'another.metadata.service'
os.environ['GCE_METADATA_ROOT'] = fake_metadata_root
reload_module(_metadata)
try:
_metadata.get(http, '')
finally:
del os.environ['GCE_METADATA_ROOT']
reload_module(_metadata)
# Verify mock.
self.assertEqual(http.requests, 1)
expected_uri = 'http://{}/computeMetadata/v1/'.format(fake_metadata_root)
self.assertEqual(http.uri, expected_uri)
