def _test_delete_agent(self):
mocked_resp = (httplib2.Response({'status': 200}), None)
self.useFixture(mockpatch.Patch(
'tempest.common.service_client.ServiceClient.delete',
return_value=mocked_resp))
self.client.delete_agent("1")
raise
except httplib.HTTPException:
conn.close()
raise
try:
response = conn.getresponse()
except (socket.error, httplib.HTTPException):
conn.close()
raise
else:
content = ''
if method == 'HEAD':
conn.close()
else:
content = response.read()
response = httplib2.Response(response)
if method != 'HEAD':
# pylint: disable=protected-access
content = httplib2._decompressContent(response, content)
return (response, content)
class HttpWithDownloadStream(httplib2.Http):
"""httplib2.Http variant that only pushes bytes through a stream.
httplib2 handles media by storing entire chunks of responses in memory, which
is undesirable particularly when multiple instances are used during
multi-threaded/multi-process copy. This class copies and then overrides some
httplib2 functions to use a streaming copy approach that uses small memory
buffers.
def _http_request(*args, **kwargs):
resp = httplib2.Response({'status': '200'})
content = simplejson.dumps({'access_token': 'bar'})
return resp, content
def _create_http_error(self, status=500, payload='Body'):
resp = httplib2.Response({'status': status})
return errors.HttpError(resp, payload.encode())
def test_authenticate_success(self):
cs = client.Client("username", "password", "project_id",
"auth_url/v2.0")
resp = {
"access": {
"token": {
"expires": "12345",
"id": "FAKE_ID",
},
"serviceCatalog": [
{
"adminURL":
"http://*****:*****@mock.patch.object(httplib2.Http, "request", mock_request)
def test_auth_call():
cs.client.authenticate()
headers = {
'User-Agent': cs.client.USER_AGENT,
'Content-Type': 'application/json',
}
body = {
'auth': {
'passwordCredentials': {
'username': cs.client.user,
'password': cs.client.password,
},
'tenantName': cs.client.projectid,
},
}
token_url = urlparse.urljoin(cs.client.auth_url, "tokens")
mock_request.assert_called_with(token_url,
"POST",
headers=headers,
body=json.dumps(body))
self.assertEqual(
cs.client.management_url, resp["access"]["serviceCatalog"][0]
['endpoints'][0]["publicURL"])
token_id = resp["access"]["token"]["id"]
self.assertEqual(cs.client.auth_token, token_id)
test_auth_call()
def Response(code, content):
return httplib2.Response({'status': str(code)}), content
from google.auth import external_account
from google.auth import identity_pool
from gslib.tests import testcase
from gslib.utils.wrapped_credentials import WrappedCredentials
import oauth2client
from six import add_move, MovedModule
add_move(MovedModule("mock", "mock", "unittest.mock"))
from six.moves import mock
ACCESS_TOKEN = "foo"
CONTENT = "content"
RESPONSE = httplib2.Response({
"content-type": "text/plain",
"status": "200",
"content-length": len(CONTENT),
})
class MockCredentials(external_account.Credentials):
def __init__(self, token=None, expiry=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self._audience = None
self.expiry = expiry
self.token = None
def side_effect(*args, **kwargs):
self.token = token
self.refresh = mock.Mock(side_effect=side_effect)
def _fake_auth_failure_response():
# the response body isn't really used in this case, but lets send it anyway
# to have a safe check in some future change on the rest client.
body = {"unauthorized": {"message": "Unauthorized", "code": "401"}}
return httplib2.Response({"status": "401"}), json.dumps(body)
import mock
import httplib2
from cloudservers.client import CloudServersClient
from nose.tools import assert_equal
fake_response = httplib2.Response({"status": 200})
fake_body = '{"hi": "there"}'
mock_request = mock.Mock(return_value=(fake_response, fake_body))
def client():
cl = CloudServersClient("username", "apikey", "auth_test")
cl.management_url = "http://example.com"
cl.auth_token = "token"
return cl
def test_get():
cl = client()
@mock.patch.object(httplib2.Http, "request", mock_request)
@mock.patch('time.time', mock.Mock(return_value=1234))
def test_get_call():
resp, body = cl.get("/hi")
mock_request.assert_called_with("http://example.com/hi?fresh=1234",
"GET",
headers={
"X-Auth-Token": "token",
"User-Agent": cl.USER_AGENT
})
# Automatic JSON parsing
def raw_request(*args, **kwargs):
self.assertNotIn(self.client.api_microversion_header_name,
kwargs['headers'])
return (httplib2.Response({'status': 200}), {})
try:
self._http.timeout = 10
response, content = self._http.request(
uri, method, body, headers)
self._http.timeout = None
self._memcache[uri] = time.time()
except Exception, e:
catch_error(e)
else:
response, content = self._http.request(uri, method, body, headers)
if DEBUG:
if response is None:
response = httplib2.Response(email.message_from_string(info))
print 'reply: %s %s from cache' % (response.status,
response.reason)
for key in response.keys():
print 'header: %s: %s' % (key.title(), response.get(key))
if response is not None and response.has_key('set-cookie'):
cookies = response.get('set-cookie')
jsessionid = re.findall(r'(JSESSIONID=[0-9A-F]+);', cookies)
if len(jsessionid) > 0:
self._jsession = jsessionid[0]
if response is not None and response.get('status') == '404':
r, _ = self._http.request(self._server)
if self._server.rstrip('/') != r.get('content-location',
def execute(self, http=None):
"""Execute all the requests as a single batched HTTP request.
Args:
http: httplib2.Http, an http object to be used in place of the one the
HttpRequest request object was constructed with. If one isn't supplied
then use a http object from the requests in this batch.
Returns:
None
Raises:
httplib2.Error if a transport error has occured.
apiclient.errors.BatchError if the response is the wrong format.
"""
# If http is not supplied use the first valid one given in the requests.
if http is None:
for request_id in self._order:
request = self._requests[request_id]
if request is not None:
http = request.http
break
if http is None:
raise ValueError("Missing a valid http object.")
self._execute(http, self._order, self._requests)
# Loop over all the requests and check for 401s. For each 401 request the
# credentials should be refreshed and then sent again in a separate batch.
redo_requests = {}
redo_order = []
for request_id in self._order:
headers, content = self._responses[request_id]
if headers['status'] == '401':
redo_order.append(request_id)
request = self._requests[request_id]
self._refresh_and_apply_credentials(request, http)
redo_requests[request_id] = request
if redo_requests:
self._execute(http, redo_order, redo_requests)
# Now process all callbacks that are erroring, and raise an exception for
# ones that return a non-2xx response? Or add extra parameter to callback
# that contains an HttpError?
for request_id in self._order:
headers, content = self._responses[request_id]
request = self._requests[request_id]
callback = self._callbacks[request_id]
response = None
exception = None
try:
r = httplib2.Response(headers)
response = request.postproc(r, content)
except HttpError, e:
exception = e
if callback is not None:
callback(request_id, response, exception)
if self._callback is not None:
self._callback(request_id, response, exception)
class BasicGoogleAPICalls(unittest.TestCase):
project = "mock_project"
zone = "mock_us_central1_a"
region = "mock_us_central1"
instance = "mock_instance_legacy"
boot_disk = "mock_boot_disk"
target_network = "mock_target_network"
target_subnetwork = "mock_target_subnetwork"
instance_template = {
"mock_instance_template": "mocking"}
internal_ip_address_body = {
"name": "example-internal-address",
"addressType": "INTERNAL",
"subnetwork": "regions/us-central1/subnetworks/my-custom-subnet",
"address": "10.128.0.12"
}
external_ip_address_body = {
"name": "example-external-address",
"address": "35.203.14.22"
}
http = HttpMock(datafile("compute_rest.json"), {
"status": "200"})
errorResponse = httplib2.Response({
"status": 404,
"reason": "HttpMock response: the resource is not found"})
successResponse = httplib2.Response({
"status": 200,
"reason": "HttpMock response: Successful"
})
def test_stop_instance_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.stop": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
stop_instance_operation = stop_instance(compute, self.project,
self.zone, self.instance)
self.assertEqual(
stop_instance_operation,
{
"foo": "bar"}
)
def test_stop_instance_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.stop": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
stop_instance(compute, self.project, self.zone, self.instance)
def test_start_instance_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.start": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
start_instance_operation = start_instance(compute, self.project,
self.zone, self.instance)
self.assertEqual(
start_instance_operation,
{
"foo": "bar"}
)
def test_start_instance_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.start": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
start_instance(compute, self.project, self.zone, self.instance)
def test_retrieve_instance_template_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.get": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
retrieve_template_operation = retrieve_instance_template(compute,
self.project,
self.zone,
self.instance)
self.assertEqual(
retrieve_template_operation,
{
"foo": "bar"}
)
def test_retrieve_instance_template_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.get": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
retrieve_instance_template(compute, self.project, self.zone,
self.instance)
def test_detach_disk_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.detachDisk": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
detach_disk_operation = detach_disk(compute, self.project,
self.zone, self.instance,
self.boot_disk)
self.assertEqual(
detach_disk_operation,
{
"foo": "bar"}
)
def test_detach_disk_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.detachDisk": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
detach_disk(compute, self.project, self.zone, self.instance,
self.boot_disk)
def test_attach_disk_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.attachDisk": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
attach_disk_operation = attach_disk(compute, self.project,
self.zone, self.instance,
self.boot_disk)
self.assertEqual(
attach_disk_operation,
{
"foo": "bar"}
)
def test_attach_disk_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.attachDisk": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
attach_disk(compute, self.project, self.zone, self.instance,
self.boot_disk)
def test_get_network_success(self):
request_builder = RequestMockBuilder(
{
"compute.networks.get": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
network_get_operation = get_network(compute, self.project,
self.target_network)
self.assertEqual(
network_get_operation,
{
"foo": "bar"}
)
def test_get_network_failure(self):
request_builder = RequestMockBuilder(
{
"compute.networks.get": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
get_network(compute, self.project, self.target_network)
def test_create_instance_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.insert": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
create_instance_operation = create_instance(compute, self.project,
self.zone,
self.instance_template)
self.assertEqual(
create_instance_operation,
{
"foo": "bar"}
)
def test_create_instance_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.insert": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
create_instance(compute, self.project, self.zone,
self.instance_template)
def test_delete_instance_success(self):
request_builder = RequestMockBuilder(
{
"compute.instances.delete": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
delete_instance_operation = delete_instance(compute, self.project,
self.zone, self.instance)
self.assertEqual(
delete_instance_operation,
{
"foo": "bar"}
)
def test_delete_instance_failure(self):
request_builder = RequestMockBuilder(
{
"compute.instances.delete": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
delete_instance(compute, self.project, self.zone,
self.instance_template)
def test_get_zone_success(self):
request_builder = RequestMockBuilder(
{
"compute.zones.get": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
get_zone_operation = get_zone(compute, self.project, self.zone)
self.assertEqual(
get_zone_operation,
{
"foo": "bar"}
)
def test_get_zone_failure(self):
request_builder = RequestMockBuilder(
{
"compute.zones.get": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
get_zone(compute, self.project, self.zone)
def test_check_network_auto_mode_success(self):
request_builder = RequestMockBuilder(
{
"compute.networks.get": (
self.successResponse, '{"autoCreateSubnetworks": true}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
auto_mode_status = check_network_auto_mode(compute, self.project,
self.zone)
self.assertEqual(auto_mode_status, True)
def test_check_network_auto_mode_failure(self):
request_builder = RequestMockBuilder(
{
"compute.networks.get": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
check_network_auto_mode(compute, self.project, self.zone)
def test_wait_for_zone_operation_success(self):
request_builder = RequestMockBuilder(
{
"compute.zoneOperations.get": (
self.successResponse, '{"status": "DONE"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
wait_response = wait_for_zone_operation(compute, self.project,
self.zone, {})
self.assertEqual(
wait_response,
{
"status": "DONE"}
)
def test_wait_for_zone_operation_failure(self):
request_builder = RequestMockBuilder(
{
"compute.zoneOperations.get": (
self.errorResponse, b'Invalid Resource')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
wait_for_zone_operation(compute, self.project, self.zone, {})
def test_wait_for_region_operation_success(self):
request_builder = RequestMockBuilder(
{
"compute.regionOperations.get": (
self.successResponse, '{"status": "DONE"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
wait_response = wait_for_region_operation(compute, self.project,
self.region, {})
self.assertEqual(
wait_response,
{
"status": "DONE"}
)
def test_wait_for_region_operation_failure(self):
request_builder = RequestMockBuilder(
{
"compute.regionOperations.get": (
self.errorResponse, b'Invalid Resource')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
wait_for_region_operation(compute, self.project, self.region, {})
def test_preserve_external_ip_success(self):
request_builder = RequestMockBuilder(
{
"compute.addresses.insert": (
self.successResponse, '{"foo": "bar"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
preserve_external_ip_address_operation = preserve_external_ip_address(
compute, self.project, self.region, self.external_ip_address_body)
self.assertEqual(
preserve_external_ip_address_operation,
{
"foo": "bar"}
)
def test_preserve_external_ip_failure(self):
request_builder = RequestMockBuilder(
{
"compute.addresses.insert": (
self.errorResponse, b"{Invalid resource}")})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(HttpError):
preserve_external_ip_address(compute, self.project, self.region,
self.external_ip_address_body)
class WaitForOperation(unittest.TestCase):
project = "mock_project"
zone = "mock_us_central1_a"
region = "mock_us_central1"
http = HttpMock(datafile("compute_rest.json"), {
"status": "200"})
errorResponse = httplib2.Response({
"status": 404,
"reason": "HttpMock response: the resource is not found"})
successResponse = httplib2.Response({
"status": 200,
"reason": "HttpMock response: Successful"
})
@timeout_decorator.timeout(3, timeout_exception=StopIteration)
def test_basic_zone_waiting(self):
request_builder = RequestMockBuilder(
{
"compute.zoneOperations.get": (
self.successResponse,
'{"status":"RUNNING"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(StopIteration):
wait_for_zone_operation(compute, self.project,
self.zone, {})
def test_error_in_zone_waiting(self):
request_builder = RequestMockBuilder(
{
"compute.zoneOperations.get": (
self.successResponse,
'{"status":"DONE", "error":"something wrong"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(ZoneOperationsError):
wait_for_zone_operation(compute, self.project,
self.zone, {})
@timeout_decorator.timeout(3, timeout_exception=StopIteration)
def test_basic_region_waiting(self):
request_builder = RequestMockBuilder(
{
"compute.regionOperations.get": (
self.successResponse,
'{"status":"RUNNING"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(StopIteration):
wait_for_region_operation(compute, self.project,
self.region, {})
def test_error_in_region_waiting(self):
request_builder = RequestMockBuilder(
{
"compute.regionOperations.get": (
self.successResponse,
'{"status":"DONE", "error":"something wrong"}')})
compute = build("compute", "v1", self.http,
requestBuilder=request_builder)
with self.assertRaises(RegionOperationsError):
wait_for_region_operation(compute, self.project,
self.region, {})
def test_authenticate_tenant_id(self):
cs = client.Client("username",
"password",
auth_url="auth_url/v2.0",
tenant_id='tenant_id',
service_type='compute')
resp = {
"access": {
"token": {
"expires": "12345",
"id": "FAKE_ID",
"tenant": {
"description": None,
"enabled": True,
"id": "tenant_id",
"name": "demo"
} # tenant associated with token
},
"serviceCatalog": [
{
"type":
"compute",
"endpoints": [
{
"region": "RegionOne",
"adminURL": "http://*****:*****@mock.patch.object(httplib2.Http, "request", mock_request)
def test_auth_call():
cs.client.authenticate()
headers = {
'User-Agent': cs.client.USER_AGENT,
'Content-Type': 'application/json',
'Accept': 'application/json',
}
body = {
'auth': {
'passwordCredentials': {
'username': cs.client.user,
'password': cs.client.password,
},
'tenantId': cs.client.tenant_id,
},
}
token_url = cs.client.auth_url + "/tokens"
mock_request.assert_called_with(token_url,
"POST",
headers=headers,
body=json.dumps(body))
endpoints = resp["access"]["serviceCatalog"][0]['endpoints']
public_url = endpoints[0]["publicURL"].rstrip('/')
self.assertEqual(cs.client.management_url, public_url)
token_id = resp["access"]["token"]["id"]
self.assertEqual(cs.client.auth_token, token_id)
tenant_id = resp["access"]["token"]["tenant"]["id"]
self.assertEqual(cs.client.tenant_id, tenant_id)
test_auth_call()
def raw_request(*args, **kwargs):
self.assertNotIn('X-OpenStack-Nova-API-Version', kwargs['headers'])
return (httplib2.Response({'status': 200}), {})
def FakeHttpResponse(
status_code: int,
body: Optional[Text] = _BODY) -> Tuple[httplib2.Response, Text]:
return (httplib2.Response({'status': status_code}), body)
def raw_request(*args, **kwargs):
self.assertIn('X-OpenStack-Nova-API-Version', kwargs['headers'])
self.assertEqual(self.client.api_microversion,
kwargs['headers']['X-OpenStack-Nova-API-Version'])
return (httplib2.Response({'status': 200}), {})
def wrapped_request(uri, method='GET', *args, **kwargs):
if uri == zoo_uri:
return httplib2.Response({'status': '200'}), zoo_contents
return original_request(uri, method=method, *args, **kwargs)
def _conn_request(self, conn, request_uri, method, body, headers):
try:
if hasattr(conn, 'sock') and conn.sock is None:
conn.connect()
conn.request(method, request_uri, body, headers)
except socket.timeout:
raise
except socket.gaierror:
conn.close()
raise httplib2.ServerNotFoundError(
'Unable to find the server at %s' % conn.host)
except httplib2.ssl.SSLError:
conn.close()
raise
except socket.error as e:
err = 0
if hasattr(e, 'args'):
err = getattr(e, 'args')[0]
else:
err = e.errno
if err == httplib2.errno.ECONNREFUSED: # Connection refused
raise
except http_client.HTTPException:
# Just because the server closed the connection doesn't apparently mean
# that the server didn't send a response.
conn.close()
raise
try:
response = conn.getresponse()
except (socket.error, http_client.HTTPException) as e:
conn.close()
raise
else:
content = ''
if method == 'HEAD':
conn.close()
response = httplib2.Response(response)
elif method == 'GET' and response.status in (http_client.OK,
http_client.PARTIAL_CONTENT):
content_length = None
if hasattr(response, 'msg'):
content_length = response.getheader('content-length')
http_stream = response
bytes_read = 0
while True:
new_data = http_stream.read(TRANSFER_BUFFER_SIZE)
if new_data:
if self.stream is None:
raise apitools_exceptions.InvalidUserInputError(
'Cannot exercise HttpWithDownloadStream with no stream')
text_util.write_to_fd(self.stream, new_data)
bytes_read += len(new_data)
else:
break
if (content_length is not None and
long(bytes_read) != long(content_length)):
# The input stream terminated before we were able to read the
# entire contents, possibly due to a network condition. Set
# content-length to indicate how many bytes we actually read.
self._logger.log(
logging.DEBUG, 'Only got %s bytes out of content-length %s '
'for request URI %s. Resetting content-length to match '
'bytes read.', bytes_read, content_length, request_uri)
response.msg['content-length'] = str(bytes_read)
response = httplib2.Response(response)
else:
# We fall back to the current httplib2 behavior if we're
# not processing download bytes, e.g., it's a redirect, an
# oauth2client POST to refresh an access token, or any HTTP
# status code that doesn't include object content.
content = response.read()
response = httplib2.Response(response)
# pylint: disable=protected-access
content = httplib2._decompressContent(response, content)
return (response, content)
def Send(self, request_path, payload='',
content_type='application/octet-stream',
timeout=None,
**kwargs):
"""Sends an RPC and returns the response.
Args:
request_path: The path to send the request to, eg /api/appversion/create.
payload: The body of the request, or None to send an empty request.
content_type: The Content-Type header to use.
timeout: timeout in seconds; default None i.e. no timeout.
(Note: for large requests on OS X, the timeout doesn't work right.)
Any keyword arguments are converted into query string parameters.
Returns:
The response body, as a string.
Raises:
AuthPermanentFail: If authorization failed in a permanent way.
urllib2.HTTPError: On most HTTP errors.
"""
self.http = httplib2.Http(cache=self.memory_cache, ca_certs=self.certpath)
self.http.follow_redirects = False
self.http.timeout = timeout
url = '%s://%s%s' % (self.scheme, self.host, request_path)
if kwargs:
url += '?' + urllib.urlencode(kwargs)
headers = {}
if self.extra_headers:
headers.update(self.extra_headers)
headers['Content-Type'] = content_type
headers['X-appcfg-api-version'] = '1'
headers['content-length'] = str(len(payload))
if self.host_override:
headers['Host'] = self.host_override
tries = 0
auth_tries = [0]
def NeedAuth():
"""Marker that we need auth; it'll actually be tried next time around."""
auth_tries[0] += 1
if auth_tries[0] > self.auth_tries:
RaiseHttpError(url, response_info, response, 'Too many auth attempts.')
while tries < self.rpc_tries:
tries += 1
self._Authenticate(self.http, auth_tries[0] > 0)
logger.debug('Sending request to %s headers=%s body=%s',
url, headers,
self.debug_data and payload or payload and 'ELIDED' or '')
try:
response_info, response = self.http.request(url, 'POST', body=payload,
headers=headers)
except client.AccessTokenRefreshError, e:
logger.info('Got access token error', exc_info=1)
response_info = httplib2.Response({'status': 401})
response_info.reason = str(e)
response = ''
status = response_info.status
if status == 200:
return response
logger.debug('Got http error %s, this is try #%s',
response_info.status, tries)
if status == 401:
NeedAuth()
continue
elif status >= 500 and status < 600:
continue
elif status == 302:
loc = response_info.get('location')
logger.debug('Got 302 redirect. Location: %s', loc)
if (loc.startswith('https://www.google.com/accounts/ServiceLogin') or
re.match(r'https://www.google.com/a/[a-z0-9.-]+/ServiceLogin',
loc)):
NeedAuth()
continue
elif loc.startswith('http://%s/_ah/login' % (self.host,)):
RaiseHttpError(url, response_info, response,
'dev_appserver login not supported')
else:
RaiseHttpError(url, response_info, response,
'Unexpected redirect to %s' % loc)
else:
logger.debug('Unexpected results: %s', response_info)
RaiseHttpError(url, response_info, response,
'Unexpected HTTP status %s' % status)
def test__create_uploads_in_chunks_and_reloads_resource(self):
resource_id = random.randint(0, 100)
name = "%s/%s" % (make_name_without_spaces("os"),
make_name_without_spaces("release"))
architecture = "%s/%s" % (make_name_without_spaces("arch"),
make_name_without_spaces("subarch"))
title = make_name_without_spaces("title")
filetype = random.choice([
boot_resources.BootResourceFileType.TGZ,
boot_resources.BootResourceFileType.DDTGZ
])
upload_uri = "/MAAS/api/2.0/boot-resources/%d/upload/1" % resource_id
# Make chunks and upload in pieces of 4, where the last piece is
# smaller than chunk size.
chunk_size = random.randint(10, 100)
data = make_string(int(chunk_size * 3.25)).encode("ascii")
sha256 = hashlib.sha256()
sha256.update(data)
sha256 = sha256.hexdigest()
size = len(data)
buf = io.BytesIO(data)
# Mock the handler calls. BootResource.read will be called after the
# upload is complete to get the updated object.
origin = make_origin()
BootResources = origin.BootResources
BootResource = origin.BootResource
BootResources._handler.uri = (
"http://localhost:5240/MAAS/api/2.0/boot-resources/")
BootResources._handler.create.return_value = {
"id": resource_id,
"type": "Uploaded",
"name": name,
"architecture": architecture,
"sets": {
"20161026": {
"version": "20161026",
"size": size,
"label": "uploaded",
"complete": False,
"files": {
"root-tgz": {
"filename": "root-tgz",
"filetype": "root-dd",
"size": size,
"sha256": sha256,
"complete": False,
"upload_uri": upload_uri,
}
}
}
}
}
BootResource._handler.read.return_value = {
"id": resource_id,
"type": "Uploaded",
"name": name,
"architecture": architecture,
"sets": {
"20161026": {
"version": "20161026",
"size": size,
"label": "uploaded",
"complete": True,
"files": {
"root-tgz": {
"filename": "root-tgz",
"filetype": "root-dd",
"size": size,
"sha256": sha256,
"complete": True,
}
}
}
}
}
# Mock signing. Test checks that its actually called.
mock_sign = self.patch(boot_resources.utils, "sign")
# Mock the httplib as the create performs PUT directly to the API.
http = MagicMock()
http.return_value.request.return_value = (httplib2.Response(
{"status": 200}), b"")
self.patch(boot_resources.httplib2, "Http", http)
# Progress handler called on each chunk.
progress_handler = MagicMock()
# Create and upload the resource.
resource = BootResources.create(name,
architecture,
buf,
title=title,
filetype=filetype,
chunk_size=chunk_size,
progress_callback=progress_handler)
# Check that returned resource is correct and updated.
self.assertThat(
resource,
MatchesStructure.byEquality(id=resource_id,
type="Uploaded",
name=name,
architecture=architecture))
self.assertTrue(resource.sets["20161026"].complete)
# Check that sign was called.
self.assertTrue(mock_sign.called)
# Check that the PUT was called for each chunk.
calls = [
call("http://localhost:5240/MAAS/api/2.0/"
"boot-resources/%d/upload/1" % resource_id,
"PUT",
body=data[0 + i:chunk_size + i],
headers={
'Content-Type': 'application/octet-stream',
'Content-Length': str(len(data[0 + i:chunk_size + i])),
}) for i in range(0, len(data), chunk_size)
]
self.assertEquals(calls, http.return_value.request.call_args_list)
# Check that progress handler was called on each chunk.
calls = [
call(len(data[:chunk_size + i]) / len(data))
for i in range(0, len(data), chunk_size)
]
self.assertEquals(calls, progress_handler.call_args_list)
def to_http_response(resp_dict):
"""Converts dict of response attributes to httplib response."""
resp = httplib2.Response(resp_dict)
for k, v in resp_dict['headers'].items():
resp[k] = v
return resp
def test__create_raises_CallError_on_chunk_upload_failure(self):
resource_id = random.randint(0, 100)
name = "%s/%s" % (make_name_without_spaces("os"),
make_name_without_spaces("release"))
architecture = "%s/%s" % (make_name_without_spaces("arch"),
make_name_without_spaces("subarch"))
title = make_name_without_spaces("title")
filetype = random.choice([
boot_resources.BootResourceFileType.TGZ,
boot_resources.BootResourceFileType.DDTGZ
])
upload_uri = "/MAAS/api/2.0/boot-resources/%d/upload/1" % resource_id
# Make chunks and upload in pieces of 4, where the last piece is
# smaller than chunk size.
chunk_size = random.randint(10, 100)
data = make_string(int(chunk_size * 3.25)).encode("ascii")
sha256 = hashlib.sha256()
sha256.update(data)
sha256 = sha256.hexdigest()
size = len(data)
buf = io.BytesIO(data)
# Mock the handler calls. BootResource.read will be called after the
# upload is complete to get the updated object.
origin = make_origin()
BootResources = origin.BootResources
BootResource = origin.BootResource
BootResources._handler.uri = (
"http://localhost:5240/MAAS/api/2.0/boot-resources/")
BootResources._handler.path = "/MAAS/api/2.0/boot-resources/"
BootResources._handler.create.return_value = {
"id": resource_id,
"type": "Uploaded",
"name": name,
"architecture": architecture,
"sets": {
"20161026": {
"version": "20161026",
"size": size,
"label": "uploaded",
"complete": False,
"files": {
"root-tgz": {
"filename": "root-tgz",
"filetype": "root-dd",
"size": size,
"sha256": sha256,
"complete": False,
"upload_uri": upload_uri,
}
}
}
}
}
# Mock signing. Test checks that its actually called.
mock_sign = self.patch(boot_resources.utils, "sign")
# Mock the httplib as the create performs PUT directly to the API.
http = MagicMock()
http.return_value.request.return_value = (httplib2.Response(
{"status": 500}), b"Error")
self.patch(boot_resources.httplib2, "Http", http)
self.assertRaises(boot_resources.CallError,
BootResources.create,
name,
architecture,
buf,
title=title,
filetype=filetype,
chunk_size=chunk_size)
class PreserveIPAddressHandler(unittest.TestCase):
MOCK_CREDENTIALS = mock.Mock(spec=google.auth.credentials.Credentials)
compute = discovery.build('compute', 'v1', credentials=MOCK_CREDENTIALS)
project = "mock_project"
zone = "mock_us_central1_a"
region = "mock_us_central1"
original_instance = "mock_instance_legacy"
new_instance = "mock_instance_new"
http = HttpMock(datafile("compute_rest.json"), {"status": "200"})
external_ip_address_body = {
"name": "example-external-address",
"address": "35.203.14.22"
}
alreadyExistsResponse = httplib2.Response({
"status":
400,
"reason":
"HttpMock response: the IP address already exists"
})
staticIPNameExistsResponse = httplib2.Response({
"status":
400,
"reason":
"HttpMock response: the IP's name already exists"
})
otherErrorResponse = httplib2.Response({
"status":
404,
"reason":
"HttpMock response: invalid request"
})
successResponse = httplib2.Response({
"status":
200,
"reason":
"HttpMock response: Successful"
})
new_network_info = {
"network":
"https://www.googleapis.com/compute/v1/projects/mock_project/global/networks/mock_target_network",
"subnetwork":
"https://www.googleapis.com/compute/v1/projects/mock_project/regions/us-central1/subnetworks/mock_target_subnetwork"
}
def test_not_preserve_external_ip(self):
request_builder = RequestMockBuilder({
"compute.addresses.insert":
(self.successResponse, '{"name": "bar"}'),
"compute.regionOperations.get":
(self.successResponse, '{"status":"DONE"}')
})
compute = build("compute",
"v1",
self.http,
requestBuilder=request_builder)
original_instance_template = read_json_file(
"sample_instance_template.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
compute, self.project, self.new_instance, self.new_network_info,
original_network_interface, self.region, False)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is ephemeral
self.assertFalse("accessConfigs" in new_network_interface)
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def test_preserve_external_ip_without_http_error(self):
request_builder = RequestMockBuilder({
"compute.addresses.insert":
(self.successResponse, '{"name": "bar"}'),
"compute.regionOperations.get":
(self.successResponse, '{"status":"DONE"}')
})
compute = build("compute",
"v1",
self.http,
requestBuilder=request_builder)
original_instance_template = read_json_file(
"sample_instance_template.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
compute, self.project, self.new_instance, self.new_network_info,
original_network_interface, self.region, True)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is unchanged
self.assertEqual(new_network_interface["accessConfigs"],
original_network_interface["accessConfigs"])
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def test_preserve_an_existing_static_external_ip(self, *mocks):
request_builder = RequestMockBuilder({
"compute.addresses.insert": (self.alreadyExistsResponse, b''),
"compute.regionOperations.get":
(self.successResponse, '{"status":"DONE"}')
})
compute = build("compute",
"v1",
self.http,
requestBuilder=request_builder)
original_instance_template = read_json_file(
"sample_instance_template.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
compute, self.project, self.new_instance, self.new_network_info,
original_network_interface, self.region, True)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is unchanged
self.assertEqual(new_network_interface["accessConfigs"],
original_network_interface["accessConfigs"])
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def test_preserve_an_existing_static_ip_name(self):
# No error raises, and the original external IP will be used
# in the new instance template
request_builder = RequestMockBuilder({
"compute.addresses.insert": (self.staticIPNameExistsResponse, b''),
"compute.regionOperations.get":
(self.successResponse, '{"status":"DONE"}')
})
compute = build("compute",
"v1",
self.http,
requestBuilder=request_builder)
original_instance_template = read_json_file(
"sample_instance_template.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
compute, self.project, self.new_instance, self.new_network_info,
original_network_interface, self.region, True)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is unchanged
self.assertEqual(new_network_interface["accessConfigs"],
original_network_interface["accessConfigs"])
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def test_other_http_error(self):
# No error raises, and the original external IP will be used
# in the new instance template
request_builder = RequestMockBuilder({
"compute.addresses.insert": (self.otherErrorResponse, b''),
"compute.regionOperations.get":
(self.successResponse, '{"status":"DONE"}')
})
compute = build("compute",
"v1",
self.http,
requestBuilder=request_builder)
original_instance_template = read_json_file(
"sample_instance_template.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
compute, self.project, self.new_instance, self.new_network_info,
original_network_interface, self.region, True)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is ephemeral
self.assertFalse("accessConfigs" in new_network_interface)
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def test_no_natIP_exists_in_original_vm(self):
# No error raises, and the original external IP will be used
# in the new instance template
original_instance_template = read_json_file(
"sample_instance_template_no_natIP.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
self.compute, self.project, self.new_instance,
self.new_network_info, original_network_interface, self.region,
True)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is ephemeral
self.assertFalse("natIP" in new_network_interface["accessConfigs"])
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def test_no_external_ip_exists_in_original_vm(self):
# No error raises, and the original external IP will be used
# in the new instance template
original_instance_template = read_json_file(
"sample_instance_template_no_external_ip.json")
original_network_interface = original_instance_template[
"networkInterfaces"][0]
new_network_interface = preserve_ip_addresses_handler(
self.compute, self.project, self.new_instance,
self.new_network_info, original_network_interface, self.region,
True)
self.assertEqual(new_network_interface["network"],
self.new_network_info["network"])
self.assertEqual(new_network_interface["subnetwork"],
self.new_network_info["subnetwork"])
# external IP is ephemeral
self.assertFalse("accessConfigs" in new_network_interface)
# internal IP is ephemeral
self.assertFalse("networkIP" in new_network_interface)
def fake_response(data, headers, reason="Ok"):
response = httplib2.Response(headers)
response.reason = reason
return response, data
def upload_from_file(self,
file_obj,
source_format,
rewind=False,
size=None,
num_retries=6,
allow_jagged_rows=None,
allow_quoted_newlines=None,
create_disposition=None,
encoding=None,
field_delimiter=None,
ignore_unknown_values=None,
max_bad_records=None,
quote_character=None,
skip_leading_rows=None,
write_disposition=None,
client=None):
"""Upload the contents of this table from a file-like object.
The content type of the upload will either be
- The value passed in to the function (if any)
- ``text/csv``.
:type file_obj: file
:param file_obj: A file handle opened in binary mode for reading.
:type source_format: str
:param source_format: one of 'CSV' or 'NEWLINE_DELIMITED_JSON'.
job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`
:type rewind: bool
:param rewind: If True, seek to the beginning of the file handle before
writing the file to Cloud Storage.
:type size: int
:param size: The number of bytes to read from the file handle.
If not provided, we'll try to guess the size using
:func:`os.fstat`. (If the file handle is not from the
filesystem this won't be possible.)
:type num_retries: int
:param num_retries: Number of upload retries. Defaults to 6.
:type allow_jagged_rows: bool
:param allow_jagged_rows: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type allow_quoted_newlines: bool
:param allow_quoted_newlines: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type create_disposition: str
:param create_disposition: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type encoding: str
:param encoding: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type field_delimiter: str
:param field_delimiter: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type ignore_unknown_values: bool
:param ignore_unknown_values: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type max_bad_records: int
:param max_bad_records: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type quote_character: str
:param quote_character: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type skip_leading_rows: int
:param skip_leading_rows: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type write_disposition: str
:param write_disposition: job configuration option; see
:meth:`google.cloud.bigquery.job.LoadJob`.
:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current dataset.
:rtype: :class:`google.cloud.bigquery.jobs.LoadTableFromStorageJob`
:returns: the job instance used to load the data (e.g., for
querying status). Note that the job is already started:
do not call ``job.begin()``.
:raises: :class:`ValueError` if ``size`` is not passed in and can not
be determined, or if the ``file_obj`` can be detected to be
a file opened in text mode.
"""
client = self._require_client(client)
connection = client.connection
content_type = 'application/octet-stream'
# Rewind the file if desired.
if rewind:
file_obj.seek(0, os.SEEK_SET)
mode = getattr(file_obj, 'mode', None)
if mode is not None and mode not in ('rb', 'r+b', 'rb+'):
raise ValueError(
"Cannot upload files opened in text mode: use "
"open(filename, mode='rb') or open(filename, mode='r+b')")
# Get the basic stats about the file.
total_bytes = size
if total_bytes is None:
if hasattr(file_obj, 'fileno'):
total_bytes = os.fstat(file_obj.fileno()).st_size
else:
raise ValueError('total bytes could not be determined. Please '
'pass an explicit size.')
headers = {
'Accept': 'application/json',
'Accept-Encoding': 'gzip, deflate',
'User-Agent': connection.USER_AGENT,
'content-type': 'application/json',
}
metadata = {
'configuration': {
'load': {
'sourceFormat': source_format,
'schema': {
'fields': _build_schema_resource(self._schema),
},
'destinationTable': {
'projectId': self._dataset.project,
'datasetId': self._dataset.name,
'tableId': self.name,
}
}
}
}
_configure_job_metadata(metadata, allow_jagged_rows,
allow_quoted_newlines, create_disposition,
encoding, field_delimiter,
ignore_unknown_values, max_bad_records,
quote_character, skip_leading_rows,
write_disposition)
upload = Upload(file_obj,
content_type,
total_bytes,
auto_transfer=False)
url_builder = _UrlBuilder()
upload_config = _UploadConfig()
# Base URL may change once we know simple vs. resumable.
base_url = connection.API_BASE_URL + '/upload'
path = '/projects/%s/jobs' % (self._dataset.project, )
upload_url = connection.build_api_url(api_base_url=base_url, path=path)
# Use apitools 'Upload' facility.
request = Request(upload_url,
'POST',
headers,
body=json.dumps(metadata))
upload.configure_request(upload_config, request, url_builder)
query_params = url_builder.query_params
base_url = connection.API_BASE_URL + '/upload'
request.url = connection.build_api_url(api_base_url=base_url,
path=path,
query_params=query_params)
try:
upload.initialize_upload(request, connection.http)
except HttpError as err_response:
faux_response = httplib2.Response(err_response.response)
raise make_exception(faux_response,
err_response.content,
error_info=request.url)
if upload.strategy == RESUMABLE_UPLOAD:
http_response = upload.stream_file(use_chunks=True)
else:
http_response = make_api_request(connection.http,
request,
retries=num_retries)
self._check_response_error(request, http_response)
response_content = http_response.content
if not isinstance(response_content,
six.string_types): # pragma: NO COVER Python3
response_content = response_content.decode('utf-8')
return client.job_from_resource(json.loads(response_content))
def ResponseToTransportResponse(self, response): """Converts a Response object to the response returned by the transport.""" headers = response.headers.copy() headers['status'] = response.status return (httplib2.Response(headers), http_encoding.Encode(response.body))
def test_successful_copy_template_with_bigger_array_fields(
self, mock_hook):
mock_hook.return_value.get_instance_template.side_effect = [
HttpError(resp=httplib2.Response({'status': 404}),
content=EMPTY_CONTENT),
GCE_INSTANCE_TEMPLATE_BODY_GET,
GCE_INSTANCE_TEMPLATE_BODY_GET_NEW,
]
op = ComputeEngineCopyInstanceTemplateOperator(
project_id=GCP_PROJECT_ID,
resource_id=GCE_INSTANCE_TEMPLATE_NAME,
task_id='id',
body_patch={
"name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
"properties": {
"disks": [
{
"kind": "compute#attachedDisk",
"type": "SCRATCH",
"licenses": [
"Updated String",
],
},
{
"kind": "compute#attachedDisk",
"type": "PERSISTENT",
"licenses": [
"Another String",
],
},
],
},
},
)
result = op.execute(None)
mock_hook.assert_called_once_with(
api_version='v1',
gcp_conn_id='google_cloud_default',
impersonation_chain=None,
)
body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
body_insert["properties"]["disks"] = [
{
"kind": "compute#attachedDisk",
"type": "SCRATCH",
"licenses": [
"Updated String",
],
},
{
"kind": "compute#attachedDisk",
"type": "PERSISTENT",
"licenses": [
"Another String",
],
},
]
mock_hook.return_value.insert_instance_template.assert_called_once_with(
project_id=GCP_PROJECT_ID,
body=body_insert,
request_id=None,
)
self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
def _CreateTestHttpNotFoundError(status, reason, body=None, url=None):
if body is None:
body = ''
response = httplib2.Response({'status': status, 'reason': reason})
return exceptions.HttpNotFoundError(response, body, url)