def _upload_file(path, data, content_type, readers):
"""Overwrites a file in GCS, makes it readable to all authorized readers.
Doesn't use streaming uploads currently. Data is limited by URL Fetch request
size (10 MB).
Args:
path: "<bucket>/<object>" string.
data: buffer with data to upload.
content_type: MIME content type of 'data', to put into GCS metadata.
readers: list of emails that should have read access to the file.
Raises:
Error if Google Storage writes fail.
"""
# We upload both metadata and body in a single request, see:
# https://cloud.google.com/storage/docs/json_api/v1/how-tos/multipart-upload.
bucket, name = path.split('/', 1)
payload, boundary = _multipart_payload(data, content_type, {
'name': name,
'acl': _gcs_acls(readers)
})
try:
net.request(
url='https://www.googleapis.com/upload/storage/v1/b/%s/o' % bucket,
method='POST',
payload=payload,
params={'uploadType': 'multipart'},
headers={
'Content-Type': 'multipart/related; boundary=%s' % boundary
},
scopes=['https://www.googleapis.com/auth/cloud-platform'],
deadline=30)
except net.Error as exc:
raise Error(str(exc))
def test_gives_up_retrying(self):
self.mock_urlfetch([
({'url': 'http://localhost/123'}, Response(500, 'server error', {})),
({'url': 'http://localhost/123'}, Response(500, 'server error', {})),
({'url': 'http://localhost/123'}, Response(200, 'response body', {})),
])
with self.assertRaises(net.Error):
net.request('http://localhost/123', max_attempts=2)
def test_405(self):
self.mock_urlfetch([
({
'url': 'http://localhost/123'
}, Response(405, 'method not allowed', {})),
])
with self.assertRaises(net.MethodNotAllowed):
net.request('http://localhost/123')
def test_404(self):
self.mock_urlfetch([
({
'url': 'http://localhost/123'
}, Response(404, 'Not found', {})),
])
with self.assertRaises(net.NotFoundError):
net.request('http://localhost/123')
def test_legitimate_cloud_endpoints_404_is_not_retried(self):
self.mock_urlfetch([
({
'url': 'http://localhost/_ah/api/blah'
}, Response(404, '{}', {'Content-Type': 'application/json'})),
])
with self.assertRaises(net.NotFoundError):
net.request('http://localhost/_ah/api/blah')
def test_403(self):
self.mock_urlfetch([
({
'url': 'http://localhost/123'
}, Response(403, 'Auth error', {})),
])
with self.assertRaises(net.AuthError):
net.request('http://localhost/123')
def test_gives_up_retrying(self):
self.mock_urlfetch([
({'url': 'http://localhost/123'}, Response(500, 'server error')),
({'url': 'http://localhost/123'}, Response(500, 'server error')),
({'url': 'http://localhost/123'}, Response(200, 'response body')),
])
with self.assertRaises(net.Error):
net.request('http://localhost/123', max_attempts=2)
def test_legitimate_cloud_endpoints_404_is_not_retried(self):
self.mock_urlfetch([
(
{'url': 'http://localhost/_ah/api/blah'},
Response(404, '{}', {'Content-Type': 'application/json'})
),
])
with self.assertRaises(net.NotFoundError):
net.request('http://localhost/_ah/api/blah')
def forward_data(data, ip):
"""Forwards the raw data to the backend.
The request contains all the required headers, incliding a special
Endpoint-Url header with the endpoint URL, and the correct
Authorization: header for that endpoint.
Args:
data (str): raw binary data to forward.
ip (str): the IP address of the data source (used for traffic split).
Raises:
AdminError when endpoint data is not entered in the admin console.
"""
lb = LoadBalancer()
module_name = lb.choose_module()
logging.info('Forwarding request (%d bytes) to module: %s',
len(data), module_name)
hostname = app_identity.get_default_version_hostname()
if utils.is_local_dev_server():
protocol = 'http'
hostname = 'localhost:808%s' % module_name[-1]
else:
protocol = 'https'
config_data = _get_config_data()
if not config_data:
raise AdminError('Endpoints are not defined')
# Make the traffic split deterministic in the source IP.
# TODO(sergeyberezin): make it truly random. Most of our sources
# are behind NAT boxes, and appear as the same IP.
random_state = random.getstate()
random.seed(ip)
if random.uniform(0, 100) < config_data.secondary_endpoint_load:
endpoint = config_data.secondary_endpoint
else:
endpoint = config_data.primary_endpoint
random.setstate(random_state)
url = '%s://%s/%s' % (protocol, hostname, module_name)
service_account_key = _get_credentials(endpoint.credentials)
headers = {
common.ENDPOINT_URL_HEADER: endpoint.url,
'Content-Type': 'application/x-protobuf',
}
headers.update(endpoint.headers)
net.request(
url=url,
method='POST',
payload=data,
headers=headers,
scopes=endpoint.scopes,
service_account_key=service_account_key)
def forward_data(data, ip):
"""Forwards the raw data to the backend.
The request contains all the required headers, incliding a special
Endpoint-Url header with the endpoint URL, and the correct
Authorization: header for that endpoint.
Args:
data (str): raw binary data to forward.
ip (str): the IP address of the data source (used for traffic split).
Raises:
AdminError when endpoint data is not entered in the admin console.
"""
lb = LoadBalancer()
module_name = lb.choose_module()
logging.info('Forwarding request (%d bytes) to module: %s', len(data),
module_name)
hostname = app_identity.get_default_version_hostname()
if utils.is_local_dev_server():
protocol = 'http'
hostname = 'localhost:808%s' % module_name[-1]
else:
protocol = 'https'
config_data = _get_config_data()
if not config_data:
raise AdminError('Endpoints are not defined')
# Make the traffic split deterministic in the source IP.
# TODO(sergeyberezin): make it truly random. Most of our sources
# are behind NAT boxes, and appear as the same IP.
random_state = random.getstate()
random.seed(ip)
if random.uniform(0, 100) < config_data.secondary_endpoint_load:
endpoint = config_data.secondary_endpoint
else:
endpoint = config_data.primary_endpoint
random.setstate(random_state)
url = '%s://%s/%s' % (protocol, hostname, module_name)
service_account_key = _get_credentials(endpoint.credentials)
headers = {
common.ENDPOINT_URL_HEADER: endpoint.url,
'Content-Type': 'application/x-protobuf',
}
headers.update(endpoint.headers)
net.request(url=url,
method='POST',
payload=data,
headers=headers,
scopes=endpoint.scopes,
service_account_key=service_account_key)
def test_project_tokens_external_service(self):
self.mock_urlfetch([
({
'deadline': 10,
'headers': {
'Authorization': 'Bearer project-id-token'
},
'method': 'GET',
'url': 'https://external.example.com/123',
}, Response(200, '', {})),
])
net.request(url='https://external.example.com/123',
project_id='project-id')
def test_request_works(self):
self.mock_urlfetch([
({
'deadline': 123,
'headers': {
'Accept': 'text/plain',
'Authorization': 'Bearer token'
},
'method': 'POST',
'payload': 'post body',
'url': 'http://localhost/123?a=%3D&b=%26',
}, Response(200, 'response body', {})),
])
response = net.request(url='http://localhost/123',
method='POST',
payload='post body',
params={
'a': '=',
'b': '&'
},
headers={'Accept': 'text/plain'},
scopes=['scope'],
service_account_key=auth.ServiceAccountKey(
'a', 'b', 'c'),
deadline=123,
max_attempts=5)
self.assertEqual('response body', response)
def test_retries_transient_errors(self):
self.mock_urlfetch([
({'url': 'http://localhost/123'}, urlfetch.Error()),
({'url': 'http://localhost/123'}, Response(408, 'clien timeout', {})),
({'url': 'http://localhost/123'}, Response(500, 'server error', {})),
({'url': 'http://localhost/123'}, Response(200, 'response body', {})),
])
response = net.request('http://localhost/123', max_attempts=4)
self.assertEqual('response body', response)
def test_retries_transient_errors(self):
self.mock_urlfetch([
({'url': 'http://localhost/123'}, urlfetch.Error()),
({'url': 'http://localhost/123'}, Response(408, 'clien timeout')),
({'url': 'http://localhost/123'}, Response(500, 'server error')),
({'url': 'http://localhost/123'}, Response(200, 'response body')),
])
response = net.request('http://localhost/123', max_attempts=4)
self.assertEqual('response body', response)
def test_crappy_cloud_endpoints_404_is_retried(self):
self.mock_urlfetch([
({
'url': 'http://localhost/_ah/api/blah'
}, Response(404, 'Not found', {})),
({
'url': 'http://localhost/_ah/api/blah'
}, Response(200, 'response body', {})),
])
response = net.request('http://localhost/_ah/api/blah')
self.assertEqual('response body', response)
def test_crappy_cloud_endpoints_404_is_retried(self):
self.mock_urlfetch([
(
{'url': 'http://localhost/_ah/api/blah'},
Response(404, 'Not found', {})
),
(
{'url': 'http://localhost/_ah/api/blah'},
Response(200, 'response body', {})
),
])
response = net.request('http://localhost/_ah/api/blah')
self.assertEqual('response body', response)
def _set_gcs_metadata(path, metadata):
"""Overwrites file metadata (including ACLs) in GCS.
Args:
path: "<bucket>/<object>" string.
metadata: the metadata dict.
Raises:
Error if Google Storage update fails.
"""
bucket, name = path.split('/', 1)
try:
net.request(
url='https://www.googleapis.com/storage/v1/b/%s/o/%s' %
(bucket, urllib.quote(name, safe='')),
method='PUT',
payload=utils.encode_to_json(metadata),
headers={'Content-Type': 'application/json; charset=UTF-8'},
scopes=['https://www.googleapis.com/auth/cloud-platform'],
deadline=30)
except net.Error as exc:
raise Error(str(exc))
def test_project_tokens_fallback(self):
@ndb.tasklet
def get_project_access_token(_project_id, _scopes):
raise auth.NotFoundError('not found')
self.mock(auth, 'get_project_access_token_async',
get_project_access_token)
self.mock_urlfetch([
(
{
'deadline': 10,
'headers': {
# Switches to own token.
'Authorization': 'Bearer own-token',
},
'method': 'GET',
'url': 'https://external.example.com/123',
},
Response(200, '', {})),
])
net.request(url='https://external.example.com/123',
project_id='project-id')
def fetch(hostname, path, **kwargs):
"""Sends request to Gerrit, returns raw response.
See 'net.request' for list of accepted kwargs.
Returns:
Response body on success.
None on 404 response.
Raises:
net.Error on communication errors.
"""
assert not path.startswith('/'), path
assert 'scopes' not in kwargs, kwargs['scopes']
try:
url = urlparse.urljoin('https://' + hostname, 'a/' + path)
return net.request(url, scopes=[AUTH_SCOPE], **kwargs)
except net.NotFoundError:
return None
def test_request_works(self):
self.mock_urlfetch([
({
'deadline': 123,
'headers': {'Accept': 'text/plain', 'Authorization': 'Bearer token'},
'method': 'POST',
'payload': 'post body',
'url': 'http://localhost/123?a=%3D&b=%26',
}, Response(200, 'response body', {})),
])
response = net.request(
url='http://localhost/123',
method='POST',
payload='post body',
params={'a': '=', 'b': '&'},
headers={'Accept': 'text/plain'},
scopes=['scope'],
service_account_key=auth.ServiceAccountKey('a', 'b', 'c'),
deadline=123,
max_attempts=5)
self.assertEqual('response body', response)
def test_request_project_token_fallback_works(self):
@ndb.tasklet
def mocked_get_project_access_token_async(*args, **kwargs):
mocked_get_project_access_token_async.params = (args, kwargs)
mocked_get_project_access_token_async.called = True
raise auth.NotFoundError('testing fallback 1')
mocked_get_project_access_token_async.called = False
@ndb.tasklet
def mocked_get_access_token_async(*args, **kwargs):
mocked_get_access_token_async.params = (args, kwargs)
mocked_get_access_token_async.called = True
raise Exception('testing fallback 2')
mocked_get_access_token_async.called = False
self.mock(auth, 'get_project_access_token_async',
mocked_get_project_access_token_async)
self.mock(auth, 'get_access_token_async',
mocked_get_access_token_async)
with self.assertRaises(Exception):
_ = net.request(url='http://localhost/123',
method='POST',
payload='post body',
params={
'a': '=',
'b': '&'
},
headers={'Accept': 'text/plain'},
scopes=['scope'],
service_account_key=auth.ServiceAccountKey(
'a', 'b', 'c'),
deadline=123,
max_attempts=5,
project_id='project1')
self.assertTrue(mocked_get_project_access_token_async.called)
self.assertTrue(mocked_get_access_token_async.called)
def test_403(self):
self.mock_urlfetch([
({'url': 'http://localhost/123'}, Response(403, 'Auth error', {})),
])
with self.assertRaises(net.AuthError):
net.request('http://localhost/123')
def test_404(self):
self.mock_urlfetch([
({'url': 'http://localhost/123'}, Response(404, 'Not found', {})),
])
with self.assertRaises(net.NotFoundError):
net.request('http://localhost/123')