def test_cached_health(self):
storage_service = StorageService()
storage_service.create_bucket()
# No health object is available in S3 bucket, yielding an error
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
response = requests.get(self.base_url + '/health/cached')
self.assertEqual(500, response.status_code)
self.assertEqual(
'ChaliceViewError: Cached health object does not exist',
response.json()['Message'])
# A successful response is obtained when all the systems are functional
self._create_mock_queues()
endpoint_states = self._endpoint_states()
app = load_app_module(self.lambda_name())
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
with self._mock_service_endpoints(helper, endpoint_states):
app.update_health_cache(MagicMock(), MagicMock())
response = requests.get(self.base_url + '/health/cached')
self.assertEqual(200, response.status_code)
# Another failure is observed when the cache health object is older than 2 minutes
future_time = time.time() + 3 * 60
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
with patch('time.time', new=lambda: future_time):
response = requests.get(self.base_url + '/health/cached')
self.assertEqual(500, response.status_code)
self.assertEqual(
'ChaliceViewError: Cached health object is stale',
response.json()['Message'])
def _mock_other_lambdas(self, helper: ResponsesHelper, up: bool):
for lambda_name in self._other_lambda_names():
helper.add(
responses.Response(method='GET',
url=config.lambda_endpoint(lambda_name) +
'/health/basic',
status=200 if up else 500,
json={'up': up}))
def _mock_service_endpoints(self, helper: ResponsesHelper,
endpoint_states: Mapping[str, bool]) -> None:
for endpoint, endpoint_up in endpoint_states.items():
helper.add(
responses.Response(method='HEAD',
url=config.service_endpoint() + endpoint,
status=200 if endpoint_up else 503,
json={}))
yield
def test_export_append_items_to_collection_raises_expired_access_token_error(
self, _dynamodb_client):
expected_collection = dict(uuid='abc', version='123')
expected_get_content_result = dict(
resume_token='rt1', items=[1, 2, 3,
4]) # NOTE: This is just for the test.
service = CartExportService()
with self.assertRaises(ExpiredAccessTokenError):
with patch.object(service,
'get_content',
side_effect=[expected_get_content_result]):
with ResponsesHelper() as helper:
url = CollectionDataAccess.endpoint_url(
'collections', expected_collection['uuid'])
helper.add(
responses.Response(responses.PATCH,
url,
status=401,
json=dict(code='abc')))
service.export(export_id='export1',
user_id='user1',
cart_id='cart1',
access_token='at1',
collection_uuid=expected_collection['uuid'],
collection_version='ver1',
resume_token='rt0')
def test_export_create_new_collection(self, _dynamodb_client):
expected_collection = dict(uuid='abc', version='123')
expected_get_content_result = dict(
resume_token='rt1', items=[1, 2, 3,
4]) # NOTE: This is just for the test.
service = CartExportService()
with patch.object(service.cart_item_manager,
'get_cart',
side_effect=[dict(CartName='abc123')]):
with patch.object(service,
'get_content',
side_effect=[expected_get_content_result]):
with ResponsesHelper() as helper:
helper.add(
responses.Response(
responses.PUT,
CollectionDataAccess.endpoint_url('collections'),
status=201,
json=expected_collection))
result = service.export(
export_id='export1',
user_id='user1',
cart_id='cart1',
access_token='at1',
collection_uuid=expected_collection['uuid'],
collection_version='ver1',
resume_token=None)
self.assertEqual(expected_collection, result['collection'])
self.assertEqual(expected_get_content_result['resume_token'],
result['resume_token'])
self.assertEqual(len(expected_get_content_result['items']),
result['exported_item_count'])
def test_health_endpoint_keys(self):
endpoint_states = self._endpoint_states()
expected = {
keys: {
'up': True,
**expected_response
}
for keys, expected_response in [
('elasticsearch', self._expected_elasticsearch(True)),
('queues', self._expected_queues(True)),
('other_lambdas', self._expected_other_lambdas(True)),
('api_endpoints',
self._expected_api_endpoints(endpoint_states)),
('progress', self._expected_progress()),
('progress,queues', {
**self._expected_progress(),
**self._expected_queues(True)
}),
]
}
self._create_mock_queues()
for keys, expected_response in expected.items():
with self.subTest(msg=keys):
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
self._mock_other_lambdas(helper, up=True)
with self._mock_service_endpoints(helper, endpoint_states):
response = requests.get(self.base_url + '/health/' +
keys)
self.assertEqual(200, response.status_code)
self.assertEqual(expected_response, response.json())
def _test(self,
endpoint_states: Mapping[str, bool],
lambdas_up: bool,
path: str = '/health/fast'):
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
self._mock_other_lambdas(helper, lambdas_up)
with self._mock_service_endpoints(helper, endpoint_states):
return requests.get(self.base_url + path)
def _shorten_query_url(self, url, expect_status=None):
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
response = requests.post(self.base_url + '/url', json={'url': url})
if expect_status is None:
response.raise_for_status()
else:
self.assertEqual(response.status_code, expect_status)
return response.json()
def test_manifest_endpoint_start_execution(self, mock_uuid,
step_function_helper):
"""
Calling start manifest generation without a token should start an
execution and return a response with Retry-After and Location in the
headers.
"""
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
for fetch in True, False:
with self.subTest(fetch=fetch):
execution_name = '6c9dfa3f-e92e-11e8-9764-ada973595c11'
mock_uuid.return_value = execution_name
step_function_helper.describe_execution.return_value = {
'status': 'RUNNING'
}
format_ = ManifestFormat.compact.value
filters = {'organ': {'is': ['lymph node']}}
params = {
'catalog': self.catalog,
'filters': json.dumps(filters),
'format': format_
}
if fetch:
response = requests.get(self.base_url +
'/fetch/manifest/files',
params=params)
response.raise_for_status()
response = response.json()
else:
response = requests.get(self.base_url +
'/manifest/files',
params=params,
allow_redirects=False)
self.assertEqual(
301,
response['Status'] if fetch else response.status_code)
self.assertIn('Retry-After',
response if fetch else response.headers)
self.assertIn('Location',
response if fetch else response.headers)
if fetch:
# The 'CommandLine' value is verified in TestManifestResponse,
# here we only need to confirm the key exists in the response.
self.assertIn('CommandLine', response)
step_function_helper.start_execution.assert_called_once_with(
config.manifest_state_machine_name,
execution_name,
execution_input=dict(catalog=self.catalog,
format=format_,
filters=filters,
object_key=None))
step_function_helper.describe_execution.assert_called_once(
)
step_function_helper.reset_mock()
def test_get_raises_retrival_error(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(responses.GET,
self.cda.endpoint_url(
'collections',
test_collection_uuid),
callback=RequestCallback(567, '{}'),
content_type='application/json'))
with self.assertRaises(RetrievalError):
self.cda.get(test_collection_uuid, test_collection_version)
def test_data_object_not_found(self):
file_uuid = 'NOT_A_GOOD_IDEA'
error_body = 'DRS should just proxy the DSS for error responses'
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
url = f'{config.dss_endpoint}/files/{file_uuid}'
helper.add(responses.Response(method=responses.GET,
body=error_body,
url=url,
status=404))
drs_response = requests.get(
dss_drs_object_url(file_uuid, base_url=self.base_url))
self.assertEqual(drs_response.status_code, 404)
self.assertEqual(drs_response.text, error_body)
def test_append_raises_update_error(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(responses.PATCH,
self.cda.endpoint_url(
'collections',
test_collection_uuid),
callback=RequestCallback(405, '{}'),
content_type='application/json'))
with self.assertRaises(UpdateError):
self.cda.append(test_collection_uuid, test_collection_version,
[])
def test_create_raises_creation_error(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
fake_dss_response = {"code": "unknown"}
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PUT,
self.cda.endpoint_url('collections'),
callback=RequestCallback(500,
json.dumps(fake_dss_response)),
content_type='application/json'))
with self.assertRaises(CreationError):
self.cda.create(test_collection_uuid, 'foo bar', 'bar',
test_collection_version, [])
def test_send_request_successful_with_auto_retry_on_http_504_timeout(self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'hello_world'}
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=RequestCallback(
200,
json.dumps(expected_response),
delay=True),
content_type='application/json'))
response = self.cda.send_request(test_collection_uuid, 'get', url,
{})
self.assertEqual(response.json(), expected_response)
def test_append_with_no_items_successful(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
expected_collection = dict(uuid=test_collection_uuid,
version=test_collection_version)
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PATCH,
self.cda.endpoint_url('collections', test_collection_uuid),
callback=RequestCallback(200,
json.dumps(expected_collection)),
content_type='application/json'))
collection = self.cda.append(test_collection_uuid,
test_collection_version, [])
self.assertEqual(collection, expected_collection)
def test_create_ok(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
expected_collection = dict(uuid=test_collection_uuid,
version=test_collection_version)
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PUT,
self.cda.endpoint_url('collections'),
callback=RequestCallback(201,
json.dumps(expected_collection)),
content_type='application/json'))
collection = self.cda.create(test_collection_uuid, 'foo bar',
'bar', test_collection_version, [])
self.assertEqual(collection, expected_collection)
def test_laziness(self):
# Note the absence of moto decorators on this test.
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
self._mock_other_lambdas(helper, up=True)
# If Health weren't lazy, it would fail due the lack of mocks for SQS.
response = requests.get(self.base_url + '/health/other_lambdas')
# The use of subTests ensures that we see the result of both
# assertions. In the case of the health endpoint, the body of a 503
# may carry a body with additional information.
self.assertEqual(200, response.status_code)
expected_response = {
'up': True,
**self._expected_other_lambdas(up=True)
}
self.assertEqual(expected_response, response.json())
def test_get_ok(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
fake_collection = {'hello': 'world'}
with ResponsesHelper() as helper:
helper.add(
responses.Response(responses.GET,
self.cda.endpoint_url(
'collections', test_collection_uuid),
json=fake_collection))
collection = self.cda.get(test_collection_uuid,
test_collection_version)
self.assertEqual(
collection,
dict(uuid=test_collection_uuid,
version=test_collection_version,
collection=fake_collection))
def test_send_request_with_unexpected_response_code_raises_unauthorized_client_access_error(
self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'mock_error'}
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=RequestCallback(
401,
json.dumps(expected_response)),
content_type='application/json'))
with self.assertRaises(UnauthorizedClientAccessError):
self.cda.send_request(test_collection_uuid,
'get',
url, {},
expected_status_code=200)
def test_send_request_successful_with_auto_retry_on_http_502(self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'hello_world'}
mock_response_sequence = [(502, {}, '{"code": "mock_error"}'),
(200, {}, json.dumps(expected_response))]
def mock_request_handler(_request):
return mock_response_sequence.pop(0)
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=mock_request_handler,
content_type='application/json'))
response = self.cda.send_request(test_collection_uuid, 'get', url,
{})
self.assertEqual(response.json(), expected_response)
def _get_data_object(self, file_uuid, file_version):
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
drs_url = dss_dos_object_url(file_uuid=file_uuid,
catalog=self.catalog,
file_version=file_version,
base_url=self.base_url)
with mock.patch('time.time', new=lambda: 1547691253.07010):
dss_url = config.dss_endpoint + '/files/7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb'
helper.add(responses.Response(method=responses.GET,
url=dss_url,
status=301,
headers={'location': dss_url}))
helper.add(responses.Response(method=responses.GET,
url=dss_url,
status=302,
headers={'location': 'gs://foo/bar'}))
drs_response = requests.get(drs_url)
drs_response.raise_for_status()
drs_response_json = drs_response.json()
data_object = drs_response_json['data_object']
return data_object
def test_drs(self):
"""
Mocks the DSS backend, then uses the DRS endpoints as a client is
expected to.
"""
file_uuid = '7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb'
file_version = '2018-11-02T113344.698028Z'
for redirects in (0, 1, 2, 6):
with self.subTest(redirects=redirects):
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
self._mock_responses(helper, redirects, file_uuid, file_version=file_version)
# Make first client request
drs_response = requests.get(
dss_drs_object_url(file_uuid,
file_version=file_version,
base_url=self.base_url))
drs_response.raise_for_status()
drs_object = drs_response.json()
expected = {
'checksums': [
{'sha1': '7ad306f154ce7de1a9a333cfd9100fc26ef652b4'},
{'sha-256': '77337cb51b2e584b5ae1b99db6c163b988cbc5b894dda2f5d22424978c3bfc7a'}
],
'created_time': '2018-11-02T11:33:44.698028Z',
'id': '7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb',
'self_uri': dss_drs_object_uri(file_uuid='7b07f99e-4a8a-4ad0-bd4f-db0d7a00c7bb',
file_version='2018-11-02T113344.698028Z'),
'size': '195142097',
'version': '2018-11-02T113344.698028Z',
}
if not redirects:
# We expect a DRS object with an access URL
expected['access_methods'] = [
{
'access_url': {'url': 'https://org-hca-dss-checkout-prod.s3.amazonaws.com/'
'blobs/307.a72.eb6?foo=bar&et=cetera'},
'type': 'https'
},
{
'access_url': {'url': 'gs://important-bucket/object/path'},
'type': 'gs'
}
]
self.assertEqual(drs_object, expected)
else:
# The access IDs are so similar because the mock tokens are the same...
expected['access_methods'] = [
{
'access_id': 'KCd7ImV4ZWN1dGlvbl9pZCI6ICI5NWIxZmNkMC01OGMyLTRmMmMtYmI0OC0xM2FkODU2YzI0Z'
'mMiLCAic3RhcnRfdGltZSI6IDE1NzUzMjQzODEuMTk4Mzg2NywgImF0dGVtcHRzIjogMH0nLC'
'AnYXdzJyk',
# ^ ...but they do differ
'type': 'https'
},
{
'access_id': 'KCd7ImV4ZWN1dGlvbl9pZCI6ICI5NWIxZmNkMC01OGMyLTRmMmMtYmI0OC0xM2FkODU2YzI0Z'
'mMiLCAic3RhcnRfdGltZSI6IDE1NzUzMjQzODEuMTk4Mzg2NywgImF0dGVtcHRzIjogMH0nLC'
'AnZ2NwJyk',
'type': 'gs'
}
]
# We must make another request with the access ID
self.assertEqual(expected, drs_object)
for method in drs_object['access_methods']:
access_id = method['access_id']
for _ in range(redirects - 1):
# The first redirect gave us the access ID, the rest are retries on 202
drs_access_url = dss_drs_object_url(file_uuid,
file_version=file_version,
base_url=self.base_url,
access_id=access_id)
drs_response = requests.get(drs_access_url)
self.assertEqual(drs_response.status_code, 202)
self.assertEqual(drs_response.text, '')
# The final request should give us just the access URL
drs_access_url = dss_drs_object_url(file_uuid,
file_version=file_version,
base_url=self.base_url,
access_id=access_id)
drs_response = requests.get(drs_access_url)
self.assertEqual(drs_response.status_code, 200)
if method['type'] == AccessMethod.https.scheme:
self.assertEqual(drs_response.json(), {'url': self.signed_url})
elif method['type'] == AccessMethod.gs.scheme:
self.assertEqual(drs_response.json(), {'url': self.gs_url})
else:
assert False, f'Access type {method["type"]} is not supported'
def test_dss_files_proxy(self, dss_direct_access_role):
dss_direct_access_role.return_value = None
self.maxDiff = None
key = ("blobs/6929799f227ae5f0b3e0167a6cf2bd683db097848af6ccde6329185212598779"
".f2237ad0a776fd7057eb3d3498114c85e2f521d7"
".7e892bf8f6aa489ccb08a995c7f017e1."
"847325b6")
bucket_name = 'org-humancellatlas-dss-checkout-staging'
s3 = boto3.client('s3')
s3.create_bucket(Bucket=bucket_name)
s3.upload_fileobj(Bucket=bucket_name, Fileobj=io.BytesIO(b'foo'), Key=key)
file_uuid = '701c9a63-23da-4978-946b-7576b6ad088a'
file_version = '2018-09-12T121154.054628Z'
organic_file_name = 'foo.txt'
file_doc = {
'name': organic_file_name,
'version': file_version,
'drs_path': None,
}
with mock.patch.object(IndexQueryService, 'get_data_file', return_value=file_doc):
dss_url = furl(
url=config.dss_endpoint,
path='/v1/files',
args={
'replica': 'aws',
'version': file_version
}).add(path=file_uuid)
dss_token = 'some_token'
dss_url_with_token = dss_url.copy().add(args={'token': dss_token})
for fetch in True, False:
for wait in None, 0, 1:
for file_name, signature in [(None, 'Wg8AqCTzZAuHpCN8AKPKWcsFHAM='),
(organic_file_name, 'Wg8AqCTzZAuHpCN8AKPKWcsFHAM=',),
('foo bar.txt', 'grbM6udwp0n/QE/L/RYfjtQCS/U='),
('foo&bar.txt', 'r4C8YxpJ4nXTZh+agBsfhZ2e7fI=')]:
with self.subTest(fetch=fetch, file_name=file_name, wait=wait):
with ResponsesHelper() as helper:
helper.add_passthru(self.base_url)
fixed_time = 1547691253.07010
expires = str(round(fixed_time + 3600))
s3_url = furl(
url=f'https://{bucket_name}.s3.amazonaws.com',
path=key,
args={
'AWSAccessKeyId': 'SOMEACCESSKEY',
'Signature': 'SOMESIGNATURE=',
'x-amz-security-token': 'SOMETOKEN',
'Expires': expires
})
helper.add(responses.Response(method='GET',
url=dss_url.url,
status=301,
headers={'Location': dss_url_with_token.url,
'Retry-After': '10'}))
azul_url = furl(
url=self.base_url,
path='/fetch/dss/files' if fetch else '/dss/files',
args={
'catalog': 'test',
'version': file_version
}).add(path=file_uuid)
if wait is not None:
azul_url.args['wait'] = str(wait)
if file_name is not None:
azul_url.args['fileName'] = file_name
def request_azul(url, expect_status):
retry_after = 1
expect_retry_after = None if wait or expect_status == 302 else retry_after
before = time.monotonic()
with mock.patch.object(type(aws), 'dss_checkout_bucket', return_value=bucket_name):
with mock.patch('time.time', new=lambda: 1547691253.07010):
response = requests.get(url, allow_redirects=False)
if wait and expect_status == 301:
self.assertLessEqual(retry_after, time.monotonic() - before)
if fetch:
self.assertEqual(200, response.status_code)
response = response.json()
self.assertEqual(expect_status, response["Status"])
else:
if response.status_code != expect_status:
response.raise_for_status()
response = dict(response.headers)
if expect_retry_after is None:
self.assertNotIn('Retry-After', response)
else:
actual_retry_after = response['Retry-After']
if fetch:
self.assertEqual(expect_retry_after, actual_retry_after)
else:
self.assertEqual(str(expect_retry_after), actual_retry_after)
return response['Location']
location = request_azul(url=azul_url.url, expect_status=301)
if file_name is None:
file_name = organic_file_name
azul_url.args['token'] = dss_token
azul_url.args['requestIndex'] = '1'
azul_url.args['fileName'] = file_name
azul_url.args['replica'] = 'aws'
self.assertUrlEqual(azul_url, location)
helper.add(responses.Response(method='GET',
url=dss_url_with_token.url,
status=302,
headers={'Location': s3_url.url}))
location = request_azul(url=location, expect_status=302)
re_pre_signed_s3_url = furl(
url=f'https://{bucket_name}.s3.amazonaws.com',
path=key,
args={
'response-content-disposition': f'attachment;filename={file_name}',
'AWSAccessKeyId': mock_access_key_id,
'Signature': signature,
'Expires': expires,
'x-amz-security-token': mock_session_token
})
self.assertUrlEqual(re_pre_signed_s3_url, location)