def export(self,
export_id: str,
user_id: str,
cart_id: str,
access_token: str,
collection_uuid: str,
collection_version: str,
resume_token: str = None):
content = self.get_content(user_id, cart_id, collection_uuid,
collection_version, resume_token)
client = CollectionDataAccess(access_token)
items = content['items']
if resume_token is None:
cart = self.cart_item_manager.get_cart(user_id, cart_id)
cart_name = cart['CartName']
description = f"Exported from Cart {cart_id} in the Data Browser"
collection = client.create(collection_uuid, cart_name, description,
collection_version, items)
else:
try:
collection = client.append(collection_uuid, collection_version,
items)
# The returning collection will contain the new version of the collection.
except UnauthorizedClientAccessError:
# DSS may deny the access to the collection API when the given
# access token expires before the cart export finishes. In this
# case, the export job can be resumed with a new access token.
logger.error(
'Export %s: DSS denied access to the collection API.',
export_id)
raise ExpiredAccessTokenError
return dict(collection=collection,
resume_token=content['resume_token'],
exported_item_count=len(items))
def test_export_create_new_collection(self, _dynamodb_client):
expected_collection = dict(uuid='abc', version='123')
expected_get_content_result = dict(
resume_token='rt1', items=[1, 2, 3,
4]) # NOTE: This is just for the test.
service = CartExportService()
with patch.object(service.cart_item_manager,
'get_cart',
side_effect=[dict(CartName='abc123')]):
with patch.object(service,
'get_content',
side_effect=[expected_get_content_result]):
with ResponsesHelper() as helper:
helper.add(
responses.Response(
responses.PUT,
CollectionDataAccess.endpoint_url('collections'),
status=201,
json=expected_collection))
result = service.export(
export_id='export1',
user_id='user1',
cart_id='cart1',
access_token='at1',
collection_uuid=expected_collection['uuid'],
collection_version='ver1',
resume_token=None)
self.assertEqual(expected_collection, result['collection'])
self.assertEqual(expected_get_content_result['resume_token'],
result['resume_token'])
self.assertEqual(len(expected_get_content_result['items']),
result['exported_item_count'])
def test_export_append_items_to_collection_raises_expired_access_token_error(
self, _dynamodb_client):
expected_collection = dict(uuid='abc', version='123')
expected_get_content_result = dict(
resume_token='rt1', items=[1, 2, 3,
4]) # NOTE: This is just for the test.
service = CartExportService()
with self.assertRaises(ExpiredAccessTokenError):
with patch.object(service,
'get_content',
side_effect=[expected_get_content_result]):
with ResponsesHelper() as helper:
url = CollectionDataAccess.endpoint_url(
'collections', expected_collection['uuid'])
helper.add(
responses.Response(responses.PATCH,
url,
status=401,
json=dict(code='abc')))
service.export(export_id='export1',
user_id='user1',
cart_id='cart1',
access_token='at1',
collection_uuid=expected_collection['uuid'],
collection_version='ver1',
resume_token='rt0')
def test_export_append_items_to_collection_ok(self, _dynamodb_client):
expected_collection = dict(uuid='abc', version='123')
expected_get_content_result = dict(resume_token='rt1',
items=[1, 2, 3, 4]) # NOTE: This is just for the test.
service = CartExportService()
with patch.object(service, 'get_content', side_effect=[expected_get_content_result]):
with responses.RequestsMock() as helper:
helper.add(responses.Response(
responses.PATCH,
CollectionDataAccess.endpoint_url('collections', expected_collection['uuid']),
json=expected_collection
))
result = service.export(export_id='export1',
user_id='user1',
cart_id='cart1',
access_token='at1',
collection_uuid=expected_collection['uuid'],
collection_version='ver1',
resume_token='rt0')
self.assertEqual(expected_collection, result['collection'])
self.assertEqual(expected_get_content_result['resume_token'], result['resume_token'])
self.assertEqual(len(expected_get_content_result['items']), result['exported_item_count'])
def test_unified_fetch_endpoint_check_status_on_succeeded_job_ok(self):
test_jwt_ttl = config.cart_export_min_access_token_ttl + 10
mock_user_id = 'user1'
mock_jwt_subject = 'fake|' + mock_user_id
mock_cart_uuid = 'mock-cart-1234'
expected_export_token = 'abc123'
mock_collection_uuid = 'ci-123'
mock_collection_ver = 'cv-123'
mock_job = dict(
status='SUCCEEDED',
user_id=mock_jwt_subject,
final=True,
last_update=dict(
state=dict(collection_uuid=mock_collection_uuid,
collection_version=mock_collection_ver)))
export_url = f'{self.base_url}/fetch/resources/carts/{mock_cart_uuid}/export?token={expected_export_token}'
# NOTE The empty side_effect is to ensure that "initiate" never get called.
with patch.object(CartExportJobManager, 'initiate', side_effect=[]):
with patch.object(CartExportJobManager,
'get',
side_effect=[mock_job]):
with AuthResponseHelper(self.base_url) as helper:
test_jwt = helper.generate_test_jwt(
'*****@*****.**',
identifier=mock_user_id,
ttl=test_jwt_ttl)
response = requests.get(
export_url,
headers=dict(Authorization=f'Bearer {test_jwt}'),
allow_redirects=False)
self.assertEqual(200, response.status_code)
response_body = response.json()
self.assertEqual(200, response_body['Status'])
prefix_expected_url = CollectionDataAccess.endpoint_url(
'collections', mock_collection_uuid)
self.assertEqual(
f'{prefix_expected_url}?version={mock_collection_ver}&replica=aws',
response_body['Location'])
def setUp(self):
fake_access_token = 'fake_access_token'
self.cda = CollectionDataAccess(fake_access_token)
class CollectionDataAccessTestCase(TestCase):
def setUp(self):
fake_access_token = 'fake_access_token'
self.cda = CollectionDataAccess(fake_access_token)
@responses.activate
def test_get_ok(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
fake_collection = {'hello': 'world'}
with ResponsesHelper() as helper:
helper.add(
responses.Response(responses.GET,
self.cda.endpoint_url(
'collections', test_collection_uuid),
json=fake_collection))
collection = self.cda.get(test_collection_uuid,
test_collection_version)
self.assertEqual(
collection,
dict(uuid=test_collection_uuid,
version=test_collection_version,
collection=fake_collection))
@responses.activate
def test_get_raises_retrival_error(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(responses.GET,
self.cda.endpoint_url(
'collections',
test_collection_uuid),
callback=RequestCallback(567, '{}'),
content_type='application/json'))
with self.assertRaises(RetrievalError):
self.cda.get(test_collection_uuid, test_collection_version)
@responses.activate
def test_create_ok(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
expected_collection = dict(uuid=test_collection_uuid,
version=test_collection_version)
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PUT,
self.cda.endpoint_url('collections'),
callback=RequestCallback(201,
json.dumps(expected_collection)),
content_type='application/json'))
collection = self.cda.create(test_collection_uuid, 'foo bar',
'bar', test_collection_version, [])
self.assertEqual(collection, expected_collection)
@responses.activate
def test_create_raises_creation_error(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
fake_dss_response = {"code": "unknown"}
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PUT,
self.cda.endpoint_url('collections'),
callback=RequestCallback(500,
json.dumps(fake_dss_response)),
content_type='application/json'))
with self.assertRaises(CreationError):
self.cda.create(test_collection_uuid, 'foo bar', 'bar',
test_collection_version, [])
@responses.activate
def test_append_with_no_items_successful(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
expected_collection = dict(uuid=test_collection_uuid,
version=test_collection_version)
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PATCH,
self.cda.endpoint_url('collections', test_collection_uuid),
callback=RequestCallback(200,
json.dumps(expected_collection)),
content_type='application/json'))
collection = self.cda.append(test_collection_uuid,
test_collection_version, [])
self.assertEqual(collection, expected_collection)
@responses.activate
def test_append_with_some_items_successful(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
expected_collection = dict(uuid=test_collection_uuid,
version=test_collection_version)
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(
responses.PATCH,
self.cda.endpoint_url('collections', test_collection_uuid),
callback=RequestCallback(200,
json.dumps(expected_collection)),
content_type='application/json'))
collection = self.cda.append(
test_collection_uuid, test_collection_version, [
dict(type='foo_1', uuid='bar_1', version='baz_1'),
dict(type='foo_2', uuid='bar_2', version='baz_2'),
dict(type='foo_n', uuid='bar_n', version='baz_n')
])
self.assertEqual(collection, expected_collection)
@responses.activate
def test_append_raises_update_error(self):
test_collection_uuid = 'abcdef123456'
test_collection_version = '1980-01-01'
with ResponsesHelper() as helper:
helper.add(
responses.CallbackResponse(responses.PATCH,
self.cda.endpoint_url(
'collections',
test_collection_uuid),
callback=RequestCallback(405, '{}'),
content_type='application/json'))
with self.assertRaises(UpdateError):
self.cda.append(test_collection_uuid, test_collection_version,
[])
@responses.activate
def test_send_request_successful_with_auto_retry_on_http_504_timeout(self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'hello_world'}
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=RequestCallback(
200,
json.dumps(expected_response),
delay=True),
content_type='application/json'))
response = self.cda.send_request(test_collection_uuid, 'get', url,
{})
self.assertEqual(response.json(), expected_response)
@responses.activate
def test_send_request_successful_with_auto_retry_on_http_502(self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'hello_world'}
mock_response_sequence = [(502, {}, '{"code": "mock_error"}'),
(200, {}, json.dumps(expected_response))]
def mock_request_handler(_request):
return mock_response_sequence.pop(0)
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=mock_request_handler,
content_type='application/json'))
response = self.cda.send_request(test_collection_uuid, 'get', url,
{})
self.assertEqual(response.json(), expected_response)
@responses.activate
def test_send_request_fails_after_too_many_retries(self):
test_collection_uuid = 'abcdef123456'
with self.assertRaises(ServerTimeoutError):
self.cda.send_request(test_collection_uuid,
'get',
'fake_url', {},
delay=64)
@responses.activate
def test_send_request_with_unexpected_response_code_raises_client_error(
self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'hello_world'}
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=RequestCallback(
201,
json.dumps(expected_response)),
content_type='application/json'))
with self.assertRaises(ClientError):
self.cda.send_request(test_collection_uuid,
'get',
url, {},
expected_status_code=200)
@responses.activate
def test_send_request_with_unexpected_response_code_raises_unauthorized_client_access_error(
self):
test_collection_uuid = 'abcdef123456'
expected_response = {'code': 'mock_error'}
with ResponsesHelper() as helper:
url = self.cda.endpoint_url(test_collection_uuid)
helper.add(
responses.CallbackResponse(responses.GET,
url,
callback=RequestCallback(
401,
json.dumps(expected_response)),
content_type='application/json'))
with self.assertRaises(UnauthorizedClientAccessError):
self.cda.send_request(test_collection_uuid,
'get',
url, {},
expected_status_code=200)