def test_next_page_is_delivered_when_next_is_in_query(self):
bundles = self.populate_search_index(self.index_document, 150)
self.check_count(smartseq2_paired_ends_vx_query, 150)
url = self.build_url()
search_obj = self.assertPostResponse(
path=url,
json_request_body=dict(es_query=smartseq2_paired_ends_vx_query),
expected_code=requests.codes.partial,
headers=get_auth_header())
found_bundles = search_obj.json['results']
next_url = self.get_next_url(search_obj.response.headers)
self.verify_next_url(next_url)
self.verify_search_result(search_obj.json,
smartseq2_paired_ends_vx_query, len(bundles),
100)
search_obj = self.assertPostResponse(
path=self.strip_next_url(next_url),
json_request_body=dict(es_query=smartseq2_paired_ends_vx_query),
expected_code=requests.codes.ok,
headers=get_auth_header())
found_bundles.extend(search_obj.json['results'])
self.verify_search_result(search_obj.json,
smartseq2_paired_ends_vx_query, len(bundles),
50)
self.verify_bundles(found_bundles, bundles)
def test_get(self):
find_uuid = self._put_subscription()
# Normal request
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(find_uuid)).add_query(
"replica", self.replica.name))
resp_obj = self.assertGetResponse(url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query,
json_response['es_query'])
self.assertEqual(self.callback_url, json_response['callback_url'])
# Forbidden request w/ previous url
with self.throw_403():
self.assertGetResponse(url,
requests.codes.forbidden,
headers=get_auth_header())
# File not found request
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(uuid.uuid4())).add_query(
"replica", self.replica.name))
self.assertGetResponse(url,
requests.codes.not_found,
headers=get_auth_header())
def test_output_format_is_raw(self):
bundles = self.populate_search_index(self.index_document, 11)
self.check_count(smartseq2_paired_ends_vx_query, 11)
url = self.build_url(url_params={
'output_format': 'raw',
'per_page': 11
})
search_obj = self.assertPostResponse(
path=url,
json_request_body=dict(es_query=smartseq2_paired_ends_vx_query),
expected_code=requests.codes.partial,
headers=get_auth_header())
results = search_obj.json['results']
next_url = self.get_next_url(search_obj.response.headers)
self.assertIsNotNone(next_url)
self.verify_search_result(search_obj.json,
smartseq2_paired_ends_vx_query, 11, 10)
search_obj = self.assertPostResponse(
path=self.strip_next_url(next_url),
json_request_body=dict(es_query=smartseq2_paired_ends_vx_query),
expected_code=requests.codes.ok,
headers=get_auth_header())
results.extend(search_obj.json['results'])
next_url = self.get_next_url(search_obj.response.headers)
self.assertIsNone(next_url)
self.verify_search_result(search_obj.json,
smartseq2_paired_ends_vx_query, 11, 1)
self.verify_bundles(results, bundles)
self.assertEqual(
sorted([
key for key in search_obj.json['results'][0]['metadata']
['manifest'].keys()
]),
sorted([key for key in self.index_document['manifest'].keys()]))
def test_get(self):
try:
find_uuid = self._put_subscription()
# Normal request
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(find_uuid))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertGetResponse(
url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query, json_response['es_query'])
self.assertEqual(self.endpoint, Endpoint.from_subscription(json_response))
self.assertEquals(self.hmac_key_id, json_response['hmac_key_id'])
self.assertNotIn('hmac_secret_key', json_response)
finally:
self._cleanup_subscription(find_uuid)
# File not found request
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(uuid.uuid4()))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
self.assertGetResponse(
url,
requests.codes.not_found,
headers=get_auth_header())
def _test_file_get_not_found(self, replica: Replica):
file_uuid = "ce55fd51-7833-469b-be0b-5da88ec0ffee"
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica", replica.name))
with override_bucket_config(BucketConfig.TEST_FIXTURE):
self.assertGetResponse(url,
requests.codes.not_found,
headers=get_auth_header(),
expected_error=ExpectedErrorFields(
code="not_found",
status=requests.codes.not_found,
expect_stacktrace=True))
version = "2017-06-16T193604.240704Z"
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica", replica.name).add_query("version", version))
with override_bucket_config(BucketConfig.TEST_FIXTURE):
self.assertGetResponse(url,
requests.codes.not_found,
headers=get_auth_header(),
expected_error=ExpectedErrorFields(
code="not_found",
status=requests.codes.not_found,
expect_stacktrace=True))
def test_access_control(self):
with self.subTest("PUT"):
uuid = str(uuid4())
self.addCleanup(self._delete_collection, uuid)
res = self.app.put("/v1/collections",
headers=get_auth_header(authorized=False),
params=dict(version=datetime.now().isoformat(),
uuid=uuid,
replica='aws'),
json=dict(name="n",
description="d",
details={},
contents=self.contents))
self.assertEqual(res.status_code, requests.codes.forbidden)
with self.subTest("GET"):
res = self.app.get("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=False),
params=dict(replica="aws"))
self.assertEqual(res.status_code, requests.codes.forbidden)
with self.subTest("PATCH"):
res = self.app.patch("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=False),
params=dict(replica="aws",
version=self.version),
json=dict(description="foo"))
self.assertEqual(res.status_code, requests.codes.forbidden)
with self.subTest("DELETE"):
res = self.app.delete("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=False),
params=dict(replica="aws"))
self.assertEqual(res.status_code, requests.codes.forbidden)
def upload_file_wait(
self: typing.Any,
source_url: str,
replica: Replica,
file_uuid: str = None,
file_version: str = None,
bundle_uuid: str = None,
timeout_seconds: int = 120,
expect_async: typing.Optional[bool] = None,
) -> DSSAssertResponse:
"""
Upload a file. If the request is being handled asynchronously, wait until the file has landed in the data
store.
"""
file_uuid = str(uuid.uuid4()) if file_uuid is None else file_uuid
bundle_uuid = str(uuid.uuid4()) if bundle_uuid is None else bundle_uuid
if expect_async is True:
expected_codes = requests.codes.accepted
elif expect_async is False:
expected_codes = requests.codes.created
else:
expected_codes = requests.codes.created, requests.codes.accepted
if file_version is None:
timestamp = datetime.datetime.utcnow()
file_version = datetime_to_version_format(timestamp)
url = UrlBuilder().set(path=f"/v1/files/{file_uuid}")
url.add_query("version", file_version)
resp_obj = self.assertPutResponse(str(url),
expected_codes,
json_request_body=dict(
bundle_uuid=bundle_uuid,
creator_uid=0,
source_url=source_url,
),
headers=get_auth_header())
if resp_obj.response.status_code == requests.codes.accepted:
# hit the GET /files endpoint until we succeed.
start_time = time.time()
timeout_time = start_time + timeout_seconds
while time.time() < timeout_time:
try:
self.assertHeadResponse(
f"/v1/files/{file_uuid}?replica={replica.name}",
requests.codes.ok,
headers=get_auth_header())
break
except AssertionError:
pass
time.sleep(1)
else:
self.fail("Could not find the output file")
return resp_obj
def test_delete(self):
uuid, version = self._put(self.contents)
with self.subTest("Delete collection"):
res = self.app.delete("/v1/collections/{}".format(uuid),
headers=get_auth_header(authorized=True),
params=dict(replica="aws"))
res.raise_for_status()
with self.subTest("Verify deleted"):
res = self.app.get("/v1/collections/{}".format(uuid),
headers=get_auth_header(authorized=True),
params=dict(replica="aws"))
self.assertEqual(res.status_code, requests.codes.not_found)
def _test_auth_errors(self,
method: str,
url: str,
skip_group_test=False,
**kwargs):
with self.subTest("Gibberish auth header"): # type: ignore
resp = self.assertResponse(method,
url,
requests.codes.unauthorized,
headers=get_auth_header(False),
**kwargs)
self.assertEqual(resp.response.headers['Content-Type'],
"application/problem+json")
self.assertEqual(resp.json['title'], 'Failed to decode token.')
with self.subTest("No auth header"): # type: ignore
resp = self.assertResponse(method, url,
requests.codes.unauthorized,
**kwargs) # type: ignore
self.assertEqual(resp.response.headers['Content-Type'],
"application/problem+json")
self.assertEqual(resp.json['title'],
'No authorization token provided')
if not skip_group_test:
with self.subTest("unauthorized group"): # type: ignore
resp = self.assertResponse(
method,
url,
requests.codes.forbidden,
headers=get_auth_header(group='someone'),
**kwargs)
self.assertEqual(resp.response.headers['Content-Type'],
"application/problem+json")
self.assertEqual(
resp.json['title'],
'User is not authorized to access this resource')
# Don't run this test for test_bundle and test_file because they don't need email
if not url.split('/')[2] in ('files', 'bundles'):
with self.subTest("no email claims"): # type: ignore
resp = self.assertResponse(method,
url,
requests.codes.unauthorized,
headers=get_auth_header(
email=False, email_claim=False),
**kwargs)
self.assertEqual(resp.response.headers['Content-Type'],
"application/problem+json")
self.assertEqual(
resp.json['title'],
'Authorization token is missing email claims.')
def test_delete(self):
find_uuid = self._put_subscription()
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + find_uuid)
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
# Authorized delete
self.assertDeleteResponse(url, requests.codes.okay, headers=get_auth_header())
# 1. Check that previous delete worked
# 2. Check that we can't delete files that don't exist
self.assertDeleteResponse(url, requests.codes.not_found, headers=get_auth_header())
def test_subscription_registration_succeeds_when_query_does_not_match_mappings(self):
# It is now possible to register a subscription query before the mapping
# of the field exists in the mappings (and may never exist in the mapppings)
es_query = {
"query": {
"bool": {
"must": [{
"match": {
"assay.fake_field": "this is a negative test"
}
}],
}
}
}
url = str(UrlBuilder()
.set(path="/v1/subscriptions")
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertPutResponse(
url,
requests.codes.created,
json_request_body=dict(
es_query=es_query,
**self.endpoint.to_dict()),
headers=get_auth_header()
)
self.assertIn('uuid', resp_obj.json)
def _get_subscription(self, uuid: str, replica: Replica):
url = str(UrlBuilder().set(path=f"/v1/subscriptions/{uuid}").add_query(
"replica", replica.name))
resp = self.assertGetResponse(url,
requests.codes.ok,
headers=get_auth_header())
return json.loads(resp.body)
def test_patch_no_version(self):
"""BAD REQUEST is returned when patching without the version."""
res = self.app.patch("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=True),
params=dict(replica="aws"),
json=dict())
self.assertEqual(res.status_code, requests.codes.bad_request)
def test_search_returns_error_when_invalid_query_used(self):
invalid_query_data = [({
"query": {
"mtch": {
"SomethingInvalid": "xxx"
}
}
}, requests.codes.bad_request),
({
"query": {
"match": ["SomethingInvalid", "xxx"]
}
}, requests.codes.bad_request)]
self.populate_search_index(self.index_document, 1)
url = self.build_url()
for bad_query, error in invalid_query_data:
with self.subTest(msg="Invalid Queries:",
bad_query=bad_query,
error=error):
self.assertPostResponse(
path=url,
headers=get_auth_header(),
json_request_body=dict(es_query=bad_query),
expected_code=error,
expected_error=ExpectedErrorFields(
code="elasticsearch_bad_request", status=error))
def test_patch_invalid_fragment(self):
"""PATCH invalid fragment reference."""
res = self.app.patch("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=True),
params=dict(version=self.version, replica="aws"),
json=dict(add_contents=[self.invalid_ptr] * 256))
self.assertEqual(res.status_code, requests.codes.unprocessable_entity)
def test_subscription_update(self, replica=Replica.aws):
"""
Test recover of subscriptions during enumeration
"""
subscription_1 = self._put_subscription(
{
'callback_url': "https://nonsense.or.whatever",
'method': "PUT",
}, replica)
subscription_2 = self._put_subscription(
{
'callback_url': "https://nonsense.or.whatever",
'method': "PUT",
}, replica)
url = str(UrlBuilder().set(path="/v1/subscriptions").add_query(
"replica", replica.name))
resp = self.assertGetResponse(url,
requests.codes.ok,
headers=get_auth_header())
subs = {
sub['uuid']: sub
for sub in json.loads(resp.body)['subscriptions']
}
self.assertIn(subscription_1['uuid'], subs)
self.assertIn(subscription_2['uuid'], subs)
for key in subscription_1:
self.assertEquals(subscription_1[key],
subs[subscription_1['uuid']][key])
for key in subscription_2:
self.assertEquals(subscription_2[key],
subs[subscription_2['uuid']][key])
def upload_file(
self: typing.Any,
source_url: str,
file_uuid: str,
bundle_uuid: str = None,
version: str = None,
expected_code: int = requests.codes.created,
):
bundle_uuid = str(uuid.uuid4()) if bundle_uuid is None else bundle_uuid
if version is None:
timestamp = datetime.datetime.utcnow()
version = timestamp.strftime("%Y-%m-%dT%H%M%S.%fZ")
urlbuilder = UrlBuilder().set(path='/v1/files/' + file_uuid)
if version != 'missing':
urlbuilder.add_query("version", version)
resp_obj = self.assertPutResponse(str(urlbuilder),
expected_code,
json_request_body=dict(
bundle_uuid=bundle_uuid,
creator_uid=0,
source_url=source_url,
),
headers=get_auth_header())
if resp_obj.response.status_code == requests.codes.created:
self.assertHeaders(resp_obj.response, {
'content-type': "application/json",
})
self.assertIn('version', resp_obj.json)
def delete_bundle(
self,
replica: Replica,
bundle_uuid: str,
bundle_version: typing.Optional[str]=None,
authorized: bool=True):
# make delete request
url_builder = UrlBuilder().set(path="/v1/bundles/" + bundle_uuid).add_query('replica', replica.name)
if bundle_version:
url_builder = url_builder.add_query('version', bundle_version)
url = str(url_builder)
json_request_body = dict(reason="reason")
if bundle_version:
json_request_body['version'] = bundle_version
expected_code = requests.codes.ok if authorized else requests.codes.forbidden
# delete and check results
return self.assertDeleteResponse(
url,
expected_code,
json_request_body=json_request_body,
headers=get_auth_header(authorized=authorized),
)
def test_get(self):
"""GET a list of all collections belonging to the user."""
res = self.app.get('/v1/collections',
headers=get_auth_header(authorized=True),
params=dict())
res.raise_for_status()
self.assertIn('collections', res.json())
def test_subscription_enumerate(self, replica=Replica.aws):
"""
Test recover of subscriptions during enumeration
"""
subscription_1 = self._put_subscription(
{
'callback_url': "https://nonsense.or.whatever",
'method': "PUT",
}, replica)
subscription_2 = self._put_subscription(
{
'callback_url': "https://nonsense.or.whatever",
'method': "PUT",
}, replica)
url = str(UrlBuilder().set(path="/v1/subscriptions").add_query(
"replica", replica.name))
resp = self.assertGetResponse(url,
requests.codes.ok,
headers=get_auth_header())
subs = {
sub['uuid']: sub
for sub in json.loads(resp.body)['subscriptions']
}
with self.subTest("Test user should own every returned subscription"):
for sub in subs.values():
self.assertEquals(self.owner, sub['owner'])
with self.subTest("Test subscriptions shuold have been returned"):
self.assertIn(subscription_1['uuid'], subs)
self.assertIn(subscription_2['uuid'], subs)
def _test_file_get_latest(self, replica: Replica):
"""
Verify we can successfully fetch the latest version of a file UUID.
"""
file_uuid = "ce55fd51-7833-469b-be0b-5da88ebebfcd"
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica", replica.name))
with override_bucket_config(BucketConfig.TEST_FIXTURE):
resp_obj = self.assertGetResponse(
url,
requests.codes.found,
headers=get_auth_header(),
redirect_follow_retries=FILE_GET_RETRY_COUNT,
min_retry_interval_header=RETRY_AFTER_INTERVAL,
override_retry_interval=1,
)
# TODO: (ttung) verify more of the headers
url = resp_obj.response.headers['Location']
sha1 = resp_obj.response.headers['X-DSS-SHA1']
data = requests.get(url)
self.assertEqual(len(data.content), 8685)
self.assertEqual(resp_obj.response.headers['X-DSS-SIZE'], '8685')
# verify that the downloaded data matches the stated checksum
hasher = hashlib.sha1()
hasher.update(data.content)
self.assertEqual(hasher.hexdigest(), sha1)
def _test_file_head(self, replica: Replica):
file_uuid = "ce55fd51-7833-469b-be0b-5da88ebebfcd"
version = "2017-06-16T193604.240704Z"
headers = {
'X-DSS-CREATOR-UID':
'4321',
'X-DSS-VERSION':
version,
'X-DSS-CONTENT-TYPE':
'text/plain',
'X-DSS-SIZE':
'11358',
'X-DSS-CRC32C':
'e16e07b9',
'X-DSS-S3-ETAG':
'3b83ef96387f14655fc854ddc3c6bd57',
'X-DSS-SHA1':
'2b8b815229aa8a61e483fb4ba0588b8b6c491890',
'X-DSS-SHA256':
'cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30',
}
url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query(
"replica", replica.name).add_query("version", version))
with override_bucket_config(BucketConfig.TEST_FIXTURE):
resp_obj = self.assertHeadResponse(url, [requests.codes.ok],
headers=get_auth_header())
self.assertHeaders(resp_obj.response, headers)
def _put(self,
contents: typing.List,
authorized: bool = True,
uuid: typing.Optional[str] = None,
version: typing.Optional[str] = None,
replica: str = 'aws') -> typing.Tuple[str, str]:
uuid = str(uuid4()) if uuid is None else uuid
version = datetime_to_version_format(
datetime.now()) if version is None else version
params = dict()
if uuid != 'missing':
params['uuid'] = uuid
if version != 'missing':
params['version'] = version
if replica != 'missing':
params['replica'] = replica
res = self.app.put("/v1/collections",
headers=get_auth_header(authorized=authorized),
params=params,
json=dict(name="n",
description="d",
details={},
contents=contents))
return res.json()["uuid"], res.json()["version"]
def test_patch(self):
col_file_item = dict(type="file",
uuid=self.s3_file_uuid,
version=self.s3_file_version)
col_ptr_item = dict(type="foo",
uuid=self.s3_file_uuid,
version=self.s3_file_version,
fragment="/foo")
contents = [col_file_item] * 8 + [col_ptr_item] * 8
uuid, version = self._put(contents, replica='aws')
self.addCleanup(self._delete_collection, uuid, replica='aws')
expected_contents = {
'contents': [col_file_item, col_ptr_item],
'description': 'd',
'details': {},
'name': 'n',
'owner': self.owner_email
}
tests = [
(dict(), None),
(dict(description="foo",
name="cn"), dict(description="foo", name="cn")),
(dict(description="bar",
details={1: 2}), dict(description="bar", details={'1': 2})),
(dict(add_contents=contents),
None), # Duplicates should be removed.
(dict(remove_contents=contents), dict(contents=[]))
]
for patch_payload, content_changes in tests:
with self.subTest(patch_payload):
res = self.app.patch("/v1/collections/{}".format(uuid),
headers=get_auth_header(authorized=True),
params=dict(version=version,
replica="aws"),
json=patch_payload)
res.raise_for_status()
self.assertNotEqual(version, res.json()["version"])
version = res.json()["version"]
res = self.app.get("/v1/collections/{}".format(uuid),
headers=get_auth_header(authorized=True),
params=dict(replica="aws", version=version))
res.raise_for_status()
collection = res.json()
if content_changes:
expected_contents.update(content_changes)
self.assertEqual(collection, expected_contents)
def test_get_uuid_latest(self):
"""GET latest version of collection."""
res = self.app.get("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=True),
params=dict(replica="aws"))
res.raise_for_status()
self.assertEqual(res.json()["contents"],
[self.s3_col_file_item, self.s3_col_ptr_item])
def _tombstone_bundle(app, replica, uuid, version=None):
url = f"/v1/bundles/{uuid}?replica={replica.name}"
if version is not None:
url += f"&version={version}"
resp = app.delete(url,
headers={** get_auth_header(), ** {'Content-Type': "application/json"}},
json=dict(reason="testing"))
resp.raise_for_status()
def _cleanup_subscription(self, uuid, subscription_type=None):
if not subscription_type:
subscription_type = 'elasticsearch'
url = (UrlBuilder()
.set(path=f"/v1/subscriptions/{uuid}")
.add_query("replica", self.replica.name)
.add_query('subscription_type', subscription_type))
self.assertDeleteResponse(url, requests.codes.okay, headers=get_auth_header())
def test_patch_excessive(self):
"""PATCH excess payload."""
res = self.app.patch("/v1/collections/{}".format(self.uuid),
headers=get_auth_header(authorized=True),
params=dict(version=self.version, replica="aws"),
json=dict(add_contents=[self.s3_col_ptr_item] *
1024))
self.assertEqual(res.status_code, requests.codes.bad_request)
def fetch_collection_paging_response(self, codes, replica: str,
per_page: int):
"""
GET /collections and iterate through the paging responses containing all of a user's collections.
If fetch_all is not True, this will return as soon as it gets one successful 206 paging reply.
"""
url = UrlBuilder().set(path="/v1/collections/")
url.add_query("replica", replica)
url.add_query("per_page", str(per_page))
resp_obj = self.assertGetResponse(
str(url), codes, headers=get_auth_header(authorized=True))
if codes == requests.codes.bad_request:
return True
link_header = resp_obj.response.headers.get('Link')
paging_response = False
while link_header:
# Make sure we're getting the expected response status code
self.assertEqual(resp_obj.response.status_code,
requests.codes.partial)
paging_response = True
link = parse_header_links(link_header)[0]
self.assertEquals(link['rel'], 'next')
parsed = urlsplit(link['url'])
url = UrlBuilder().set(path=parsed.path,
query=parse_qsl(parsed.query),
fragment=parsed.fragment)
self.assertEqual(resp_obj.response.headers['X-OpenAPI-Pagination'],
'true')
self.assertEqual(
resp_obj.response.headers['X-OpenAPI-Paginated-Content-Key'],
'collections')
resp_obj = self.assertGetResponse(
str(url),
expected_code=codes,
headers=get_auth_header(authorized=True))
link_header = resp_obj.response.headers.get('Link')
self.assertEqual(resp_obj.response.headers['X-OpenAPI-Pagination'],
'false')
self.assertEqual(resp_obj.response.status_code, requests.codes.ok)
return paging_response
def test_checkout():
# assert 302 and verify checksum on checkout completion
api_get = self.assertGetResponse(url,
requests.codes.found,
headers=get_auth_header(),
redirect_follow_retries=0)
file_get = requests.get(api_get.response.headers['Location'])
self.assertTrue(file_get.ok)
self.assertEquals(file_get.content, src_data)
