def test_next_page_is_delivered_when_next_is_in_query(self): bundles = self.populate_search_index(self.index_document, 150) self.check_count(smartseq2_paired_ends_vx_query, 150) url = self.build_url() search_obj = self.assertPostResponse( path=url, json_request_body=dict(es_query=smartseq2_paired_ends_vx_query), expected_code=requests.codes.partial, headers=get_auth_header()) found_bundles = search_obj.json['results'] next_url = self.get_next_url(search_obj.response.headers) self.verify_next_url(next_url) self.verify_search_result(search_obj.json, smartseq2_paired_ends_vx_query, len(bundles), 100) search_obj = self.assertPostResponse( path=self.strip_next_url(next_url), json_request_body=dict(es_query=smartseq2_paired_ends_vx_query), expected_code=requests.codes.ok, headers=get_auth_header()) found_bundles.extend(search_obj.json['results']) self.verify_search_result(search_obj.json, smartseq2_paired_ends_vx_query, len(bundles), 50) self.verify_bundles(found_bundles, bundles)
def test_get(self): find_uuid = self._put_subscription() # Normal request url = str(UrlBuilder().set(path="/v1/subscriptions/" + str(find_uuid)).add_query( "replica", self.replica.name)) resp_obj = self.assertGetResponse(url, requests.codes.okay, headers=get_auth_header()) json_response = resp_obj.json self.assertEqual(self.sample_percolate_query, json_response['es_query']) self.assertEqual(self.callback_url, json_response['callback_url']) # Forbidden request w/ previous url with self.throw_403(): self.assertGetResponse(url, requests.codes.forbidden, headers=get_auth_header()) # File not found request url = str(UrlBuilder().set(path="/v1/subscriptions/" + str(uuid.uuid4())).add_query( "replica", self.replica.name)) self.assertGetResponse(url, requests.codes.not_found, headers=get_auth_header())
def test_output_format_is_raw(self): bundles = self.populate_search_index(self.index_document, 11) self.check_count(smartseq2_paired_ends_vx_query, 11) url = self.build_url(url_params={ 'output_format': 'raw', 'per_page': 11 }) search_obj = self.assertPostResponse( path=url, json_request_body=dict(es_query=smartseq2_paired_ends_vx_query), expected_code=requests.codes.partial, headers=get_auth_header()) results = search_obj.json['results'] next_url = self.get_next_url(search_obj.response.headers) self.assertIsNotNone(next_url) self.verify_search_result(search_obj.json, smartseq2_paired_ends_vx_query, 11, 10) search_obj = self.assertPostResponse( path=self.strip_next_url(next_url), json_request_body=dict(es_query=smartseq2_paired_ends_vx_query), expected_code=requests.codes.ok, headers=get_auth_header()) results.extend(search_obj.json['results']) next_url = self.get_next_url(search_obj.response.headers) self.assertIsNone(next_url) self.verify_search_result(search_obj.json, smartseq2_paired_ends_vx_query, 11, 1) self.verify_bundles(results, bundles) self.assertEqual( sorted([ key for key in search_obj.json['results'][0]['metadata'] ['manifest'].keys() ]), sorted([key for key in self.index_document['manifest'].keys()]))
def test_get(self): try: find_uuid = self._put_subscription() # Normal request url = str(UrlBuilder() .set(path="/v1/subscriptions/" + str(find_uuid)) .add_query("replica", self.replica.name) .add_query("subscription_type", "elasticsearch")) resp_obj = self.assertGetResponse( url, requests.codes.okay, headers=get_auth_header()) json_response = resp_obj.json self.assertEqual(self.sample_percolate_query, json_response['es_query']) self.assertEqual(self.endpoint, Endpoint.from_subscription(json_response)) self.assertEquals(self.hmac_key_id, json_response['hmac_key_id']) self.assertNotIn('hmac_secret_key', json_response) finally: self._cleanup_subscription(find_uuid) # File not found request url = str(UrlBuilder() .set(path="/v1/subscriptions/" + str(uuid.uuid4())) .add_query("replica", self.replica.name) .add_query("subscription_type", "elasticsearch")) self.assertGetResponse( url, requests.codes.not_found, headers=get_auth_header())
def _test_file_get_not_found(self, replica: Replica): file_uuid = "ce55fd51-7833-469b-be0b-5da88ec0ffee" url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query( "replica", replica.name)) with override_bucket_config(BucketConfig.TEST_FIXTURE): self.assertGetResponse(url, requests.codes.not_found, headers=get_auth_header(), expected_error=ExpectedErrorFields( code="not_found", status=requests.codes.not_found, expect_stacktrace=True)) version = "2017-06-16T193604.240704Z" url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query( "replica", replica.name).add_query("version", version)) with override_bucket_config(BucketConfig.TEST_FIXTURE): self.assertGetResponse(url, requests.codes.not_found, headers=get_auth_header(), expected_error=ExpectedErrorFields( code="not_found", status=requests.codes.not_found, expect_stacktrace=True))
def test_access_control(self): with self.subTest("PUT"): uuid = str(uuid4()) self.addCleanup(self._delete_collection, uuid) res = self.app.put("/v1/collections", headers=get_auth_header(authorized=False), params=dict(version=datetime.now().isoformat(), uuid=uuid, replica='aws'), json=dict(name="n", description="d", details={}, contents=self.contents)) self.assertEqual(res.status_code, requests.codes.forbidden) with self.subTest("GET"): res = self.app.get("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=False), params=dict(replica="aws")) self.assertEqual(res.status_code, requests.codes.forbidden) with self.subTest("PATCH"): res = self.app.patch("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=False), params=dict(replica="aws", version=self.version), json=dict(description="foo")) self.assertEqual(res.status_code, requests.codes.forbidden) with self.subTest("DELETE"): res = self.app.delete("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=False), params=dict(replica="aws")) self.assertEqual(res.status_code, requests.codes.forbidden)
def upload_file_wait( self: typing.Any, source_url: str, replica: Replica, file_uuid: str = None, file_version: str = None, bundle_uuid: str = None, timeout_seconds: int = 120, expect_async: typing.Optional[bool] = None, ) -> DSSAssertResponse: """ Upload a file. If the request is being handled asynchronously, wait until the file has landed in the data store. """ file_uuid = str(uuid.uuid4()) if file_uuid is None else file_uuid bundle_uuid = str(uuid.uuid4()) if bundle_uuid is None else bundle_uuid if expect_async is True: expected_codes = requests.codes.accepted elif expect_async is False: expected_codes = requests.codes.created else: expected_codes = requests.codes.created, requests.codes.accepted if file_version is None: timestamp = datetime.datetime.utcnow() file_version = datetime_to_version_format(timestamp) url = UrlBuilder().set(path=f"/v1/files/{file_uuid}") url.add_query("version", file_version) resp_obj = self.assertPutResponse(str(url), expected_codes, json_request_body=dict( bundle_uuid=bundle_uuid, creator_uid=0, source_url=source_url, ), headers=get_auth_header()) if resp_obj.response.status_code == requests.codes.accepted: # hit the GET /files endpoint until we succeed. start_time = time.time() timeout_time = start_time + timeout_seconds while time.time() < timeout_time: try: self.assertHeadResponse( f"/v1/files/{file_uuid}?replica={replica.name}", requests.codes.ok, headers=get_auth_header()) break except AssertionError: pass time.sleep(1) else: self.fail("Could not find the output file") return resp_obj
def test_delete(self): uuid, version = self._put(self.contents) with self.subTest("Delete collection"): res = self.app.delete("/v1/collections/{}".format(uuid), headers=get_auth_header(authorized=True), params=dict(replica="aws")) res.raise_for_status() with self.subTest("Verify deleted"): res = self.app.get("/v1/collections/{}".format(uuid), headers=get_auth_header(authorized=True), params=dict(replica="aws")) self.assertEqual(res.status_code, requests.codes.not_found)
def _test_auth_errors(self, method: str, url: str, skip_group_test=False, **kwargs): with self.subTest("Gibberish auth header"): # type: ignore resp = self.assertResponse(method, url, requests.codes.unauthorized, headers=get_auth_header(False), **kwargs) self.assertEqual(resp.response.headers['Content-Type'], "application/problem+json") self.assertEqual(resp.json['title'], 'Failed to decode token.') with self.subTest("No auth header"): # type: ignore resp = self.assertResponse(method, url, requests.codes.unauthorized, **kwargs) # type: ignore self.assertEqual(resp.response.headers['Content-Type'], "application/problem+json") self.assertEqual(resp.json['title'], 'No authorization token provided') if not skip_group_test: with self.subTest("unauthorized group"): # type: ignore resp = self.assertResponse( method, url, requests.codes.forbidden, headers=get_auth_header(group='someone'), **kwargs) self.assertEqual(resp.response.headers['Content-Type'], "application/problem+json") self.assertEqual( resp.json['title'], 'User is not authorized to access this resource') # Don't run this test for test_bundle and test_file because they don't need email if not url.split('/')[2] in ('files', 'bundles'): with self.subTest("no email claims"): # type: ignore resp = self.assertResponse(method, url, requests.codes.unauthorized, headers=get_auth_header( email=False, email_claim=False), **kwargs) self.assertEqual(resp.response.headers['Content-Type'], "application/problem+json") self.assertEqual( resp.json['title'], 'Authorization token is missing email claims.')
def test_delete(self): find_uuid = self._put_subscription() url = str(UrlBuilder() .set(path="/v1/subscriptions/" + find_uuid) .add_query("replica", self.replica.name) .add_query("subscription_type", "elasticsearch")) # Authorized delete self.assertDeleteResponse(url, requests.codes.okay, headers=get_auth_header()) # 1. Check that previous delete worked # 2. Check that we can't delete files that don't exist self.assertDeleteResponse(url, requests.codes.not_found, headers=get_auth_header())
def test_subscription_registration_succeeds_when_query_does_not_match_mappings(self): # It is now possible to register a subscription query before the mapping # of the field exists in the mappings (and may never exist in the mapppings) es_query = { "query": { "bool": { "must": [{ "match": { "assay.fake_field": "this is a negative test" } }], } } } url = str(UrlBuilder() .set(path="/v1/subscriptions") .add_query("replica", self.replica.name) .add_query("subscription_type", "elasticsearch")) resp_obj = self.assertPutResponse( url, requests.codes.created, json_request_body=dict( es_query=es_query, **self.endpoint.to_dict()), headers=get_auth_header() ) self.assertIn('uuid', resp_obj.json)
def _get_subscription(self, uuid: str, replica: Replica): url = str(UrlBuilder().set(path=f"/v1/subscriptions/{uuid}").add_query( "replica", replica.name)) resp = self.assertGetResponse(url, requests.codes.ok, headers=get_auth_header()) return json.loads(resp.body)
def test_patch_no_version(self): """BAD REQUEST is returned when patching without the version.""" res = self.app.patch("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=True), params=dict(replica="aws"), json=dict()) self.assertEqual(res.status_code, requests.codes.bad_request)
def test_search_returns_error_when_invalid_query_used(self): invalid_query_data = [({ "query": { "mtch": { "SomethingInvalid": "xxx" } } }, requests.codes.bad_request), ({ "query": { "match": ["SomethingInvalid", "xxx"] } }, requests.codes.bad_request)] self.populate_search_index(self.index_document, 1) url = self.build_url() for bad_query, error in invalid_query_data: with self.subTest(msg="Invalid Queries:", bad_query=bad_query, error=error): self.assertPostResponse( path=url, headers=get_auth_header(), json_request_body=dict(es_query=bad_query), expected_code=error, expected_error=ExpectedErrorFields( code="elasticsearch_bad_request", status=error))
def test_patch_invalid_fragment(self): """PATCH invalid fragment reference.""" res = self.app.patch("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=True), params=dict(version=self.version, replica="aws"), json=dict(add_contents=[self.invalid_ptr] * 256)) self.assertEqual(res.status_code, requests.codes.unprocessable_entity)
def test_subscription_update(self, replica=Replica.aws): """ Test recover of subscriptions during enumeration """ subscription_1 = self._put_subscription( { 'callback_url': "https://nonsense.or.whatever", 'method': "PUT", }, replica) subscription_2 = self._put_subscription( { 'callback_url': "https://nonsense.or.whatever", 'method': "PUT", }, replica) url = str(UrlBuilder().set(path="/v1/subscriptions").add_query( "replica", replica.name)) resp = self.assertGetResponse(url, requests.codes.ok, headers=get_auth_header()) subs = { sub['uuid']: sub for sub in json.loads(resp.body)['subscriptions'] } self.assertIn(subscription_1['uuid'], subs) self.assertIn(subscription_2['uuid'], subs) for key in subscription_1: self.assertEquals(subscription_1[key], subs[subscription_1['uuid']][key]) for key in subscription_2: self.assertEquals(subscription_2[key], subs[subscription_2['uuid']][key])
def upload_file( self: typing.Any, source_url: str, file_uuid: str, bundle_uuid: str = None, version: str = None, expected_code: int = requests.codes.created, ): bundle_uuid = str(uuid.uuid4()) if bundle_uuid is None else bundle_uuid if version is None: timestamp = datetime.datetime.utcnow() version = timestamp.strftime("%Y-%m-%dT%H%M%S.%fZ") urlbuilder = UrlBuilder().set(path='/v1/files/' + file_uuid) if version != 'missing': urlbuilder.add_query("version", version) resp_obj = self.assertPutResponse(str(urlbuilder), expected_code, json_request_body=dict( bundle_uuid=bundle_uuid, creator_uid=0, source_url=source_url, ), headers=get_auth_header()) if resp_obj.response.status_code == requests.codes.created: self.assertHeaders(resp_obj.response, { 'content-type': "application/json", }) self.assertIn('version', resp_obj.json)
def delete_bundle( self, replica: Replica, bundle_uuid: str, bundle_version: typing.Optional[str]=None, authorized: bool=True): # make delete request url_builder = UrlBuilder().set(path="/v1/bundles/" + bundle_uuid).add_query('replica', replica.name) if bundle_version: url_builder = url_builder.add_query('version', bundle_version) url = str(url_builder) json_request_body = dict(reason="reason") if bundle_version: json_request_body['version'] = bundle_version expected_code = requests.codes.ok if authorized else requests.codes.forbidden # delete and check results return self.assertDeleteResponse( url, expected_code, json_request_body=json_request_body, headers=get_auth_header(authorized=authorized), )
def test_get(self): """GET a list of all collections belonging to the user.""" res = self.app.get('/v1/collections', headers=get_auth_header(authorized=True), params=dict()) res.raise_for_status() self.assertIn('collections', res.json())
def test_subscription_enumerate(self, replica=Replica.aws): """ Test recover of subscriptions during enumeration """ subscription_1 = self._put_subscription( { 'callback_url': "https://nonsense.or.whatever", 'method': "PUT", }, replica) subscription_2 = self._put_subscription( { 'callback_url': "https://nonsense.or.whatever", 'method': "PUT", }, replica) url = str(UrlBuilder().set(path="/v1/subscriptions").add_query( "replica", replica.name)) resp = self.assertGetResponse(url, requests.codes.ok, headers=get_auth_header()) subs = { sub['uuid']: sub for sub in json.loads(resp.body)['subscriptions'] } with self.subTest("Test user should own every returned subscription"): for sub in subs.values(): self.assertEquals(self.owner, sub['owner']) with self.subTest("Test subscriptions shuold have been returned"): self.assertIn(subscription_1['uuid'], subs) self.assertIn(subscription_2['uuid'], subs)
def _test_file_get_latest(self, replica: Replica): """ Verify we can successfully fetch the latest version of a file UUID. """ file_uuid = "ce55fd51-7833-469b-be0b-5da88ebebfcd" url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query( "replica", replica.name)) with override_bucket_config(BucketConfig.TEST_FIXTURE): resp_obj = self.assertGetResponse( url, requests.codes.found, headers=get_auth_header(), redirect_follow_retries=FILE_GET_RETRY_COUNT, min_retry_interval_header=RETRY_AFTER_INTERVAL, override_retry_interval=1, ) # TODO: (ttung) verify more of the headers url = resp_obj.response.headers['Location'] sha1 = resp_obj.response.headers['X-DSS-SHA1'] data = requests.get(url) self.assertEqual(len(data.content), 8685) self.assertEqual(resp_obj.response.headers['X-DSS-SIZE'], '8685') # verify that the downloaded data matches the stated checksum hasher = hashlib.sha1() hasher.update(data.content) self.assertEqual(hasher.hexdigest(), sha1)
def _test_file_head(self, replica: Replica): file_uuid = "ce55fd51-7833-469b-be0b-5da88ebebfcd" version = "2017-06-16T193604.240704Z" headers = { 'X-DSS-CREATOR-UID': '4321', 'X-DSS-VERSION': version, 'X-DSS-CONTENT-TYPE': 'text/plain', 'X-DSS-SIZE': '11358', 'X-DSS-CRC32C': 'e16e07b9', 'X-DSS-S3-ETAG': '3b83ef96387f14655fc854ddc3c6bd57', 'X-DSS-SHA1': '2b8b815229aa8a61e483fb4ba0588b8b6c491890', 'X-DSS-SHA256': 'cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30', } url = str(UrlBuilder().set(path="/v1/files/" + file_uuid).add_query( "replica", replica.name).add_query("version", version)) with override_bucket_config(BucketConfig.TEST_FIXTURE): resp_obj = self.assertHeadResponse(url, [requests.codes.ok], headers=get_auth_header()) self.assertHeaders(resp_obj.response, headers)
def _put(self, contents: typing.List, authorized: bool = True, uuid: typing.Optional[str] = None, version: typing.Optional[str] = None, replica: str = 'aws') -> typing.Tuple[str, str]: uuid = str(uuid4()) if uuid is None else uuid version = datetime_to_version_format( datetime.now()) if version is None else version params = dict() if uuid != 'missing': params['uuid'] = uuid if version != 'missing': params['version'] = version if replica != 'missing': params['replica'] = replica res = self.app.put("/v1/collections", headers=get_auth_header(authorized=authorized), params=params, json=dict(name="n", description="d", details={}, contents=contents)) return res.json()["uuid"], res.json()["version"]
def test_patch(self): col_file_item = dict(type="file", uuid=self.s3_file_uuid, version=self.s3_file_version) col_ptr_item = dict(type="foo", uuid=self.s3_file_uuid, version=self.s3_file_version, fragment="/foo") contents = [col_file_item] * 8 + [col_ptr_item] * 8 uuid, version = self._put(contents, replica='aws') self.addCleanup(self._delete_collection, uuid, replica='aws') expected_contents = { 'contents': [col_file_item, col_ptr_item], 'description': 'd', 'details': {}, 'name': 'n', 'owner': self.owner_email } tests = [ (dict(), None), (dict(description="foo", name="cn"), dict(description="foo", name="cn")), (dict(description="bar", details={1: 2}), dict(description="bar", details={'1': 2})), (dict(add_contents=contents), None), # Duplicates should be removed. (dict(remove_contents=contents), dict(contents=[])) ] for patch_payload, content_changes in tests: with self.subTest(patch_payload): res = self.app.patch("/v1/collections/{}".format(uuid), headers=get_auth_header(authorized=True), params=dict(version=version, replica="aws"), json=patch_payload) res.raise_for_status() self.assertNotEqual(version, res.json()["version"]) version = res.json()["version"] res = self.app.get("/v1/collections/{}".format(uuid), headers=get_auth_header(authorized=True), params=dict(replica="aws", version=version)) res.raise_for_status() collection = res.json() if content_changes: expected_contents.update(content_changes) self.assertEqual(collection, expected_contents)
def test_get_uuid_latest(self): """GET latest version of collection.""" res = self.app.get("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=True), params=dict(replica="aws")) res.raise_for_status() self.assertEqual(res.json()["contents"], [self.s3_col_file_item, self.s3_col_ptr_item])
def _tombstone_bundle(app, replica, uuid, version=None): url = f"/v1/bundles/{uuid}?replica={replica.name}" if version is not None: url += f"&version={version}" resp = app.delete(url, headers={** get_auth_header(), ** {'Content-Type': "application/json"}}, json=dict(reason="testing")) resp.raise_for_status()
def _cleanup_subscription(self, uuid, subscription_type=None): if not subscription_type: subscription_type = 'elasticsearch' url = (UrlBuilder() .set(path=f"/v1/subscriptions/{uuid}") .add_query("replica", self.replica.name) .add_query('subscription_type', subscription_type)) self.assertDeleteResponse(url, requests.codes.okay, headers=get_auth_header())
def test_patch_excessive(self): """PATCH excess payload.""" res = self.app.patch("/v1/collections/{}".format(self.uuid), headers=get_auth_header(authorized=True), params=dict(version=self.version, replica="aws"), json=dict(add_contents=[self.s3_col_ptr_item] * 1024)) self.assertEqual(res.status_code, requests.codes.bad_request)
def fetch_collection_paging_response(self, codes, replica: str, per_page: int): """ GET /collections and iterate through the paging responses containing all of a user's collections. If fetch_all is not True, this will return as soon as it gets one successful 206 paging reply. """ url = UrlBuilder().set(path="/v1/collections/") url.add_query("replica", replica) url.add_query("per_page", str(per_page)) resp_obj = self.assertGetResponse( str(url), codes, headers=get_auth_header(authorized=True)) if codes == requests.codes.bad_request: return True link_header = resp_obj.response.headers.get('Link') paging_response = False while link_header: # Make sure we're getting the expected response status code self.assertEqual(resp_obj.response.status_code, requests.codes.partial) paging_response = True link = parse_header_links(link_header)[0] self.assertEquals(link['rel'], 'next') parsed = urlsplit(link['url']) url = UrlBuilder().set(path=parsed.path, query=parse_qsl(parsed.query), fragment=parsed.fragment) self.assertEqual(resp_obj.response.headers['X-OpenAPI-Pagination'], 'true') self.assertEqual( resp_obj.response.headers['X-OpenAPI-Paginated-Content-Key'], 'collections') resp_obj = self.assertGetResponse( str(url), expected_code=codes, headers=get_auth_header(authorized=True)) link_header = resp_obj.response.headers.get('Link') self.assertEqual(resp_obj.response.headers['X-OpenAPI-Pagination'], 'false') self.assertEqual(resp_obj.response.status_code, requests.codes.ok) return paging_response
def test_checkout(): # assert 302 and verify checksum on checkout completion api_get = self.assertGetResponse(url, requests.codes.found, headers=get_auth_header(), redirect_follow_retries=0) file_get = requests.get(api_get.response.headers['Location']) self.assertTrue(file_get.ok) self.assertEquals(file_get.content, src_data)