def test_validation(self):
with self.subTest("Missing URL"):
self._put_subscription(expect_code=400, endpoint={})
with self.subTest("Invalid form field value"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(
callback_url=self.endpoint.callback_url,
form_fields={'foo': 1}))
with self.subTest("Invalid encoding"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(
callback_url=self.endpoint.callback_url,
encoding='foo'))
with self.subTest("Invalid method"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(
callback_url=self.endpoint.callback_url,
method='foo'))
with self.subTest("Invalid attachment type"):
self._put_subscription(
expect_code=400,
attachments={'foo': dict(type='foo', expression='bar')})
with self.subTest("Invalid attachment expression"):
self._put_subscription(
expect_code=400,
attachments={'foo': dict(type='jmespath', expression='')})
def test_get(self):
try:
find_uuid = self._put_subscription()
# Normal request
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(find_uuid))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertGetResponse(
url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query, json_response['es_query'])
self.assertEqual(self.endpoint, Endpoint.from_subscription(json_response))
self.assertEquals(self.hmac_key_id, json_response['hmac_key_id'])
self.assertNotIn('hmac_secret_key', json_response)
finally:
self._cleanup_subscription(find_uuid)
# File not found request
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(uuid.uuid4()))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
self.assertGetResponse(
url,
requests.codes.not_found,
headers=get_auth_header())
def test_get(self):
find_uuid = self._put_subscription()
# Normal request
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(find_uuid)).add_query(
"replica", self.replica.name))
resp_obj = self.assertGetResponse(url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query,
json_response['es_query'])
self.assertEqual(self.endpoint,
Endpoint.from_subscription(json_response))
# Forbidden request w/ previous url
with self.throw_403():
self.assertGetResponse(url,
requests.codes.forbidden,
headers=get_auth_header())
# File not found request
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(uuid.uuid4())).add_query(
"replica", self.replica.name))
self.assertGetResponse(url,
requests.codes.not_found,
headers=get_auth_header())
def setUp(self):
super().setUp()
self.alias_name = dss.Config.get_es_alias_name(dss.ESIndexType.docs, self.replica)
self.sub_index_name = dss.Config.get_es_index_name(dss.ESIndexType.subscriptions, self.replica)
shape_identifier = self.index_document._get_shape_descriptor()
shape_identifier = hashlib.sha1(f"{shape_identifier}".encode("utf-8")).hexdigest()
self.doc_index_name = dss.Config.get_es_index_name(dss.ESIndexType.docs, self.replica, shape_identifier)
es_client = ElasticsearchClient.get()
IndexManager.create_index(es_client, self.replica, self.doc_index_name)
es_client.index(index=self.doc_index_name,
doc_type=dss.ESDocType.doc.name,
id=str(uuid.uuid4()),
body=self.index_document,
refresh=True)
self.endpoint = Endpoint(callback_url="https://example.com",
method="POST",
encoding="application/json",
form_fields={'foo': 'bar'},
payload_form_field='baz')
self.sample_percolate_query = smartseq2_paired_ends_vx_query
self.hmac_key_id = 'dss_test'
self.hmac_secret_key = '23/33'
def _notify_subscriber(self, doc: BundleDocument, subscription: dict):
transaction_id = str(uuid.uuid4())
subscription_id = subscription['id']
endpoint = Endpoint.from_subscription(subscription)
payload = dict(transaction_id=transaction_id,
subscription_id=subscription_id,
es_query=subscription['es_query'],
match=dict(bundle_uuid=doc.fqid.uuid,
bundle_version=doc.fqid.version))
definitions = subscription.get('attachments')
# Only mention attachments in the notification if the subscription does, too.
if definitions is not None:
payload['attachments'] = attachment.select(definitions, doc)
if endpoint.encoding == 'application/json':
body = payload
elif endpoint.encoding == 'multipart/form-data':
body = endpoint.form_fields.copy()
body[endpoint.payload_form_field] = json.dumps(payload)
else:
raise ValueError(f"Encoding {endpoint.encoding} is not supported")
try:
hmac_key = subscription['hmac_secret_key']
except KeyError:
hmac_key = None
hmac_key_id = None
else:
hmac_key = hmac_key.encode()
hmac_key_id = subscription.get('hmac_key_id', "hca-dss:" + subscription_id)
notification = Notification.create(notification_id=transaction_id,
subscription_id=subscription_id,
url=endpoint.callback_url,
method=endpoint.method,
encoding=endpoint.encoding,
body=body,
hmac_key=hmac_key,
hmac_key_id=hmac_key_id,
correlation_id=str(doc.fqid))
if self.notifier:
logger.info(f"Queueing asynchronous notification {notification} for bundle {doc.fqid}")
self.notifier.enqueue(notification)
else:
logger.info(f"Synchronously sending notification {notification} about bundle {doc.fqid}")
notification.deliver_or_raise()
def test_find(self):
num_additions = 25
for _ in range(num_additions):
self._put_subscription()
url = str(UrlBuilder().set(path="/v1/subscriptions").add_query(
"replica", self.replica.name))
resp_obj = self.assertGetResponse(url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query,
json_response['subscriptions'][0]['es_query'])
self.assertEqual(
self.endpoint,
Endpoint.from_subscription(json_response['subscriptions'][0]))
self.assertEqual(num_additions, len(json_response['subscriptions']))
def test_find(self):
try:
num_additions = 3
uuids = list()
for _ in range(num_additions):
uuids.append(self._put_subscription())
url = str(UrlBuilder()
.set(path="/v1/subscriptions")
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertGetResponse(
url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query, json_response['subscriptions'][0]['es_query'])
self.assertEqual(self.hmac_key_id, json_response['subscriptions'][0]['hmac_key_id'])
self.assertEqual(self.endpoint, Endpoint.from_subscription(json_response['subscriptions'][0]))
self.assertNotIn('hmac_secret_key', json_response['subscriptions'][0])
self.assertEqual(num_additions, len(json_response['subscriptions']))
finally:
for _uuid in uuids:
self._cleanup_subscription(_uuid)
class TestSubscriptionsBase(ElasticsearchTestCase, TestAuthMixin, DSSAssertMixin):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.app = ThreadedLocalServer()
cls.app.start()
dss.Config.set_config(dss.BucketConfig.TEST)
with open(os.path.join(os.path.dirname(__file__), "sample_vx_index_doc.json"), "r") as fh:
cls.index_document = BundleDocument(cls.replica, get_bundle_fqid())
cls.index_document.update(json.load(fh))
@classmethod
def tearDownClass(cls):
cls.app.shutdown()
super().tearDownClass()
def setUp(self):
super().setUp()
self.alias_name = dss.Config.get_es_alias_name(dss.ESIndexType.docs, self.replica)
self.sub_index_name = dss.Config.get_es_index_name(dss.ESIndexType.subscriptions, self.replica)
shape_identifier = self.index_document._get_shape_descriptor()
shape_identifier = hashlib.sha1(f"{shape_identifier}".encode("utf-8")).hexdigest()
self.doc_index_name = dss.Config.get_es_index_name(dss.ESIndexType.docs, self.replica, shape_identifier)
es_client = ElasticsearchClient.get()
IndexManager.create_index(es_client, self.replica, self.doc_index_name)
es_client.index(index=self.doc_index_name,
doc_type=dss.ESDocType.doc.name,
id=str(uuid.uuid4()),
body=self.index_document,
refresh=True)
self.endpoint = Endpoint(callback_url="https://example.com",
method="POST",
encoding="application/json",
form_fields={'foo': 'bar'},
payload_form_field='baz')
self.sample_percolate_query = smartseq2_paired_ends_vx_query
self.hmac_key_id = 'dss_test'
self.hmac_secret_key = '23/33'
def test_auth_errors(self):
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(uuid.uuid4()))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
self._test_auth_errors('get', url)
def test_put(self):
try:
uuid_ = self._put_subscription()
es_client = ElasticsearchClient.get()
response = es_client.get(index=self.doc_index_name,
doc_type=dss.ESDocType.query.name,
id=uuid_)
registered_query = response['_source']
self.assertEqual(self.sample_percolate_query, registered_query)
finally:
self._cleanup_subscription(uuid_)
@eventually(timeout=5, interval=1, errors={AssertionError})
def assert_owner_subscription_count(self, owner, replica, subscription_count):
self.assertEquals(count_subscriptions_for_owner(Replica[replica], owner), subscription_count)
def test_db_count_subscriptions_for_owner(self):
"""Test dynamoDB helper functions used to store and retrieve subscription information."""
owner = '*****@*****.**'
subscription_uuid = str(uuid.uuid4())
for r in Replica:
replica = r.name
subscription_doc = {SubscriptionData.OWNER: owner,
SubscriptionData.UUID: subscription_uuid,
SubscriptionData.REPLICA: replica}
with self.subTest(f'DynamoDB: Counting initial (zero) subscriptions for {owner} in {replica}.'):
self.assert_owner_subscription_count(owner, replica, subscription_count=0)
with self.subTest(f'DynamoDB: put_subscription for {owner} in {replica} and check count is one.'):
put_subscription(subscription_doc)
self.assert_owner_subscription_count(owner, replica, subscription_count=1)
with self.subTest(f'DynamoDB: Counting (zero) subscriptions for {owner} in {replica} after deletion.'):
delete_subscription(Replica[replica], owner=owner, uuid=subscription_uuid)
self.assert_owner_subscription_count(owner, replica, subscription_count=0)
def test_validation(self):
with self.subTest("Missing URL"):
self._put_subscription(expect_code=400,
endpoint={})
with self.subTest("Invalid form field value"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(callback_url=self.endpoint.callback_url,
form_fields={'foo': 1}))
with self.subTest("Invalid encoding"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(callback_url=self.endpoint.callback_url,
encoding='foo'))
with self.subTest("Invalid method"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(callback_url=self.endpoint.callback_url,
method='foo'))
with self.subTest("Invalid attachment type"):
self._put_subscription(expect_code=400,
attachments={'foo': dict(type='foo', expression='bar')})
with self.subTest("Invalid attachment expression"):
self._put_subscription(expect_code=400,
attachments={'foo': dict(type='jmespath', expression='')})
def test_subscription_registration_succeeds_when_query_does_not_match_mappings(self):
# It is now possible to register a subscription query before the mapping
# of the field exists in the mappings (and may never exist in the mapppings)
es_query = {
"query": {
"bool": {
"must": [{
"match": {
"assay.fake_field": "this is a negative test"
}
}],
}
}
}
url = str(UrlBuilder()
.set(path="/v1/subscriptions")
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertPutResponse(
url,
requests.codes.created,
json_request_body=dict(
es_query=es_query,
**self.endpoint.to_dict()),
headers=get_auth_header()
)
self.assertIn('uuid', resp_obj.json)
def test_get(self):
try:
find_uuid = self._put_subscription()
# Normal request
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(find_uuid))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertGetResponse(
url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query, json_response['es_query'])
self.assertEqual(self.endpoint, Endpoint.from_subscription(json_response))
self.assertEquals(self.hmac_key_id, json_response['hmac_key_id'])
self.assertNotIn('hmac_secret_key', json_response)
finally:
self._cleanup_subscription(find_uuid)
# File not found request
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + str(uuid.uuid4()))
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
self.assertGetResponse(
url,
requests.codes.not_found,
headers=get_auth_header())
def test_find(self):
try:
num_additions = 3
uuids = list()
for _ in range(num_additions):
uuids.append(self._put_subscription())
url = str(UrlBuilder()
.set(path="/v1/subscriptions")
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
resp_obj = self.assertGetResponse(
url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query, json_response['subscriptions'][0]['es_query'])
self.assertEqual(self.hmac_key_id, json_response['subscriptions'][0]['hmac_key_id'])
self.assertEqual(self.endpoint, Endpoint.from_subscription(json_response['subscriptions'][0]))
self.assertNotIn('hmac_secret_key', json_response['subscriptions'][0])
self.assertEqual(num_additions, len(json_response['subscriptions']))
finally:
for _uuid in uuids:
self._cleanup_subscription(_uuid)
def test_delete(self):
find_uuid = self._put_subscription()
url = str(UrlBuilder()
.set(path="/v1/subscriptions/" + find_uuid)
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
# Authorized delete
self.assertDeleteResponse(url, requests.codes.okay, headers=get_auth_header())
# 1. Check that previous delete worked
# 2. Check that we can't delete files that don't exist
self.assertDeleteResponse(url, requests.codes.not_found, headers=get_auth_header())
def _put_subscription(self, endpoint=None, expect_code=None, attachments=None):
url = str(UrlBuilder()
.set(path="/v1/subscriptions")
.add_query("replica", self.replica.name)
.add_query("subscription_type", "elasticsearch"))
if endpoint is None:
endpoint = self.endpoint
if isinstance(endpoint, Endpoint):
endpoint = endpoint.to_dict()
json_request_body = dict(endpoint, es_query=self.sample_percolate_query, hmac_key_id=self.hmac_key_id,
hmac_secret_key=self.hmac_secret_key)
if attachments is not None:
json_request_body['attachments'] = attachments
resp_obj = self.assertPutResponse(
url,
expect_code or requests.codes.created,
json_request_body=json_request_body,
headers=get_auth_header()
)
return resp_obj.json.get('uuid')
def _cleanup_subscription(self, uuid, subscription_type=None):
if not subscription_type:
subscription_type = 'elasticsearch'
url = (UrlBuilder()
.set(path=f"/v1/subscriptions/{uuid}")
.add_query("replica", self.replica.name)
.add_query('subscription_type', subscription_type))
self.assertDeleteResponse(url, requests.codes.okay, headers=get_auth_header())
class TestSubscriptionsBase(ElasticsearchTestCase, DSSAssertMixin):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.app = ThreadedLocalServer()
cls.app.start()
dss.Config.set_config(dss.BucketConfig.TEST)
with open(
os.path.join(os.path.dirname(__file__),
"sample_v3_index_doc.json"), "r") as fh:
cls.index_document = BundleDocument(cls.replica, get_bundle_fqid())
cls.index_document.update(json.load(fh))
@classmethod
def tearDownClass(cls):
cls.app.shutdown()
super().tearDownClass()
def setUp(self):
super().setUp()
self.alias_name = dss.Config.get_es_alias_name(dss.ESIndexType.docs,
self.replica)
self.sub_index_name = dss.Config.get_es_index_name(
dss.ESIndexType.subscriptions, self.replica)
shape_identifier = self.index_document.get_shape_descriptor()
self.doc_index_name = dss.Config.get_es_index_name(
dss.ESIndexType.docs, self.replica, shape_identifier)
es_client = ElasticsearchClient.get()
IndexManager.create_index(es_client, self.replica, self.doc_index_name)
es_client.index(index=self.doc_index_name,
doc_type=dss.ESDocType.doc.name,
id=str(uuid.uuid4()),
body=self.index_document,
refresh=True)
self.endpoint = Endpoint(callback_url="https://example.com",
method="POST",
encoding="application/json",
form_fields={'foo': 'bar'},
payload_form_field='baz')
self.sample_percolate_query = smartseq2_paired_ends_v2_or_v3_query
def test_auth_errors(self):
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(uuid.uuid4())).add_query(
"replica", self.replica.name))
# Unauthorized email
with self.throw_403():
resp_obj = self.assertGetResponse(url,
requests.codes.forbidden,
headers=get_auth_header())
self.assertEqual(resp_obj.response.headers['Content-Type'],
"application/problem+json")
# Gibberish auth header
resp_obj = self.assertGetResponse(url,
requests.codes.unauthorized,
headers=get_auth_header(False))
self.assertEqual(resp_obj.response.headers['Content-Type'],
"application/problem+json")
# No auth header
self.assertGetResponse(url, requests.codes.unauthorized)
def test_put(self):
uuid_ = self._put_subscription()
es_client = ElasticsearchClient.get()
response = es_client.get(index=self.doc_index_name,
doc_type=dss.ESDocType.query.name,
id=uuid_)
registered_query = response['_source']
self.assertEqual(self.sample_percolate_query, registered_query)
def test_validation(self):
with self.subTest("Missing URL"):
self._put_subscription(expect_code=400, endpoint={})
with self.subTest("Invalid form field value"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(
callback_url=self.endpoint.callback_url,
form_fields={'foo': 1}))
with self.subTest("Invalid encoding"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(
callback_url=self.endpoint.callback_url,
encoding='foo'))
with self.subTest("Invalid method"):
self._put_subscription(expect_code=400,
endpoint=Endpoint(
callback_url=self.endpoint.callback_url,
method='foo'))
with self.subTest("Invalid attachment type"):
self._put_subscription(
expect_code=400,
attachments={'foo': dict(type='foo', expression='bar')})
with self.subTest("Invalid attachment expression"):
self._put_subscription(
expect_code=400,
attachments={'foo': dict(type='jmespath', expression='')})
def test_subscription_registration_succeeds_when_query_does_not_match_mappings(
self):
# It is now possible to register a subscription query before the mapping
# of the field exists in the mappings (and may never exist in the mapppings)
es_query = {
"query": {
"bool": {
"must": [{
"match": {
"assay.fake_field": "this is a negative test"
}
}],
}
}
}
url = str(UrlBuilder().set(path="/v1/subscriptions").add_query(
"replica", self.replica.name))
resp_obj = self.assertPutResponse(url,
requests.codes.created,
json_request_body=dict(
es_query=es_query,
**self.endpoint.to_dict()),
headers=get_auth_header())
self.assertIn('uuid', resp_obj.json)
def test_get(self):
find_uuid = self._put_subscription()
# Normal request
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(find_uuid)).add_query(
"replica", self.replica.name))
resp_obj = self.assertGetResponse(url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query,
json_response['es_query'])
self.assertEqual(self.endpoint,
Endpoint.from_subscription(json_response))
# Forbidden request w/ previous url
with self.throw_403():
self.assertGetResponse(url,
requests.codes.forbidden,
headers=get_auth_header())
# File not found request
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
str(uuid.uuid4())).add_query(
"replica", self.replica.name))
self.assertGetResponse(url,
requests.codes.not_found,
headers=get_auth_header())
def test_find(self):
num_additions = 25
for _ in range(num_additions):
self._put_subscription()
url = str(UrlBuilder().set(path="/v1/subscriptions").add_query(
"replica", self.replica.name))
resp_obj = self.assertGetResponse(url,
requests.codes.okay,
headers=get_auth_header())
json_response = resp_obj.json
self.assertEqual(self.sample_percolate_query,
json_response['subscriptions'][0]['es_query'])
self.assertEqual(
self.endpoint,
Endpoint.from_subscription(json_response['subscriptions'][0]))
self.assertEqual(num_additions, len(json_response['subscriptions']))
def test_delete(self):
find_uuid = self._put_subscription()
url = str(UrlBuilder().set(path="/v1/subscriptions/" +
find_uuid).add_query(
"replica", self.replica.name))
# Forbidden delete request
with self.throw_403():
self.assertDeleteResponse(url,
requests.codes.forbidden,
headers=get_auth_header())
# Authorized delete
self.assertDeleteResponse(url,
requests.codes.okay,
headers=get_auth_header())
# 1. Check that previous delete worked
# 2. Check that we can't delete files that don't exist
self.assertDeleteResponse(url,
requests.codes.not_found,
headers=get_auth_header())
def _put_subscription(self,
endpoint=None,
expect_code=None,
attachments=None):
url = str(UrlBuilder().set(path="/v1/subscriptions").add_query(
"replica", self.replica.name))
if endpoint is None:
endpoint = self.endpoint
if isinstance(endpoint, Endpoint):
endpoint = endpoint.to_dict()
json_request_body = dict(endpoint,
es_query=self.sample_percolate_query)
if attachments is not None:
json_request_body['attachments'] = attachments
resp_obj = self.assertPutResponse(url,
expect_code
or requests.codes.created,
json_request_body=json_request_body,
headers=get_auth_header())
return resp_obj.json.get('uuid')
@contextmanager
def throw_403(self):
orig_testing_403 = connexion.apis.abstract.Operation.testing_403
try:
connexion.apis.abstract.Operation.testing_403 = True
yield
finally:
connexion.apis.abstract.Operation.testing_403 = orig_testing_403