def test_custom_email_claims(self):
self.addCleanup(self.restore_email_claims,
os.environ.pop('OIDC_EMAIL_CLAIM', 'EMPTY'))
email = '*****@*****.**'
email_claim = '*****@*****.**'
tests = [({
'email': email,
Config.get_OIDC_email_claim(): email_claim
}, email_claim),
({
Config.get_OIDC_email_claim(): email_claim
}, email_claim), ({
'email': email
}, email)]
for param, result in tests:
with self.subTest(f"no custom claim {param}"):
self.assertEqual(security.get_token_email(param), result)
os.environ['OIDC_EMAIL_CLAIM'] = 'TEST_CLAIM'
for param, result in tests:
with self.subTest(f"custom claim {param}"):
self.assertEqual(security.get_token_email(param), result)
with self.subTest("missing claim"):
with self.assertRaises(DSSException) as ex:
security.get_token_email({})
self.assertEqual(ex.exception.status, 401)
self.assertEqual(ex.exception.message,
'Authorization token is missing email claims.')
def get(uuid: str, replica: str, version: str = None):
authenticated_user_email = security.get_token_email(request.token_info)
collection_body = get_impl(uuid=uuid, replica=replica, version=version)
if collection_body["owner"] != authenticated_user_email:
raise DSSException(requests.codes.forbidden, "forbidden",
f"Collection access denied")
return collection_body
def delete(uuid: str, replica: str):
authenticated_user_email = security.get_token_email(request.token_info)
uuid = uuid.lower()
tombstone_key = CollectionTombstoneID(uuid, version=None).to_key()
tombstone_object_data = dict(email=authenticated_user_email)
owner = get_impl(uuid=uuid, replica=replica)["owner"]
if owner != authenticated_user_email:
raise DSSException(requests.codes.forbidden, "forbidden",
f"Collection access denied")
created, idempotent = idempotent_save(
Config.get_blobstore_handle(Replica[replica]), Replica[replica].bucket,
tombstone_key,
json.dumps(tombstone_object_data).encode("utf-8"))
if not idempotent:
raise DSSException(
requests.codes.conflict, f"collection_tombstone_already_exists",
f"collection tombstone with UUID {uuid} already exists")
status_code = requests.codes.ok
response_body = dict() # type: dict
# update dynamoDB
owner_lookup.delete_collection_uuid(owner=authenticated_user_email,
uuid=uuid)
return jsonify(response_body), status_code
def get(uuid: str, replica: str):
owner = security.get_token_email(request.token_info)
es_client = ElasticsearchClient.get()
try:
response = es_client.get(index=Config.get_es_index_name(
ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name,
id=uuid)
except NotFoundError:
raise DSSException(requests.codes.not_found, "not_found",
"Cannot find subscription!")
source = response['_source']
source['uuid'] = uuid
source['replica'] = replica
if 'hmac_key_id' in response:
source['hmac_key_id'] = response['hmac_key_id']
if 'hmac_secret_key' in source:
source.pop('hmac_secret_key')
if source['owner'] != owner:
# common_error_handler defaults code to capitalized 'Forbidden' for Werkzeug exception. Keeping consistent.
raise DSSException(requests.codes.forbidden, "Forbidden",
"Your credentials can't access this subscription!")
return jsonify(source), requests.codes.okay
def delete(uuid: str, replica: str):
owner = security.get_token_email(request.token_info)
es_client = ElasticsearchClient.get()
try:
response = es_client.get(index=Config.get_es_index_name(
ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name,
id=uuid)
except NotFoundError:
raise DSSException(requests.codes.not_found, "not_found",
"Cannot find subscription!")
stored_metadata = response['_source']
if stored_metadata['owner'] != owner:
# common_error_handler defaults code to capitalized 'Forbidden' for Werkzeug exception. Keeping consistent.
raise DSSException(requests.codes.forbidden, "Forbidden",
"Your credentials can't access this subscription!")
_delete_subscription(es_client, uuid)
timestamp = datetime.datetime.utcnow()
time_deleted = timestamp.strftime("%Y-%m-%dT%H%M%S.%fZ")
return jsonify({'timeDeleted': time_deleted}), requests.codes.okay
def find(replica: str):
owner = security.get_token_email(request.token_info)
subscriptions = get_subscriptions_for_owner(Replica[replica], owner)
for subscription in subscriptions:
subscription['replica'] = Replica[replica].name
if 'hmac_secret_key' in subscription:
subscription.pop('hmac_secret_key')
return {'subscriptions': subscriptions}, requests.codes.ok
def get(uuid: str, replica: str):
owner = security.get_token_email(request.token_info)
subscription = get_subscription(Replica[replica], owner, uuid)
if subscription is None or owner != subscription[SubscriptionData.OWNER]:
raise DSSException(404, "not_found", "Cannot find subscription!")
if 'hmac_secret_key' in subscription:
subscription.pop('hmac_secret_key')
return subscription, requests.codes.ok
def delete(uuid: str, replica: str):
owner = security.get_token_email(request.token_info)
subscription = get_subscription(Replica[replica], owner, uuid)
if subscription is None or owner != subscription[SubscriptionData.OWNER]:
raise DSSException(404, "not_found", "Cannot find subscription!")
delete_subscription(Replica[replica], owner, uuid)
timestamp = datetime.datetime.utcnow()
time_deleted = timestamp.strftime("%Y-%m-%dT%H%M%S.%fZ")
return jsonify({'timeDeleted': time_deleted}), requests.codes.okay
def put(json_request_body: dict, replica: str):
owner = security.get_token_email(request.token_info)
if count_subscriptions_for_owner(Replica[replica], owner) > SUBSCRIPTION_LIMIT:
raise DSSException(requests.codes.not_acceptable, "not_acceptable",
f"Users cannot exceed {SUBSCRIPTION_LIMIT} subscriptions!")
subscription_doc = json_request_body.copy()
subscription_doc[SubscriptionData.OWNER] = security.get_token_email(request.token_info)
subscription_uuid = str(uuid4())
subscription_doc[SubscriptionData.UUID] = subscription_uuid
subscription_doc[SubscriptionData.REPLICA] = Replica[replica].name
if subscription_doc.get(SubscriptionData.JMESPATH_QUERY) is not None:
try:
jmespath.compile(subscription_doc[SubscriptionData.JMESPATH_QUERY])
except JMESPathError:
raise DSSException(
requests.codes.bad_request,
"invalid_jmespath",
"JMESPath query is invalid"
)
# validate attachment JMESPath if present
attachments = subscription_doc.get(SubscriptionData.ATTACHMENTS)
if attachments is not None:
for name, definition in attachments.items():
if name.startswith('_'):
raise DSSException(requests.codes.bad_request,
"invalid_attachment_name",
f"Attachment names must not start with underscore ({name})")
type_ = definition['type']
if type_ == 'jmespath':
expression = definition['expression']
try:
jmespath.compile(expression)
except JMESPathError as e:
raise DSSException(requests.codes.bad_request,
"invalid_attachment_expression",
f"Unable to compile JMESPath expression for attachment {name}") from e
else:
assert False, type_
put_subscription(subscription_doc)
return subscription_doc, requests.codes.created
def find(replica: str):
owner = security.get_token_email(request.token_info)
es_client = ElasticsearchClient.get()
search_obj = Search(using=es_client,
index=Config.get_es_index_name(
ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name)
search = search_obj.query({'bool': {'must': [{'term': {'owner': owner}}]}})
responses = [{
'uuid': hit.meta.id,
'replica': replica,
'owner': owner,
**{k: v
for k, v in hit.to_dict().items() if k != 'hmac_secret_key'}
} for hit in search.scan()]
full_response = {'subscriptions': responses}
return jsonify(full_response), requests.codes.okay
def patch(uuid: str, json_request_body: dict, replica: str, version: str):
authenticated_user_email = security.get_token_email(request.token_info)
uuid = uuid.lower()
owner = get_impl(uuid=uuid, replica=replica)["owner"]
if owner != authenticated_user_email:
raise DSSException(requests.codes.forbidden, "forbidden",
f"Collection access denied")
handle = Config.get_blobstore_handle(Replica[replica])
try:
cur_collection_blob = handle.get(
Replica[replica].bucket,
CollectionFQID(uuid, version).to_key())
except BlobNotFoundError:
raise DSSException(
404, "not_found",
"Could not find collection for UUID {}".format(uuid))
collection = json.loads(cur_collection_blob)
for field in "name", "description", "details":
if field in json_request_body:
collection[field] = json_request_body[field]
remove_contents_set = set(
map(hashabledict, json_request_body.get("remove_contents", [])))
collection["contents"] = [
i for i in collection["contents"]
if hashabledict(i) not in remove_contents_set
]
verify_collection(json_request_body.get("add_contents", []),
Replica[replica], handle)
collection["contents"].extend(json_request_body.get("add_contents", []))
collection["contents"] = _dedpuplicate_contents(collection["contents"])
timestamp = datetime.datetime.utcnow()
new_collection_version = datetime_to_version_format(timestamp)
handle.upload_file_handle(
Replica[replica].bucket,
CollectionFQID(uuid, new_collection_version).to_key(),
io.BytesIO(json.dumps(collection).encode("utf-8")))
return jsonify(dict(uuid=uuid,
version=new_collection_version)), requests.codes.ok
def put(json_request_body: dict, replica: str, uuid: str, version: str):
authenticated_user_email = security.get_token_email(request.token_info)
collection_body = dict(json_request_body, owner=authenticated_user_email)
uuid = uuid.lower()
handle = Config.get_blobstore_handle(Replica[replica])
collection_body["contents"] = _dedpuplicate_contents(
collection_body["contents"])
verify_collection(collection_body["contents"], Replica[replica], handle)
collection_uuid = uuid if uuid else str(uuid4())
collection_version = version
# update dynamoDB; used to speed up lookup time; will not update if owner already associated w/uuid
owner_lookup.put_collection(owner=authenticated_user_email,
collection_fqid=str(
CollectionFQID(collection_uuid,
collection_version)))
# add the collection file to the bucket
handle.upload_file_handle(
Replica[replica].bucket,
CollectionFQID(collection_uuid, collection_version).to_key(),
io.BytesIO(json.dumps(collection_body).encode("utf-8")))
return jsonify(dict(uuid=collection_uuid,
version=collection_version)), requests.codes.created
def delete(uuid: str,
replica: str,
json_request_body: dict,
version: str = None):
email = security.get_token_email(request.token_info)
if email not in ADMIN_USER_EMAILS:
raise DSSForbiddenException(
"You can't delete bundles with these credentials!")
uuid = uuid.lower()
tombstone_id = BundleTombstoneID(uuid=uuid, version=version)
bundle_prefix = tombstone_id.to_key_prefix()
tombstone_object_data = _create_tombstone_data(
email=email,
reason=json_request_body.get('reason'),
version=version,
)
handle = Config.get_blobstore_handle(Replica[replica])
if not test_object_exists(handle,
Replica[replica].bucket,
bundle_prefix,
test_type=ObjectTest.PREFIX):
raise DSSException(404, "not_found", "Cannot find bundle!")
created, idempotent = idempotent_save(
handle, Replica[replica].bucket, tombstone_id.to_key(),
json.dumps(tombstone_object_data).encode("utf-8"))
if not idempotent:
raise DSSException(
requests.codes.conflict,
f"bundle_tombstone_already_exists",
f"bundle tombstone with UUID {uuid} and version {version} already exists",
)
return dict(), requests.codes.ok
def list_collections(per_page: int, start_at: int = 0):
"""
Return a list of a user's collections.
Collection uuids are indexed and called by the user's email in a dynamoDB table.
:param int per_page: # of collections returned per paged response.
:param int start_at: Where the next chunk of paged response should start at.
:return: A dictionary containing a list of dictionaries looking like:
{'collections': [{'uuid': uuid, 'version': version}, {'uuid': uuid, 'version': version}, ... , ...]}
"""
# TODO: Replica is unused, so this does not use replica. Appropriate?
owner = security.get_token_email(request.token_info)
collections = []
for collection in owner_lookup.get_collection_fqids_for_owner(owner):
fqid = CollectionFQID.from_key(f'{COLLECTION_PREFIX}/{collection}')
collections.append({'uuid': fqid.uuid, 'version': fqid.version})
# paged response
if len(collections) - start_at > per_page:
next_url = UrlBuilder(request.url)
next_url.replace_query("start_at", str(start_at + per_page))
collection_page = collections[start_at:start_at + per_page]
response = make_response(jsonify({'collections': collection_page}),
requests.codes.partial)
response.headers['Link'] = f"<{next_url}>; rel='next'"
response.headers['X-OpenAPI-Pagination'] = 'true'
# single response returning all collections (or those remaining)
else:
collection_page = collections[start_at:]
response = make_response(jsonify({'collections': collection_page}),
requests.codes.ok)
response.headers['X-OpenAPI-Pagination'] = 'false'
response.headers['X-OpenAPI-Paginated-Content-Key'] = 'collections'
return response
def put(json_request_body: dict, replica: str):
uuid = str(uuid4())
es_query = json_request_body['es_query']
owner = security.get_token_email(request.token_info)
attachment.validate(json_request_body.get('attachments', {}))
es_client = ElasticsearchClient.get()
index_mapping = {
"mappings": {
ESDocType.subscription.name: {
"properties": {
"owner": {
"type": "string",
"index": "not_analyzed"
},
"es_query": {
"type": "object",
"enabled": "false"
}
}
}
}
}
# Elasticsearch preprocesses inputs by splitting strings on punctuation.
# So for [email protected], if I searched for people with the email address [email protected],
# [email protected] would show up because elasticsearch matched example w/ example.
# By including "index": "not_analyzed", Elasticsearch leaves all owner inputs alone.
index_name = Config.get_es_index_name(ESIndexType.subscriptions,
Replica[replica])
IndexManager.get_subscription_index(es_client, index_name, index_mapping)
# get all indexes that use current alias
alias_name = Config.get_es_alias_name(ESIndexType.docs, Replica[replica])
doc_indexes = _get_indexes_by_alias(es_client, alias_name)
search_obj = Search(using=es_client,
index=index_name,
doc_type=ESDocType.subscription.name)
search = search_obj.query({'bool': {'must': [{'term': {'owner': owner}}]}})
if search.count() > SUBSCRIPTION_LIMIT:
raise DSSException(
requests.codes.not_acceptable, "not_acceptable",
f"Users cannot exceed {SUBSCRIPTION_LIMIT} subscriptions!")
# try to subscribe query to each of the indexes.
subscribed_indexes = []
for doc_index in doc_indexes:
try:
percolate_registration = _register_percolate(
es_client, doc_index, uuid, es_query, replica)
except ElasticsearchException as ex:
logger.debug(
f"Exception occured when registering a document to an index. Exception: {ex}"
)
last_ex = ex
else:
logger.debug(
f"Percolate query registration succeeded:\n{percolate_registration}"
)
subscribed_indexes.append(doc_index)
# Queries are unlikely to fit in all of the indexes, therefore errors will almost always occur. Only return an error
# if no queries are successfully indexed.
if doc_indexes and not subscribed_indexes:
logger.critical(
f"Percolate query registration failed: owner: {owner}, uuid: {uuid}, "
f"replica: {replica}, es_query: {es_query}, Exception: {last_ex}")
raise DSSException(
requests.codes.internal_server_error, "elasticsearch_error",
"Unable to register elasticsearch percolate query!") from last_ex
json_request_body['owner'] = owner
try:
subscription_registration = _register_subscription(
es_client, uuid, json_request_body, replica)
logger.info(
f"Event Subscription succeeded:\n{subscription_registration}, owner: {owner}, uuid: {uuid}, "
f"replica: {replica}")
except ElasticsearchException as ex:
logger.critical(
f"Event Subscription failed: owner: {owner}, uuid: {uuid}, "
f"replica: {replica}, Exception: {ex}")
# Delete percolate query to make sure queries and subscriptions are in sync.
_unregister_percolate(es_client, uuid)
raise DSSException(
requests.codes.internal_server_error, "elasticsearch_error",
"Unable to register subscription! Rolling back percolate query.")
return jsonify(dict(uuid=uuid)), requests.codes.created