def _register_subscription(es_client: Elasticsearch, uuid: str,
json_request_body: dict, replica: str):
index_name = Config.get_es_index_name(ESIndexType.subscriptions,
Replica[replica])
return es_client.index(index=index_name,
doc_type=ESDocType.subscription.name,
id=uuid,
body=json_request_body,
refresh=True)
def _prepare_index(self, dryrun):
shape_descriptor = self['shape_descriptor']
if shape_descriptor is not None:
hashed_shape_descriptor = hashlib.sha1(
str(shape_descriptor).encode("utf-8")).hexdigest()
else:
hashed_shape_descriptor = ""
index_name = Config.get_es_index_name(ESIndexType.docs, self.replica,
hashed_shape_descriptor)
es_client = ElasticsearchClient.get()
if not dryrun:
IndexManager.create_index(es_client, self.replica, index_name)
return index_name
def find(replica: str):
owner = request.token_info['email']
es_client = ElasticsearchClient.get()
search_obj = Search(using=es_client,
index=Config.get_es_index_name(
ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name)
search = search_obj.query({'match': {'owner': owner}})
responses = [{
'uuid': hit.meta.id,
'replica': replica,
'owner': owner,
'callback_url': hit.callback_url,
'es_query': hit.es_query.to_dict()
} for hit in search.scan()]
full_response = {'subscriptions': responses}
return jsonify(full_response), requests.codes.okay
def _get_subscription(self, doc: BundleDocument, subscription_id: str) -> dict:
subscription_query = {
'query': {
'ids': {
'type': ESDocType.subscription.name,
'values': [subscription_id]
}
}
}
response = ElasticsearchClient.get().search(
index=Config.get_es_index_name(ESIndexType.subscriptions, doc.replica),
body=subscription_query)
hits = response['hits']['hits']
assert len(hits) == 1
hit = hits[0]
assert hit['_id'] == subscription_id
subscription = hit['_source']
assert 'id' not in subscription
subscription['id'] = subscription_id
return subscription
def find(replica: str):
owner = security.get_token_email(request.token_info)
es_client = ElasticsearchClient.get()
search_obj = Search(using=es_client,
index=Config.get_es_index_name(
ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name)
search = search_obj.query({'bool': {'must': [{'term': {'owner': owner}}]}})
responses = [{
'uuid': hit.meta.id,
'replica': replica,
'owner': owner,
**{k: v
for k, v in hit.to_dict().items() if k != 'hmac_secret_key'}
} for hit in search.scan()]
full_response = {'subscriptions': responses}
return jsonify(full_response), requests.codes.okay
def get(uuid: str, replica: str):
owner = request.token_info['email']
es_client = ElasticsearchClient.get()
try:
response = es_client.get(index=Config.get_es_index_name(
ESIndexType.subscriptions, Replica[replica]),
doc_type=ESDocType.subscription.name,
id=uuid)
except NotFoundError as ex:
raise DSSException(requests.codes.not_found, "not_found",
"Cannot find subscription!")
source = response['_source']
source['uuid'] = uuid
source['replica'] = replica
if source['owner'] != owner:
# common_error_handler defaults code to capitalized 'Forbidden' for Werkzeug exception. Keeping consistent.
raise DSSException(requests.codes.forbidden, "Forbidden",
"Your credentials can't access this subscription!")
return jsonify(source), requests.codes.okay
def put(json_request_body: dict, replica: str):
uuid = str(uuid4())
es_query = json_request_body['es_query']
owner = request.token_info['email']
es_client = ElasticsearchClient.get()
index_mapping = {
"mappings": {
ESDocType.subscription.name: {
"properties": {
"owner": {
"type": "string",
"index": "not_analyzed"
},
"es_query": {
"type": "object",
"enabled": "false"
}
}
}
}
}
# Elasticsearch preprocesses inputs by splitting strings on punctuation.
# So for [email protected], if I searched for people with the email address [email protected],
# [email protected] would show up because elasticsearch matched example w/ example.
# By including "index": "not_analyzed", Elasticsearch leaves all owner inputs alone.
index_name = Config.get_es_index_name(ESIndexType.subscriptions,
Replica[replica])
IndexManager.get_subscription_index(es_client, index_name, index_mapping)
# get all indexes that use current alias
alias_name = Config.get_es_alias_name(ESIndexType.docs, Replica[replica])
doc_indexes = _get_indexes_by_alias(es_client, alias_name)
# try to subscribe query to each of the indexes.
subscribed_indexes = []
for doc_index in doc_indexes:
try:
percolate_registration = _register_percolate(
es_client, doc_index, uuid, es_query, replica)
except ElasticsearchException as ex:
logger.debug(
f"Exception occured when registering a document to an index. Exception: {ex}"
)
last_ex = ex
else:
logger.debug(
f"Percolate query registration succeeded:\n{percolate_registration}"
)
subscribed_indexes.append(doc_index)
# Queries are unlikely to fit in all of the indexes, therefore errors will almost always occur. Only return an error
# if no queries are successfully indexed.
if doc_indexes and not subscribed_indexes:
logger.critical(
f"Percolate query registration failed: owner: {owner}, uuid: {uuid}, "
f"replica: {replica}, es_query: {es_query}, Exception: {last_ex}")
raise DSSException(
requests.codes.internal_server_error, "elasticsearch_error",
"Unable to register elasticsearch percolate query!") from last_ex
json_request_body['owner'] = owner
try:
subscription_registration = _register_subscription(
es_client, uuid, json_request_body, replica)
logger.debug(
f"Event Subscription succeeded:\n{subscription_registration}")
except ElasticsearchException as ex:
logger.critical(
f"Event Subscription failed: owner: {owner}, uuid: {uuid}, "
f"replica: {replica}, Exception: {ex}")
# Delete percolate query to make sure queries and subscriptions are in sync.
doc_indexes = _get_indexes_by_alias(es_client, alias_name)
_unregister_percolate(es_client, doc_indexes, uuid)
raise DSSException(
requests.codes.internal_server_error, "elasticsearch_error",
"Unable to register subscription! Rolling back percolate query.")
return jsonify(dict(uuid=uuid)), requests.codes.created
