def _prepare_index(self, dryrun):
shape_descriptor = self.get_shape_descriptor()
index_name = Config.get_es_index_name(ESIndexType.docs, self.replica, shape_descriptor)
es_client = ElasticsearchClient.get()
if not dryrun:
IndexManager.create_index(es_client, self.replica, index_name)
return index_name
def _prepare_index(self, dryrun):
shape_descriptor = self['shape_descriptor']
if shape_descriptor is not None:
hashed_shape_descriptor = hashlib.sha1(
str(shape_descriptor).encode("utf-8")).hexdigest()
else:
hashed_shape_descriptor = ""
index_name = Config.get_es_index_name(ESIndexType.docs, self.replica,
hashed_shape_descriptor)
es_client = ElasticsearchClient.get()
if not dryrun:
IndexManager.create_index(es_client, self.replica, index_name)
return index_name
def setUp(self):
super().setUp()
self.alias_name = dss.Config.get_es_alias_name(dss.ESIndexType.docs,
self.replica)
self.sub_index_name = dss.Config.get_es_index_name(
dss.ESIndexType.subscriptions, self.replica)
shape_identifier = self.index_document.get_shape_descriptor()
self.doc_index_name = dss.Config.get_es_index_name(
dss.ESIndexType.docs, self.replica, shape_identifier)
es_client = ElasticsearchClient.get()
IndexManager.create_index(es_client, self.replica, self.doc_index_name)
es_client.index(index=self.doc_index_name,
doc_type=dss.ESDocType.doc.name,
id=str(uuid.uuid4()),
body=self.index_document,
refresh=True)
self.callback_url = "https://example.com"
self.sample_percolate_query = smartseq2_paired_ends_v2_or_v3_query
def setUp(self):
super().setUp()
self.alias_name = dss.Config.get_es_alias_name(dss.ESIndexType.docs, self.replica)
self.sub_index_name = dss.Config.get_es_index_name(dss.ESIndexType.subscriptions, self.replica)
shape_identifier = self.index_document._get_shape_descriptor()
shape_identifier = hashlib.sha1(f"{shape_identifier}".encode("utf-8")).hexdigest()
self.doc_index_name = dss.Config.get_es_index_name(dss.ESIndexType.docs, self.replica, shape_identifier)
es_client = ElasticsearchClient.get()
IndexManager.create_index(es_client, self.replica, self.doc_index_name)
es_client.index(index=self.doc_index_name,
doc_type=dss.ESDocType.doc.name,
id=str(uuid.uuid4()),
body=self.index_document,
refresh=True)
self.endpoint = Endpoint(callback_url="https://example.com",
method="POST",
encoding="application/json",
form_fields={'foo': 'bar'},
payload_form_field='baz')
self.sample_percolate_query = smartseq2_paired_ends_vx_query
self.hmac_key_id = 'dss_test'
self.hmac_secret_key = '23/33'
def put(json_request_body: dict, replica: str):
uuid = str(uuid4())
es_query = json_request_body['es_query']
owner = request.token_info['email']
es_client = ElasticsearchClient.get()
index_mapping = {
"mappings": {
ESDocType.subscription.name: {
"properties": {
"owner": {
"type": "string",
"index": "not_analyzed"
},
"es_query": {
"type": "object",
"enabled": "false"
}
}
}
}
}
# Elasticsearch preprocesses inputs by splitting strings on punctuation.
# So for [email protected], if I searched for people with the email address [email protected],
# [email protected] would show up because elasticsearch matched example w/ example.
# By including "index": "not_analyzed", Elasticsearch leaves all owner inputs alone.
index_name = Config.get_es_index_name(ESIndexType.subscriptions,
Replica[replica])
IndexManager.get_subscription_index(es_client, index_name, index_mapping)
# get all indexes that use current alias
alias_name = Config.get_es_alias_name(ESIndexType.docs, Replica[replica])
doc_indexes = _get_indexes_by_alias(es_client, alias_name)
# try to subscribe query to each of the indexes.
subscribed_indexes = []
for doc_index in doc_indexes:
try:
percolate_registration = _register_percolate(
es_client, doc_index, uuid, es_query, replica)
except ElasticsearchException as ex:
logger.debug(
f"Exception occured when registering a document to an index. Exception: {ex}"
)
last_ex = ex
else:
logger.debug(
f"Percolate query registration succeeded:\n{percolate_registration}"
)
subscribed_indexes.append(doc_index)
# Queries are unlikely to fit in all of the indexes, therefore errors will almost always occur. Only return an error
# if no queries are successfully indexed.
if doc_indexes and not subscribed_indexes:
logger.critical(
f"Percolate query registration failed: owner: {owner}, uuid: {uuid}, "
f"replica: {replica}, es_query: {es_query}, Exception: {last_ex}")
raise DSSException(
requests.codes.internal_server_error, "elasticsearch_error",
"Unable to register elasticsearch percolate query!") from last_ex
json_request_body['owner'] = owner
try:
subscription_registration = _register_subscription(
es_client, uuid, json_request_body, replica)
logger.debug(
f"Event Subscription succeeded:\n{subscription_registration}")
except ElasticsearchException as ex:
logger.critical(
f"Event Subscription failed: owner: {owner}, uuid: {uuid}, "
f"replica: {replica}, Exception: {ex}")
# Delete percolate query to make sure queries and subscriptions are in sync.
doc_indexes = _get_indexes_by_alias(es_client, alias_name)
_unregister_percolate(es_client, doc_indexes, uuid)
raise DSSException(
requests.codes.internal_server_error, "elasticsearch_error",
"Unable to register subscription! Rolling back percolate query.")
return jsonify(dict(uuid=uuid)), requests.codes.created
def setUp(self):
super().setUp()
self.dss_index_name = dss.Config.get_es_index_name(
dss.ESIndexType.docs, self.replica)
es_client = ElasticsearchClient.get()
IndexManager.create_index(es_client, self.replica, self.dss_index_name)