def update_project_indices(self, serializer, project_obj):
''' add new_index included in the request to the relevant project object '''
index_to_add = serializer.validated_data['new_index']
from texta_elastic.core import ElasticCore
ec = ElasticCore()
ec.create_index(index_to_add)
index, is_open = Index.objects.get_or_create(name=index_to_add)
project_obj.indices.add(index)
project_obj.save()
def create(self, request, **kwargs):
data = IndexSerializer(data=request.data)
data.is_valid(raise_exception=True)
es = ElasticCore()
index = data.validated_data["name"]
is_open = data.validated_data["is_open"]
description = data.validated_data["description"]
added_by = data.validated_data["added_by"]
test = data.validated_data["test"]
source = data.validated_data["source"]
client = data.validated_data["client"]
domain = data.validated_data["domain"]
# Using get_or_create to avoid unique name constraints on creation.
if es.check_if_indices_exist([index]):
# Even if the index already exists, create the index object just in case
index, is_created = Index.objects.get_or_create(name=index)
if is_created:
utc_time = es.get_index_creation_date(index)
index.is_open = is_open
index.description = description
index.added_by = added_by
index.test = test
index.source = source
index.client = client
index.domain = domain
index.created_at = utc_time
index.save()
raise ElasticIndexAlreadyExists()
else:
es.create_index(index=index)
if not is_open:
es.close_index(index)
index, is_created = Index.objects.get_or_create(name=index)
if is_created:
utc_time = es.get_index_creation_date(index)
index.is_open = is_open
index.description = description
index.added_by = added_by
index.test = test
index.source = source
index.client = client
index.domain = domain
index.created_at = utc_time
index.save()
return Response(
{"message": f"Added index {index} into Elasticsearch!"},
status=status.HTTP_201_CREATED)
def reindex_test_dataset(query: dict = None, from_index: Optional[str] = None, hex_size=20) -> str:
"""
Reindexes the master test dataset into isolated pieces.
:param from_index: Index from which to reindex.
:param query: Query you want to limit the reindex to.
:param hex_size: How many random characters should there be in the new indexes name.
:return: Name of the newly generated index.
"""
from texta_elastic.core import ElasticCore
from toolkit.test_settings import TEST_INDEX
from_index = from_index if from_index else TEST_INDEX
ec = ElasticCore()
new_test_index_name = f"ttk_test_{uuid.uuid4().hex[:hex_size]}"
ec.create_index(index=new_test_index_name)
ec.add_texta_facts_mapping(new_test_index_name)
from_scan = elasticsearch_dsl.Search() if query is None else elasticsearch_dsl.Search.from_dict(query)
from_scan = from_scan.index(from_index).using(ec.es)
from_scan = from_scan.scan()
def doc_actions(generator):
for document in generator:
yield {
"_index": new_test_index_name,
"_type": "_doc",
"_source": document.to_dict(),
"retry_on_conflict": 3
}
actions = doc_actions(from_scan)
from elasticsearch.helpers import bulk
bulk(actions=actions, client=ec.es, refresh="wait_for")
return new_test_index_name
