def post(self, request, project_pk: int):
"""
Returns existing fact names and values from Elasticsearch.
"""
serializer = ProjectGetFactsSerializer(data=request.data)
if not serializer.is_valid():
raise SerializerNotValid(detail=serializer.errors)
indices = serializer.validated_data["indices"]
indices = [index["name"] for index in indices]
# retrieve and validate project indices
project = get_object_or_404(Project, pk=project_pk)
self.check_object_permissions(request, project)
project_indices = project.get_available_or_all_project_indices(indices) # Gives all if n one, the default, is entered.
if not project_indices:
return Response([])
vals_per_name = serializer.validated_data['values_per_name']
include_values = serializer.validated_data['include_values']
fact_name = serializer.validated_data['fact_name']
include_doc_path = serializer.validated_data['include_doc_path']
exclude_zero_spans = serializer.validated_data['exclude_zero_spans']
mlp_doc_path = serializer.validated_data['mlp_doc_path']
aggregator = ElasticAggregator(indices=project_indices)
if mlp_doc_path and exclude_zero_spans:
# If exclude_zerp_spans is enabled and mlp_doc_path specified, the other values don't have any effect -
# this behaviour might need to change at some point
fact_map = aggregator.facts(size=1, include_values=True, include_doc_path=True, exclude_zero_spans=exclude_zero_spans)
else:
fact_map = aggregator.facts(size=vals_per_name, include_values=include_values, filter_by_fact_name=fact_name, include_doc_path=include_doc_path, exclude_zero_spans=exclude_zero_spans)
if fact_name:
fact_map_list = [v for v in fact_map]
elif mlp_doc_path and exclude_zero_spans:
# Return only fact names where doc_path contains mlp_doc_path as a parent field and facts have spans.
# NB! Doesn't take into account the situation where facts have the same name, but different doc paths! Could happen!
fact_map_list = [k for k, v in fact_map.items() if v and mlp_doc_path == v[0]["doc_path"].rsplit(".", 1)[0]]
elif include_values:
fact_map_list = [{'name': k, 'values': v} for k, v in fact_map.items()]
else:
fact_map_list = [v for v in fact_map]
return Response(fact_map_list, status=status.HTTP_200_OK)
def validate_pos_label(data):
""" For Tagger, TorchTagger and BertTagger.
Checks if the inserted pos label is present in the fact values.
"""
fact_name = data.get("fact_name")
# If fact name is not selected, the value for pos label doesn't matter
if not fact_name:
return data
indices = [index.get("name") for index in data.get("indices")]
pos_label = data.get("pos_label")
serializer_query = data.get("query")
try:
# If query is passed as a JSON string
query = json.loads(serializer_query)
except Exception as e:
# if query is passed as a JSON dict
query = serializer_query
ag = ElasticAggregator(indices=indices, query=query)
fact_values = ag.facts(size=10, filter_by_fact_name=fact_name, include_values=True)
# If there exists exactly two possible values for the selected fact, check if pos label
# is selected and if it is present in corresponding fact values.
if len(fact_values) == 2:
if not pos_label:
raise ValidationError(f"The fact values corresponding to the selected query and fact '{fact_name}' are binary. You must specify param 'pos_label' for evaluation purposes. Allowed values for 'pos_label' are: {fact_values}")
elif pos_label not in fact_values:
raise ValidationError(f"The specified pos label '{pos_label}' is NOT one of the fact values for fact '{fact_name}'. Please select an existing fact value. Allowed fact values are: {fact_values}")
return data
def _get_tags(self, fact_name, min_count=50, max_count=None, query={}):
"""Finds possible tags for training by aggregating active project's indices."""
active_indices = self.tagger_object.get_indices()
es_a = ElasticAggregator(indices=active_indices, query=query)
# limit size to 10000 unique tags
tag_values = es_a.facts(filter_by_fact_name=fact_name,
min_count=min_count,
max_count=max_count,
size=10000)
return tag_values
def get_tags(self,
fact_name,
active_project,
min_count=1000,
max_count=None,
indices=None):
"""Finds possible tags for training by aggregating active project's indices."""
active_indices = list(
active_project.get_indices()) if indices is None else indices
es_a = ElasticAggregator(indices=active_indices)
# limit size to 10000 unique tags
tag_values = es_a.facts(filter_by_fact_name=fact_name,
min_count=min_count,
max_count=max_count,
size=10000)
return tag_values
def validate_fact_value(indices: List[str], query: dict, fact: str,
fact_value: str):
""" Check if given fact value exists under given fact. """
# Fact value is allowed to be empty
if not fact_value:
return True
ag = ElasticAggregator(indices=indices, query=deepcopy(query))
fact_values = ag.facts(size=choices.DEFAULT_MAX_AGGREGATION_SIZE,
filter_by_fact_name=fact,
include_values=True)
if fact_value not in fact_values:
raise ValidationError(
f"Fact value '{fact_value}' not in the list of fact values for fact '{fact}'."
)
return True
def evaluate_tags_task(object_id: int,
indices: List[str],
query: dict,
es_timeout: int = 10,
scroll_size: int = 100):
try:
logging.getLogger(INFO_LOGGER).info(
f"Starting evaluator task for Evaluator with ID {object_id}.")
evaluator_object = Evaluator.objects.get(pk=object_id)
progress = ShowProgress(evaluator_object.task, multiplier=1)
# Retreieve facts and sklearn average function from the model
true_fact = evaluator_object.true_fact
pred_fact = evaluator_object.predicted_fact
true_fact_value = evaluator_object.true_fact_value
pred_fact_value = evaluator_object.predicted_fact_value
average = evaluator_object.average_function
add_individual_results = evaluator_object.add_individual_results
searcher = ElasticSearcher(indices=indices,
field_data=["texta_facts"],
query=query,
output=ElasticSearcher.OUT_RAW,
timeout=f"{es_timeout}m",
callback_progress=progress,
scroll_size=scroll_size)
# Binary
if true_fact_value and pred_fact_value:
logging.getLogger(INFO_LOGGER).info(
f"Starting binary evaluation. Comparing following fact and fact value pairs: TRUE: ({true_fact}: {true_fact_value}), PREDICTED: ({pred_fact}: {pred_fact_value})."
)
# Set the evaluation type in the model
evaluator_object.evaluation_type = "binary"
true_set = {true_fact_value, "other"}
pred_set = {pred_fact_value, "other"}
classes = ["other", true_fact_value]
n_total_classes = len(classes)
# Multilabel/multiclass
else:
logging.getLogger(INFO_LOGGER).info(
f"Starting multilabel evaluation. Comparing facts TRUE: '{true_fact}', PRED: '{pred_fact}'."
)
# Make deepcopy of the query to avoid modifying Searcher's query.
es_aggregator = ElasticAggregator(indices=indices,
query=deepcopy(query))
# Get all fact values corresponding to true and predicted facts to construct total set of labels
# needed for confusion matrix, individual score calculations and memory imprint calculations
true_fact_values = es_aggregator.facts(
size=choices.DEFAULT_MAX_AGGREGATION_SIZE,
filter_by_fact_name=true_fact)
pred_fact_values = es_aggregator.facts(
size=choices.DEFAULT_MAX_AGGREGATION_SIZE,
filter_by_fact_name=pred_fact)
true_set = set(true_fact_values)
pred_set = set(pred_fact_values)
classes = list(true_set.union(pred_set))
n_total_classes = len(classes)
# Add dummy classes for missing labels
classes.extend(
[choices.MISSING_TRUE_LABEL, choices.MISSING_PRED_LABEL])
## Set the evaluation type in the model
evaluator_object.evaluation_type = "multilabel"
classes.sort(key=lambda x: x[0].lower())
# Get number of documents in the query to estimate memory imprint
n_docs = searcher.count()
evaluator_object.task.total = n_docs
evaluator_object.task.save()
logging.getLogger(INFO_LOGGER).info(
f"Number of documents: {n_docs} | Number of classes: {len(classes)}"
)
# Get the memory buffer value from core variables
core_memory_buffer_value_gb = get_core_setting(
"TEXTA_EVALUATOR_MEMORY_BUFFER_GB")
# Calculate the value based on given ratio if the core variable is empty
memory_buffer_gb = calculate_memory_buffer(
memory_buffer=core_memory_buffer_value_gb,
ratio=EVALUATOR_MEMORY_BUFFER_RATIO,
unit="gb")
required_memory = get_memory_imprint(
n_docs=n_docs,
n_classes=len(classes),
eval_type=evaluator_object.evaluation_type,
unit="gb",
int_size=64)
enough_memory = is_enough_memory_available(
required_memory=required_memory,
memory_buffer=memory_buffer_gb,
unit="gb")
# Enable scoring after each scroll if there isn't enough memory
# for calculating the scores for the whole set of documents at once.
score_after_scroll = False if enough_memory else True
# If scoring after each scroll is enabled and scores are averaged after each scroll
# the results for each averaging function besides `micro` are imprecise
scores_imprecise = True if (score_after_scroll
and average != "micro") else False
# Store document counts, labels' class counts and indicatior if scores are imprecise
evaluator_object.document_count = n_docs
evaluator_object.n_true_classes = len(true_set)
evaluator_object.n_predicted_classes = len(pred_set)
evaluator_object.n_total_classes = n_total_classes
evaluator_object.scores_imprecise = scores_imprecise
evaluator_object.score_after_scroll = score_after_scroll
# Save model updates
evaluator_object.save()
logging.getLogger(INFO_LOGGER).info(
f"Enough available memory: {enough_memory} | Score after scroll: {score_after_scroll}"
)
# Get number of batches for the logger
n_batches = math.ceil(n_docs / scroll_size)
# Scroll and score tags
scores, bin_scores = scroll_and_score(
generator=searcher,
evaluator_object=evaluator_object,
true_fact=true_fact,
pred_fact=pred_fact,
true_fact_value=true_fact_value,
pred_fact_value=pred_fact_value,
classes=classes,
average=average,
score_after_scroll=score_after_scroll,
n_batches=n_batches,
add_individual_results=add_individual_results)
logging.getLogger(INFO_LOGGER).info(f"Final scores: {scores}")
for conn in connections.all():
conn.close_if_unusable_or_obsolete()
confusion = scores["confusion_matrix"]
confusion = np.asarray(confusion, dtype="int64")
if len(classes) <= choices.DEFAULT_MAX_CONFUSION_CLASSES:
# Delete empty rows and columns corresponding to missing pred/true labels from the confusion matrix
confusion, classes = delete_empty_rows_and_cols(confusion, classes)
scores["confusion_matrix"] = confusion.tolist()
# Generate confusion matrix plot and save it
image_name = f"{secrets.token_hex(15)}.png"
evaluator_object.plot.save(image_name,
create_confusion_plot(
scores["confusion_matrix"], classes),
save=False)
image_path = pathlib.Path(MEDIA_URL) / image_name
evaluator_object.plot.name = str(image_path)
# Add final scores to the model
evaluator_object.precision = scores["precision"]
evaluator_object.recall = scores["recall"]
evaluator_object.f1_score = scores["f1_score"]
evaluator_object.accuracy = scores["accuracy"]
evaluator_object.confusion_matrix = json.dumps(
scores["confusion_matrix"])
evaluator_object.individual_results = json.dumps(
remove_not_found(bin_scores), ensure_ascii=False)
evaluator_object.add_misclassified_examples = False
evaluator_object.save()
evaluator_object.task.complete()
return True
except Exception as e:
logging.getLogger(ERROR_LOGGER).exception(e)
error_message = f"{str(e)[:100]}..." # Take first 100 characters in case the error message is massive.
evaluator_object.task.add_error(error_message)
evaluator_object.task.update_status(Task.STATUS_FAILED)