def post(self, request, project_pk: int): """ Returns existing fact names and values from Elasticsearch. """ serializer = ProjectFactAggregatorSerializer(data=request.data) if not serializer.is_valid(): raise SerializerNotValid(detail=serializer.errors) indices = serializer.validated_data["indices"] indices = [index["name"] for index in indices] # retrieve and validate project indices project = get_object_or_404(Project, pk=project_pk) self.check_object_permissions(request, project) project_indices = project.get_available_or_all_project_indices(indices) # Gives all if n one, the default, is entered. if not project_indices: return Response([]) key_field = serializer.validated_data["key_field"] value_field = serializer.validated_data["value_field"] filter_by_key = serializer.validated_data["filter_by_key"] max_count = serializer.validated_data["max_count"] query = serializer.validated_data["query"] if isinstance(query, str): query = json.loads(query) aggregator = ElasticAggregator(indices=project_indices, query=query) results = aggregator.facts_abstract(key_field=key_field, value_field=value_field, filter_by_key=filter_by_key, size=max_count) return Response(results, status=status.HTTP_200_OK)
def validate_evaluation_type(indices: List[str], query: dict, evaluation_type: str, true_fact: str, pred_fact: str, true_value: str, pred_value: str): """ Checks if the chosen facts (and values) are applicable for the chosen evaluation type. """ if evaluation_type == "binary": if not true_value or not pred_value: raise ValidationError( f"Please specify true and predicted values for evaluation type 'binary'." ) #elif evaluation_type == "multilabel": # if true_value or pred_value: # raise ValidationError(f"Please leave true and predicted values unspeficied for evaluation type 'multilabel'.") elif evaluation_type == "entity": if true_value or pred_value: raise ValidationError( f"Please leave true and predicted values unspeficied for evaluation type 'entity'." ) ag = ElasticAggregator(indices=indices, query=deepcopy(query)) true_fact_results = ag.facts_abstract(key_field="fact", value_field="spans", filter_by_key=true_fact, size=5) pred_fact_results = ag.facts_abstract(key_field="fact", value_field="spans", filter_by_key=pred_fact, size=5) if len(true_fact_results) == 1: spans = json.loads(true_fact_results[0]) if not spans[0] or (spans[0][0] == 0 and spans[0][1] == 0): raise ValidationError( f"Did not find non-zero spans for selected true fact '{true_fact}'. Please make sure to use facts with existing spans for evaluation_type 'entity'." ) if len(pred_fact_results) == 1: spans = json.loads(pred_fact_results[0]) if not spans[0] or (spans[0][0] == 0 and spans[0][1] == 0): raise ValidationError( f"Did not find non-zero spans for selected predicted fact '{pred_fact}'. Please make sure to use facts with existing spans for evaluation_type 'entity'." ) return True
def validate_entity_facts(indices: List[str], query: dict, true_fact: str, pred_fact: str, doc_path: str): """ Check if facts chosen for entity evaluation follow all the necessary requirements. """ ag = ElasticAggregator(indices=indices, query=deepcopy(query)) true_fact_doc_paths = ag.facts_abstract(key_field="fact", value_field="doc_path", filter_by_key=true_fact) pred_fact_doc_paths = ag.facts_abstract(key_field="fact", value_field="doc_path", filter_by_key=pred_fact) if doc_path: if doc_path not in true_fact_doc_paths: raise ValidationError( f"The selected true_fact ('{true_fact}') doesn't contain any instances corresponding to the selected field('{doc_path}')." ) if doc_path not in pred_fact_doc_paths: raise ValidationError( f"The selected predicted_fact ('{pred_fact}') doesn't contain any instances corresponding to the selected field('{doc_path}')." ) if not doc_path: if set(true_fact_doc_paths) != set(pred_fact_doc_paths): raise ValidationError( f"The doc paths for true and predicted facts are different (true = {true_fact_doc_paths}; predicted = {pred_fact_doc_paths}). Please make sure you are evaluating facts based on the same fields." ) if len(true_fact_doc_paths) > 1: raise ValidationError( f"Selected true fact ({true_fact}) is related to two or more fields {true_fact_doc_paths}, but the value for parameter 'field' isn't defined. Please define parameter 'field'." ) if len(pred_fact_doc_paths) > 1: raise ValidationError( f"Selected predicted fact ({pred_fact}) is related to two or more fields {pred_fact_doc_paths}, but the value for parameter 'field' isn't defined. Please define parameter 'field'." ) return True
def evaluate_entity_tags_task(object_id: int, indices: List[str], query: dict, es_timeout: int = 10, scroll_size: int = 100): try: logging.getLogger(INFO_LOGGER).info( f"Starting entity evaluator task for Evaluator with ID {object_id}." ) evaluator_object = Evaluator.objects.get(pk=object_id) progress = ShowProgress(evaluator_object.task, multiplier=1) true_fact = evaluator_object.true_fact pred_fact = evaluator_object.predicted_fact add_misclassified_examples = evaluator_object.add_misclassified_examples token_based = evaluator_object.token_based # If the user hasn't defined a field, retrieve it automatically if not evaluator_object.field: es_aggregator = ElasticAggregator(indices=indices, query=deepcopy(query)) true_fact_doc_paths = es_aggregator.facts_abstract( key_field="fact", value_field="doc_path", filter_by_key=true_fact) doc_path = true_fact_doc_paths[0] else: doc_path = evaluator_object.field searcher = ElasticSearcher(indices=indices, field_data=[doc_path, "texta_facts"], query=query, output=ElasticSearcher.OUT_RAW, timeout=f"{es_timeout}m", callback_progress=progress, scroll_size=scroll_size) # Get number of documents n_docs = searcher.count() evaluator_object.task.total = n_docs evaluator_object.task.save() evaluator_object.document_count = n_docs evaluator_object.scores_imprecise = False evaluator_object.score_after_scroll = False evaluator_object.add_individual_results = False # Save model updates evaluator_object.save() # Get number of batches for the logger n_batches = math.ceil(n_docs / scroll_size) scores, misclassified = scroll_and_score_entity( searcher, evaluator_object, true_fact, pred_fact, doc_path, token_based, n_batches, add_misclassified_examples) logging.getLogger(INFO_LOGGER).info(f"Final scores: {scores}") for conn in connections.all(): conn.close_if_unusable_or_obsolete() # Generate confusion matrix plot and save it image_name = f"{secrets.token_hex(15)}.png" classes = ["other", true_fact] evaluator_object.plot.save(image_name, create_confusion_plot( scores["confusion_matrix"], classes), save=False) image_path = pathlib.Path(MEDIA_URL) / image_name evaluator_object.plot.name = str(image_path) evaluator_object.save() evaluator_object.task.complete() return True except Exception as e: logging.getLogger(ERROR_LOGGER).exception(e) error_message = f"{str(e)[:100]}..." # Take first 100 characters in case the error message is massive. evaluator_object.task.add_error(error_message) evaluator_object.task.update_status(Task.STATUS_FAILED)