def _save_annotation(annotator_data: Dict, user, field_value_id=None) -> Dict: """ Add a new annotation / document field value. Accepts JSON structure generated by annotator.js. """ doc = Document.objects.get(pk=annotator_data['document_id']) document_field = DocumentField.objects.get( pk=annotator_data.get('field_id')) value = annotator_data.get('value') selection_range = annotator_data['ranges'][0] location_start = selection_range['startOffset'] location_end = selection_range['endOffset'] location_text = doc.full_text[location_start:location_end] field_type = FIELD_TYPES_REGISTRY.get(document_field.type) if field_value_id: field_value = DocumentFieldValue.objects.get(pk=field_value_id) field_value = field_type.update(field_value, doc, document_field, location_start, location_end, location_text, value, user) else: field_value = field_type.save_value(doc, document_field, location_start, location_end, location_text, value, user, True) _trigger_retraining_model(doc) return _to_dto(field_value)
def get(self, request, subscription_id, content_format, **_kwargs): send_email = as_bool(request.GET, self.PARAM_SEND, False) subscription = DocumentNotificationSubscription.objects.get(pk=subscription_id) document_type = subscription.document_type document_id = as_int(request.GET, self.PARAM_DOCUMENT, None) if document_id: document = Document.objects.filter(document_type=document_type, pk=document_id).first() if not document: return HttpResponseBadRequest('Document with id = {0} not found or has wrong type.'.format(document_id)) else: document = Document.objects.filter(document_type=document_type).first() if not document: return HttpResponseBadRequest('Document id not provided and ' 'there are no example documents of type {0}.'.format(document_type.code)) document_id = document.pk field_handlers = build_field_handlers(document_type, include_suggested_fields=False) field_values = get_document_field_values(document_type, document_id, handlers=field_handlers) example_changes = dict() if subscription.event in {DocumentAssignedEvent.code, DocumentChangedEvent.code} and field_values: for h in field_handlers: if random.random() > 0.3: continue field_type = FIELD_TYPES_REGISTRY.get(h.field_type) # type: FieldType field = DocumentField.objects.filter(code=h.field_code).first() if not field: continue example_value = field_type.example_python_value(field=field) example_changes[h.field_code] = (example_value, field_values.get(h.field_code)) notification = render_notification(already_sent_user_ids=set(), subscription=subscription, document=document, field_handlers=field_handlers, field_values=field_values, changes=example_changes, changed_by_user=request.user ) if not notification: return HttpResponse('Notification contains no data.', status=200) if content_format == self.FORMAT_HTML: content = notification.html content_type = 'text/html' else: content = notification.txt content_type = 'text/plain' if send_email: log = ErrorCollectingLogger() notification.send(log=log) error = log.get_error() if error: return HttpResponseServerError(content=error, content_type='application/json') return HttpResponse(content=content, content_type=content_type, status=200)
def _save_annotation(annotator_data: Dict, user, field_value_id=None) -> Dict: """ Add a new annotation / document field value. Accepts JSON structure generated by annotator.js. """ doc = Document.objects.get(pk=annotator_data['document_id']) document_field = DocumentField.objects.get( pk=annotator_data.get('field_id')) value = annotator_data.get('value') selection_range = annotator_data['ranges'][0] location_start = selection_range['startOffset'] location_end = selection_range['endOffset'] location_text = doc.full_text[location_start:location_end] field_type = FIELD_TYPES_REGISTRY.get(document_field.type) sentence_text_unit = TextUnit.objects.filter( document=doc, unit_type='sentence', location_start__lte=location_end, location_end__gte=location_start).first() field_value = field_type.save_value(doc, document_field, location_start, location_end, location_text, sentence_text_unit, value, user, True) _trigger_retraining_model(doc, document_field) return _to_dto(field_value)
def load_field_values(task: ExtendedTask, document: Document, document_fields: dict, field_owners: dict) -> Dict: document_type = document.document_type fields_to_values = {} if not document_type: return fields_to_values field_code_aliases = document_type.field_code_aliases field_codes_to_fields = { f.code.lower(): f for f in document_type.fields.all() } if field_code_aliases: field_codes_to_fields.update({ field_alias.lower(): field_codes_to_fields.get(field_code.lower()) for field_alias, field_code in field_code_aliases.items() if field_alias and field_code }) def _maybe_add_val(f, v, owner): if v is None: return v = DetectedFieldValue(f, v, user=owner) prev = fields_to_values.get(f) if not prev: fields_to_values[f] = [v] else: prev.append(v) for field_alias, field_value_text in document_fields.items(): if field_value_text is None: continue field = field_codes_to_fields.get(field_alias.lower()) if not field: task.log_warn( 'Field alias "{0}" not found for document type {1}'.format( field_alias, document_type.code)) continue field_type_adapter = FIELD_TYPES_REGISTRY.get( field.type) # type: FieldType field_owner = field_owners.get(field_alias) if type(field_value_text) is list: for possible_value_text in list(field_value_text): maybe_value = field_type_adapter.extract_from_possible_value_text( field, possible_value_text) _maybe_add_val(field, maybe_value, field_owner) else: maybe_value = field_type_adapter.extract_from_possible_value_text( field, field_value_text) _maybe_add_val(field, maybe_value, field_owner) return fields_to_values
def detect_field_values(cls, log: ProcessLogger, doc: Document, field: DocumentField) -> List[DetectedFieldValue]: depends_on_fields = list(field.depends_on_fields.all()) qs_document_field_values = doc.documentfieldvalue_set \ .filter(removed_by_user=False) \ .filter(field__in=depends_on_fields) field_code_to_value = merge_document_field_values_to_python_value( list(qs_document_field_values)) field_code_to_value = { f.code: field_code_to_value.get(f.code) for f in depends_on_fields } document_type = doc.document_type field_detectors = DocumentFieldDetector.objects.filter( document_type=document_type, field=field) field_type_adapter = FIELD_TYPES_REGISTRY.get( field.type) # type: FieldType detected_values = list() # type: List[DetectedFieldValue] for depends_on_value in field_code_to_value.values(): if not depends_on_value: continue depends_on_value = str(depends_on_value) for field_detector in field_detectors: if field_detector.matches(depends_on_value): value = field_detector.detected_value hint_name = None if field_type_adapter.value_aware: hint_name = field_detector.extraction_hint or ValueExtractionHint.TAKE_FIRST.name value, hint_name = field_type_adapter \ .get_or_extract_value(doc, field, value, hint_name, depends_on_value) if value is None: continue detected_values.append( DetectedFieldValue(field, value, None, hint_name)) if not (field_type_adapter.multi_value or field.is_choice_field()): break if detected_values and not (field_type_adapter.multi_value or field.is_choice_field()): break return detected_values
def detect_field_values(cls, log: ProcessLogger, doc: Document, field: DocumentField) -> List[DetectedFieldValue]: depends_on_full_text = doc.full_text detected_with_stop_words, detected_values = detect_with_stop_words_by_field_and_full_text( field, depends_on_full_text) if detected_with_stop_words: return detected_values or list() qs_text_units = TextUnit.objects \ .filter(document=doc) \ .filter(unit_type=field.text_unit_type) \ .order_by('location_start', 'pk') document_type = doc.document_type field_detectors = DocumentFieldDetector.objects.filter( document_type=document_type, field=field) field_type_adapter = FIELD_TYPES_REGISTRY.get( field.type) # type: FieldType detected_values = list() # type: List[DetectedFieldValue] for text_unit in qs_text_units.iterator(): for field_detector in field_detectors: if field_detector.matches(text_unit.text): value = field_detector.detected_value hint_name = None if field_type_adapter.value_aware: hint_name = field_detector.extraction_hint or ValueExtractionHint.TAKE_FIRST.name value, hint_name = field_type_adapter \ .get_or_extract_value(doc, field, value, hint_name, text_unit.text) if value is None: continue detected_values.append( DetectedFieldValue(field, value, text_unit, hint_name)) if not (field_type_adapter.multi_value or field.is_choice_field()): break if detected_values and not (field_type_adapter.multi_value or field.is_choice_field()): break return detected_values
def delete(self, request, *args, pk): """ Delete an annotation. """ field_value = DocumentFieldValue.objects.get(pk=pk) doc = field_value.document field = field_value.field field_type = FIELD_TYPES_REGISTRY.get(field.type) field_type.delete(field_value) _trigger_retraining_model(doc, field, request.user.id) return JsonResponse(_to_dto(field_value))
def detect_with_stop_words_by_field_and_full_text(field: DocumentField, full_text: str) -> Tuple[bool, Optional[List]]: if field.requires_text_annotations: return False, None stop_words = compile_stop_words(field.stop_words) if not stop_words: return False, None field_type_adapter = FIELD_TYPES_REGISTRY.get(field.type) # type: FieldType detected, possible_value = detect_value_with_stop_words(stop_words, full_text) if not detected: return False, None if possible_value is None: return True, None else: possible_value = field_type_adapter.extract_from_possible_value_text(field, possible_value) return True, [DetectedFieldValue(field, possible_value)]
def post(self, request, *args, **kwargs): """ Suggest field value before creating an annotation. Accepts the same JSON structure as annotation saving endpoint. Returns suggested field value of the specified field possibly found in the provided text. """ annotator_data = request.data doc = Document.objects.get(pk=annotator_data['document_id']) document_field = DocumentField.objects.get(pk=annotator_data.get('field_id')) location_text = annotator_data['quote'] field_type = FIELD_TYPES_REGISTRY.get(document_field.type) field_value = field_type.suggest_value(doc, document_field, location_text) return Response({'suggested_value': field_value})
def delete(self, request, *args, pk): """ Delete an annotation. """ field_value = DocumentFieldValue.objects.get(pk=pk) doc = field_value.document field = field_value.field field_type = FIELD_TYPES_REGISTRY.get(field.type) with transaction.atomic(): if field_value.created_by_id is not None: field_type.delete(field_value) else: field_type.mark_removed_by_user(field_value) DocumentTypeField.objects.set_dirty_for_value(field_value) _trigger_retraining_model(doc, field, request.user.id) return Response(_to_dto(field_value))
def detect_field_values_with_regexps(document: Document, field: DocumentField, sentence_text_units: List[TextUnit], do_not_write: bool) -> int: document_type = document.document_type field_detectors = DocumentFieldDetector.objects.filter( document_type=document_type, field=field) field_type_adapter = FIELD_TYPES_REGISTRY.get( field.type) # type: FieldType detected_values = list() # type: List[DetectedFieldValue] for text_unit in sentence_text_units: for field_detector in field_detectors: if field_detector.matches(text_unit.text): value = field_detector.detected_value hint_name = None if field_type_adapter.value_aware: hint_name = field_detector.extraction_hint or ValueExtractionHint.TAKE_FIRST.name value, hint_name = field_type_adapter \ .get_or_extract_value(document, field, value, hint_name, text_unit.text) if value is None: continue detected_values.append( DetectedFieldValue(text_unit, value, hint_name)) if not (field_type_adapter.multi_value or field.is_choice_field()): break return DetectFieldValues.save_detected_values(document, field, field_type_adapter, detected_values, do_not_write)
def detect_initial_field_values_for_document_field(document, field, sentence_text_units, do_not_write, main_task) -> int: document_type = document.document_type field_detectors = DocumentFieldDetector.objects.filter( document_type=document_type, field=field) field_type_adapter = FIELD_TYPES_REGISTRY.get(field.type) detected_values = list() for text_unit in sentence_text_units: for field_detector in field_detectors: if field_detector.matches(text_unit.text): value = field_detector.detected_value hint = None if field_type_adapter.value_aware: hint = field_detector.extraction_hint or ValueExtractionHint.TAKE_FIRST.name value, hint = field_type_adapter \ .get_or_extract_value(document, field, value, hint, text_unit.text) if value is None: continue detected_values.append((text_unit, value, hint)) if not (field_type_adapter.multi_value or field.is_choice_field()): break return DetectFieldValues.save_detected_values(document, field, field_type_adapter, detected_values, do_not_write)
def destroy(self, request, *args, **kwargs): instance = self.get_object() adapter = FIELD_TYPES_REGISTRY.get(instance.field.type) adapter.delete(instance) return Response(status=status.HTTP_204_NO_CONTENT)
def save(self, **kwargs): field = self.validated_data['field'] self.adapter = FIELD_TYPES_REGISTRY.get(field.type) return super().save(**kwargs)
def detect_field_values(cls, log: ProcessLogger, doc: Document, field: DocumentField) -> List[DetectedFieldValue]: depends_on_fields = list(field.depends_on_fields.all()) qs_document_field_values = doc.documentfieldvalue_set \ .filter(removed_by_user=False) \ .filter(field__in=depends_on_fields) field_code_to_value = merge_document_field_values_to_python_value( list(qs_document_field_values)) field_code_to_value = { f.code: field_code_to_value.get(f.code) for f in depends_on_fields } if field.stop_words: depends_on_full_text = '\n'.join( [str(v) for v in field_code_to_value.values()]) detected_with_stop_words, detected_values \ = detect_with_stop_words_by_field_and_full_text(field, depends_on_full_text) if detected_with_stop_words: return detected_values or list() field_detectors = DocumentFieldDetector.objects.filter(field=field) field_type_adapter = FIELD_TYPES_REGISTRY.get( field.type) # type: FieldType detected_values = list() # type: List[DetectedFieldValue] for depends_on_value in field_code_to_value.values(): if not depends_on_value: continue depends_on_value = str(depends_on_value) for field_detector in field_detectors: # type: DocumentFieldDetector matching_string = field_detector.matching_string( depends_on_value, text_is_sentence=False) if matching_string is not None: value = field_detector.get_validated_detected_value(field) hint_name = None if field_type_adapter.requires_value: hint_name = field_detector.extraction_hint or ValueExtractionHint.TAKE_FIRST.name value, hint_name = field_type_adapter \ .get_or_extract_value(doc, field, value, hint_name, matching_string) if value is None: continue detected_values.append( DetectedFieldValue(field, value, None, hint_name)) if not (field_type_adapter.multi_value or field.is_choice_field()): break if detected_values and not (field_type_adapter.multi_value or field.is_choice_field()): break return detected_values