def save(self, log: ProcessLogger, user_id):
try:
with transaction.atomic():
if self.processed_text_unit_ids:
if not self.document_initial_load:
TextUnitTag.objects.filter(text_unit_id__in=self.processed_text_unit_ids).delete()
for entity_class in self.processed_usage_entity_classes:
entity_class.objects.filter(text_unit_id__in=self.processed_text_unit_ids).delete()
tag_models = list()
from apps.document.app_vars import LOCATE_TEXTUNITTAGS
tags_saved = 0
if LOCATE_TEXTUNITTAGS.val:
for text_unit_id, tags in self.tags.items():
for tag in tags:
tag_models.append(TextUnitTag(user_id=user_id,
text_unit_id=text_unit_id,
tag=tag))
tags_saved = SafeBulkCreate.bulk_create(TextUnitTag.objects.bulk_create, tag_models)
# save "_usage" objects
count = 0
for entity_class, entities in self.located_usage_entities.items(): # type: Type[Usage], List[Usage]
if not entities:
continue
count += SafeBulkCreate.bulk_create(entity_class.objects, entities)
log.info(
'Stored {0} usage entities and {1} tags for {2} text units'.format(
count, tags_saved, len(self.processed_text_unit_ids)))
except Exception as e:
entities_str = '\n'.join([str(e) for e in self.processed_usage_entity_classes])
log.error(f'Unable to store location results.\n'
f'Text unit ids: {self.processed_text_unit_ids}\n'
f'Usage models caused the problem:\n{entities_str}', exc_info=e)
def save(self, log: ProcessLogger, user_id):
try:
with transaction.atomic():
if self.processed_text_unit_ids:
TextUnitTag.objects.filter(text_unit_id__in=self.processed_text_unit_ids).delete()
for entity_class in self.processed_usage_entity_classes:
entity_class.objects.filter(text_unit_id__in=self.processed_text_unit_ids).delete()
count = 0
for entity_class, entities in self.located_usage_entities.items(): # type: Type[Usage], List[Usage]
if entities:
entity_class.objects.bulk_create(entities, ignore_conflicts=True)
count += len(entities)
tag_models = list()
from apps.document.app_vars import LOCATE_TEXTUNITTAGS
if LOCATE_TEXTUNITTAGS.val:
for text_unit_id, tags in self.tags.items():
for tag in tags:
tag_models.append(TextUnitTag(user_id=user_id,
text_unit_id=text_unit_id,
tag=tag))
TextUnitTag.objects.bulk_create(tag_models, ignore_conflicts=True)
log.info(
'Stored {0} usage entities and {1} tags for {2} text units'.format(
count, len(tag_models), len(self.processed_text_unit_ids)))
except Exception as e:
entities_str = '\n'.join([str(e) for e in self.processed_usage_entity_classes])
log.error(f'Unable to store location results.\n'
f'Text unit ids: {self.processed_text_unit_ids}\n'
f'Usage models caused the problem:\n{entities_str}', exc_info=e)
self.save_summary(log, user_id)
def save(self, log: ProcessLogger, user_id):
try:
with transaction.atomic():
if self.processed_text_unit_ids:
TextUnitTag.objects.filter(
text_unit_id__in=self.processed_text_unit_ids).delete(
)
for entity_class in self.processed_usage_entity_classes:
entity_class.objects.filter(
text_unit_id__in=self.processed_text_unit_ids
).delete()
count = 0
for entity_class, entities in self.located_usage_entities.items(
): # type: Type[Usage], List[Usage]
if entities:
entity_class.objects.bulk_create(entities,
ignore_conflicts=True)
count += len(entities)
tag_models = list()
for text_unit_id, tags in self.tags.items():
for tag in tags:
tag_models.append(
TextUnitTag(user_id=user_id,
text_unit_id=text_unit_id,
tag=tag))
TextUnitTag.objects.bulk_create(tag_models,
ignore_conflicts=True)
log.info(
'Stored {0} usage entities and {1} tags for {2} text units'
.format(count, len(tag_models),
len(self.processed_text_unit_ids)))
except:
msg = render_error(
'Unable to store location results.\n'
'Text unit ids: {text_unit_ids}\n'
'Usage models caused the problem:\n{entities}'.format(
text_unit_ids=self.processed_text_unit_ids,
entities='\n'.join([
str(e) for e in self.processed_usage_entity_classes
])))
log.error(msg)
def build_text_unit_tag_record(
values: Dict[str, Any]) -> Optional[TextUnitTag]:
unit_id = self.text_unit_ids[values['text_unit_id']]
if TextUnitTag.objects.filter(text_unit_id=unit_id).count():
return None
record = TextUnitTag()
record.text_unit_id = unit_id
record.tag = values['tag']
if not pd.isnull(values['timestamp']):
record.timestamp = values['timestamp']
if not pd.isnull(values['user_id']):
record.user = self.target_user
return record