def check_document_field_values(task: ExtendedTask, dfv_ids: Set, delete_broken: bool = False):
for dfv in DocumentFieldValue.objects \
.filter(pk__in=dfv_ids) \
.select_related('field'): # type: DocumentFieldValue
try:
temp_value = dfv.python_value
if temp_value is not None:
field = dfv.field
if field.is_choice_field() and not field.is_choice_value(temp_value):
raise ValueError('Field value {0} is not in list of its choice values:\n{1}'
.format(temp_value, field.choices))
except:
if delete_broken:
dfv.delete()
msg = render_error('Found broken document field value.\n'
'Document field value id: {0}\n'
'DB value: {1}\n'
'The document field value has been deleted.\n'
.format(dfv.pk, dfv.value))
else:
content_type = ContentType.objects.get_for_model(DocumentFieldValue)
dfv_admin_url = reverse("admin:%s_%s_change" %
(content_type.app_label, content_type.model), args=(dfv.pk,))
msg = render_error('Found broken document field value.\n'
'Document field value id: {0}\n'
'DB value: {1}\n'
'Admin URL: {2}\n'.format(dfv.pk, dfv.value, dfv_admin_url))
task.log_info(msg)
def process(self, **kwargs):
if Task.objects \
.exclude(id=self.request.id) \
.filter(name=IManageSynchronization.name, status__in=UNREADY_STATES) \
.exists():
self.log_info('Previous IManage Synchronization task is still running. Exiting.')
return
imanage_config_dict = kwargs.get('imanage_config')
auto = kwargs.get('auto')
if auto:
qr = IManageConfig.objects.raw('''select * from "{table_name}" where enabled = True
and (last_sync_start is null
or (last_sync_start + (sync_frequency_minutes::text||\' minute\')::INTERVAL) <= now())'''
.format(table_name=IManageConfig._meta.db_table))
else:
if imanage_config_dict:
qr = IManageConfig.objects.filter(pk=imanage_config_dict['pk'], enabled=True)
else:
qr = IManageConfig.objects.filter(enabled=True)
found = False
for imanage_config in list(qr):
try:
found = True
self.sync_imanage_config(imanage_config)
except:
self.log_error(render_error('Unable to synchronize iManage config "{0}"'.format(imanage_config.code)))
return
if not found:
self.log_info('No enabled iManage configs matching the specified criteria found.')
def _build_insert_clause(log: ProcessLogger, table_name: str,
handlers: List[field_handlers.FieldHandler],
document: Document,
fields_to_python_values: Dict[str, Any]) -> SQLClause:
insert_clauses = list()
for handler in handlers: # type: field_handlers.FieldHandler
python_value = fields_to_python_values.get(handler.field_code)
try:
insert_clause = handler.get_pg_sql_insert_clause(
document.language, python_value) # type: SQLInsertClause
insert_clauses.append(insert_clause)
except Exception as ex:
msg = render_error('Unable to cache field values.\n'
'Document: {0} (#{1}).\n'
'Field: {2}'.format(document.name, document.id,
handler.field_code),
caused_by=ex)
log.error(msg)
columns_clause, values_clause = SQLInsertClause.join(insert_clauses)
insert_clause = format_clause(
'insert into "{table_name}" ({columns}) '
'values ({values}) on conflict ({column_document_id}) '
'do update set ({columns}) = ({values})',
table_name=table_name,
columns=columns_clause,
values=values_clause,
column_document_id=FIELD_CODE_DOC_ID)
return insert_clause
def send_email(log: ProcessLogger, dst_user, subject: str, txt: str, html: str, image_dir: str, cc: Set[str] = None):
if not dst_user.email:
log.error('Destination user {0} has no email assigned'.format(dst_user.get_full_name()))
return
try:
email = EmailMultiAlternatives(subject=subject,
body=txt,
cc=list(cc) if cc else None,
from_email=settings.DEFAULT_FROM_EMAIL,
to=['"{0}" <{1}>'.format(dst_user.get_full_name(), dst_user.email)])
if html:
images = [m.group(3) for m in RE_SRC_ATTACHMENT.finditer(html)]
email_html = RE_SRC_ATTACHMENT.sub(r'\1cid:\3\4', html)
email.attach_alternative(email_html, 'text/html')
for image_fn in images:
data = get_notification_template_resource(os.path.join(image_dir, image_fn))
img = MIMEImage(data)
img.add_header('Content-Id', '<' + image_fn + '>')
img.add_header("Content-Disposition", "inline", filename=image_fn)
email.attach(img)
email.send(fail_silently=False)
except Exception as caused_by:
msg = render_error('Unable to send email to user "{0}" (#{1})'.format(dst_user.get_full_name(), dst_user.pk),
caused_by=caused_by)
log.error(msg)
def __init__(self, message: str = None, caused_by: Exception = None, http_status_code: int = 400) -> None:
super().__init__(message)
self.http_status_code = http_status_code
self.message = message
if caused_by:
self.details = render_error(message, caused_by=caused_by)
else:
self.details = None
def _register_app_dump_models(plugin_attr_name: str, dst_collection: Dict[Type[Model], Callable]):
app_dump_models = collect_plugins_in_apps(STR_APP_DUMP_MODELS, plugin_attr_name)
for app_name, models in app_dump_models.items():
try:
models = dict(models)
dst_collection.update(models)
except Exception as e:
logging.error(render_error('Unable to register app dump models from app {0}.\n'
'Check {0}.app_dump_models.{1}'
.format(app_name, plugin_attr_name), caused_by=e))
def save(self, log: ProcessLogger, user_id):
try:
with transaction.atomic():
if self.processed_text_unit_ids:
TextUnitTag.objects.filter(
text_unit_id__in=self.processed_text_unit_ids).delete(
)
for entity_class in self.processed_usage_entity_classes:
entity_class.objects.filter(
text_unit_id__in=self.processed_text_unit_ids
).delete()
count = 0
for entity_class, entities in self.located_usage_entities.items(
): # type: Type[Usage], List[Usage]
if entities:
entity_class.objects.bulk_create(entities,
ignore_conflicts=True)
count += len(entities)
tag_models = list()
from apps.document.app_vars import LOCATE_TEXTUNITTAGS
if LOCATE_TEXTUNITTAGS.val:
for text_unit_id, tags in self.tags.items():
for tag in tags:
tag_models.append(
TextUnitTag(user_id=user_id,
text_unit_id=text_unit_id,
tag=tag))
TextUnitTag.objects.bulk_create(tag_models,
ignore_conflicts=True)
log.info(
'Stored {0} usage entities and {1} tags for {2} text units'
.format(count, len(tag_models),
len(self.processed_text_unit_ids)))
except:
msg = render_error(
'Unable to store location results.\n'
'Text unit ids: {text_unit_ids}\n'
'Usage models caused the problem:\n{entities}'.format(
text_unit_ids=self.processed_text_unit_ids,
entities='\n'.join([
str(e) for e in self.processed_usage_entity_classes
])))
log.error(msg)
def try_parsing(self, log: ProcessLogger, locate_results: LocationResults,
text: str, text_unit_id: int, text_unit_lang: str,
**kwargs):
try:
parse_results = self.parse(text, text_unit_id, text_unit_lang,
**kwargs) # type: ParseResults
locate_results.collect(self, text_unit_id, parse_results)
except:
msg = render_error(
'Exception caught while trying to run locator on a text unit.\n'
'Locator: {locator}\n'
'Text unit id: {text_unit_id}\n'
'Text: {text}\n'
'Text unit language: {text_unit_lang}\n'.format(
locator=self.__class__.__name__,
text_unit_id=text_unit_id,
text=text[:1024],
text_unit_lang=text_unit_lang))
log.error(msg)
def detect_and_cache_field_values_for_document(log: ProcessLogger,
document: Document,
save: bool = True,
clear_old_values: bool = True,
changed_by_user: User = None,
system_fields_changed: bool = False,
generic_fields_changed: bool = False,
document_initial_load: bool = False,
ignore_field_codes: Set[str] = None,
updated_field_codes: List[str] = None):
"""
Detects field values for a document and stores their DocumentFieldValue objects as well as Document.field_value.
These two should always be consistent.
:param log:
:param document:
:param save:
:param clear_old_values:
:param changed_by_user
:param system_fields_changed
:param generic_fields_changed
:param document_initial_load
:param updated_field_codes - if set, we search for changed and dependent fields only
:return:
"""
save_cache = save
save_detected = save
if save and document.status and not document.status.is_active:
log.info('Forbidden storing detected field values for document with "completed"'
' status, document #{} ({})'.format(document.id, document.name))
save_detected = False
document_type = document.document_type # type: DocumentType
all_fields = document_type.fields \
.all() \
.prefetch_related(Prefetch('depends_on_fields', queryset=DocumentField.objects.only('uid').all()))
all_fields = list(all_fields)
fields_and_deps = [(f.code, f.get_depends_on_codes() or set()) for f in all_fields]
required_fields = get_dependent_fields(fields_and_deps, set(updated_field_codes)) \
if updated_field_codes else None
sorted_codes = order_field_detection(fields_and_deps)
all_fields_code_to_field = {f.code: f for f in all_fields} # type: Dict[str, DocumentField]
res = list()
for field_code in sorted_codes:
if ignore_field_codes and field_code in ignore_field_codes:
continue
if required_fields and field_code not in required_fields:
continue
field = all_fields_code_to_field[field_code] # type: DocumentField
field_detection_strategy = FIELD_DETECTION_STRATEGY_REGISTRY[
field.value_detection_strategy] # type: FieldDetectionStrategy
try:
field_vals = field_value_cache.cache_field_values(document, None, save=False)
detected_values = field_detection_strategy.detect_field_values(log,
document,
field,
field_vals) # type: List[DetectedFieldValue]
except Exception as e:
msg = '''Unable to detect field value.
Document type: {0}
Document: {1}
Field: {2}'''.format(document_type.code, document.pk, field.code)
log.error(render_error(msg, e))
raise e
if save_detected and clear_old_values:
# Delete previously detected values
# to avoid accumulating garbage on each iteration.
DocumentFieldValue.objects \
.filter(document=document,
field=field,
removed_by_user=False,
created_by__isnull=True,
modified_by__isnull=True) \
.exclude(field__value_detection_strategy=DocumentField.VD_DISABLED) \
.delete()
if detected_values:
res.extend(detected_values)
if save_detected:
save_detected_values(document, field, detected_values)
if save_cache:
field_value_cache.cache_field_values(document, suggested_field_values=res,
save=True, log=log,
changed_by_user=changed_by_user,
system_fields_changed=system_fields_changed,
generic_fields_changed=generic_fields_changed,
document_initial_load=document_initial_load)
return res
