def check_document_field_values(task: ExtendedTask, dfv_ids: Set, delete_broken: bool = False): for dfv in DocumentFieldValue.objects \ .filter(pk__in=dfv_ids) \ .select_related('field'): # type: DocumentFieldValue try: temp_value = dfv.python_value if temp_value is not None: field = dfv.field if field.is_choice_field() and not field.is_choice_value(temp_value): raise ValueError('Field value {0} is not in list of its choice values:\n{1}' .format(temp_value, field.choices)) except: if delete_broken: dfv.delete() msg = render_error('Found broken document field value.\n' 'Document field value id: {0}\n' 'DB value: {1}\n' 'The document field value has been deleted.\n' .format(dfv.pk, dfv.value)) else: content_type = ContentType.objects.get_for_model(DocumentFieldValue) dfv_admin_url = reverse("admin:%s_%s_change" % (content_type.app_label, content_type.model), args=(dfv.pk,)) msg = render_error('Found broken document field value.\n' 'Document field value id: {0}\n' 'DB value: {1}\n' 'Admin URL: {2}\n'.format(dfv.pk, dfv.value, dfv_admin_url)) task.log_info(msg)
def process(self, **kwargs): if Task.objects \ .exclude(id=self.request.id) \ .filter(name=IManageSynchronization.name, status__in=UNREADY_STATES) \ .exists(): self.log_info('Previous IManage Synchronization task is still running. Exiting.') return imanage_config_dict = kwargs.get('imanage_config') auto = kwargs.get('auto') if auto: qr = IManageConfig.objects.raw('''select * from "{table_name}" where enabled = True and (last_sync_start is null or (last_sync_start + (sync_frequency_minutes::text||\' minute\')::INTERVAL) <= now())''' .format(table_name=IManageConfig._meta.db_table)) else: if imanage_config_dict: qr = IManageConfig.objects.filter(pk=imanage_config_dict['pk'], enabled=True) else: qr = IManageConfig.objects.filter(enabled=True) found = False for imanage_config in list(qr): try: found = True self.sync_imanage_config(imanage_config) except: self.log_error(render_error('Unable to synchronize iManage config "{0}"'.format(imanage_config.code))) return if not found: self.log_info('No enabled iManage configs matching the specified criteria found.')
def _build_insert_clause(log: ProcessLogger, table_name: str, handlers: List[field_handlers.FieldHandler], document: Document, fields_to_python_values: Dict[str, Any]) -> SQLClause: insert_clauses = list() for handler in handlers: # type: field_handlers.FieldHandler python_value = fields_to_python_values.get(handler.field_code) try: insert_clause = handler.get_pg_sql_insert_clause( document.language, python_value) # type: SQLInsertClause insert_clauses.append(insert_clause) except Exception as ex: msg = render_error('Unable to cache field values.\n' 'Document: {0} (#{1}).\n' 'Field: {2}'.format(document.name, document.id, handler.field_code), caused_by=ex) log.error(msg) columns_clause, values_clause = SQLInsertClause.join(insert_clauses) insert_clause = format_clause( 'insert into "{table_name}" ({columns}) ' 'values ({values}) on conflict ({column_document_id}) ' 'do update set ({columns}) = ({values})', table_name=table_name, columns=columns_clause, values=values_clause, column_document_id=FIELD_CODE_DOC_ID) return insert_clause
def send_email(log: ProcessLogger, dst_user, subject: str, txt: str, html: str, image_dir: str, cc: Set[str] = None): if not dst_user.email: log.error('Destination user {0} has no email assigned'.format(dst_user.get_full_name())) return try: email = EmailMultiAlternatives(subject=subject, body=txt, cc=list(cc) if cc else None, from_email=settings.DEFAULT_FROM_EMAIL, to=['"{0}" <{1}>'.format(dst_user.get_full_name(), dst_user.email)]) if html: images = [m.group(3) for m in RE_SRC_ATTACHMENT.finditer(html)] email_html = RE_SRC_ATTACHMENT.sub(r'\1cid:\3\4', html) email.attach_alternative(email_html, 'text/html') for image_fn in images: data = get_notification_template_resource(os.path.join(image_dir, image_fn)) img = MIMEImage(data) img.add_header('Content-Id', '<' + image_fn + '>') img.add_header("Content-Disposition", "inline", filename=image_fn) email.attach(img) email.send(fail_silently=False) except Exception as caused_by: msg = render_error('Unable to send email to user "{0}" (#{1})'.format(dst_user.get_full_name(), dst_user.pk), caused_by=caused_by) log.error(msg)
def __init__(self, message: str = None, caused_by: Exception = None, http_status_code: int = 400) -> None: super().__init__(message) self.http_status_code = http_status_code self.message = message if caused_by: self.details = render_error(message, caused_by=caused_by) else: self.details = None
def _register_app_dump_models(plugin_attr_name: str, dst_collection: Dict[Type[Model], Callable]): app_dump_models = collect_plugins_in_apps(STR_APP_DUMP_MODELS, plugin_attr_name) for app_name, models in app_dump_models.items(): try: models = dict(models) dst_collection.update(models) except Exception as e: logging.error(render_error('Unable to register app dump models from app {0}.\n' 'Check {0}.app_dump_models.{1}' .format(app_name, plugin_attr_name), caused_by=e))
def save(self, log: ProcessLogger, user_id): try: with transaction.atomic(): if self.processed_text_unit_ids: TextUnitTag.objects.filter( text_unit_id__in=self.processed_text_unit_ids).delete( ) for entity_class in self.processed_usage_entity_classes: entity_class.objects.filter( text_unit_id__in=self.processed_text_unit_ids ).delete() count = 0 for entity_class, entities in self.located_usage_entities.items( ): # type: Type[Usage], List[Usage] if entities: entity_class.objects.bulk_create(entities, ignore_conflicts=True) count += len(entities) tag_models = list() from apps.document.app_vars import LOCATE_TEXTUNITTAGS if LOCATE_TEXTUNITTAGS.val: for text_unit_id, tags in self.tags.items(): for tag in tags: tag_models.append( TextUnitTag(user_id=user_id, text_unit_id=text_unit_id, tag=tag)) TextUnitTag.objects.bulk_create(tag_models, ignore_conflicts=True) log.info( 'Stored {0} usage entities and {1} tags for {2} text units' .format(count, len(tag_models), len(self.processed_text_unit_ids))) except: msg = render_error( 'Unable to store location results.\n' 'Text unit ids: {text_unit_ids}\n' 'Usage models caused the problem:\n{entities}'.format( text_unit_ids=self.processed_text_unit_ids, entities='\n'.join([ str(e) for e in self.processed_usage_entity_classes ]))) log.error(msg)
def try_parsing(self, log: ProcessLogger, locate_results: LocationResults, text: str, text_unit_id: int, text_unit_lang: str, **kwargs): try: parse_results = self.parse(text, text_unit_id, text_unit_lang, **kwargs) # type: ParseResults locate_results.collect(self, text_unit_id, parse_results) except: msg = render_error( 'Exception caught while trying to run locator on a text unit.\n' 'Locator: {locator}\n' 'Text unit id: {text_unit_id}\n' 'Text: {text}\n' 'Text unit language: {text_unit_lang}\n'.format( locator=self.__class__.__name__, text_unit_id=text_unit_id, text=text[:1024], text_unit_lang=text_unit_lang)) log.error(msg)
def detect_and_cache_field_values_for_document(log: ProcessLogger, document: Document, save: bool = True, clear_old_values: bool = True, changed_by_user: User = None, system_fields_changed: bool = False, generic_fields_changed: bool = False, document_initial_load: bool = False, ignore_field_codes: Set[str] = None, updated_field_codes: List[str] = None): """ Detects field values for a document and stores their DocumentFieldValue objects as well as Document.field_value. These two should always be consistent. :param log: :param document: :param save: :param clear_old_values: :param changed_by_user :param system_fields_changed :param generic_fields_changed :param document_initial_load :param updated_field_codes - if set, we search for changed and dependent fields only :return: """ save_cache = save save_detected = save if save and document.status and not document.status.is_active: log.info('Forbidden storing detected field values for document with "completed"' ' status, document #{} ({})'.format(document.id, document.name)) save_detected = False document_type = document.document_type # type: DocumentType all_fields = document_type.fields \ .all() \ .prefetch_related(Prefetch('depends_on_fields', queryset=DocumentField.objects.only('uid').all())) all_fields = list(all_fields) fields_and_deps = [(f.code, f.get_depends_on_codes() or set()) for f in all_fields] required_fields = get_dependent_fields(fields_and_deps, set(updated_field_codes)) \ if updated_field_codes else None sorted_codes = order_field_detection(fields_and_deps) all_fields_code_to_field = {f.code: f for f in all_fields} # type: Dict[str, DocumentField] res = list() for field_code in sorted_codes: if ignore_field_codes and field_code in ignore_field_codes: continue if required_fields and field_code not in required_fields: continue field = all_fields_code_to_field[field_code] # type: DocumentField field_detection_strategy = FIELD_DETECTION_STRATEGY_REGISTRY[ field.value_detection_strategy] # type: FieldDetectionStrategy try: field_vals = field_value_cache.cache_field_values(document, None, save=False) detected_values = field_detection_strategy.detect_field_values(log, document, field, field_vals) # type: List[DetectedFieldValue] except Exception as e: msg = '''Unable to detect field value. Document type: {0} Document: {1} Field: {2}'''.format(document_type.code, document.pk, field.code) log.error(render_error(msg, e)) raise e if save_detected and clear_old_values: # Delete previously detected values # to avoid accumulating garbage on each iteration. DocumentFieldValue.objects \ .filter(document=document, field=field, removed_by_user=False, created_by__isnull=True, modified_by__isnull=True) \ .exclude(field__value_detection_strategy=DocumentField.VD_DISABLED) \ .delete() if detected_values: res.extend(detected_values) if save_detected: save_detected_values(document, field, detected_values) if save_cache: field_value_cache.cache_field_values(document, suggested_field_values=res, save=True, log=log, changed_by_user=changed_by_user, system_fields_changed=system_fields_changed, generic_fields_changed=generic_fields_changed, document_initial_load=document_initial_load) return res