def process(self, **kwargs): if Task.objects \ .exclude(id=self.request.id) \ .filter(name=IManageSynchronization.name, status__in=UNREADY_STATES) \ .exists(): self.log_info('Previous IManage Synchronization task is still running. Exiting.') return imanage_config_dict = kwargs.get('imanage_config') auto = kwargs.get('auto') if auto: qr = IManageConfig.objects.raw('''select * from "{table_name}" where enabled = True and (last_sync_start is null or (last_sync_start + (sync_frequency_minutes::text||\' minute\')::INTERVAL) <= now())''' .format(table_name=IManageConfig._meta.db_table)) else: if imanage_config_dict: qr = IManageConfig.objects.filter(pk=imanage_config_dict['pk']) else: qr = IManageConfig.objects.all() found = False for imanage_config in list(qr): try: found = True self.sync_imanage_config(imanage_config) except: self.log_error(render_error('Unable to synchronize iManage config "{0}"'.format(imanage_config.code))) return if not found: self.log_info('No enabled iManage configs matching the specified criteria found.')
def has_problems_with_field(cls, field: DocumentField) -> Optional[str]: if not field.mlflow_model_uri: return f'MLFlow model uri is not set for field {field.code}' try: output = cls.test_model(field.mlflow_model_uri) output = output[0] if output is not None else None tf = TypedField.by(field) if not tf.is_python_field_value_ok(output): return f'MLFlow model returned value which does not match the field type.\n' \ f'Returned value (shortened up to 100 chars): {str(output)[:100]}.\n ' \ f'Example value: {tf.example_python_value()}.' return None except Exception as e: from apps.common.errors import render_error render_error('MLFlow model has thrown exception when testing ' '(input = 1-row DataFrame with text = "Hello World")', e)
def get(self, request, config_id, content_format, **kwargs): dst_user_id_or_name = request.GET[self.PARAM_DST_USER] send_email = as_bool(request.GET, self.PARAM_SEND, False) emulate_no_docs = as_bool(request.GET, self.PARAM_EMULATE_NO_DOCS, False) config = DocumentDigestConfig.objects.get(pk=config_id) if dst_user_id_or_name: try: dst_user = User.objects.get(pk=int(dst_user_id_or_name)) except ValueError: dst_user = User.objects.get(username=str(dst_user_id_or_name)) else: dst_user = request.user run_date = datetime.now(tz=dst_user.timezone or get_localzone()) try: digest = render_digest(config=config, dst_user=dst_user, run_date=run_date, emulate_no_docs=emulate_no_docs) except Exception as e: return HttpResponse(render_error( 'Exception caught while trying to render digest', e), status=500, content_type='text/plain') if not digest: return HttpResponse('Notification contains no data.', status=200) if content_format == self.FORMAT_HTML: content = digest.html content_type = 'text/html' else: content = digest.txt content_type = 'text/plain' if send_email: log = ErrorCollectingLogger() digest.send(log) log.raise_if_error() return HttpResponse(content=content, content_type=content_type, status=200)
def sync_imanage_document(task: ExtendedTask, imanage_config_id: int, imanage_doc_id: str): task.log_info('Synchronizing iManage document #{0} or config #{1}'.format(imanage_doc_id, imanage_config_id)) imanage_doc = IManageDocument.objects \ .filter(imanage_config_id=imanage_config_id, imanage_doc_id=imanage_doc_id) \ .select_related('imanage_config').get() file_storage = get_file_storage() try: imanage_config = imanage_doc.imanage_config log = CeleryTaskLogger(task) project = imanage_config.resolve_dst_project(imanage_doc.imanage_doc_data, log) project_id = project.pk assignee = imanage_config.resolve_assignee(imanage_doc.imanage_doc_data, log) assignee_id = assignee.pk if assignee else None task.log_info('Assignee resolved to: {0}'.format(assignee.get_full_name() if assignee else '<no assignee>')) task.log_info('Downloading iManage document contents into a temp file...') auth_token = imanage_config.login() filename, response = imanage_config.load_document(auth_token, imanage_doc_id) upload_session_id = str(uuid.uuid4()) filename = get_valid_filename(filename) rel_filepath = os.path.join(upload_session_id, filename) _, ext = os.path.splitext(filename) if filename else None with buffer_contents_into_temp_file(response, ext) as temp_fn: # upload file to file storage with open(temp_fn, 'rb') as f: file_storage.mk_doc_dir(upload_session_id) file_storage.write_document(rel_filepath, f) kwargs = { 'document_type_id': imanage_config.document_type_id, 'project_id': project_id, 'assignee_id': assignee_id, 'user_id': get_main_admin_user().pk, 'propagate_exception': True, 'run_standard_locators': True, 'metadata': {}, 'do_not_check_exists': True } pre_defined_fields = None if imanage_doc.imanage_doc_data and imanage_config.imanage_to_contraxsuite_field_binding: pre_defined_fields = dict() for imanage_field_code, contraxsuite_field_code \ in dict(imanage_config.imanage_to_contraxsuite_field_binding).items(): imanage_field_value = imanage_doc.imanage_doc_data.get(imanage_field_code) if imanage_field_value: pre_defined_fields[contraxsuite_field_code] = imanage_field_value task.log_info('Assigning iManage field {0} to Contraxsuite field {1}: {2}' .format(imanage_field_code, contraxsuite_field_code, imanage_field_value)) else: task.log_info('iManage field {0} has no value assigned.' .format(imanage_field_code)) else: task.log_info('No binding of iManage fields to Contraxsuite fields.') document_id = LoadDocuments \ .create_document_local(task, temp_fn, rel_filepath, kwargs, return_doc_id=True, pre_defined_doc_fields_code_to_python_val=pre_defined_fields) if document_id: task.log_info('Created Contraxsuite document #{0}'.format(document_id)) imanage_doc.document_id = document_id imanage_doc.last_sync_date = timezone.now() imanage_doc.save(update_fields=['document_id', 'last_sync_date']) else: task.log_error('Unable to create Contraxsuite document for ' 'iManage document #{0}'.format(imanage_doc_id)) raise RuntimeError('No document loaded.') except Exception as ex: msg = render_error('Unable to synchronize iManage document #{0}'.format(imanage_doc_id), ex) task.log_error(msg) imanage_doc.import_problem = True imanage_doc.save(update_fields=['import_problem'])
def get(self, request, subscription_id, content_format, **_kwargs): send_email = as_bool(request.GET, self.PARAM_SEND, False) subscription = DocumentNotificationSubscription.objects.get( pk=subscription_id) document_type = subscription.document_type document_id = as_int(request.GET, self.PARAM_DOCUMENT, None) if document_id: document = Document.objects.filter(document_type=document_type, pk=document_id).first() if not document: return HttpResponseBadRequest( 'Document with id = {0} not found or has wrong type.'. format(document_id)) else: document = Document.objects.filter( document_type=document_type).first() if not document: return HttpResponseBadRequest( 'Document id not provided and ' 'there are no example documents of type {0}.'.format( document_type.code)) document_id = document.pk field_handlers = build_field_handlers(document_type, include_annotation_fields=False) field_values = get_document_field_values(document_type, document_id, handlers=field_handlers) example_changes = dict() if subscription.event in { DocumentAssignedEvent.code, DocumentChangedEvent.code } and field_values: for h in field_handlers: if random.random() > 0.3: continue field = DocumentField.objects.filter(code=h.field_code).first() if not field: continue typed_field = TypedField.by(field) example_value = typed_field.example_python_value() example_changes[h.field_code] = (example_value, field_values.get( h.field_code)) try: notification = NotificationRenderer.render_notification( uuid.uuid4().hex, subscription, DocumentNotificationSource(document=document, field_handlers=field_handlers, field_values=field_values, changes=example_changes, changed_by_user=request.user)) except Exception as e: return HttpResponse(render_error( 'Exception caught while trying to render notification', e), status=500, content_type='text/plain') if not notification: return HttpResponse('Notification contains no data.', status=200) if content_format == self.FORMAT_HTML: content = notification.html content_type = 'text/html' else: content = notification.txt content_type = 'text/plain' if send_email: log = ErrorCollectingLogger() notification.send(log=log) error = log.get_error() if error: return HttpResponseServerError(content=error, content_type='application/json') return HttpResponse(content=content, content_type=content_type, status=200)
def process(self, **kwargs): if self.PARAM_CONFIG in kwargs: config_id = kwargs[self.PARAM_CONFIG]['pk'] else: config_id = kwargs[self.PARAM_CONFIG_ID] if self.PARAM_USER in kwargs: user_ids = {kwargs[self.PARAM_USER]['pk']} else: user_ids = kwargs.get(self.PARAM_USER_IDS) run_date = kwargs.get(self.PARAM_RUN_DATE) run_date_specified = run_date is not None if isinstance(run_date, str): run_date = parse(run_date) run_date = run_date or datetime.datetime.now( tz=tzlocal.get_localzone()) run_even_if_not_enabled = bool( kwargs.get(self.PARAM_RUN_EVEN_IF_NOT_ENABLED)) config = DocumentDigestConfig.objects \ .filter(pk=config_id).select_related('for_role', 'for_user').first() # type: DocumentDigestConfig if not config: self.log_error('{1} not found: #{0}'.format( config_id, DocumentDigestConfig.__name__)) return if not config.enabled and not run_even_if_not_enabled: self.log_info('{1} #{0} is disabled.'.format( config_id, DocumentDigestConfig.__name__)) return tz_msg = ' at timezone {0}'.format( run_date.tzname()) if run_date_specified else '' self.log_info( 'Rendering and sending {what} #{pk} ({doc_filter}) for date "{run_date}" to {n} users{tz_msg}' .format(what=DocumentDigestConfig.__name__, pk=config.pk, doc_filter=config.documents_filter, n=len(user_ids), run_date=run_date, tz_msg=tz_msg)) if user_ids: users_qr = User.objects.filter(pk__in=user_ids) elif config.for_role_id is not None: users_qr = User.objects.filter(role_id=config.for_role_id) elif config.for_user_id is not None: users_qr = User.objects.get(pk=config.for_user_id) else: self.log_error( '{what} #{config_id} specifies neither for_user nor for_role.'. format(what=DocumentDigestConfig.__name__, config_id=config.pk)) return log = CeleryTaskLogger(self) for user in users_qr: # type: User if config.for_user_id != user.id and ( config.for_role_id is None or config.for_role_id != user.role_id): self.log_error( '{what} #{what_id} is not applicable for user {user_name} (#{user_id})' .format(what=DocumentDigestConfig.__name__, what_id=config.pk, user_name=user.get_full_name(), user_id=user.pk)) continue try: digest = render_digest(config=config, dst_user=user, run_date=run_date) if digest: digest.send(log) except: msg = render_error('Unable to send {what}.\n' 'Config: #{config_id}\n' 'Dst user: {user_name} #{user_id}\n' 'Run date: {run_date}'.format( what=RenderedDigest, config_id=config.pk, user_name=user.get_full_name(), user_id=user.pk, run_date=run_date)) self.log_error(msg)