def sync_imanage_document(task: ExtendedTask, imanage_config_id: int, imanage_doc_id: str): task.log_info('Synchronizing iManage document #{0} or config #{1}'.format(imanage_doc_id, imanage_config_id)) imanage_doc = IManageDocument.objects \ .filter(imanage_config_id=imanage_config_id, imanage_doc_id=imanage_doc_id) \ .select_related('imanage_config').get() file_storage = get_file_storage() try: imanage_config = imanage_doc.imanage_config log = CeleryTaskLogger(task) project = imanage_config.resolve_dst_project(imanage_doc.imanage_doc_data, log) project_id = project.pk assignee = imanage_config.resolve_assignee(imanage_doc.imanage_doc_data, log) assignee_id = assignee.pk if assignee else None task.log_info('Assignee resolved to: {0}'.format(assignee.get_full_name() if assignee else '<no assignee>')) task.log_info('Downloading iManage document contents into a temp file...') auth_token = imanage_config.login() filename, response = imanage_config.load_document(auth_token, imanage_doc_id) upload_session_id = str(uuid.uuid4()) filename = get_valid_filename(filename) rel_filepath = os.path.join(upload_session_id, filename) _, ext = os.path.splitext(filename) if filename else None with buffer_contents_into_temp_file(response, ext) as temp_fn: # upload file to file storage with open(temp_fn, 'rb') as f: file_storage.mk_doc_dir(upload_session_id) file_storage.write_document(rel_filepath, f) kwargs = { 'document_type_id': imanage_config.document_type_id, 'project_id': project_id, 'assignee_id': assignee_id, 'user_id': get_main_admin_user().pk, 'propagate_exception': True, 'run_standard_locators': True, 'metadata': {}, 'do_not_check_exists': True } pre_defined_fields = None if imanage_doc.imanage_doc_data and imanage_config.imanage_to_contraxsuite_field_binding: pre_defined_fields = dict() for imanage_field_code, contraxsuite_field_code \ in dict(imanage_config.imanage_to_contraxsuite_field_binding).items(): imanage_field_value = imanage_doc.imanage_doc_data.get(imanage_field_code) if imanage_field_value: pre_defined_fields[contraxsuite_field_code] = imanage_field_value task.log_info('Assigning iManage field {0} to Contraxsuite field {1}: {2}' .format(imanage_field_code, contraxsuite_field_code, imanage_field_value)) else: task.log_info('iManage field {0} has no value assigned.' .format(imanage_field_code)) else: task.log_info('No binding of iManage fields to Contraxsuite fields.') document_id = LoadDocuments \ .create_document_local(task, temp_fn, rel_filepath, kwargs, return_doc_id=True, pre_defined_doc_fields_code_to_python_val=pre_defined_fields) if document_id: task.log_info('Created Contraxsuite document #{0}'.format(document_id)) imanage_doc.document_id = document_id imanage_doc.last_sync_date = timezone.now() imanage_doc.save(update_fields=['document_id', 'last_sync_date']) else: task.log_error('Unable to create Contraxsuite document for ' 'iManage document #{0}'.format(imanage_doc_id)) raise RuntimeError('No document loaded.') except Exception as ex: msg = render_error('Unable to synchronize iManage document #{0}'.format(imanage_doc_id), ex) task.log_error(msg) imanage_doc.import_problem = True imanage_doc.save(update_fields=['import_problem'])
def import_document_type(json_bytes: bytes, save: bool, auto_fix_validation_errors: bool, remove_missed_in_dump_objects: bool, task: ExtendedTask) -> DocumentType: tasks = Task.objects \ .get_active_user_tasks() \ .exclude(pk=task.task.pk) \ .distinct('name') \ .order_by('name') \ .values_list('name', flat=True) tasks = list(tasks) if tasks: msg = 'The following user tasks are running: {0}. This import can cause their crashing because of document' \ ' type / field structure changes.'.format(', '.join(tasks)) raise RuntimeError(msg) objects = serializers.deserialize("json", json_bytes.decode("utf-8")) document_type = None pk_to_field = {} field_detectors = [] other_objects = [] logger = CeleryTaskLogger(task) for deserialized_object in objects: obj = deserialized_object.object if isinstance(obj, DocumentType): if document_type is not None: raise RuntimeError('More than one document types was detected') document_type = DeserializedDocumentType( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, remove_missed_in_dump_objects=remove_missed_in_dump_objects, logger=logger) elif isinstance(obj, DocumentField): field = DeserializedDocumentField( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, remove_missed_in_dump_objects=remove_missed_in_dump_objects, logger=logger) pk_to_field[field.pk] = field elif isinstance(obj, DocumentFieldDetector): field_detector = DeserializedDocumentFieldDetector( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, logger=logger) field_detectors.append(field_detector) elif isinstance(obj, DocumentFieldCategory): category = DeserializedDocumentFieldCategory( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, logger=logger) other_objects.append(category) else: raise RuntimeError('Unknown model') if document_type is None: raise RuntimeError('Unable to find document type') conflicting_document_type = DocumentType.objects \ .filter(code=document_type.object.code) \ .exclude(pk=document_type.pk) \ .first() if conflicting_document_type is not None: err_msg = 'Unable to import document type #{0} "{1}". Database already contains a document type #{2}' \ ' with code "{3}"'.format(document_type.pk, document_type.object.code, conflicting_document_type.pk, conflicting_document_type.code) raise RuntimeError(err_msg) for field_detector in field_detectors: field = pk_to_field.get(field_detector.field_pk) if field is not None: field.add_dependent_object(field_detector) else: raise RuntimeError('Unknown field #{0}'.format( field_detector.field_pk)) for field in pk_to_field.values(): if field.document_type_pk == document_type.pk: document_type.add_dependent_object(field) else: raise RuntimeError('Unknown document type #{0}'.format( document_type.pk)) for obj in other_objects: document_type.add_dependent_object(obj) logger.info('Validation of {0} ...'.format(document_type.object.code)) validation_errors = document_type.validate() logger.info('Validation of {0} is finished'.format( document_type.object.code)) if validation_errors: task.log_error( '{0} VALIDATION ERRORS HAS OCCURRED DURING VALIDATION OF {1}.'. format(len(validation_errors), document_type.object.code)) for index, validation_error in enumerate(validation_errors): # for different timestamps sleep(0.001) task.log_error('VALIDATION ERROR {0}. {1}'.format( index + 1, str(validation_error))) raise ValidationError( 'Validation errors has occurred during import of {0}'.format( document_type.object.code)) if save: logger.info('Import of {0} ...'.format(document_type.object.code)) with transaction.atomic(): document_type.save() logger.info('Import of {0} is finished'.format( document_type.object.code)) return document_type.object
def import_document_type(json_bytes: bytes, save: bool, auto_fix_validation_errors: bool, remove_missed_in_dump_objects: bool, source_version: int, task: ExtendedTask) -> DocumentType: tasks = Task.objects \ .get_active_user_tasks() \ .exclude(pk=task.task.pk) \ .exclude(name__in=[task_names.TASK_NAME_REFRESH_MATERIALIZED_VIEW, task_names.TASK_NAME_CLEAN_ALL_TASKS, task_names.TASK_NAME_CHECK_EMAIL_POOL]) \ .distinct('name') \ .order_by('name') \ .values_list('name', flat=True) tasks = list(tasks) if tasks: msg = f'The following user tasks are running: {", ".join(tasks)}. ' + \ 'This import can cause their crashing because of document ' + \ 'type / field structure changes.' raise RuntimeError(msg) # check data contains version json_str = json_bytes.decode('utf-8') json_dict = json.loads(json_str) sm = SchemeMigration() if isinstance(json_dict, dict): # {"version":"75","data":[{"model": ... ]} version = json_dict.get('version') records = sm.migrate_model_records(json_dict['data'], int(version), CURRENT_VERSION) json_str = json.dumps(records) elif source_version != CURRENT_VERSION: json_str = sm.migrate_json(json_str, source_version, CURRENT_VERSION) for doc_type_subclass in DESERIALIZED_OBJECT_CLASSES: doc_type_subclass.init_static() objects = serializers.deserialize("json", json_str) document_type = None pk_to_field = {} field_detectors = [] other_objects = [] logger = CeleryTaskLogger(task) for deserialized_object in objects: obj = deserialized_object.object if isinstance(obj, DocumentType): if document_type is not None: raise RuntimeError('More than one document types was detected') document_type = DeserializedDocumentType( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, remove_missed_in_dump_objects=remove_missed_in_dump_objects, logger=logger) elif isinstance(obj, DocumentField): field = DeserializedDocumentField( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, remove_missed_in_dump_objects=remove_missed_in_dump_objects, logger=logger) pk_to_field[field.pk] = field elif isinstance(obj, DocumentFieldDetector): field_detector = DeserializedDocumentFieldDetector( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, logger=logger) field_detectors.append(field_detector) elif isinstance(obj, DocumentFieldCategory): category = DeserializedDocumentFieldCategory( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, logger=logger) other_objects.append(category) elif isinstance(obj, DocumentFieldFamily): family = DeserializedDocumentFieldFamily( deserialized_object, auto_fix_validation_errors=auto_fix_validation_errors, logger=logger) other_objects.append(family) else: raise RuntimeError('Unknown model') if document_type is None: raise RuntimeError('Unable to find document type') conflicting_document_type = DocumentType.objects \ .filter(code=document_type.object.code) \ .exclude(pk=document_type.pk) \ .first() if conflicting_document_type is not None: err_msg = f'Unable to import document type #{document_type.pk} "{document_type.object.code}". ' +\ f'Database already contains a document type #{conflicting_document_type.pk} ' + \ f'with code "{conflicting_document_type.code}"' raise RuntimeError(err_msg) for field_detector in field_detectors: field = pk_to_field.get(field_detector.field_pk) if field is not None: field.add_dependent_object(field_detector) else: raise RuntimeError(f'Unknown field #{field_detector.field_pk}') for field in pk_to_field.values(): if field.document_type_pk == document_type.pk: document_type.add_dependent_object(field) else: raise RuntimeError(f'Unknown document type #{document_type.pk}') for obj in other_objects: document_type.add_dependent_object(obj) logger.info(f'Validation of {document_type.object.code} ...') validation_errors = document_type.validate() logger.info(f'Validation of {document_type.object.code} is finished') if validation_errors: task.log_error( f'{len(validation_errors)} VALIDATION ERRORS HAS OCCURRED DURING VALIDATION OF {document_type.object.code}.' ) for index, validation_error in enumerate(validation_errors): # for different timestamps sleep(0.001) task.log_error(f'VALIDATION ERROR {index + 1}. {validation_error}') raise ValidationError( f'Validation errors has occurred during import of {document_type.object.code}' ) if save: logger.info(f'Import of {document_type.object.code} ...') with transaction.atomic(): document_type.save() logger.info(f'Import of {document_type.object.code} is finished') return document_type.object