def process(self, document_type_config_csv_file: Dict, action: str, update_cache: bool, **kwargs): if action == 'validate': save = False auto_fix_validation_errors = False remove_missed_objects = False elif action == 'validate|import': save = True auto_fix_validation_errors = False remove_missed_objects = False elif action == 'import|auto_fix|retain_missing_objects': save = True auto_fix_validation_errors = True remove_missed_objects = False elif action == 'import|auto_fix|remove_missing_objects': save = True auto_fix_validation_errors = True remove_missed_objects = True else: raise RuntimeError('Unknown action') try: json_bytes = DbCache.get(document_type_config_csv_file['cache_key']) document_type = import_document_type(json_bytes=json_bytes, save=save, auto_fix_validation_errors=auto_fix_validation_errors, remove_missed_in_dump_objects=remove_missed_objects, task=self) finally: DbCache.clean_cache(document_type_config_csv_file['cache_key']) if not (save and update_cache): return from apps.rawdb.app_vars import APP_VAR_DISABLE_RAW_DB_CACHING if not APP_VAR_DISABLE_RAW_DB_CACHING.val: self.log_info('Adapting RawDB table structure after import ...') adapt_table_structure(CeleryTaskLogger(self), document_type, force=False) ids = Document.all_objects.filter(document_type=document_type).values_list('pk', flat=True) ids = list(ids) self.log_info('Caching document field values ...') for chunk in chunks(ids, 50): self.run_sub_tasks('Cache field values for a set of documents', ImportDocumentType.cache_document_fields_for_doc_ids, [(list(chunk),)])
def process(self, document_field: Dict, config_csv_file: Dict, drop_previous_field_detectors: bool, update_field_choice_values: bool, csv_contains_regexps: bool, **kwargs): try: self.log_info('Going to configure simple field detection config...') document_field = DocumentField.objects.get(pk=document_field['pk']) csv_bytes = DbCache.get(config_csv_file['cache_key']) apply_simple_config(CeleryTaskLogger(self), document_field, csv_bytes, drop_previous_field_detectors, update_field_choice_values, csv_contains_regexps=csv_contains_regexps) finally: DbCache.clean_cache(config_csv_file['cache_key'])
def download_task_attached_file( document_import_file: Dict[str, Any]) -> Generator[str, None, None]: if 'cache_key' in document_import_file: # download from DB cache zip_bytes = DbCache.get(document_import_file['cache_key']) ext = os.path.splitext( document_import_file['file_name'])[1][1:].lower() _fd, fn = tempfile.mkstemp(suffix=ext) try: with open(fn, 'wb') as fw: fw.write(zip_bytes) yield fn # TODO: fix yield ... finally: DbCache.clean_cache(document_import_file['cache_key']) else: # download from file storage cache file_ref_id = document_import_file['file_ref_id'] file_ref = ExportFile.objects.get(pk=file_ref_id) # type: ExportFile storage = get_file_storage() with storage.get_as_local_fn(file_ref.file_path) as f_path: yield f_path[0]