def get_term_config(project_id=None): res = None if project_id is not None: key = CACHE_KEY_TERM_STEMS_PROJECT_PTN.format(project_id) res = DbCache.get(key) if res is None: res = DbCache.get(CACHE_KEY_TERM_STEMS) return res
def make_records_untyped(_apps, _schema_editor): for cache_key in CACHE_KEYS: untyped = [] records = DbCache.get(cache_key) for record in records: if record.__class__.__name__ != 'DictionaryEntry': untyped.append(record) continue try: aliases = [(a.alias, a.language, a.is_abbreviation, a.alias_id, a.normalized_alias) for a in record.aliases] rec = ( record.id, record.name, record.priority, aliases, ) untyped.append(rec) except Exception as e: print( f'Unable to cast a DictionaryEntry in "{cache_key}" to a tuple: {e}' ) DbCache.put_to_db(cache_key, untyped) if DbCache.INSTANCE: DbCache.INSTANCE.stop_watching()
def make_records_typed(_apps, _schema_editor): for cache_key in CACHE_KEYS: typed = [] # type: List[DictionaryEntry] records = DbCache.get(cache_key) for record in records: if record.__class__.__name__ == 'DictionaryEntry': typed.append(record) continue try: aliases = [ DictionaryEntryAlias(alias, lang, is_abbr, alias_id, norm_als) for alias, lang, is_abbr, alias_id, norm_als in record[3] ] rec = DictionaryEntry(record[0], record[1], priority=record[2], aliases=aliases) typed.append(rec) except Exception as e: print( f'Unable to cast a record in "{cache_key}" to DictionaryEntry: {e}' ) DbCache.put_to_db(cache_key, typed) if DbCache.INSTANCE: DbCache.INSTANCE.stop_watching()
def process(self, document_type_config_csv_file: Dict, action: str, update_cache: bool, **kwargs): if action == 'validate': save = False auto_fix_validation_errors = False remove_missed_objects = False elif action == 'validate|import': save = True auto_fix_validation_errors = False remove_missed_objects = False elif action == 'import|auto_fix|retain_missing_objects': save = True auto_fix_validation_errors = True remove_missed_objects = False elif action == 'import|auto_fix|remove_missing_objects': save = True auto_fix_validation_errors = True remove_missed_objects = True else: raise RuntimeError('Unknown action') try: json_bytes = DbCache.get(document_type_config_csv_file['cache_key']) document_type = import_document_type(json_bytes=json_bytes, save=save, auto_fix_validation_errors=auto_fix_validation_errors, remove_missed_in_dump_objects=remove_missed_objects, task=self) finally: DbCache.clean_cache(document_type_config_csv_file['cache_key']) if not (save and update_cache): return from apps.rawdb.app_vars import APP_VAR_DISABLE_RAW_DB_CACHING if not APP_VAR_DISABLE_RAW_DB_CACHING.val: self.log_info('Adapting RawDB table structure after import ...') adapt_table_structure(CeleryTaskLogger(self), document_type, force=False) ids = Document.all_objects.filter(document_type=document_type).values_list('pk', flat=True) ids = list(ids) self.log_info('Caching document field values ...') for chunk in chunks(ids, 50): self.run_sub_tasks('Cache field values for a set of documents', ImportDocumentType.cache_document_fields_for_doc_ids, [(list(chunk),)])
def process(self, document_field: Dict, config_csv_file: Dict, drop_previous_field_detectors: bool, update_field_choice_values: bool, csv_contains_regexps: bool, **kwargs): try: self.log_info('Going to configure simple field detection config...') document_field = DocumentField.objects.get(pk=document_field['pk']) csv_bytes = DbCache.get(config_csv_file['cache_key']) apply_simple_config(CeleryTaskLogger(self), document_field, csv_bytes, drop_previous_field_detectors, update_field_choice_values, csv_contains_regexps=csv_contains_regexps) finally: DbCache.clean_cache(config_csv_file['cache_key'])
def download_task_attached_file( document_import_file: Dict[str, Any]) -> Generator[str, None, None]: if 'cache_key' in document_import_file: # download from DB cache zip_bytes = DbCache.get(document_import_file['cache_key']) ext = os.path.splitext( document_import_file['file_name'])[1][1:].lower() _fd, fn = tempfile.mkstemp(suffix=ext) try: with open(fn, 'wb') as fw: fw.write(zip_bytes) yield fn # TODO: fix yield ... finally: DbCache.clean_cache(document_import_file['cache_key']) else: # download from file storage cache file_ref_id = document_import_file['file_ref_id'] file_ref = ExportFile.objects.get(pk=file_ref_id) # type: ExportFile storage = get_file_storage() with storage.get_as_local_fn(file_ref.file_path) as f_path: yield f_path[0]
def get_court_config(): return DbCache.get(CACHE_KEY_COURT_CONFIG)
def get_geo_config(): return DbCache.get(CACHE_KEY_GEO_CONFIG)
def get_term_config(): return DbCache.get(CACHE_KEY_TERM_STEMS)