def get_term_config(project_id=None):
res = None
if project_id is not None:
key = CACHE_KEY_TERM_STEMS_PROJECT_PTN.format(project_id)
res = DbCache.get(key)
if res is None:
res = DbCache.get(CACHE_KEY_TERM_STEMS)
return res
def make_records_untyped(_apps, _schema_editor):
for cache_key in CACHE_KEYS:
untyped = []
records = DbCache.get(cache_key)
for record in records:
if record.__class__.__name__ != 'DictionaryEntry':
untyped.append(record)
continue
try:
aliases = [(a.alias, a.language, a.is_abbreviation, a.alias_id,
a.normalized_alias) for a in record.aliases]
rec = (
record.id,
record.name,
record.priority,
aliases,
)
untyped.append(rec)
except Exception as e:
print(
f'Unable to cast a DictionaryEntry in "{cache_key}" to a tuple: {e}'
)
DbCache.put_to_db(cache_key, untyped)
if DbCache.INSTANCE:
DbCache.INSTANCE.stop_watching()
def make_records_typed(_apps, _schema_editor):
for cache_key in CACHE_KEYS:
typed = [] # type: List[DictionaryEntry]
records = DbCache.get(cache_key)
for record in records:
if record.__class__.__name__ == 'DictionaryEntry':
typed.append(record)
continue
try:
aliases = [
DictionaryEntryAlias(alias, lang, is_abbr, alias_id,
norm_als)
for alias, lang, is_abbr, alias_id, norm_als in record[3]
]
rec = DictionaryEntry(record[0],
record[1],
priority=record[2],
aliases=aliases)
typed.append(rec)
except Exception as e:
print(
f'Unable to cast a record in "{cache_key}" to DictionaryEntry: {e}'
)
DbCache.put_to_db(cache_key, typed)
if DbCache.INSTANCE:
DbCache.INSTANCE.stop_watching()
def process(self,
document_type_config_csv_file: Dict,
action: str,
update_cache: bool,
**kwargs):
if action == 'validate':
save = False
auto_fix_validation_errors = False
remove_missed_objects = False
elif action == 'validate|import':
save = True
auto_fix_validation_errors = False
remove_missed_objects = False
elif action == 'import|auto_fix|retain_missing_objects':
save = True
auto_fix_validation_errors = True
remove_missed_objects = False
elif action == 'import|auto_fix|remove_missing_objects':
save = True
auto_fix_validation_errors = True
remove_missed_objects = True
else:
raise RuntimeError('Unknown action')
try:
json_bytes = DbCache.get(document_type_config_csv_file['cache_key'])
document_type = import_document_type(json_bytes=json_bytes,
save=save,
auto_fix_validation_errors=auto_fix_validation_errors,
remove_missed_in_dump_objects=remove_missed_objects,
task=self)
finally:
DbCache.clean_cache(document_type_config_csv_file['cache_key'])
if not (save and update_cache):
return
from apps.rawdb.app_vars import APP_VAR_DISABLE_RAW_DB_CACHING
if not APP_VAR_DISABLE_RAW_DB_CACHING.val:
self.log_info('Adapting RawDB table structure after import ...')
adapt_table_structure(CeleryTaskLogger(self), document_type, force=False)
ids = Document.all_objects.filter(document_type=document_type).values_list('pk', flat=True)
ids = list(ids)
self.log_info('Caching document field values ...')
for chunk in chunks(ids, 50):
self.run_sub_tasks('Cache field values for a set of documents',
ImportDocumentType.cache_document_fields_for_doc_ids,
[(list(chunk),)])
def process(self,
document_field: Dict,
config_csv_file: Dict,
drop_previous_field_detectors: bool,
update_field_choice_values: bool,
csv_contains_regexps: bool,
**kwargs):
try:
self.log_info('Going to configure simple field detection config...')
document_field = DocumentField.objects.get(pk=document_field['pk'])
csv_bytes = DbCache.get(config_csv_file['cache_key'])
apply_simple_config(CeleryTaskLogger(self),
document_field,
csv_bytes,
drop_previous_field_detectors,
update_field_choice_values,
csv_contains_regexps=csv_contains_regexps)
finally:
DbCache.clean_cache(config_csv_file['cache_key'])
def download_task_attached_file(
document_import_file: Dict[str, Any]) -> Generator[str, None, None]:
if 'cache_key' in document_import_file:
# download from DB cache
zip_bytes = DbCache.get(document_import_file['cache_key'])
ext = os.path.splitext(
document_import_file['file_name'])[1][1:].lower()
_fd, fn = tempfile.mkstemp(suffix=ext)
try:
with open(fn, 'wb') as fw:
fw.write(zip_bytes)
yield fn # TODO: fix yield ...
finally:
DbCache.clean_cache(document_import_file['cache_key'])
else:
# download from file storage cache
file_ref_id = document_import_file['file_ref_id']
file_ref = ExportFile.objects.get(pk=file_ref_id) # type: ExportFile
storage = get_file_storage()
with storage.get_as_local_fn(file_ref.file_path) as f_path:
yield f_path[0]
def get_court_config():
return DbCache.get(CACHE_KEY_COURT_CONFIG)
def get_geo_config():
return DbCache.get(CACHE_KEY_GEO_CONFIG)
def get_term_config():
return DbCache.get(CACHE_KEY_TERM_STEMS)
