async def model_create(ClassOrm, ClassPyd, data_model, soft_insert):
"""
Универсальное создание моделей с использованием пидантика
"""
general_response = {'status': False, 'data': None}
# Шаг 1 валидация
try:
obj_pyd = ClassPyd(**data_model)
except ValidationError as exp:
if soft_insert:
general_response['data'] = jsl(exp.json())
return general_response
else:
raise PydanticCreateError(exp.json())
# Шаг 2 Создание
try:
orm_obj = await ClassOrm.create(**obj_pyd.dict())
except Exception as exp:
if soft_insert:
general_response['data'] = str(exp)
return general_response
else:
raise ORMCreateError(str(exp))
# Шаг 3 Ответ
back_proc = await ClassPyd.from_tortoise_orm(orm_obj)
general_response['data'] = back_proc.dict()
general_response['status'] = True
return general_response
def conf_one_msg_convertor(msg):
return {
'topic': msg.topic(),
'partition': msg.partition(),
'offset': msg.offset(),
'key': msg.key().decode('utf-8'),
'value': jsl(msg.value().decode('utf-8'))
}
async def test_processing_delete():
"""
Тестирование удаления витрины данных
"""
with open(TEST_DEL_FILE, 'r', encoding='utf-8') as file:
input_data = jsl(fp=file)
result = await smart_delete_showcases(json_data=input_data)
with open('z_send_delete_showcase.json', 'w', encoding='utf-8') as fb:
jsd(result, fb, ensure_ascii=False, indent=4)
async def test_processing_post_create():
"""
Тестирование создания/обновления клиентов
"""
with open(TEST_POST_FILE, 'r', encoding='utf-8') as file:
input_data = jsl(fp=file)
result = await smart_create_client(json_data=input_data)
with open('z_send_insert_showcase.json', 'w', encoding='utf-8') as fb:
jsd(result, fb, ensure_ascii=False, indent=4)
async def test_processing_post_create():
"""
Тестирование создания словаря
"""
with open(TEST_POST_FILE, 'r', encoding='utf-8') as file:
input_data = jsl(fp=file)
result = await bridge_smart_create_dictionaries(data_json=input_data)
with open('z_send_insert_dict.json', 'w', encoding='utf-8') as fb:
jsd(result, fb, ensure_ascii=False, indent=4)
async def test_processing_delete():
"""
Тестирование удаления словаря
"""
with open(TEST_DEL_FILE, 'r', encoding='utf-8') as file:
input_data = jsl(fp=file)
result = await bridge_smart_delete_dictionaries(json_data=input_data)
with open('z_send_delete_dict.json', 'w', encoding='utf-8') as fb:
jsd(result, fb, ensure_ascii=False, indent=4)
def aio_msg_processing(list_data):
"""
Конвертор данных в универсальный генератор
:param list_data:
:return:
"""
return ({
'topic': i.topic,
'partition': i.partition,
'offset': i.offset,
'key': i.key.decode('utf-8'),
'value': jsl(i.value.decode('utf-8'))
} for i in list_data)
def get_urls_from_json(
json_config_file=sys.argv[1]
): #получаем конфигурационный файл из аргументов командной строки
with open(json_config_file) as file:
return jsl(file) # загружаем данные в json load и возвращаем
async def process_metadata(json_in):
"""
Обработка метаданных. Главный обработчик
:param json_in:
:return:
"""
array_out = []
transform_add = {}
# Выясняем надо ли нам использовать хардкод
PATTERN_META_CONV_FOLDER = getattr(GeneralConfig,
'PATTERN_META_CONV_FOLDER', None)
if PATTERN_META_CONV_FOLDER:
for conv_name, conv_file in DEFAULT_FILE_META_CONV.items():
try:
with open(PATTERN_META_CONV_FOLDER / conv_file,
'r',
encoding='utf-8') as file:
transform_add[conv_name] = jsl(fp=file)
except Exception:
continue
conv_name = transform_add.get('conv_name', {})
conv_rel_dict = transform_add.get('conv_dict', {})
# Преобразование водных данных для получения метаданных
for schema in json_in:
try:
ip = schema['ip']
port = schema['port']
data_bases = schema['data_bases']
name_hosts = f'{ip}:{port}'
type = schema.get('type_db', 'ycl')
storage_names = conv_name.get(name_hosts, {})
storage_rels = conv_rel_dict.get(name_hosts, {})
out_data_bases = []
out_schema = {
'ip': ip,
'port': port,
'type': type,
'_name': storage_names.get('_', name_hosts),
'name_schema': schema.get('name', name_hosts),
'data_bases': out_data_bases,
}
# Каждую базу данных обраатываем отдельно
for db in data_bases:
try:
login = db.get('user', 'default')
db['user'] = login
pswd = db.get('pswd', '')
db['pswd'] = ''
db_name = db.get('name', 'default')
db['name'] = 'default'
db_tables = db.get('tables', '')
db['tables'] = ''
data_base_names = storage_names.get(db_name, {})
data_base_rels = storage_rels.get(db_name, {})
out_data_base = {
'name': db_name,
'_name': data_base_names.get('_', db_name)
}
if metadata_settings.DataBaseTypes(
type) == metadata_settings.DataBaseTypes.psql:
# Постгрес в кликхаус
data_scheme = await psql.get_tables_from_database(
host=ip,
port=port,
user=login,
password=pswd,
database=db_name,
table_filter=db_tables)
out_tables = psql_cl_convertations.main_psql_ycl_json_converter(
metadata_psql=data_scheme,
transform_add={
'conv_name': data_base_names,
'conv_dict': data_base_rels
})
if metadata_settings.DataBaseTypes(
type) == metadata_settings.DataBaseTypes.ycl:
# Кликхаус в кликхаус
data_scheme = await ycl.get_tables_from_database(
host=ip,
port=port,
user=login,
password=pswd,
database=db_name,
table_filter=db_tables)
out_tables = ycl_ycl_convertations.main_ycl_ycl_json_converter(
metadata_ycl=data_scheme,
transform_add={
'conv_name': data_base_names,
'conv_dict': data_base_rels
})
out_data_base['tables'] = out_tables
out_data_bases.append(out_data_base)
except Exception as exp:
out_data_bases.append({
'invalid_data': db,
'error': str(exp)
})
# Массив ответов
array_out.append(out_schema)
except Exception as exp:
array_out.append({'invalid_data': schema, 'error': str(exp)})
return array_out
"""
Черновик
"""
from json import dump as jsd, load as jsl, dumps as str_jsd
from pathlib import Path
DEFAULT_JSON_METADATA_PATH = Path().cwd() / Path(
'../documents/front_back_interaction/json_sdmx_ex2.json')
with open(DEFAULT_JSON_METADATA_PATH, 'r') as fobj:
data_settings_json = jsl(fobj)
print(data_settings_json)
with open(DEFAULT_JSON_METADATA_PATH, 'w') as fobj:
jsd(data_settings_json, fobj, indent=4)
