def baidu_nlp(id):
"""
百度自然语言处理
:param id:
:return:
"""
print(111111111111)
ne_list = AudioLexerNeModel().dao_get_codes()
result = []
nlp = NLP()
file_dir = 'D:/AIData/音频转写'
file_name = '{id}.txt'.format(id=id)
file_lexer_name = '{id}_lexer.txt'.format(id=id)
with open(os.path.join(file_dir, id, file_name), 'r') as f:
for line in f.readlines():
asr_json = json.loads(line.strip('\n').replace("'", "\""))
lexer_res = nlp.lexer(text=asr_json.get('onebest', ''),
ne_list=ne_list)
lexer_json, errors = LexerRes.LexerResSchema().dump(lexer_res)
Assert.is_true(is_empty(errors), errors)
print(lexer_json)
asr_json.update({'items': lexer_json.get('items', [])})
result.append(asr_json)
with open(os.path.join(file_dir, id, file_lexer_name), 'a') as f:
for info in result:
print(info)
f.writelines(str(info) + '\n')
return 'ok'
def xunfei_asr_progress(id, task_id, ts, signa):
"""
讯飞音频转写结果查询
:param id:
:param task_id:
:param ts:
:param signa:
:return:
"""
audio = Audio()
audio.init_asr(task_id, ts, signa)
asr_progress = audio.get_asr_progress()
print(asr_progress)
if asr_progress.ok == codes.success and asr_progress.data.status == codes.asr_success:
asr_result = audio.get_asr_result()
Assert.is_true(asr_result.ok == codes.success, asr_result.failed)
with open(os.path.join('D:/AIData/音频转写', id, '{id}.txt'.format(id=id)),
'a') as f:
for info in json.loads(asr_result.data):
print(info)
f.writelines(str(info) + '\n')
else:
print(asr_progress.failed
if asr_progress.ok != codes.success else asr_progress.data.desc)
return 'ok'
def dao_find_page(self, page, error_out=False):
"""
分页条件查询
:param page:
:param error_out:
:return:
"""
# 条件查询
filter = []
if is_not_empty(self.img_data_id):
filter.append(ImgDetailModel.img_data_id == self.img_data_id)
if is_not_empty(self.is_handle):
filter.append(ImgDetailModel.is_handle == self.is_handle)
pagination = self.query.filter(*filter).\
order_by(ImgDetailModel.create_date.desc()).\
paginate(page=page.page, per_page=page.page_size, error_out=error_out)
page.init_pagination(pagination)
# 数据序列化 json
img_detail_dict, errors = ImgDetailSchema(
only=ImgDetailSchema().dump_only_page()).dump(page.data, many=True)
Assert.is_true(is_empty(errors), errors)
page.data = img_detail_dict
return page, pagination
def mvsi(self, url, image_bs64):
"""
机动车销售发票OCR识别
:param url: OCR接口地址
:param image_bs64: 图片base64数据, base64编码后大小不超过10M。图片最小边不小于15像素，最长边不超过4096像素,
支持JPG/PNG/BMP/TIFF格式
:return:
"""
headers = {
'Content-Type': 'application/json',
'charset': 'UTF-8',
'X-Auth-Token': self.token
}
res = requests.post(url=url, data=json.dumps(obj={'image': image_bs64}), headers=headers)
res.encoding = encodes.Unicode.UTF_8.value
print(json.dumps(res.json(), indent=4, ensure_ascii=False))
if res is None or res.status_code != codes.ok:
mvsi_error, errors = MvsiResultSchema(only=MvsiResultSchema().only_error()).load(res.json())
Assert.is_true(is_empty(errors), errors)
return mvsi_error
mvsi_result, errors = MvsiResultSchema(only=MvsiResultSchema().only_success()).load(res.json().get('result'))
Assert.is_true(is_empty(errors), errors)
return mvsi_result
def img_files_flow_page(page, img_detail):
"""
图片文件流加载
:param page:
:param img_detail:
:return:
"""
page, pagination = img_detail.dao_find_page(page)
flow_img_details = []
for img_detail in pagination.items:
url = url_for('file.file_download',
id=img_detail.file_id,
md5_id=img_detail.file_md5)
if not img_detail.is_handle:
alt = '待处理'
elif is_not_empty(img_detail.img_type):
alt = img_detail.img_type.type_explain
else:
alt = img_detail.err_msg
flow_info = FlowInfo(url, url, alt, img_detail.file_id)
flow_img_details.append(flow_info)
flow_img_details_dict, errors = FlowInfoSchema().dump(flow_img_details,
many=True)
Assert.is_true(is_empty(errors), errors)
page_dict, page_errors = PageSchema().dump(page)
Assert.is_true(is_empty(page_errors), page_errors)
page_dict['data'] = flow_img_details_dict
return render_info(page_dict)
def init_log(project_name, lever, log_dir_name='logs'):
"""
初始化日志
:param lever: 日志级别
:param project_name: 项目名，不可为空
:param log_dir_name: 日志父目录名
:return:
"""
Assert.is_true(is_not_empty(project_name), '初始化日志。项目名不可为空.')
log_file_name = 'logger'
log_file_folder = os.path.abspath(
os.path.join(
os.path.dirname(__file__), os.pardir,
os.pardir)) + os.sep + log_dir_name + os.sep + project_name
FileUtil.creat_dirs(log_file_folder)
log_file = log_file_folder + os.sep + log_file_name
file_handler = MyLoggerHandler(log_file,
when='D',
encoding=Unicode.UTF_8.value,
suffix='_%Y-%m-%d.log',
extMatch=r'^_\d{4}-\d{2}-\d{2}.log$')
file_handler.setLevel(lever) # 日志输出级别
fmt = '%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s'
formatter = logging.Formatter(fmt)
file_handler.setFormatter(formatter)
return file_handler
def dao_find_page(self, page, error_out=False):
"""
分页条件查询
:param page:
:param error_out:
:return:
"""
# 条件查询
filter = []
if is_not_empty(self.type):
filter.append(SysDict.type == self.type)
if is_not_empty(self.description):
filter.append(
SysDict.description.like(
'%{description}%'.format(description=self.description)))
pagination = self.query.filter(*filter).\
order_by(SysDict.sort.asc(), SysDict.type.asc(), SysDict.create_date.asc()).\
paginate(page=page.page, per_page=page.page_size, error_out=error_out)
page.init_pagination(pagination)
# 数据序列化 json
data_dict, errors = SysDictSchema().dump(page.data, many=True)
Assert.is_true(is_empty(errors), errors)
page.data = data_dict
return page, pagination
def add_menu(menu):
"""
添加系统菜单
:param menu:
:return:
"""
parent_menu = Menu().dao_get(menu.parent_id)
Assert.is_true(is_not_empty(parent_menu),
'无效的父级菜单ID：{0}'.format(menu.parent_id))
menu.dao_add()
return render_info(MyResponse(msg='添加成功'))
def exempt_views(self, views):
"""
标记要从LoginManager保护中排除的视图或蓝图
:param views:
:return:
"""
Assert.is_true(
isinstance(views, tuple) or isinstance(views, list),
'the parameters "views" must be lists or tuples.')
for view in views:
self.exempt_view(view)
def put_menu(menu):
"""
更新系统菜单
:param menu:
:return:
"""
to_do_menu = Menu().dao_get(menu.id)
Assert.is_true(is_not_empty(to_do_menu), '无效的菜单ID：{0}'.format(menu.id))
parent_menu = Menu().dao_get(menu.parent_id)
Assert.is_true(is_not_empty(parent_menu),
'无效的父级菜单ID：{0}'.format(menu.parent_id))
to_do_menu.dao_put(menu)
return render_info(MyResponse(msg='更新成功'))
def dao_get_all(self, is_dump=False):
"""
获取全部菜单列表 包括功能菜单
:param is_dump:
:return:
"""
menus = self.query.order_by(Menu.sort.asc()).all()
if is_dump:
if is_empty(menus):
return []
menus, errors = MenuSchema().dump(menus, many=True)
Assert.is_true(is_empty(errors), errors)
return menus
def get_img_data(img_data, id):
"""
图片流水查询
:param img_data:
:param id:
:return:
"""
# 查询图片流水
img_data = ImgDataModel().dao_get(img_data.id) # type: ImgDataModel
Assert.is_true(is_not_empty(img_data), '查无此数据', codes.no_data)
img_data_dict, errors = ImgDataSchema(
only=ImgDataSchema().dump_only_get()).dump(img_data)
Assert.is_true(is_empty(errors), errors)
return render_info(MyResponse(msg='查询成功', imgData=img_data_dict))
def asr_nlp(id):
"""
音频转写加词性分析
:param id:
:return:
"""
result = []
with open(
os.path.join('D:/AIData/音频转写', id, '{id}_lexer.txt'.format(id=id)),
'r') as f:
for line in f.readlines():
result.append(json.loads(line.strip('\n').replace("'", "\"")))
asr_nlp_datas, errors = AudioAsrNlp.AudioAsrNlpSchema().load(result,
many=True)
Assert.is_true(is_empty(errors), errors)
# 查询识别出的词性列表, 单词性
# ne_list = []
# for asr_nlp_data in asr_nlp_datas:
# if is_not_empty(asr_nlp_data.ne_list):
# ne_list.extend(asr_nlp_data.ne_list)
# audio_lexers = AudioLexerNeModel().dao_get_by_codes(ne_list)
# ***********************************
items = []
for asr_nlp_data in asr_nlp_datas:
for item in asr_nlp_data.items:
items.append(item)
audio_lexers = AudioLexerNeModel().dao_get_all()
for audio_lexer in audio_lexers:
audio_lexer.items = []
for item in items:
if audio_lexer.code == item.ne:
audio_lexer.items.append(item)
# 移除 item 为空的标签栏
for audio_lexer in audio_lexers.copy():
if is_empty(audio_lexer.items):
audio_lexers.remove(audio_lexer)
return render_template('audio/asr_nlp.html',
audio_src=url_for(
'static',
filename='songs/{id}.wav'.format(id=id)),
asr_nlp_datas=asr_nlp_datas,
audio_lexers=audio_lexers)
def dao_get_sub_menus(self, id, is_dump=False):
"""
根据ID获取全部子级菜单
:param id:
:param is_dump: 是否序列化字典(序列化尽量写在同一个方法，因为relationship的懒加载机制，结合缓存会报Session异常)
:return:
"""
menus = self.query.filter(Menu.parent_id == id).order_by(Menu.sort.asc()).all()
if is_dump:
if is_empty(menus):
return []
menus, errors = MenuSchema().dump(menus, many=True)
Assert.is_true(is_empty(errors), errors)
return menus
def files_handle_html(img_data, id):
"""
文件处理页面
:param img_data:
:param id: 图片流水号
:return:
"""
source_files = img_data.dao_get_source_files(img_data.id)
Assert.is_true(is_not_empty(source_files), '查无此数据', codes.no_data)
for source_file in source_files:
children = ImgDetailModel().dao_get_children(source_file.id)
source_file.img_page = 0 if is_empty(children) else len(children)
return render_template('ai/img/files_handle.html',
img_data_id=img_data.id,
source_files=source_files)
def get_img(img_data, id):
"""
根据 ID 流水号查询图片流水(附带路径)
:param img_data:
:param id:
:return:
"""
# 查询图片流水
img_data = ImgDataModel.query.get(img_data.id) # type: ImgDataModel
Assert.is_true(is_not_empty(img_data), '查无此数据', codes.no_data)
img_data_dict, errors = ImgDataSchema().dump(img_data)
Assert.is_true(is_empty(errors), errors)
# 过滤图片明细字典字段
ImgDataSchema().filter_img_details(img_data_dict.get('imgDetails', []),
['fileData'])
return render_info(MyResponse(msg='查询成功', imgData=img_data_dict))
def login_validate(user):
"""
登录验证
:param user:
:return:
"""
if current_user.is_authenticated:
return render_info(MyResponse(msg='登录成功'))
real_user = User().dao_get_by_login_name(user.login_name) # type: User
Assert.is_true(is_not_empty(real_user),
assert_code=codes.login_fail,
assert_msg='账号未注册')
if real_user.validate_password(user.password):
# 密码验证成功
login_user(real_user, False)
return render_info(MyResponse(msg='登录成功'))
return render_info(MyResponse(code=codes.login_fail, msg='用户名或密码不正确'))
def add_img(img_data):
"""
图片文件入库
:param ImgDataModel img_data:
:return:
"""
img_data.dao_create()
app_sys = get_app_sys(img_data.app_sys_code)
Assert.is_true(is_not_empty(app_sys),
'无效的应用系统：{0}'.format(img_data.app_sys_code),
codes.unprocessable)
# 通过外键添加
img_data.app_sys_id = app_sys.id
# 创建资料目录
loan_dir = FileUtil.path_join(current_app.config.get('DATA_DIR'),
img_data.id)
# 信息入库
handle_info = img_data.dao_add_info(loan_dir)
return render_info(MyResponse(msg='接收资料成功', handle_info=handle_info))
def patch_img_detail_type(img_detail):
"""
更新图片明细类型
:param ImgDetailModel img_detail:
:return:
"""
img_type = ImgTypeModel().dao_get_by_code(img_detail.img_type_code)
Assert.is_true(is_not_empty(img_type),
'无效的图片类型：{0}'.format(img_detail.img_type_code),
codes.unprocessable)
img_detail_id = img_detail.id
# 接口提交的更新者
update_by = img_detail.update_by
img_detail = ImgDetailModel().dao_get(
img_detail_id) # type: ImgDetailModel
Assert.is_true(is_not_empty(img_detail),
'无查无此数据：{0}'.format(img_detail_id), codes.no_data)
img_detail.update_by = update_by
img_detail.dao_update_type(img_type.id)
return render_info(MyResponse('更新成功'))
def img_source_files_flow_page(img_data, id):
"""
源文件流加载
:param img_data:
:param id:
:return:
"""
source_files = img_data.dao_get_source_files(img_data.id)
Assert.is_true(is_not_empty(source_files), '查无此数据', codes.no_data)
file_datas = []
for source_file in source_files:
url = url_for('file.file_download',
id=source_file.id,
md5_id=source_file.md5_id)
if source_file.file_format.upper() == FileFormat.PDF.value:
url = url + '/' + source_file.id
flow_info = FlowInfo(url, url, source_file.file_name, source_file.id,
source_file.file_format)
file_datas.append(flow_info)
file_datas_dict, errors = FlowInfoSchema().dump(file_datas, many=True)
Assert.is_true(is_empty(errors), errors)
return render_info(MyResponse('查询成功', data=file_datas_dict))
def ocr(ocrFile):
"""
机动车销售发票 OCR识别
:return:
"""
app_sys = get_app_sys(ocrFile.app_sys_code)
Assert.is_true(is_not_empty(app_sys),
'无效的应用系统：{0}'.format(ocrFile.app_sys_code),
codes.unprocessable)
# 通过外键添加
ocrFile.app_sys_id = app_sys.id
file_model = FileModel()
file_model.dao_create()
# 资料写入磁盘
file_path = encodes.base64_to_file(ocrFile.file_data.file_base64,
current_app.config.get('OCR_FILE_DIR'),
file_model.id,
ocrFile.file_data.file_format)
# OCR识别
token = SysData().dao_get_key(system.SysKey.HUAWEI_CLOUD_TOKEN.value).value
mvsi_result = OCR(token=token)\
.mvsi(image_bs64=img.ImgUtil.img_compress(path=file_path, threshold=5))
Assert.is_true(
is_empty(mvsi_result.error_code),
'OCR FAILED: 【{0}】【{1}】'.format(mvsi_result.error_code,
mvsi_result.error_msg))
mvsi_result_json, errors = MvsiResultSchema(
only=MvsiResultSchema().only_success()).dump(mvsi_result)
Assert.is_true(is_empty(errors), errors)
# 信息入库
mvsi = Mvsi(**mvsi_result_json)
mvsi.dao_add(ocrFile, file_model, file_path)
# json 序列化
mvsi_json, errors = MvsiSchema().dump(mvsi)
Assert.is_true(is_empty(errors), errors)
return render_info(MyResponse('OCR SUCCESS', results=mvsi_json))