def get(request: Request, host):
kwargs = dict(request.query_params)
zk_path = kwargs.get('p', '/mipush/cfg')
mode = kwargs.get('m', 'yaml')
return get_zk_config(zk_path, host, mode)
def send_feishu(body, hook_url=get_zk_config('/mipush/bot')['ann']):
"""
:param body: {"title": "x", "text": "xx"}
:param hook_url:
:return:
"""
logger.info(body)
return requests.post(hook_url, json=body).json()
def text_match_flag(image_result, w, h):
cp2flags = get_zk_config('/mipush/ocr')
for text_loc, (text, _) in image_result:
text = text.strip().lower()
for cp, flags in cp2flags.items():
for flag in flags:
if text.__contains__(flag):
text_loc = np.array(text_loc).mean(0)
text_loc_ = text_loc - (w / 2, h / 2)
return cp, text, text_loc_.tolist()
def send_email(subject="",
msg: Union[str, pd.DataFrame] = "",
receivers: Union[str, list] = '*****@*****.**',
_subtype='html',
msg_prefix='',
msg_suffix='',
msg_fn=lambda x: x,
date=date_difference(format='%Y-%m-%d %H:%M:%S'),
**kwargs):
"""
:param subject: 主题
:param msg:
:param receivers:
:param _subtype:
:param msg_prefix:
:param msg_suffix:
:param msg_fn:
:param kwargs:
:return:
"""
# init
token = get_zk_config("/mipush/email_token")
# logger.info(token)
host, sender = list(token.items())[0]
smtp = smtplib.SMTP(host, 25)
# 主题+内容
subject = f"👉{subject}📅{date}"
if isinstance(msg, pd.DataFrame):
msg_fn = lambda df: better_df2html(df, subject)
msg = f"{msg_prefix}{msg_fn(msg)}{msg_suffix}"
message = MIMEText(msg, _subtype, 'utf-8')
message['Subject'] = Header(subject, 'utf-8')
message['From'] = sender
if isinstance(receivers, str) and receivers.__contains__("@"):
receivers = [receivers]
message['To'] = ",".join(receivers)
try:
smtp.sendmail(sender, receivers, message.as_string())
logger.info("邮件发送成功")
except smtplib.SMTPException as e:
logger.warning(f"{e}: 无法发送邮件")
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Project : MeUtils. # @File : get_dsl_conf # @Time : 2021/1/25 7:54 下午 # @Author : yuanjie # @Email : [email protected] # @Software : PyCharm # @Description : from meutils.pipe import * from meutils.zk_utils import get_zk_config data = get_zk_config('/mipush/dsl', mode='').decode().strip() data = [ line.strip().split() for line in data.split('\r\n') if line != '' and not line.startswith('#') ] columns = [ 'fea_id', 'fea_name', 'component', 'fea_type', 'fea_dim', 'isOut', 'dnn_group' ] df = pd.DataFrame(data, columns=columns) ### VeryGood!!! df = pd.read_csv('dsl.txt', sep='\s+', names=columns, comment='#')
from meutils.pipe import *
os.environ['TF_KERAS'] = '1'
from meutils.zk_utils import get_zk_config
from bert4keras.backend import keras, search_layer, K
from bert4keras.tokenizers import Tokenizer
from bert4keras.models import build_transformer_model
from bert4keras.optimizers import Adam
from bert4keras.snippets import sequence_padding, DataGenerator
from bert4keras.snippets import open
from tensorflow.keras.layers import Lambda, Dense
from tensorflow.keras.utils import to_categorical
cfg = get_zk_config('/mipush/nh_model')
vocab_url = cfg['vocab_url']
model_url = cfg['nh_content_bert_model_url']
if not Path('vocab.txt').exists():
download(vocab_url, 'vocab.txt')
tokenizer = Tokenizer('vocab.txt', do_lower_case=True)
model_name = 'model.h5'
if not Path(model_name).exists():
download(model_url, model_name) #
model = keras.models.load_model(model_name, compile=False)
maxlen = 64
from meutils.zk_utils import get_zk_config
from appzoo import App
import tensorflow as tf
keras = tf.keras
os.environ['TF_KERAS'] = '1'
from bert4keras.models import build_transformer_model
from bert4keras.tokenizers import Tokenizer
from bert4keras.snippets import sequence_padding
# BERT_DIR
BERT_DIR = './chinese_simbert_L-12_H-768_A-12'
fds_url = get_zk_config("/mipush/cfg")['fds_url']
if not os.path.exists(BERT_DIR):
url = f"{fds_url}/data/bert/chinese_simbert_L-12_H-768_A-12.zip"
os.system(f"wget {url} && unzip chinese_simbert_L-12_H-768_A-12.zip")
config_path = f'{BERT_DIR}/bert_config.json'
checkpoint_path = f'{BERT_DIR}/bert_model.ckpt'
dict_path = f'{BERT_DIR}/vocab.txt'
# 建立分词器
tokenizer = Tokenizer(dict_path, do_lower_case=True)
# 建立加载模型
bert = build_transformer_model(
config_path,
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Project : AppZoo. # @File : image_utils # @Time : 2020/11/10 11:52 上午 # @Author : yuanjie # @Email : [email protected] # @Software : PyCharm # @Description : import os from PIL import Image from paddleocr import draw_ocr from meutils.zk_utils import get_zk_config fds_url = get_zk_config('/mipush/cfg')['fds_url'] def get_image_size(image): img = Image.open(image) return img.size # (w, h) def ocr_result_image(result, input_image, output_image='output_image.png'): if not os.path.exists('./simfang.ttf'): os.system(f"wget -q {fds_url}/data/simfang.ttf") image = Image.open(input_image).convert('RGB') boxes = [line[0] for line in result] txts = [line[1][0] for line in result] scores = [line[1][1] for line in result]
# @Project : tql-App. # @File : ocr_app # @Time : 2020/11/4 4:05 下午 # @Author : yuanjie # @Email : [email protected] # @Software : PyCharm # @Description : from meutils.common import * from meutils.zk_utils import get_zk_config from appzoo import App from paddleocr import PaddleOCR from PIL import Image ac_url = get_zk_config('/mipush/cfg')['ac_url'] ocr = PaddleOCR(use_angle_cls=True, lang="ch") def request(url, json=None, method='get'): r = requests.request(method, url, json=json) r.encoding = r.apparent_encoding return r.json() def get_ocr_result(**kwargs): image_urls = kwargs.get('image_urls', []) results = [] for image_url in image_urls:
# @Email : [email protected] # @Software : PyCharm # @Description : https://docs.streamlit.io/api.html?highlight=markdown#display-progress-and-status # https://docs.streamlit.io/cli.html # streamlit run your_script.py --server.port 80 # 8051 # https://github.com/streamlit/streamlit/blob/develop/examples/file_uploader.py from meutils.common import * from meutils.zk_utils import get_zk_config import streamlit as st from mi.db import Mongo, ANN from bson import json_util simbert_url = get_zk_config('/mipush/bert')['simbert_url'] # db mongo = Mongo() ann = ANN() # side st.sidebar.markdown('**SideBar**') biz = st.sidebar.selectbox('业务场景', ('xindao_ann', 'other'), index=0) mongo_collection = mongo.db.__getitem__(f"{biz}") ## mongo st.sidebar.markdown('---') st.sidebar.markdown('**Mongo Find**') filter_ = st.sidebar.text_input('filter', {'xindaoid': 44517}) # **kwargs _ = mongo_collection.find_one(eval(filter_))
if biz == 'ImageUrl':
ImageUrl = st.text_input(
"ImageUrl",
"https://i1.mifile.cn/f/i/mioffice/img/slogan_5.png?1604383825042")
input_image = 'image.png'
os.system(f"wget -q {ImageUrl} -O {input_image}")
st.markdown("## 文字识别")
image_result = ocr.ocr(input_image, cls=True)
output_image = ocr_result_image(image_result, input_image)
st.image(output_image)
# st.json(image_result)
st.markdown("## 水印识别")
w, h = Image.open(input_image).size
st.json({"水印识别": text_match_flag(image_result, w, h)})
st.markdown("## 水印召回词")
st.json({"zk已配置水印召回词": get_zk_config('/mipush/ocr')})
os.system(f"rm {input_image}")
elif biz == 'ImageFile':
input_image = file_uploader(st)
if input_image:
result = ocr.ocr(input_image, cls=True)
output_image = ocr_result_image(result, input_image)
st.image(output_image)
st.json(result)
os.system(f"rm {input_image}")
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Project : tql-App. # @File : ann_search # @Time : 2020/11/5 4:04 下午 # @Author : yuanjie # @Email : [email protected] # @Software : PyCharm # @Description : from meutils.common import * from meutils.zk_utils import get_zk_config from appzoo import App cfg = get_zk_config("/mipush/cfg") simbert_url = cfg['simbert_url'] xindao_search_url = cfg['xindao_search_url'] @lru_cache(256) def get_bert_vector(text): return requests.get(f"{simbert_url}?texts=['{text}']").json()['vectors'] def xindao_search(**kwargs): text = kwargs.get('text', '') topk = kwargs.get('topk', 5) return_ids = int(kwargs.get('return_ids', 0)) query_embedding = get_bert_vector(text) body = {