def write_db(labels):
# 写入数据库
db = Mysql('192.168.1.11', 3306, 'root', '123456', 'face_reid')
sql = "update t_cluster2 set cluster_id = %s where id = %s"
for i, label in enumerate(labels):
result = db.update(sql, (str(label), str(844 + i)))
print(result)
db.commit()
sup = os.path.dirname(sup)
if sup not in sys.path:
sys.path.append(sup)
from .error import *
from .param_tool import check_param_key, update_param, check_param_value, G_RE
from util import Log
from util import Mysql
logger = Log('freq', 'logs/')
config = configparser.ConfigParser()
config.read('./app.config')
db = Mysql(host=config.get('db', 'host'),
port=config.getint('db', 'port'),
user=config.get('db', 'user'),
password=config.get('db', 'password'),
db=config.get('db', 'db'),
charset=config.get('db', 'charset'))
db.set_logger(logger)
bp_freq = Blueprint('freq', __name__)
@bp_freq.route('/freq', methods=['POST'])
def freq():
'''
频次查询，查表
:return:
'''
start = time.time()
data = request.data.decode('utf-8')
def __init__(self):
self.mysql = Mysql.Mysql()
def process(lock):
# 1.从kafka消费消息
kafka = Kafka(bootstrap_servers=config.get('kafka', 'boot_servers'))
db = Mysql(host=config.get('db', 'host'),
port=config.getint('db', 'port'),
user=config.get('db', 'user'),
password=config.get('db', 'password'),
db=config.get('db', 'db'),
charset=config.get('db', 'charset'))
db.set_logger(logger)
topics = config.get('camera', 'topic')
consumer = kafka.get_consumer(topics, 'default')
face_tool = Face(config.get('api', 'face_server'))
searcher = Faiss(config.get('api', 'faiss_host'),
config.getint('api', 'faiss_port'))
count = 0
global re_cluster
for msg in consumer:
if not re_cluster:
logger.info('get msg: ', msg)
msg = msg.value
start = round(time.time())
msg = json.loads(msg)
b64 = download(msg['url'])
landmark = msg['landmark']
feature = face_tool.extract([b64], [landmark])
feature = feature['feature'][0]
# get one unique cluster_id
cluster_id = str(hash(uuid.uuid4()))
# 1. 进行预分类
search_result = searcher.search(1, topk, [feature])
if search_result is not None and search_result['rtn'] == 0:
logger.info('index search successfully, return result: ',
search_result)
elif search_result is None:
logger.warning('index server is not accessible, please check')
else:
logger.info('index search failed, return result: ',
search_result)
# 4. 计算cluster的平均相似度，然后合并
dist_lable = list(
map(lambda x, y: (x, y),
search_result['results']['distances'][0],
search_result['results']['lables'][0]))
# 过滤掉低于给定阈值相似度的向量
dist_lable = list(filter(lambda x: x[0] > threshold, dist_lable))
distances = list(map(lambda x: x[0], dist_lable))
lables = list(map(lambda x: x[1], dist_lable))
logger.info('real topk is:', len(lables))
if len(lables) == 0:
# 新建一个类
logger.info('new a cluster, cluster_id:', cluster_id)
# 插入数据库和索引中
sql = "insert into `t_cluster` (`cluster_id`, `uri`, `timestamp`, `camera_id`) values (%s, %s, %s, %s)"
sample_id = db.insert(
sql, (cluster_id, msg['url'], time_to_date(int(
msg['time'])), msg['camera_id']))
if sample_id == -1:
logger.info(
'sample insert failed, mysql maybe not available')
continue
db.commit()
else:
logger.info('start merge into another cluster')
# 统计属于哪个cluster
sql = "select `cluster_id` from t_cluster where id in {} order by field (id, {}) " \
.format(trans_sqlin(lables), trans_sqlin(lables)[1:-1])
select_result = db.select(sql)
if select_result is not None and len(select_result) > 0:
logger.info('select from t_cluster successfully, size: ',
len(select_result))
clusters = list(map(lambda x: x[0], select_result))
assert len(clusters) == len(
distances), 'cluster select error'
cluster_dic = {} # cluster-> [distance1, distance2...]
for j in range(len(clusters)):
if clusters[j] not in cluster_dic:
cluster_dic[clusters[j]] = [distances[j]]
else:
cluster_dic[clusters[j]].append(distances[j])
for k, v in cluster_dic.items():
cluster_dic[k] = sum(v) / len(
v) # 每个cluster和目标对应的平均相似度
# 按照value的值从大到小排序, 取最大的cluster的相似度
cluster_sim = sorted(list(cluster_dic.items()),
key=lambda x: x[1],
reverse=True)[0]
# 合并成一个类
# 插入数据库和索引中
sql = "insert into `t_cluster` (`cluster_id`, `uri`, `timestamp`, `camera_id`) values (%s, %s, %s, %s)"
sample_id = db.insert(
sql,
(cluster_sim[0], msg['url'],
time_to_date(int(msg['time'])), msg['camera_id']))
if sample_id == -1:
logger.info(
'sample insert failed, mysql maybe not available')
else:
logger.info('merge into the same cluster, %s -> %s' %
(sample_id, cluster_sim[0]))
db.commit()
else:
logger.info(
'select from t_cluster failed, mysql maybe not available'
)
logger.info('discard this image')
continue
# 添加特征到索引中
logger.info('add feature to index server')
add_result = searcher.add(1, [sample_id], [feature])
if add_result is not None and add_result['rtn'] == 0:
logger.info('index add successfully, return result:',
add_result)
else:
logger.info('index add failed, return result:', add_result)
lantency = round(time.time()) - start
count += 1
logger.info('process lantency: %d, count = %d' % (lantency, count))
else:
logger.info('waiting re-cluster task over')
lock.wait(3600) # re-cluster 任务最大执行时长
re_cluster = False # 恢复运行