def __init__(self, videoinfo, logfile=None, isShow=False, savepic=False):
"""初始化
Arguments:
videoinfo {视频信息} -- {'name','datetime','descrption'}
Keyword Arguments:
logfile {string} -- 可选，log文件名 (default: {None})
isShow {bool} -- 是否显示处理过程 (default: {False})
"""
BasicPart.__init__(self, logfile, isShow)
self.videoinfo = videoinfo
self.__mkdirs()
self.dbhandler = DBHandler()
self.item_list = []
# 人物识别
self.pf = PersonFace(True)
fi = FeatureIndex(True)
fi.load_person_index(["Person"])
self.pf.setFeatureIndex(fi)
self.savepic = savepic
# solr
self.solrhandler = MainSolr(logfile, isShow)
self.save_video()
pass
def __init__(self, featureIndex=None, randomSubset=False):
if featureIndex == None: #Case when a FeatureIndex is not provided. Creates a default instance of FeatureIndex
self.featureIndex = FeatureIndex(randomSubset=randomSubset)
else:
self.featureIndex = featureIndex
self.attributeIndex = -1 #Index into the list of attributes that could be used for splitting the data
self.rs = randomSubset
self.letters = ['A', 'C', 'T', 'G'] #possible letters
self.letterToNum = dict()
for i in range(4): #Mapping ACTG to Integers
self.letterToNum[self.letters[i]] = i
def __init__(self,
max_len=1000, logfile=None, isShow=False):
BasicPart.__init__(self, logfile, isShow)
self.imagename = ''
self.facerecog = FaceRecog(logfile=logfile, isShow=isShow)
self.objectdetect = ObjectDet(logfile, single_pic_process=True,isShow=isShow, picShow=False)
self.dbhandler = DBHandler()
self.searchfeature = FeatureIndex(logfile=logfile, isShow=isShow)
self.personface = PersonFace(logfile=logfile, isShow=isShow)
self.solrobj = MainSolr(logfile=logfile, isShow=isShow)
self.max_len = max_len
# 默认阈值
self.setThreshold(800, 1000)
def splitOnIndex(self, parentNode, givenSplitIndex=None):
geneLists = []
for i in range(
4
): #Generating lists of list in order to hold the list of Genes for each possible child.
geneLists.append([])
if givenSplitIndex == None: #Case for iterating through all possible indicies in order to compute information gained.
splitIndex = self.featureIndex.getNextValue()
else: #Case using a particular attribute to split a Node.
splitIndex = givenSplitIndex
parentNode.setSplitIndex(
splitIndex
) #Setting which index a node was split on. Used for traversal.
for gene in parentNode.geneList: #Getting the letter at a given index then using the letterToNum to map it to 1 of 4 lists.
geneLists[self.letterToNum[gene.geneSequence[splitIndex]]].append(
gene) #Adding the gene to the corresponding list.
nodeList = list() #List of child Nodes to return.
for i in range(
4
): #Iterating through each possible geneList. Each geneList contains the Genes that had the same letter at splitIndex
if geneLists[i] != []:
node = Node(False, geneLists[i], parentNode,
i) #Creating new Nodes
if givenSplitIndex != None: #Removing the splitIndex from list of possible indicies to consider splitting on.
node.attributeGetter.featureIndex = self.featureIndex.createNewWithMaxValueRemoved(
givenSplitIndex)
if self.rs: #Creating a new random subset of attributes, if using random subsets.
node.attributeGetter.featureIndex = FeatureIndex(
randomSubset=True)
nodeList.append(node)
return nodeList
class MainSearch(BasicPart):
def __init__(self,
max_len=1000, logfile=None, isShow=False):
BasicPart.__init__(self, logfile, isShow)
self.imagename = ''
self.facerecog = FaceRecog(logfile=logfile, isShow=isShow)
self.objectdetect = ObjectDet(logfile, single_pic_process=True,isShow=isShow, picShow=False)
self.dbhandler = DBHandler()
self.searchfeature = FeatureIndex(logfile=logfile, isShow=isShow)
self.personface = PersonFace(logfile=logfile, isShow=isShow)
self.solrobj = MainSolr(logfile=logfile, isShow=isShow)
self.max_len = max_len
# 默认阈值
self.setThreshold(800, 1000)
def setThreshold(self, faceThreshhold, contentThreshold):
"""设置阈值,阈值越大搜索到的结果更多
Arguments:
faceThreshhold {int} -- 脸部搜索阈值
contentThreshold {int} -- 内容搜索阈值
"""
self.faceThreshhold = faceThreshhold
self.contentThreshold = contentThreshold
def load_index(self,prefixlist,person_prefix_list):
self.searchfeature.load_index(prefixlist,person_prefix_list)
self.personface.setFeatureIndex(self.searchfeature)
def set_image(self,imagename):
"""设置要搜索的图片路径
Arguments:
imagename {string} -- 图片路径
"""
self.imagename = imagename
def read_config(self):
# TODO:配置文件
self.thumb_prefix = self.config.get("search","thumb_prefix")
self.thumb_web_prefix = self.config.get("search","thumb_web_prefix")
self.thumb_size = self.config.get("search","thumb_size")
self.personinfo_dir = self.config.get("datadir","person_info")
self.thumb_info = (self.thumb_size, self.thumb_prefix, self.thumb_web_prefix)
def __get_db_info(self, id_type, id_list):
"""根据id查询数据库
id_type = 'face' or 'content'
"""
results = []
if id_type == 'face':
for faceid in id_list:
v_s_info = self.dbhandler.search_scene_video_info_by_faceid(int(faceid))
results += [SceneInfo(v_s_item) for v_s_item in v_s_info]
else:
for picid in id_list:
v_s_info = self.dbhandler.search_scene_video_info_by_picid(int(picid))
results += [SceneInfo(v_s_item) for v_s_item in v_s_info]
return set(results)
def get_face_to_video_sceneinfo(self, faceidlist):
results = self.__get_db_info('face', faceidlist)
return results
def get_content_to_video_sceneinfo(self, picidlist):
results = self.__get_db_info('content', picidlist)
return results
def create_indexs(self, perfix, featidlist=[], isSave=True):
"""创建特征的索引文件
"""
# 创建文件名:
facefeatlist = [id + "_ff.pkl" for id in featidlist]
contfeatlist = [id + "_sf.pkl" for id in featidlist]
self.searchfeature.create_facefeat_index(perfix,isSave,facefeatlist)
self.searchfeature.create_contentfeat_index(perfix,isSave,contfeatlist)
def selectByDistance(self, distance, thresh):
"""按照阈值选择符合条件的结果
Arguments:
distance {list} -- 距离列表
thresh {int} -- 距离阈值
"""
max_item = 0
for index, dist in enumerate(distance):
if dist >= thresh:
max_item = index
break
else:
max_item = index
# 截取
return max_item
def search_face(self):
pic_face_dic = self.facerecog.extract_image_face_feature(self.imagename)
num_faces = len(pic_face_dic['feats'])
if num_faces == 0:
self.lg("Face not found")
return [],[]
# 只搜索第一个脸
query_feat = pic_face_dic['feats'][0]
# print(self.prefix)
query_result,distance = self.searchfeature.queryFace(query_feat)
# 选取
max_index = self.selectByDistance(distance, self.faceThreshhold)
query_result = query_result[:max_index]
distance = distance[:max_index]
sceenid_unique = set(query_result)
return query_result, distance
def search_pic(self):
obj_ts = time.time()
image_obj_dic = self.objectdetect.extract_image_feature(self.imagename)
query_feat = image_obj_dic['feat']
tag_name = image_obj_dic['tag_name']
query_feat = np.array(query_feat)
obj_te = time.time()
feat_ts = time.time()
query_result,distance = self.searchfeature.queryContent(query_feat)
feat_te = time.time()
# 选取
max_index = self.selectByDistance(distance, self.contentThreshold)
query_result = query_result[:max_index]
distance = distance[:max_index]
# TODO: 按照sceneid 去重
query_result_unique = set(query_result)
return query_result,tag_name,distance,feat_te-feat_ts,obj_te-obj_ts
# print(result)
def show_pics(self, results):
image_names = []
for list_index,re in enumerate(results):
re = re.dic
timestamp = re['starttime']
sceneid = re['sceneid']
full_videoname = re['videoname']
videoname = full_videoname.split("/")[-1]
image_name = 'Data/Tmp/%s_s%d.jpg'%(videoname,sceneid)
timestamp = round(timestamp,1) + 0.3
cmd = '''
ffmpeg -ss %d -i %s -y -f image2 -vframes 1 -s 800x600 %s
'''%(timestamp,full_videoname,image_name)
a = subprocess.getoutput(cmd)
Image.open(image_name).show()
image_names.append(image_name)
return image_names
def searchKeywords(self, text):
"""使用solr搜索关键词
Returns:
result_json:
{
keywords{string},
video_num{int},
video_list{list[{'videoname','videopath','thumb'}]}
scene_num{int}
scene_list{list[{'videoname','videopath','thumb',starttime,sceneid,length}]}
}
"""
v_num, v_list, s_num, s_list = self.solrobj.queryKeywords(text)
# 创建略缩图
json_video_list = to_json_video(self.thumb_info, v_list)
json_scene_list = to_json_scene(self.thumb_info, s_list,False)
result_json = {}
result_json['keywords'] = text
result_json['video_num'] = len(json_video_list)
result_json['video_list'] = json_video_list
result_json['scene_num'] = len(json_scene_list)
result_json['scene_list'] = json_scene_list
return result_json
def joinsearch(self, image, keywords):
"""图片文字联合搜索
Arguments:
image {string} -- 图片路径
keywords {string} -- 关键词
"""
keywords_json = self.searchKeywords(keywords)
self.set_image(image)
image_json = self.searchImage()
def searchImage(self):
"""返回json格式的图片检索结果
Returns:
result_json:
{
keywords{string},
video_num{int},
video_list{list[{'videoname','videopath','thumb'}]},
face_scene_num{int},
face_scene_list{list[scene},
face_dist_list{list[double]},
content_scene_num{int},
content_scene_list{list[scene]},
content_dist_list{list[double]},
both_video_num{int}
both_scene_num{int}
both_scene_list{list[scene]}
personid,personname,personinfo
object_num,object_list
}
TODO: 完成物体搜索结果
"""
fs = time.time()
face_idlist, face_distance = self.search_face()[:self.max_len]
face_results = self.get_face_to_video_sceneinfo(face_idlist)
self.lg('searchImage FACE:' + str(len(face_idlist)))
fe = time.time()
cont_idlist, object_list, cont_distance,ft,ot = self.search_pic()
cont_idlist = cont_idlist[:self.max_len]
cont_results = self.get_content_to_video_sceneinfo(cont_idlist)
self.lg('searchImage CONT:' + str(len(cont_idlist)))
ts = time.time()
result_json = {}
result_json['face_scene_num'] = len(face_idlist)
result_json['face_scene_list'] = to_json_scene(self.thumb_info, face_results)
result_json['face_dist_list'] = list(face_distance)
result_json['content_scene_num'] = len(cont_idlist)
result_json['content_scene_list'] = to_json_scene(self.thumb_info, cont_results)
result_json['content_dist_list'] = list(cont_distance)
# 交集
fsids, fvids = extrace_ids(face_results)
csids, cvids = extrace_ids(cont_results)
both_scene_list = []
# both_scene_list = set(face_results) & set(cont_results)
both_video_list = set(fvids) | set(cvids)
result_json['both_video_num'] = len(both_video_list)
result_json['both_scene_num'] = len(both_scene_list)
result_json['both_scene_list'] = to_json_scene(self.thumb_info, both_scene_list)
te = time.time()
ps = time.time()
# 识别人物
pid, pname = self.personface.identify_pic_person(self.imagename)
# 读取存储的人物简介
pinfo = ''
if pid != -1:
pinfo = read_person_info(self.personinfo_dir, pid)
result_json['personid'] = pid
result_json['personname'] = pname
result_json['personinfo'] = pinfo
# 物体集合
result_json['object_num'] = len(object_list)
result_json['object_list'] = object_list
pe = time.time()
print("%.4f, %.4f, %.4f, %.4f,%.4f"%(fe-fs, ot, ft, pe-ts, pe-fs))
return result_json
def initFI(self):
"""初始化索引模块
"""
from FeatureIndex import FeatureIndex
prefixs = ['Person']
self.fi = FeatureIndex(index_prefixs=prefixs,isShow = self.isShow)
class PersonFace(BasicPart):
"""PersonFace，人物人脸录入及比对模块
Arguments:
BasicPart {[type]} -- [description]
"""
def __init__(self,
threshold = None,
logfile = None,
picShow = False,
isShow = False):
'''
threshold: 人脸大小阈值
logfile: 日志文件路径
isShow: 显示图片处理过程
'''
BasicPart.__init__(self, logfile=logfile, isShow=isShow)
self.picShow = picShow
from DBHandler import DBHandler
self.handler = DBHandler()
self.initFR()
# 缓存
self.dbcache = {}
# self.initFI()
def initFI(self):
"""初始化索引模块
"""
from FeatureIndex import FeatureIndex
prefixs = ['Person']
self.fi = FeatureIndex(index_prefixs=prefixs,isShow = self.isShow)
def initFR(self):
"""初始化人脸特征提取模块
"""
from FaceRecog import FaceRecog
self.fr = FaceRecog(isShow = self.isShow, picShow = self.picShow)
def read_config(self):
"""读配置文件
"""
self.dir = {}
self.maxdistance = self.config.getint('facerecog','max_distance')
self.thresh = self.config.getint('facerecog','threshold')
self.dir['faces'] = self.config.get('datadir','faces')
self.dir['faces_feat'] = self.config.get('datadir','faces_feat')
self.dir['faces_sample'] = self.config.get('datadir','faces_sample')
def storePersonToDB(self):
"""将人物存到PersonInfo表，返回id
"""
self.person_ids = self.handler.addmanyPerson(self.person_names)
def index_person(self, person_list = [], prefix="Person"):
"""对sample文件夹下所有人物人脸图片建立索引
或只建立某个人的索引
"""
if len(person_list) == 0:
self.person_names = os.listdir(self.dir['faces_sample'])
else:
self.person_names = person_list
# 存数据库，获取id
self.storePersonToDB()
self.person_pic_feats = []
self.person_pic_ids = []
for index, person_name in enumerate(self.person_names):
# 每一个人物对应一个文件夹
full_path = self.dir['faces_sample'] + '/' + person_name
pic_list = os.listdir(full_path)
num_faces = len(pic_list)
for face_pic in pic_list:
face_pic = full_path + '/' + face_pic
pic_face_dic = self.fr.extract_image_face_feature(face_pic)
feat = pic_face_dic['feats'][0]
self.person_pic_feats.append(feat)
self.person_pic_ids.append(self.person_ids[index])
# 建立索引
self.initFI()
self.fi.create_person_index(self.person_pic_feats, self.person_pic_ids, prefix)
def setFeatureIndex(self, fi):
"""设置FI对象，用于查询时，从外部导入已载入索引的索引对象
Arguments:
fi {FeatureIndex} -- 索引检索对象
"""
self.fi = fi
def identify_pic_person(self, imagename):
"""确定图片中人物名字以及id
Arguments:
imagename {string} -- 图片名
Returns:
personid, personname
"""
face_dic = self.fr.extract_image_face_feature(imagename)
if len(face_dic['feats']) > 0:
personid, personname = self.idenity(face_dic['feats'][0])
else:
personid=-1
personname="无人物"
return personid, personname
# print(result)
def idenity(self, facefeat):
"""确定人物身份，返回人物名及人物id
Arguments:
facefeat {data} -- 待确定的人脸特征
-1, unknown 未识别人物
"""
# 首先进行query
results, distance = self.fi.queryPerson(facefeat)
# 若高于最远距离,则为未知人物,返回None,''
if distance[0] > self.maxdistance:
personid = -1
personname = 'unknown'
else:
# 确定人物身份:
# kmean
max_count_id = pd.value_counts(results, sort=True).index[0]
# 计算每个结果的得分
personid = int(max_count_id)
# 首先查询缓存,若miss后再查询数据库
if personid in self.dbcache.keys():
personname = self.dbcache[personid]
else:
personname = self.handler.queryPersonById(personid)[0][1]
self.dbcache[personid] = personname
return personid, personname
# 若高于最远距离,则为未知人物,返回None,''
if distance[0] > self.maxdistance:
personid = -1
personname = 'unknown'
else:
# 确定人物身份:
# kmean
max_count_id = pd.value_counts(results, sort=True).index[0]
# 计算每个结果的得分
personid = int(max_count_id)
# 首先查询缓存,若miss后再查询数据库
if personid in self.dbcache.keys():
personname = self.dbcache[personid]
else:
personname = self.handler.queryPersonById(personid)[0][1]
self.dbcache[personid] = personname
return personid, personname
if __name__ == '__main__':
from FeatureIndex import FeatureIndex
pf = PersonFace(True)
fi = FeatureIndex(True)
fi.load_person_index()
pf.setFeatureIndex(fi)
pid, name = pf.identify_pic_person('Data/Tmp/t1.jpg')
print(pid)
print(name)
# PersonFace(isShow=True).index_person(person_list=["赵乐际","汪洋","栗战书","俞正声"],prefix="Person2")
# PersonFace(isShow=True).index_person(person_list=[],prefix="Person")
class MainSearch(BasicPart):
def __init__(self, max_len=1000, logfile=None, isShow=False):
BasicPart.__init__(self, logfile, isShow)
self.imagename = ''
self.facerecog = FaceRecog(logfile=logfile, isShow=isShow)
self.objectdetect = ObjectDet(logfile,
single_pic_process=True,
isShow=isShow,
picShow=False)
self.dbhandler = DBHandler()
self.searchfeature = FeatureIndex(logfile=logfile, isShow=isShow)
self.personface = PersonFace(logfile=logfile, isShow=isShow)
self.solrobj = MainSolr(logfile=logfile, isShow=isShow)
self.max_len = max_len
# 默认阈值
# self.setThreshold(800, 1000) # nmslib
self.setThreshold(800, 300000) # faiss
def setThreshold(self, faceThreshhold, contentThreshold):
"""设置阈值,阈值越大搜索到的结果更多
Arguments:
faceThreshhold {int} -- 脸部搜索阈值
contentThreshold {int} -- 内容搜索阈值
"""
self.faceThreshhold = faceThreshhold
self.contentThreshold = contentThreshold
def load_index(self, prefixlist, person_prefix_list):
self.searchfeature.load_index(self.face_index_method,
self.content_index_method, prefixlist,
person_prefix_list)
self.personface.setFeatureIndex(self.searchfeature)
def set_image(self, imagename):
"""设置要搜索的图片路径
Arguments:
imagename {string} -- 图片路径
"""
self.imagename = imagename
def read_config(self):
# TODO:配置文件
self.thumb_prefix = self.config.get("search", "thumb_prefix")
self.thumb_web_prefix = self.config.get("search", "thumb_web_prefix")
self.thumb_size = self.config.get("search", "thumb_size")
self.personinfo_dir = self.config.get("datadir", "person_info")
self.thumb_info = (self.thumb_size, self.thumb_prefix,
self.thumb_web_prefix)
self.content_distance_discount = self.config.getint(
"search", "content_distance_discount")
# 索引方法
self.content_index_method = self.config.get("search",
"content_index_method")
self.face_index_method = self.config.get("search", "face_index_method")
def __get_db_info(self, id_type, id_list, distance):
"""根据id查询数据库
id_type = 'face' or 'content'
"""
results = []
result_distance = []
if id_type == 'face':
for index, faceid in enumerate(id_list):
v_s_info = self.dbhandler.search_scene_video_info_by_faceid(
int(faceid))
# 字典中增加distance
for i in range(len(v_s_info)):
v_s_info[i][id_type + '_distance'] = distance[index]
results += [SceneInfo(v_s_item) for v_s_item in v_s_info]
for i in range(len(v_s_info)):
result_distance.append(distance[index])
else:
for index, picid in enumerate(id_list):
v_s_info = self.dbhandler.search_scene_video_info_by_picid(
int(picid))
# 字典中增加distance
for i in range(len(v_s_info)):
v_s_info[i][id_type + '_distance'] = distance[index]
results += [SceneInfo(v_s_item) for v_s_item in v_s_info]
for i in range(len(v_s_info)):
result_distance.append(distance[index])
return results, result_distance
def get_face_to_video_sceneinfo(self, faceidlist, distance):
results, re_distance = self.__get_db_info('face', faceidlist, distance)
return results, re_distance
def get_content_to_video_sceneinfo(self, picidlist, distance):
results, re_distance = self.__get_db_info('content', picidlist,
distance)
return results, re_distance
def create_indexs(self, perfix, featidlist=[], isSave=True):
"""创建特征的索引文件
"""
# 创建文件名:
facefeatlist = [id + "_ff.pkl" for id in featidlist]
contfeatlist = [id + "_sf.pkl" for id in featidlist]
self.searchfeature.create_facefeat_index(self.face_index_method,
perfix, isSave, facefeatlist)
self.searchfeature.create_contentfeat_index(self.content_index_method,
perfix, isSave,
contfeatlist)
def selectByDistance(self, distance, thresh):
"""按照阈值选择符合条件的结果
Arguments:
distance {list} -- 距离列表
thresh {int} -- 距离阈值
"""
max_item = 0
for index, dist in enumerate(distance):
if dist >= thresh:
max_item = index
break
else:
max_item = index
# 截取
return max_item
def search_face(self):
pic_face_dic = self.facerecog.extract_image_face_feature(
self.imagename)
num_faces = len(pic_face_dic['feats'])
if num_faces == 0:
self.lg("Face not found")
return [], []
# 只搜索第一个脸
query_feat = pic_face_dic['feats'][0]
# print(self.prefix)
query_result, distance = self.searchfeature.queryFace(
self.face_index_method, ['all'], query_feat)
# 选取
max_index = self.selectByDistance(distance, self.faceThreshhold)
query_result = query_result[:max_index]
distance = distance[:max_index]
return query_result, distance
def search_pic(self):
image_obj_dic = self.objectdetect.extract_image_feature(self.imagename)
query_feat = image_obj_dic['feat']
tag_name = image_obj_dic['tag_name']
query_feat = np.array(query_feat)
query_result, distance = self.searchfeature.queryContent(
self.content_index_method, ['all'], query_feat)
# print(len(distance))
distance = [i / self.content_distance_discount for i in distance]
# 选取
max_index = self.selectByDistance(distance, self.contentThreshold)
query_result = query_result[:max_index]
distance = distance[:max_index]
return query_result, tag_name, distance
# print(result)
def show_pics(self, results):
image_names = []
for list_index, re in enumerate(results):
re = re.dic
timestamp = re['starttime']
sceneid = re['sceneid']
full_videoname = re['videoname']
videoname = full_videoname.split("/")[-1]
image_name = 'Data/Tmp/%s_s%d.jpg' % (videoname, sceneid)
timestamp = round(timestamp, 1) + 0.3
cmd = '''
ffmpeg -ss %d -i %s -y -f image2 -vframes 1 -s 800x600 %s
''' % (timestamp, full_videoname, image_name)
a = subprocess.getoutput(cmd)
Image.open(image_name).show()
image_names.append(image_name)
return image_names
def searchKeywords(self, text, keepObjFormat=False):
"""使用solr搜索关键词
Returns:
result_dic:
{
keywords{string},
video_num{int},
video_list{list[{'videoname','videopath','thumb'}]}
scene_num{int}
scene_list{list[{'videoname','videopath','thumb',starttime,sceneid,length}]}
}
"""
v_num, v_list, s_num, s_list = self.solrobj.queryKeywords(text)
# 创建略缩图
json_video_list = make_video_thumb(self.thumb_info, v_list)
ns_list = []
for scene in s_list:
scene['sceneid'] = int(scene['sceneid'])
scene['videoid'] = int(scene['videoid'][0])
ns_list.append(scene)
s_list = ns_list
if not keepObjFormat:
json_scene_list = make_scene_thumb(self.thumb_info, s_list, False)
else:
# 转换为SceneInfo
json_scene_list = [SceneInfo(dic) for dic in s_list]
result_dic = {}
result_dic['keywords'] = text
result_dic['video_num'] = len(json_video_list)
result_dic['video_list'] = json_video_list
result_dic['scene_num'] = len(json_scene_list)
result_dic['scene_list'] = json_scene_list
return result_dic
def searchJoint(self, image, keywords):
"""图片文字联合搜索
Arguments:
image {string} -- 图片路径
keywords {string} -- 关键词
Returns:
result_dic:
{
keywords{string},
face_scene_num{int},
face_scene_list{list[scene},
face_dist_list{list[double]},
content_scene_num{int},
content_scene_list{list[scene]},
content_dist_list{list[double]},
smart_scene_num{int},
smart_scene_list{list[scene]},
keywords_scene_num{int},
keywords_scene_list{list[scene]},
both_video_num{int}
both_scene_num{int}
both_scene_list{list[scene]}
personid,personname,personinfo
object_num,object_list
}
"""
keywords_scenes = self.searchKeywords(keywords, keepObjFormat=True)
image_scenes = self.searchImage(image, keepObjFormat=True)
# 结果综合 TODO
smart_scene_list, both_scene_list = self.smartsort(
keywords_scenes['scene_list'], image_scenes['content_scene_list'],
image_scenes['face_scene_list'], image_scenes['content_dist_list'],
image_scenes['face_dist_list'])
result_dic = image_scenes
# 更新both
result_dic['both_scene_num'] = len(both_scene_list)
result_dic['both_scene_list'] = make_scene_thumb(
self.thumb_info, both_scene_list)
result_dic['smart_scene_num'] = len(smart_scene_list)
result_dic['smart_scene_list'] = make_scene_thumb(
self.thumb_info, smart_scene_list)
result_dic['keywords_scene_num'] = len(keywords_scenes['scene_list'])
result_dic['keywords_scene_list'] = make_scene_thumb(
self.thumb_info, keywords_scenes['scene_list'])
result_dic['face_scene_list'] = make_scene_thumb(
self.thumb_info, result_dic['face_scene_list'])
result_dic['content_scene_list'] = make_scene_thumb(
self.thumb_info, result_dic['content_scene_list'])
result_dic['keywords'] = keywords_scenes['keywords']
return result_dic
def compute_score(self, list, keyname, decay):
# 从前到后,按照100, 100-(100/size), 100-2*(100/size)...计算得分
# 加权距离的指数 log(1/distance)
# 总分 =
list_size = len(list)
if list_size > 0:
delta = 100 / list_size
delta = decay
scores = [100 - i * delta for i in range(list_size)]
for index, item in enumerate(list):
item.dic[keyname] = scores[index]
return list
def smartsort(self, klist, clist, flist, cdistance, fdistance):
"""多来源结果智能排序
Arguments:
klist {list} -- 关键词搜索结果
clist {list} -- content搜索结果
flist {list} -- face搜索结果
cdistance {list} -- content搜索距离
fdistance {list} -- face搜索距离
Returns:
list -- 智能排序结果
"""
klist = self.compute_score(klist, 'keyword_scores', 0.44)
clist = self.compute_score(clist, 'content_scores', 0.42)
flist = self.compute_score(flist, 'face_scores', 0.4)
Kset = set(klist)
Cset = set(clist)
Fset = set(flist)
smart_list = []
keyword_weight = 0.7
content_weight = 0.78
face_weight = 0.8
for content_item in clist:
sum_score_item = deepcopy(content_item)
sum_score_item.dic[
'sum_scores'] = content_weight * sum_score_item.dic[
'content_scores']
if sum_score_item in Fset:
face_score = flist[flist.index(
sum_score_item)].dic['face_scores']
sum_score_item.dic['sum_scores'] += face_weight * face_score
if sum_score_item in Kset:
key_score = klist[klist.index(
sum_score_item)].dic['keyword_scores']
sum_score_item.dic['sum_scores'] += keyword_weight * key_score
smart_list.append(sum_score_item)
Sset = set(smart_list)
for face_item in flist:
if face_item in Sset:
continue
sum_score_item = deepcopy(face_item)
sum_score_item.dic[
'sum_scores'] = face_weight * sum_score_item.dic['face_scores']
if sum_score_item in Cset:
content_score = clist[clist.index(
sum_score_item)].dic['content_scores']
sum_score_item.dic[
'sum_scores'] += content_weight * content_score
if sum_score_item in Kset:
key_score = klist[klist.index(
sum_score_item)].dic['keyword_scores']
sum_score_item.dic['sum_scores'] += keyword_weight * key_score
smart_list.append(sum_score_item)
Sset = set(smart_list)
for key_item in klist:
if key_item in Sset:
continue
sum_score_item = deepcopy(key_item)
sum_score_item.dic[
'sum_scores'] = keyword_weight * sum_score_item.dic[
'keyword_scores']
if sum_score_item in Cset:
content_score = clist[clist.index(
sum_score_item)].dic['content_scores']
sum_score_item.dic[
'sum_scores'] += content_weight * content_score
if sum_score_item in Fset:
face_score = flist[flist.index(
sum_score_item)].dic['face_scores']
sum_score_item.dic['sum_scores'] += face_weight * face_score
smart_list.append(sum_score_item)
# 对smart_list按照 sum_scores 排序
smart_list = sorted(smart_list,
key=lambda x: x.dic['sum_scores'],
reverse=True)
if len(klist) == 0:
both_list = list(Cset & Fset)
else:
both_list = list(Kset & Cset & Fset)
return smart_list, both_list
def searchImage(self, imagename, keepObjFormat=False):
"""返回json格式的图片检索结果
Returns:
result_dic:
{
video_num{int},
video_list{list[{'videoname','videopath','thumb'}]},
face_scene_num{int},
face_scene_list{list[scene},
face_dist_list{list[double]},
content_scene_num{int},
content_scene_list{list[scene]},
content_dist_list{list[double]},
both_video_num{int}
both_scene_num{int}
both_scene_list{list[scene]}
personid,personname,personinfo
object_num,object_list
}
TODO: 完成物体搜索结果
"""
self.imagename = imagename
# 由于relate加入,所以需要对返回结果进行处理
face_idlist, face_distance = self.search_face()
face_results, face_distance = self.get_face_to_video_sceneinfo(
face_idlist, face_distance)
cont_idlist, object_list, cont_distance = self.search_pic()
cont_idlist = cont_idlist
cont_results, cont_distance = self.get_content_to_video_sceneinfo(
cont_idlist, cont_distance)
result_dic = {}
# face
result_dic['face_scene_num'] = len(face_results)
if not keepObjFormat:
result_dic['face_scene_list'] = make_scene_thumb(
self.thumb_info, face_results)
else:
result_dic['face_scene_list'] = face_results
result_dic['face_dist_list'] = list(face_distance)
# content
result_dic['content_scene_num'] = len(cont_results)
if not keepObjFormat:
result_dic['content_scene_list'] = make_scene_thumb(
self.thumb_info, cont_results)
else:
result_dic['content_scene_list'] = cont_results
result_dic['content_dist_list'] = list(cont_distance)
# 交集
_, fvids = extrace_ids(face_results)
_, cvids = extrace_ids(cont_results)
both_video_list = set(fvids) | set(cvids)
result_dic['both_video_num'] = len(both_video_list)
smart_scene_list, both_list = self.smartsort([], cont_results,
face_results,
cont_distance,
face_distance)
result_dic['both_scene_num'] = len(both_list)
if not keepObjFormat:
result_dic['both_scene_list'] = make_scene_thumb(
self.thumb_info, both_list)
else:
result_dic['both_scene_list'] = both_list
result_dic['smart_scene_num'] = len(smart_scene_list)
if not keepObjFormat:
result_dic['smart_scene_list'] = make_scene_thumb(
self.thumb_info, smart_scene_list)
else:
result_dic['smart_scene_list'] = smart_scene_list
# 识别人物
pid, pname = self.personface.identify_pic_person(self.imagename)
# 读取存储的人物简介
pinfo = ''
if pid != -1:
pinfo = read_person_info(self.personinfo_dir, pid)
result_dic['personid'] = pid
result_dic['personname'] = pname
result_dic['personinfo'] = pinfo
# 物体集合
result_dic['object_num'] = len(object_list)
result_dic['object_list'] = object_list
return result_dic
class AttributeGetter():
def __init__(self, featureIndex=None, randomSubset=False):
if featureIndex == None: #Case when a FeatureIndex is not provided. Creates a default instance of FeatureIndex
self.featureIndex = FeatureIndex(randomSubset=randomSubset)
else:
self.featureIndex = featureIndex
self.attributeIndex = -1 #Index into the list of attributes that could be used for splitting the data
self.rs = randomSubset
self.letters = ['A', 'C', 'T', 'G'] #possible letters
self.letterToNum = dict()
for i in range(4): #Mapping ACTG to Integers
self.letterToNum[self.letters[i]] = i
"""
Determines if the Node should stop splitting. First checks if there is only one classification
in the Node's geneList. Next checks to see if the list of possible attributes to split on has
been exhausted. Then if using a random subset of features checks if the max depth has been exceeded.
The arguements are:
node: The Node that is being evaluated to either continue or stop splitting.
"""
def shouldStopSplitting(self, node):
stopSplitting = False
classSum = node.numN + node.numEI + node.numIE
if classSum == node.numN or classSum == node.numEI or classSum == node.numIE:
stopSplitting = True
if len(node.attributeGetter.featureIndex.attributes) == 0:
stopSplitting = True
if self.rs and node.depth >= 10:
stopSplitting = True
if stopSplitting:
node.isLeaf = True
node.getMaxClass()
#Need to change to CHI-Squared model
"""
This function is used for partitioning a Node's list of Gene's into child nodes.
This is done both for constructing the tree and for finding the best attribute to
split on based on information gained.
The arguements are:
parentNode: The Node from which the child Nodes are being created.
givenSplitIndex: the index on which to split parentNode.
"""
def splitOnIndex(self, parentNode, givenSplitIndex=None):
geneLists = []
for i in range(
4
): #Generating lists of list in order to hold the list of Genes for each possible child.
geneLists.append([])
if givenSplitIndex == None: #Case for iterating through all possible indicies in order to compute information gained.
splitIndex = self.featureIndex.getNextValue()
else: #Case using a particular attribute to split a Node.
splitIndex = givenSplitIndex
parentNode.setSplitIndex(
splitIndex
) #Setting which index a node was split on. Used for traversal.
for gene in parentNode.geneList: #Getting the letter at a given index then using the letterToNum to map it to 1 of 4 lists.
geneLists[self.letterToNum[gene.geneSequence[splitIndex]]].append(
gene) #Adding the gene to the corresponding list.
nodeList = list() #List of child Nodes to return.
for i in range(
4
): #Iterating through each possible geneList. Each geneList contains the Genes that had the same letter at splitIndex
if geneLists[i] != []:
node = Node(False, geneLists[i], parentNode,
i) #Creating new Nodes
if givenSplitIndex != None: #Removing the splitIndex from list of possible indicies to consider splitting on.
node.attributeGetter.featureIndex = self.featureIndex.createNewWithMaxValueRemoved(
givenSplitIndex)
if self.rs: #Creating a new random subset of attributes, if using random subsets.
node.attributeGetter.featureIndex = FeatureIndex(
randomSubset=True)
nodeList.append(node)
return nodeList
"""
Used for travering the tree. If the letter of the Gene at givenSplitIndex is one
of ACTG then return the Integer mapping of that letter. If the letter of the Gene
at givenSplitIndex is one of DNRS then return a negative value.
The arguements are:
gene: The Gene that is traversing the tree to be classified.
givenSplitIndex: The index of the letter to be returned.
"""
def getLetterForTraversal(self, gene, givenSplitIndex):
letter = gene.geneSequence[givenSplitIndex]
if letter in self.letters:
return self.letterToNum[gene.geneSequence[givenSplitIndex]]
else:
if letter == 'D':
return -1
elif letter == 'N':
return -2
elif letter == 'R':
return -3
else:
return -4
"""
This method iterates through all available attributes, splits the data on that
attribute, then stores the result of the information gaines in a dictionary.
After all available values have been considers it returns the argMax.
The arguements are:
node: The Node that is being split by all available attributes to find the
the attribute that yields the most information gained.
"""
def getBestAttribute(self, node):
for i in range(len(self.featureIndex.attributes)):
nodesToTest = self.splitOnIndex(
node
) #Nodes resulting from splitting the parameter Node the next index.
nodesToTestValues = [
[n.numEI, n.numIE, n.numN] for n in nodesToTest
] #Seperating the counts of each class to pass to info gained
ig = InformationGained(
nodesToTestValues
) #Creating a new InformationGained object with a given impurity function.
self.featureIndex.infoGained[self.featureIndex.attributes[
self.featureIndex.currentIndex]] = ig.get_ig(
) #Storing the info gained in a dict
maxIndex = self.featureIndex.argMax(
) #Getting the index that maximized information gain.
self.featureIndex.reset() #Clearing the dict
return maxIndex