class ImgStatusChk(object):
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"],
conf["db_pass"])
def run(self):
sql = "select pic_dir, fname, `md5` from images where status = '1' "
res = self.db.query(sql)
i = 0
for one in res:
i += 1
imgpath = ""
try:
imgpath = self.conf["img_path"] + one[0] + "/" + one[1]
tmpimg = cv.LoadImage(imgpath)
print i, cv.GetSize(tmpimg)
sql = "update images set status = '2' where `md5` = '%s'" % (
one[2])
self.db.execute(sql)
except:
sql = "update images set status = 'z' where `md5` = '%s'" % (
one[2])
print sql
self.db.execute(sql)
try:
if not imgpath:
os.remove(imgpath)
except:
1
def testPrint(self):
print "Hello World!"
def __init__(self):
super(MainDev, self).__init__()
self._logger = logging.getLogger(str(__name__))
interval = SysConfig.config["servers"]["interval"]
if not interval:
interval = 200
self._main_proc = MainProc(self)
self.addProcess(self._main_proc, interval)
mailProcCount = SysConfig.config["mail"]["mailProcCount"]
self._mail_procs = []
for i in range(mailProcCount):
mailProc = MailProc(self)
mailProc.setIndex(i)
self._mail_procs.append(mailProc)
self.addProcess(mailProc, interval)
self._work_procs = []
workProcCount = SysConfig.config['servers']['workProcCount']
for i in range(workProcCount):
workProc = WorkProc(self)
self._work_procs.append(workProc)
self.addProcess(workProc, interval)
host = SysConfig.config['db']['host']
port = SysConfig.config['db']['port']
user = SysConfig.config['db']['user']
passwd = SysConfig.config['db']['passwd']
dbName = SysConfig.config['db']['db']
charset = SysConfig.config['db']['charset']
self._db = Mysql(host, port, user, passwd, dbName, charset)
pass
def exactor(self, one):
faces = one[1].split(';')
face_dir = one[-1]
for face in faces:
all_finished = True
if face:
face_path = face_dir + one[2] + '/' + face
cmd = "../cmodel/featurebr/build/feature %s %s " % (
"FaceRecognition", face_path)
print cmd
res = exeCmd(cmd)
data = res.strip(',').split(',')
ldata = len(data)
if ldata != self.conf['item_len']:
print "len of data error: %d" % ldata
all_finished = False
continue
data = self.indexData(data)
data['hash_id'] = one[0]
data['path'] = face_path
data['uri'] = one[3]
res = self.es.index(index="face",
doc_type="feature",
body=data)
if all_finished:
db = Mysql(self.conf["db_host"], self.conf["db_database"],
self.conf["db_user"], self.conf["db_pass"])
sql = "update images set status = '4' where `md5` = '%s' " % one[
0]
db.execute(sql)
db.close()
return
class DataChecker(object):
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"],
conf["db_pass"])
self.rc = []
def getOneRecord(self):
if len(self.rc) <= 0:
sql = "select spec, count(*), pic_dir from images where pic_dir is null and (type = 'star_female' or type = 'star_male') group by spec order by spec limit 50"
self.rc = self.db.query(sql)
if len(self.rc) <= 0:
print "empty"
exit()
one = self.rc.pop()
print one[0], one[1], one[2]
return one[0]
def run(self):
spec = ""
while (True):
spec = self.getOneRecord()
cmd = raw_input()
sql = ""
if cmd == "exit":
break
else:
sql = "update images set pic_dir = '%s' where pic_dir is null and spec = '%s'" % (
cmd, spec)
self.db.execute(sql)
print sql
return
def testPrint(self):
print "Hello World!"
def finishPic(self, pic): # opencv
face_cascade = cv2.CascadeClassifier(self.conf["cascade"])
imgpath = self.conf['img_path'] + pic[0] + '/' + pic[1]
img = cv2.imread(imgpath)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.5, 7)
i = 0
faces_str = ""
for (x,y,w,h) in faces:
faces_str += pic[1] + ".face_"+str(i)+".jpg;"
face_path = pic[3] + pic[1] + ".face_"+str(i)+".jpg"
tmpimg = cv.LoadImage(imgpath)
cv.SetImageROI(tmpimg, (x, y, w, int(h*1.15)))
cv.SaveImage(face_path,tmpimg)
print face_path
i += 1
db = Mysql(self.conf["db_host"], self.conf["db_database"], self.conf["db_user"], self.conf["db_pass"])
sql = "update images set status='3', face_dir='%s' where `md5`='%s'" % (faces_str, pic[2])
db.execute(sql)
db.close()
return
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.es = Elasticsearch(conf["nodes"], timeout=10000)
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"],
conf["db_pass"])
class IndexCreator(object):
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.es = Elasticsearch(conf["nodes"], timeout=10000)
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"],
conf["db_pass"])
def groovyCode(self):
print "groovy code is: "
code = "-("
for i in range(self.conf["item_len"]):
code += "abs(doc['f%d'].value-f%d) + " % (i, i)
code += "0)"
print code
writeToFile(self.conf["groovy_path"] + self.conf['groovy_file'], code)
return
def indexData(self, data):
index_data = {}
for i in range(len(data)):
index_data["f" + str(i)] = int(data[i])
return index_data
def loadDatas(self):
sql = "select `md5`, face_dir, pic_dir, uri from images where status = '3' limit 1000"
res = self.db.query(sql)
return res
def exactor(self, one):
faces = one[1].split(';')
face_dir = one[-1]
for face in faces:
all_finished = True
if face:
face_path = face_dir + one[2] + '/' + face
cmd = "../cmodel/featurebr/build/feature %s %s " % (
"FaceRecognition", face_path)
print cmd
res = exeCmd(cmd)
data = res.strip(',').split(',')
ldata = len(data)
if ldata != self.conf['item_len']:
print "len of data error: %d" % ldata
all_finished = False
continue
data = self.indexData(data)
data['hash_id'] = one[0]
data['path'] = face_path
data['uri'] = one[3]
res = self.es.index(index="face",
doc_type="feature",
body=data)
if all_finished:
db = Mysql(self.conf["db_host"], self.conf["db_database"],
self.conf["db_user"], self.conf["db_pass"])
sql = "update images set status = '4' where `md5` = '%s' " % one[
0]
db.execute(sql)
db.close()
return
def run(self):
self.es.indices.delete("face")
res = self.es.indices.create(index="face")
print res
self.es.indices.flush("face")
face_dir = self.conf["img_path"] + 'face/'
th = WorkerManager(self.conf["thread"])
while True:
ls = self.loadDatas()
if len(ls) <= 0:
break
for one in ls:
one.append(face_dir)
# self.exactor(one)
th.add_job(self.exactor, [one])
th.wait_for_complete()
return
def test(self):
cmd = '../cmodel/featurebr/build/feature ' + sys.argv[1]
res = exeCmd(cmd)
data = res.strip(',').split(',')
ldata = len(data)
if ldata != self.conf['item_len']:
print "len of data error: %d" % ldata
exit()
data = self.indexData(data)
print data
q = {
"query": {
"function_score": {
"functions": [{
"script_score": {
"params": data,
"script": "distance_face"
}
}],
"query": {
"match_all": {}
},
"score_mode":
"first"
}
},
"_source": ["uri", "path", "hash_id"],
"from": 0,
"size": self.conf["search_limit"],
}
res = self.es.search(index="face", body=q, timeout=100000)
printJson(res)
return res
def testPrint(self):
print "Hello World!"
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"])
class Detecter(object):
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"])
def test(self):
if len(sys.argv) < 2:
self.help()
#print dir(cv2)
face_cascade = cv2.CascadeClassifier(sys.argv[1])
imgpath = sys.argv[2]
img = cv2.imread(imgpath)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.1, 7)
i = 0
for (x,y,w,h) in faces:
tmpimg = cv.LoadImage(imgpath)
cv.SetImageROI(tmpimg, (x, y, w, int(h*1.15)))
cv.SaveImage("face_"+str(i)+".jpg",tmpimg)
return
def help(self):
print "Usage:", sys.argv[0], "cascade.xml pic.jpg"
exit()
def loadDatas(self):
sql = "select pic_dir, fname, `md5` from images where status = '2' \
and (type like 'star_%' or type like 'meinv') \
limit 1000"
res = self.db.query(sql)
return res
def finishPic(self, pic): # opencv
face_cascade = cv2.CascadeClassifier(self.conf["cascade"])
imgpath = self.conf['img_path'] + pic[0] + '/' + pic[1]
img = cv2.imread(imgpath)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.5, 7)
i = 0
faces_str = ""
for (x,y,w,h) in faces:
faces_str += pic[1] + ".face_"+str(i)+".jpg;"
face_path = pic[3] + pic[1] + ".face_"+str(i)+".jpg"
tmpimg = cv.LoadImage(imgpath)
cv.SetImageROI(tmpimg, (x, y, w, int(h*1.15)))
cv.SaveImage(face_path,tmpimg)
print face_path
i += 1
db = Mysql(self.conf["db_host"], self.conf["db_database"], self.conf["db_user"], self.conf["db_pass"])
sql = "update images set status='3', face_dir='%s' where `md5`='%s'" % (faces_str, pic[2])
db.execute(sql)
db.close()
return
def run(self):
face_path = self.conf["img_path"] + 'face/'
if not os.path.exists(face_path):
os.mkdir(face_path)
th = WorkerManager(self.conf["thread"])
while True:
ls = self.loadDatas()
if len(ls) <= 0:
break
for one in ls:
face_path = self.conf["img_path"] + 'face/' + one[0] + '/'
if not os.path.exists(face_path):
os.mkdir(face_path)
one.append(face_path)
th.add_job(self.finishPic, [one])
th.wait_for_complete()
return
def testPrint(self):
print "Hello World!"
class ImgSet(object):
def __init__(self, conf):
self.conf = {}
self.conf = conf
self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"],
conf["db_pass"])
def loadsData(self, data):
print "loadsData should be rewrite!"
return data
def uriTemplate(self, uri_tmp, uri_referer, data_key):
self.uri = uri_tmp
self.referer = uri_referer
self.data_key = data_key
return
def setUriParam(self, data):
self.s = data
for k in data.keys():
self.uri = self.uri.replace("<%s>" % (k), urllib.quote(data[k]))
self.referer = self.referer.replace("<%s>" % (k),
urllib.quote(data[k]))
return
def fetchSet(self, start=0, limit=60):
res = {}
uri = self.uri.replace("<start>", str(start))
uri = uri.replace("<limit>", str(limit))
#print uri
#print self.referer
text = fetchUrl(uri, param={"referer": self.referer})
res = self.loadsData(text)
return res
def save(self, data, loop_i):
sql = "select `md5`, tags from images where `md5` = '%s' limit 1" % (
data["md5"])
res = self.db.query(sql)
#print res
tag = data["tags"]
if len(res) > 0: # update
tags = res[0][1].split(",")
tags_dic = {}
for t in tags:
tags_dic[md5(t)] = 1
if not tags_dic.has_key(md5(tag)):
sql = "update images set tags = concat(tags, ',%s') where `md5` = '%s'" % (
tag, data["md5"])
#print sql
self.db.execute(sql)
else: # insert
keys = ""
values = ""
for k in data.keys():
keys += " `%s`, " % (k)
values += " '%s' ," % (data[k])
keys = keys.strip(" ,") + ", createtime, fetch_loop "
values = values.strip(" ,") + ", now(), %s " % (str(loop_i))
sql = "insert into images ( %s ) values( %s )" % (keys, values)
self.db.execute(sql)
#print keys, values
#print sql
#exit()
return
def testPrint(self):
print "Hello World!"
class MainDev(SDevice):
_logger = None
_main_proc = None
_mail_procs = None
_work_procs = None
_db = None
instance = None
mutex = threading.Lock()
def __init__(self):
super(MainDev, self).__init__()
self._logger = logging.getLogger(str(__name__))
interval = SysConfig.config["servers"]["interval"]
if not interval:
interval = 200
self._main_proc = MainProc(self)
self.addProcess(self._main_proc, interval)
mailProcCount = SysConfig.config["mail"]["mailProcCount"]
self._mail_procs = []
for i in range(mailProcCount):
mailProc = MailProc(self)
mailProc.setIndex(i)
self._mail_procs.append(mailProc)
self.addProcess(mailProc, interval)
self._work_procs = []
workProcCount = SysConfig.config['servers']['workProcCount']
for i in range(workProcCount):
workProc = WorkProc(self)
self._work_procs.append(workProc)
self.addProcess(workProc, interval)
host = SysConfig.config['db']['host']
port = SysConfig.config['db']['port']
user = SysConfig.config['db']['user']
passwd = SysConfig.config['db']['passwd']
dbName = SysConfig.config['db']['db']
charset = SysConfig.config['db']['charset']
self._db = Mysql(host, port, user, passwd, dbName, charset)
pass
@staticmethod
def GetInstance():
if(MainDev.instance == None):
MainDev.mutex.acquire()
if(MainDev.instance == None):
MainDev.instance = MainDev()
MainDev.mutex.release()
return MainDev.instance
pass
def getMainProc(self):
return self._main_proc
pass
def getRedis(self):
return self._redis
pass
def start(self):
super(MainDev, self).start()
row = self._db.query("select num_iid from ftxia_items order by id desc")
for r in row:
AppQueue.numiidQueue.put(r[0])
self._logger.debug("numiid count %d"%(AppQueue.numiidQueue.qsize()))
self._logger.debug("server start ........")
def close(self):
self._logger.debug("server close ........")
super(MainDev,self).close()
self._db.close()
def getDb(self):
return self._db
def getFactory(self):
return self._factory
pass
