class ImgStatusChk(object): def __init__(self, conf): self.conf = {} self.conf = conf self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"]) def run(self): sql = "select pic_dir, fname, `md5` from images where status = '1' " res = self.db.query(sql) i = 0 for one in res: i += 1 imgpath = "" try: imgpath = self.conf["img_path"] + one[0] + "/" + one[1] tmpimg = cv.LoadImage(imgpath) print i, cv.GetSize(tmpimg) sql = "update images set status = '2' where `md5` = '%s'" % ( one[2]) self.db.execute(sql) except: sql = "update images set status = 'z' where `md5` = '%s'" % ( one[2]) print sql self.db.execute(sql) try: if not imgpath: os.remove(imgpath) except: 1 def testPrint(self): print "Hello World!"
def __init__(self): super(MainDev, self).__init__() self._logger = logging.getLogger(str(__name__)) interval = SysConfig.config["servers"]["interval"] if not interval: interval = 200 self._main_proc = MainProc(self) self.addProcess(self._main_proc, interval) mailProcCount = SysConfig.config["mail"]["mailProcCount"] self._mail_procs = [] for i in range(mailProcCount): mailProc = MailProc(self) mailProc.setIndex(i) self._mail_procs.append(mailProc) self.addProcess(mailProc, interval) self._work_procs = [] workProcCount = SysConfig.config['servers']['workProcCount'] for i in range(workProcCount): workProc = WorkProc(self) self._work_procs.append(workProc) self.addProcess(workProc, interval) host = SysConfig.config['db']['host'] port = SysConfig.config['db']['port'] user = SysConfig.config['db']['user'] passwd = SysConfig.config['db']['passwd'] dbName = SysConfig.config['db']['db'] charset = SysConfig.config['db']['charset'] self._db = Mysql(host, port, user, passwd, dbName, charset) pass
def exactor(self, one): faces = one[1].split(';') face_dir = one[-1] for face in faces: all_finished = True if face: face_path = face_dir + one[2] + '/' + face cmd = "../cmodel/featurebr/build/feature %s %s " % ( "FaceRecognition", face_path) print cmd res = exeCmd(cmd) data = res.strip(',').split(',') ldata = len(data) if ldata != self.conf['item_len']: print "len of data error: %d" % ldata all_finished = False continue data = self.indexData(data) data['hash_id'] = one[0] data['path'] = face_path data['uri'] = one[3] res = self.es.index(index="face", doc_type="feature", body=data) if all_finished: db = Mysql(self.conf["db_host"], self.conf["db_database"], self.conf["db_user"], self.conf["db_pass"]) sql = "update images set status = '4' where `md5` = '%s' " % one[ 0] db.execute(sql) db.close() return
class DataChecker(object): def __init__(self, conf): self.conf = {} self.conf = conf self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"]) self.rc = [] def getOneRecord(self): if len(self.rc) <= 0: sql = "select spec, count(*), pic_dir from images where pic_dir is null and (type = 'star_female' or type = 'star_male') group by spec order by spec limit 50" self.rc = self.db.query(sql) if len(self.rc) <= 0: print "empty" exit() one = self.rc.pop() print one[0], one[1], one[2] return one[0] def run(self): spec = "" while (True): spec = self.getOneRecord() cmd = raw_input() sql = "" if cmd == "exit": break else: sql = "update images set pic_dir = '%s' where pic_dir is null and spec = '%s'" % ( cmd, spec) self.db.execute(sql) print sql return def testPrint(self): print "Hello World!"
def finishPic(self, pic): # opencv face_cascade = cv2.CascadeClassifier(self.conf["cascade"]) imgpath = self.conf['img_path'] + pic[0] + '/' + pic[1] img = cv2.imread(imgpath) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray, 1.5, 7) i = 0 faces_str = "" for (x,y,w,h) in faces: faces_str += pic[1] + ".face_"+str(i)+".jpg;" face_path = pic[3] + pic[1] + ".face_"+str(i)+".jpg" tmpimg = cv.LoadImage(imgpath) cv.SetImageROI(tmpimg, (x, y, w, int(h*1.15))) cv.SaveImage(face_path,tmpimg) print face_path i += 1 db = Mysql(self.conf["db_host"], self.conf["db_database"], self.conf["db_user"], self.conf["db_pass"]) sql = "update images set status='3', face_dir='%s' where `md5`='%s'" % (faces_str, pic[2]) db.execute(sql) db.close() return
def __init__(self, conf): self.conf = {} self.conf = conf self.es = Elasticsearch(conf["nodes"], timeout=10000) self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"])
class IndexCreator(object): def __init__(self, conf): self.conf = {} self.conf = conf self.es = Elasticsearch(conf["nodes"], timeout=10000) self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"]) def groovyCode(self): print "groovy code is: " code = "-(" for i in range(self.conf["item_len"]): code += "abs(doc['f%d'].value-f%d) + " % (i, i) code += "0)" print code writeToFile(self.conf["groovy_path"] + self.conf['groovy_file'], code) return def indexData(self, data): index_data = {} for i in range(len(data)): index_data["f" + str(i)] = int(data[i]) return index_data def loadDatas(self): sql = "select `md5`, face_dir, pic_dir, uri from images where status = '3' limit 1000" res = self.db.query(sql) return res def exactor(self, one): faces = one[1].split(';') face_dir = one[-1] for face in faces: all_finished = True if face: face_path = face_dir + one[2] + '/' + face cmd = "../cmodel/featurebr/build/feature %s %s " % ( "FaceRecognition", face_path) print cmd res = exeCmd(cmd) data = res.strip(',').split(',') ldata = len(data) if ldata != self.conf['item_len']: print "len of data error: %d" % ldata all_finished = False continue data = self.indexData(data) data['hash_id'] = one[0] data['path'] = face_path data['uri'] = one[3] res = self.es.index(index="face", doc_type="feature", body=data) if all_finished: db = Mysql(self.conf["db_host"], self.conf["db_database"], self.conf["db_user"], self.conf["db_pass"]) sql = "update images set status = '4' where `md5` = '%s' " % one[ 0] db.execute(sql) db.close() return def run(self): self.es.indices.delete("face") res = self.es.indices.create(index="face") print res self.es.indices.flush("face") face_dir = self.conf["img_path"] + 'face/' th = WorkerManager(self.conf["thread"]) while True: ls = self.loadDatas() if len(ls) <= 0: break for one in ls: one.append(face_dir) # self.exactor(one) th.add_job(self.exactor, [one]) th.wait_for_complete() return def test(self): cmd = '../cmodel/featurebr/build/feature ' + sys.argv[1] res = exeCmd(cmd) data = res.strip(',').split(',') ldata = len(data) if ldata != self.conf['item_len']: print "len of data error: %d" % ldata exit() data = self.indexData(data) print data q = { "query": { "function_score": { "functions": [{ "script_score": { "params": data, "script": "distance_face" } }], "query": { "match_all": {} }, "score_mode": "first" } }, "_source": ["uri", "path", "hash_id"], "from": 0, "size": self.conf["search_limit"], } res = self.es.search(index="face", body=q, timeout=100000) printJson(res) return res def testPrint(self): print "Hello World!"
def __init__(self, conf): self.conf = {} self.conf = conf self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"])
class Detecter(object): def __init__(self, conf): self.conf = {} self.conf = conf self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"]) def test(self): if len(sys.argv) < 2: self.help() #print dir(cv2) face_cascade = cv2.CascadeClassifier(sys.argv[1]) imgpath = sys.argv[2] img = cv2.imread(imgpath) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray, 1.1, 7) i = 0 for (x,y,w,h) in faces: tmpimg = cv.LoadImage(imgpath) cv.SetImageROI(tmpimg, (x, y, w, int(h*1.15))) cv.SaveImage("face_"+str(i)+".jpg",tmpimg) return def help(self): print "Usage:", sys.argv[0], "cascade.xml pic.jpg" exit() def loadDatas(self): sql = "select pic_dir, fname, `md5` from images where status = '2' \ and (type like 'star_%' or type like 'meinv') \ limit 1000" res = self.db.query(sql) return res def finishPic(self, pic): # opencv face_cascade = cv2.CascadeClassifier(self.conf["cascade"]) imgpath = self.conf['img_path'] + pic[0] + '/' + pic[1] img = cv2.imread(imgpath) gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray, 1.5, 7) i = 0 faces_str = "" for (x,y,w,h) in faces: faces_str += pic[1] + ".face_"+str(i)+".jpg;" face_path = pic[3] + pic[1] + ".face_"+str(i)+".jpg" tmpimg = cv.LoadImage(imgpath) cv.SetImageROI(tmpimg, (x, y, w, int(h*1.15))) cv.SaveImage(face_path,tmpimg) print face_path i += 1 db = Mysql(self.conf["db_host"], self.conf["db_database"], self.conf["db_user"], self.conf["db_pass"]) sql = "update images set status='3', face_dir='%s' where `md5`='%s'" % (faces_str, pic[2]) db.execute(sql) db.close() return def run(self): face_path = self.conf["img_path"] + 'face/' if not os.path.exists(face_path): os.mkdir(face_path) th = WorkerManager(self.conf["thread"]) while True: ls = self.loadDatas() if len(ls) <= 0: break for one in ls: face_path = self.conf["img_path"] + 'face/' + one[0] + '/' if not os.path.exists(face_path): os.mkdir(face_path) one.append(face_path) th.add_job(self.finishPic, [one]) th.wait_for_complete() return def testPrint(self): print "Hello World!"
class ImgSet(object): def __init__(self, conf): self.conf = {} self.conf = conf self.db = Mysql(conf["db_host"], conf["db_database"], conf["db_user"], conf["db_pass"]) def loadsData(self, data): print "loadsData should be rewrite!" return data def uriTemplate(self, uri_tmp, uri_referer, data_key): self.uri = uri_tmp self.referer = uri_referer self.data_key = data_key return def setUriParam(self, data): self.s = data for k in data.keys(): self.uri = self.uri.replace("<%s>" % (k), urllib.quote(data[k])) self.referer = self.referer.replace("<%s>" % (k), urllib.quote(data[k])) return def fetchSet(self, start=0, limit=60): res = {} uri = self.uri.replace("<start>", str(start)) uri = uri.replace("<limit>", str(limit)) #print uri #print self.referer text = fetchUrl(uri, param={"referer": self.referer}) res = self.loadsData(text) return res def save(self, data, loop_i): sql = "select `md5`, tags from images where `md5` = '%s' limit 1" % ( data["md5"]) res = self.db.query(sql) #print res tag = data["tags"] if len(res) > 0: # update tags = res[0][1].split(",") tags_dic = {} for t in tags: tags_dic[md5(t)] = 1 if not tags_dic.has_key(md5(tag)): sql = "update images set tags = concat(tags, ',%s') where `md5` = '%s'" % ( tag, data["md5"]) #print sql self.db.execute(sql) else: # insert keys = "" values = "" for k in data.keys(): keys += " `%s`, " % (k) values += " '%s' ," % (data[k]) keys = keys.strip(" ,") + ", createtime, fetch_loop " values = values.strip(" ,") + ", now(), %s " % (str(loop_i)) sql = "insert into images ( %s ) values( %s )" % (keys, values) self.db.execute(sql) #print keys, values #print sql #exit() return def testPrint(self): print "Hello World!"
class MainDev(SDevice): _logger = None _main_proc = None _mail_procs = None _work_procs = None _db = None instance = None mutex = threading.Lock() def __init__(self): super(MainDev, self).__init__() self._logger = logging.getLogger(str(__name__)) interval = SysConfig.config["servers"]["interval"] if not interval: interval = 200 self._main_proc = MainProc(self) self.addProcess(self._main_proc, interval) mailProcCount = SysConfig.config["mail"]["mailProcCount"] self._mail_procs = [] for i in range(mailProcCount): mailProc = MailProc(self) mailProc.setIndex(i) self._mail_procs.append(mailProc) self.addProcess(mailProc, interval) self._work_procs = [] workProcCount = SysConfig.config['servers']['workProcCount'] for i in range(workProcCount): workProc = WorkProc(self) self._work_procs.append(workProc) self.addProcess(workProc, interval) host = SysConfig.config['db']['host'] port = SysConfig.config['db']['port'] user = SysConfig.config['db']['user'] passwd = SysConfig.config['db']['passwd'] dbName = SysConfig.config['db']['db'] charset = SysConfig.config['db']['charset'] self._db = Mysql(host, port, user, passwd, dbName, charset) pass @staticmethod def GetInstance(): if(MainDev.instance == None): MainDev.mutex.acquire() if(MainDev.instance == None): MainDev.instance = MainDev() MainDev.mutex.release() return MainDev.instance pass def getMainProc(self): return self._main_proc pass def getRedis(self): return self._redis pass def start(self): super(MainDev, self).start() row = self._db.query("select num_iid from ftxia_items order by id desc") for r in row: AppQueue.numiidQueue.put(r[0]) self._logger.debug("numiid count %d"%(AppQueue.numiidQueue.qsize())) self._logger.debug("server start ........") def close(self): self._logger.debug("server close ........") super(MainDev,self).close() self._db.close() def getDb(self): return self._db def getFactory(self): return self._factory pass