def start_thread(kw): """执行任务 """ #注册数据库 mysqlwrap.setup_db('default',config.CONFIG['mysqld']) #启动数据连接池守护进程 mysqlwrap.pool_monitor() thread_count = kw.pop('thread_count',THREAD_COUNT) biz_flag = kw.pop('biz_flag') exit_flag = kw.get('exit_flag') urls = kw.get('urls','') if biz_flag not in datamodel.SF_DIST.keys(): raise Exception("illegal biz_flag!") mod = kw.pop('mod') #设置进程名 title = "%s %s %s" %(datamodel.proc_title,mod,biz_flag) setproctitle.setproctitle(title) logger.info("%s start..." % title) prox_queue = Queue() try: md = __import__(mod) md.biz_flag = biz_flag if urls: for k,url in urls.items(): exec("md.%s='%s'" % (k,url)) print(urls) thrd = [ExecThread(md,prox_queue) for i in range(0,thread_count)] #thrd.append(Thread(target=exec_main,args=(md,))) for i in range(0,len(thrd)): thrd[i].setDaemon(True) thrd[i].start() except Exception as e: traceback.print_exc() s_time = time.time() while 1: #填充代理ip队列 push_proxy_queue(prox_queue,md.ille_proxy_ip) if exit_flag.is_set(): logger.info("%s exit..." % title) datamodel.g_exit = True for i in range(0,len(thrd)): #print(thrd[i].name + "exit....") thrd[i].stop() thrd[i].join() break for i in range(0,len(thrd)): if not thrd[i].isAlive(): #print("%s is death try reload..." % thrd[i].name) thrd[i] = ExecThread(md,prox_queue) thrd[i].setDaemon(True) thrd[i].start() if int(time.time())-s_time > REPORT_TIME: logger.info("%s %s S:%s N:%s F:%s" % (mod,biz_flag,md.Ok_num,md.Null_num,md.False_num)) s_time = time.time() time.sleep(3)
def init(self): global KWORDS if self._input is None: raise Exception("not set input config!") if self._kwsource is None and self._keywords is None: raise Exception("not set keywords!") if self._dbsource is None: raise Exception("not set dbsource!") #注册数据库连接 conn_rd = get_conn_hash() info = conn_rd.get(self._input['dbserver'])['info'] info['dbname']=self._input['data']['dbname'] mysqlwrap.setup_db(self._input['dbserver'],info) info = conn_rd.get(self._kwsource['dbserver'])['info'] info['dbname']=self._kwsource['data']['dbname'] mysqlwrap.setup_db(self._kwsource['dbserver'],info) info = conn_rd.get(self._output['dbserver'])['info'] info['dbname']=self._output['dbname'] mysqlwrap.setup_db(self._output['dbserver'],info) mysqlwrap.pool_monitor() #生成keywords if self._keywords: self.keywords = self._keywords KWORDS = self.keywords else: db = mysqlwrap.get_db(self._kwsource['dbserver']) res,desc = db.query(self._kwsource['data']) if res == -1 or not desc: print(res,desc) raise Exception("no keywords!") for row in desc: if row['k_word']: self.keywords.append(row['k_word']) self.keyweight[row['k_word']]=row['k_weight']
sconf.BIZ[f[4:-5]] = json.loads(open("./conf/%s" % f).read().replace("\n", "").replace("\t", "")) except: logger.error("config file %s not load.please check." % f) # 加载zk zkwrap.setup("default", sconf.SYS["zk"]) # 加载数据库 mysqlwrap.setup_db("default", sconf.SYS["mysql"]) mysqlwrap.pool_monitor() # rediswrap.setup_redis('default',sconf.SYS['redis']['host'],sconf.SYS['redis']['port']) rediswrap.setup_redis("cache", sconf.SYS["redis"]["host"], sconf.SYS["redis"]["port"], decode_flag=False) sconf.SYS["root_dir"] = sys.path[0] G_exit = False mpexit = mp.Event() # 启动定时任务 # from stat_daemon import cron