def queue_mnt(): """公司称名数据分配进程。 公司名称的基本数据表命名格式 base_省份简称。 自动提取以base_为前缀的公司基本表中的数据当对应的队列元素小于 设定的阀值queue_threshold时。 """ db = mysqlwrap.get_db() index_rd = rediswrap.get_hash('wbsp:gov:baseindex') while 1: try: sql = "SHOW TABLES LIKE 'base_%'" res, desc = db.query(sql) if res==0 and desc: for row in desc: tb = list(row.values())[0] q = rediswrap.get_queue('wbsp:gov:queue.%s'%tb) #print('tb:%s q_len:%s' % (tb,q.__len__())) if q.__len__() >= queue_threshold: continue index = index_rd.get(tb,0) #print('tb:%s index:%s' % (tb,index)) sql = "select id,name from %s where id >%s limit %s" % (tb,index,rows_limit) #print(sql) rs, ds = db.query(sql) if rs==0 and ds: id = 0 for d in ds: id = d['id'] q.push(d['name']) index_rd.set(tb,id) except Exception as e: traceback.print_exc() time.sleep(10)
def doworke(): db = mysqlwrap.get_db() sql_item = {} sql_item['table']= 'data_total' sql_item['fields']="id,gov_url" sql_item['where']=" isnull(reg_no)" sql_item['limit']=50 sql_item['order']="id asc" id = 0 pool = eventlet.GreenPool(60) i=0 while True: sql_item['where']+=" and id >%s" %id res,desc = db.query(sql_item) #print(res,desc) urls = [] if res==0 and desc: for row in desc: urls.append(row['gov_url']) id = row['id'] urls=[row['gov_url'] for row in desc ] if urls: for res in pool.imap(anhui.format_html,urls): i+=1 if res and 'name' in res: print("total_num:%s" % i,id) print(updata(res,'ah')) else: print(res)
def savedata(data,table): """保存数据到mysql数据库 """ db = mysqlwrap.get_db() if type(data) != list: data = [data] for item in data: fields = [] values = [] for k,v in item.items(): fields.append(k) v=mysqlwrap.addslashes(v) values.append('"%s"' % v) sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values)) res,desc = db.query(sql,1) #print(sql,res,desc) #数据入库失败处理 if res == -1: #print('insert_error:',desc) if desc.args[0] in [1064,1062]: return (0,()) item['table'] = 'data_%s'%table datamodel.get_tmp_queue().push(item) #插入dat_total总表 #print(item) if 'reg_status' in item and item['reg_status'] in ['存续','在业'] : fields.append('prov_tb') values.append('"%s"' % table) t_table = 'data_total' sql = "insert into %s(%s)values(%s)" % (t_table,','.join(fields),','.join(values)) rs,ds = db.query(sql,1) #print("insert total tb",rs,ds) return (res,desc)
def update_group(gkey,data): """更新 """ stat_db = mysqldb.get_db() data.pop('gid') res,group = get_groups([gkey]) if res == -1: return res,group if res ==0 and not group: return [-3,"%s 未注册."%gkey] if data: res,desc = stat_db.update("stat_item_group",data,"gkey='%s'" %gkey) if res ==0 and desc: #成功更新后检查是否有变更item表和history表 if "items_mrk" in data and data['items_mrk'] != group['items_mrk'] \ or "history_mrk" in data and data['history_mrk'] != group['history_mrk']: if "name" not in data: data['name'] = group['name'] if "pid" not in data: data['pid'] = group['pid'] reg_group(gkey,data) reg_items2redis(gkey) return res,desc elif res ==-1: return err_handle.db_err return [0,[]]
def set(gkey,data): """上传值,先从redis缓存中取出item的id,再匹配itemkey入history库 参数说明: gkey group 的gkey data 数据集 最好一次不要超过100个 {'itemkey':[值,生成时间默认为当前时间戳]} """ stat_db = mysqldb.get_db() rdb = rediswrap.get_redis() kys = list(data.keys()) #加入表标识以确定目标表 kys.append('mrk') itm = rdb.hmget(RD_ITM_KEY_PRFX+gkey,kys) itm_dict = {} if itm: mrk=itm.pop().split(',') hst_tb = get_hst_name(mrk[1]) value = [] for i in range(0,len(itm)): if itm[i] : tmp = data[kys[i]] clock = tmp[1] if tmp[1] else int(time.time()) value.append("('%s','%s','%s')" %(itm[i],tmp[0],clock)) if value: res,desc = stat_db.query("insert into %s(itemid,val,clock)values%s" % (hst_tb,','.join(value)),1) if res ==-1 : return [res,str(desc)] return res,desc else: return [-1,"no items find."] return [-2,'items not in redis.']
def sys_log(contxt): """写入系统日志表 """ stat_db = mysqldb.get_db() sql_item={'table':'stat_syslog','contxt':contxt} stat_db.query(sql_item)
def reg_items(gkey,itm,prefix=''): """注册items 参数说明: gkey group的标识 itm items组[[name,key]] prefix key的前缀 """ #stat_db = dbclass(sconf.SYS['mysql']) #stat_db.connect() stat_db = mysqldb.get_db() #取group信息 res,desc = get_groups([gkey]) if res==0 and desc: itm_tb = "stat_item_%s" % desc[0]['items_mrk'] if desc[0]['items_mrk'] else "stat_item" value = [] for k in itm: value.append("('%s','%s%s',%s)" % (k[0],prefix,k[1],desc[0]['gid'])) sql = "insert into %s (name,itemkey,gid)values %s " %(itm_tb, ",".join(value)) res,desc = stat_db.query(sql,1) if res == -1: if desc[0] == 1062: return err_handle.item_exis return err_handle.db_err else: return [0,''] else: return res,desc
def get(gkey,itm=[],start_time=0,stop_time=0,sort='clock asc',groupby=0,page=None): """ 获取统计数据 参数说明 itm itemid列表 为空时提取整个group的记录 start_time 开始时间戮 stop_time 结构时间戮 sort 排序方式 groupby 分组方式 page 分页参数集 {'site':每页数据量,'num':页码} 默认返回所有记录 """ stat_db = mysqldb.get_db() rdb = rediswrap.get_redis() sql_item = {'fields':'*'} r_itmkey = RD_ITM_KEY_PRFX+gkey if itm: itm.append('mrk') itmids = rdb.hmget(r_itmkey,itm) mrk = itmids.pop().split(',') else: mrk = rdb.hget(r_itmkey,'mrk') itmids = rdb.hvals(r_itmkey) itmids.remove(mrk) mrk = mrk.split(',') ids = [k for k in itmids if k ] sql_item['table'] = get_hst_name(mrk[1]) sql_item['where'] = " itemid in (%s) " % ",".join(ids) start_time = utils.timestamp(start_time) if start_time else utils.timestamp(0,'d') stop_time = utils.timestamp(stop_time) if stop_time else int(time.time()) sql_item['where'] += " and clock>=%s and clock <%s" % (start_time,stop_time) sql_item['order'] = sort if groupby: if groupby ==1: sql_item['group'] = 'itemid' elif groupby == 2: sql_item['group'] = 'clock' else: sql_item['group'] = 'itemid,clock' sql_item['fields'] = "itemid,sum(val) as val,clock" #分页这个mark一下。待定 if page: s = page['num']*page['site'] sql_item['limit'] = "%s,%s" %(s,page['site']) res,desc = stat_db.query(sql_item) #取得items的名称 item_lab = {} if res == 0 and desc: itm_tb = "stat_item_" + mrk[0] if mrk[0] else "stat_item" rs, ds = stat_db.query("select name,id from %s where id in(%s)" %(itm_tb,",".join(ids) )) if rs==0 and ds: for row in ds : item_lab[row['id']]=row['name'] return 0,[item_lab,desc] return 0,[{},[]]
def update_item_name(gkey,itemkey,name): stat_db = mysqldb.get_db() res,desc = get_groups([gkey]) if res==0 and desc: itm_tb = get_itm_name(desc[0]['items_mrk']) res,desc = stat_db.update(itm_tb,{"name":name},"itemkey='%s'" % itemkey) if res ==-1 : return err_handle.db_err return res,desc
def exist_corp(name,table): """检查公司数据是否已存在 """ db = mysqlwrap.get_db() sql = "select id from %s where name='%s'" %(table,name) res,desc = db.query(sql) if res == 0 and desc: return True return False
def check_exist_corp(biz): """对比数据表 """ res,desc = mysqlwrap.get_db().query("select max(id) as mx,min(id) as mi from base_%s" % biz) mx = desc[0]['mx'] mi = desc[0]['mi'] while mi <=mx: step = mi + 1000 sql = "select name from base_%s where id >%s and id <%s" %(biz,mi,step) res,desc = mysqlwrap.get_db().query(sql) for row in desc: s = "select id from data_%s where name='%s'" % (biz,row['name']) rs,ds = mysqlwrap.get_db().query(s) if rs==0 and ds: sql = "update base_%s set flag=1 where name='%s'" % (biz,row['name']) result = mysqlwrap.get_db().query(sql,1) #print(result) mi=step
def run(self): db = mysqlwrap.get_db() queue = datamodel.get_tmp_queue() while 1: try: row = queue.pop(3) if row: tb = row.pop('table')[-2:] res,desc = savedata(row,tb) if desc: print(tb,res,desc) except: traceback.print_exc()
def reg_items2redis(gkey,itm=[]): """将items存入redis缓存 """ rdb = rediswrap.get_redis() stat_db = mysqldb.get_db() hash_tb = rediswrap. get_hash(gkey, system='default',serialized_type='string') res,desc = get_groups([gkey]) if res==0 and desc: itm_tb = get_itm_name(desc[0]['items_mrk']) rdb.hsetnx(RD_ITM_KEY_PRFX+gkey,'mrk',"%s,%s" %(desc[0]['items_mrk'],desc[0]['history_mrk'])) gid = desc[0]['gid'] i=0 #设置了itemkey列表,依据itemkey取值 if itm: itm_len = len(itm) while True: j = i+50 if i+50 < itm_len else itm_len if i == itm_len: break key = ['"%s"' % k for k in itm[i:j]] sql = "select id,itemkey from %s where itemkey in (%s)" % (itm_tb, ",".join(key)) res,desc = stat_db.query(sql) new_dict = {} if res ==0 and desc: for row in desc: new_dict[row['itemkey']]=row['id'] rs = rdb.hmset(RD_ITM_KEY_PRFX+gkey,new_dict) i = j #未设置itemkey列表,依据gkey取出该组所有的items else: res, desc = stat_db.query("select min(id) as mnid,max(id) as mxid from %s where gid = %s" % (itm_tb,gid)) mnid = desc[0]['mnid'] mxid = desc[0]['mxid'] limit = 50 while True: if mnid == mxid: break sql = "select id,itemkey from %s where id > %s and gid=%s limit %s" % (itm_tb,mnid,gid,limit) res,desc = stat_db.query(sql) new_dict = {} if res ==0 and desc: for row in desc: new_dict[row['itemkey']]=row['id'] mnid = row['id'] rdb.hmset(RD_ITM_KEY_PRFX+gkey,new_dict)
def savedata(data,table): """保存数据到mysql数据库 """ db = mysqlwrap.get_db() fields = [] values = [] for k,v in data.items(): fields.append(k) #v = db.escape_string(v) v=v.replace('"'," ") values.append('"%s"' % v) sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values)) res,desc = db.query(sql,1) return (res,desc)
def update_item_key(gkey,oldkey,newkey): stat_db = mysqldb.get_db() res,desc = get_groups([gkey]) rdb = rediswrap.get_redis() if res==0 and desc: itm_tb = get_itm_name(desc[0]['items_mrk']) res,des = stat_db.update(itm_tb,{"itemkey":newkey},"itemkey='%s'" % oldkey) if res ==0 and des: id = rdb.hget(RD_ITM_KEY_PRFX+gkey,oldkey) if id : rdb.hset(RD_ITM_KEY_PRFX+gkey,newkey,id) elif res == -1: return err_handle.db_err return res,des return res,desc
def output_data(self,queue,exit_flag): """输出结果到数据库 """ kword_stat={} kcount = 0 out_db = mysqlwrap.get_db(self._output['dbserver']) #插入sql模版,如果记录存在则更新 tb = self._output['table'] insert_sql = "insert into " + tb + "(rowid,contents,tb_name,k_weight) values('%s','%s','" + self._input['data']['table'] + "','%s') on DUPLICATE KEY UPDATE contents='%s',k_weight='%s'" while 1: if exit_flag.is_set() and queue.qsize()==0: import os print("out_db exit...",os.getpid()) #sql = "insert into illegal_log(kcount,taskid,contents)values('%s','%s','%s')" % \ #(kcount,self.taskid,mysqlwrap.addslashes(json.dumps(kword_stat))) #res,desc = out_db.query(sql,1) get_result_queue(self.pid).push(kword_stat) break #print("im waiting for data.....",queue.qsize()) try: try: row = queue.get(timeout=5) except: continue id = row.pop('id') weight = row.pop('weight') #小于或等设置的输出权重,刚放弃 if weight <= self._output['weight']: continue #统计关键匹配数量 for ks in row['content'].values(): for k in ks: kcount+=1 if k in kword_stat: kword_stat[k]+=1 else: kword_stat[k]=1 content = mysqlwrap.addslashes(json.dumps(row.get('content'))) res, desc = out_db.query(insert_sql % (id, content, weight,content, weight)) except Exception as e: logger.error(str(e))
def stat_tabls_rows(tb_prefix): "统计同一前缀的表记录数" db = mysqlwrap.get_db() sql = "SHOW TABLES LIKE '"+tb_prefix+"%'" res, desc = db.query(sql) totel=0 if res==0 and desc: for row in desc: tb = list(row.values())[0] sql = "select count(0) as n from %s" % tb rs, ds = db.query(sql) if rs ==0 and ds: totel+=ds[0]['n'] print(tb,ds[0]['n']) print("totel:%s" % totel)
def get_items(gkey,item=[]): """提取items """ stat_db = mysqldb.get_db() res,desc = get_groups([gkey]) if res==0 and desc: itm_tb = get_itm_name(desc[0]['items_mrk']) sql_item={"table":itm_tb,"fields":"*",'where':"gid=%s"%desc[0]['gid']} if item: itms = ["'%s'" % k for k in item] sql_item['where']+=" and itemkey in (%s)" % ",".join(itms) res,desc = stat_db.query(sql_item) if res ==-1: return err_handle.db_err return [res,desc] else: return res,desc
def get_groups(gkey=[]): """提取group信息 返回值:[status,info] status: 0 成功 -1 gkey未设置 -100 数据库错误 """ stat_db = mysqldb.get_db() item = {"table":"stat_item_group","fields":"*"} if gkey and type(gkey) in (list,tuple): gk = ["'%s'" % k for k in gkey] item['where'] = "gkey in (%s)" % ",".join(gk) elif type(gkey) == str and gkey !='all' or not gkey: return err_handle.gkey_not_set res,desc = stat_db.query(item) if res == -1: return err_handle.db_err return res,desc
def updata(data,table): """更新数据 """ db = mysqlwrap.get_db() if type(data) != list: data = [data] for item in data: for k,v in item.items(): v=mysqlwrap.addslashes(v) item[k]=v where = "name='%s'" % item['name'] res,desc = db.update("data_%s" % table,item,where) #数据入库失败处理 if 'reg_status' in item and item['reg_status'] in ['存续','在业','存续(在营、开业、在'] : item['prov_tb']='"%s"' % table t_table = 'data_total' res,desc = db.update(t_table,item,where) print("insert total tb",res,desc) return (res,desc)
def init(self): global KWORDS if self._input is None: raise Exception("not set input config!") if self._kwsource is None and self._keywords is None: raise Exception("not set keywords!") if self._dbsource is None: raise Exception("not set dbsource!") #注册数据库连接 conn_rd = get_conn_hash() info = conn_rd.get(self._input['dbserver'])['info'] info['dbname']=self._input['data']['dbname'] mysqlwrap.setup_db(self._input['dbserver'],info) info = conn_rd.get(self._kwsource['dbserver'])['info'] info['dbname']=self._kwsource['data']['dbname'] mysqlwrap.setup_db(self._kwsource['dbserver'],info) info = conn_rd.get(self._output['dbserver'])['info'] info['dbname']=self._output['dbname'] mysqlwrap.setup_db(self._output['dbserver'],info) mysqlwrap.pool_monitor() #生成keywords if self._keywords: self.keywords = self._keywords KWORDS = self.keywords else: db = mysqlwrap.get_db(self._kwsource['dbserver']) res,desc = db.query(self._kwsource['data']) if res == -1 or not desc: print(res,desc) raise Exception("no keywords!") for row in desc: if row['k_word']: self.keywords.append(row['k_word']) self.keyweight[row['k_word']]=row['k_weight']
def savedata(data,table): """保存数据到mysql数据库 """ db = mysqlwrap.get_db() fields = [] values = [] for k,v in data.items(): fields.append(k) #v = db.escape_string(v) v=v.replace('"'," ") values.append('"%s"' % v) sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values)) res,desc = db.query(sql,1) #数据入库失败处理 if res == -1: if desc.args[0] == 1064: print(desc) return data['table'] = 'data_%s'%table datamodel.get_tmp_queue().push(data)
def run(self): manager = mp.Manager() queue = manager.Queue() in_queue = manager.Queue() out_db_exit = mp.Event() input_db = mysqlwrap.get_db(self._input['dbserver']) sql_item = self._input['data'] #设置id范围 if 'range' in sql_item and sql_item['range']: minid, maxid = sql_item.pop('range') else: res, desc = input_db.query("select min(id) as min,max(id) as max from %s" % sql_item['table']) minid, maxid = desc[0]['min'], desc[0]['max'] sql_where = sql_item.pop('where','') #字段权重 weight = sql_item.pop('weight') if sql_where: sql_where+= " and " #定义运行级别取得进程数 #pool = mp.Pool(get_process_count(self.powerlevel)) """以守护的方式运行""" pnum = get_process_count(self.powerlevel) pool = [] q_pool=[] for i in range(0,pnum): q = manager.Queue() q_pool.append(q) pool.append(mp.Process(target=fielt_srv, args=(self.keywords,self.keyweight,weight,q,queue,self.exit_flag))) pool[i].daemon = True pool[i].start() """-""" p = None #启动输出线程 if self._output: p=mp.Process(target=self.output_data, args=(queue,out_db_exit)) p.start() #pool.append(p) #pool.apply_async(output_data,(self,queue,self.exit_flag)) #output_data(self,queue,self.exit_flag) p_start = time.time() #e_pool = eventlet.GreenPool(10) item=[] while minid < maxid: #检查退出状态 if self.exit_flag.is_set() or global_list.G_EXIT: pool.close() pool.join() self.close() break #检查任务状态 if global_list.TASK_STAUS[self.taskid] == 1: #暂停状态 time.sleep(1) continue if global_list.TASK_STAUS[self.taskid] == 2: #退出状态 #pool.close() #pool.join() #self.close() break stop_id = minid + self.rowstep if stop_id > maxid : stop_id = maxid _sql_item=sql_item.copy() _sql_item['where'] = "%s id >= %s and id < %s" % (sql_where, minid, stop_id) res, desc = input_db.query(_sql_item) """这里sql_item存在同步问题,有共享冲突""" #e_pool.spawn_n(self.data_source,input_db,q_pool,_sql_item) #item.append(_sql_item) minid = stop_id if res == -1: print(desc) time.sleep(3) continue #result = [] q_index = 0 for row in desc: #ps = pool.apply_async(func=fielt,args=(row,self.keywords,self.keyweight,weight,queue)) #fielt(row,self.keywords,self.keyweight,weight,queue) #if self._return: result.append(ps) #数据队列模式 if q_index == len(q_pool):q_index=0 q_pool[q_index].put(row) q_index+=1 """返回结果的代码 if 1:#self._return: for i in result: _res = i.get() print(_res) if _res: self.result.append(_res) """ #任务完成 #pool.close() #pool.join() self.close() #if p: p.join() #import pdb #pdb.set_trace() print(len(pool)) for i in range(0,len(pool)): print('join..',pool[i].pid) if global_list.TASK_STAUS[self.taskid] == 2: pool[i].terminate() else: pool[i].join() print("is done") out_db_exit.set() if p: p.join() print("finish job user :" ,time.time()-p_start) jobstatus.done(self.taskid)
def sumdelay(): """ 检测redis中的统计缓存,根据stat_item_group.sumdelay 将满足条件的统计数据入库,并清除缓存 """ rdb = rediswrap.get_redis() db = mysqlwrap.get_db() now = time.localtime() fields = rdb.keys(stat_base.RD_ITM_HST_TMP_PRFX+'*') if fields: gkeys = [f.split(':')[-1] for f in fields] todo_gkeys=[] step = 20 #初始化时间 clock = int(time.time())-120 for i in range(0,len(gkeys),step): #每次提取step条group记录 ks = ["'%s'" % k for k in gkeys[i:i+step]] sql = "select gkey,sumdelay,history_mrk from stat_item_group where gkey in (%s)" % ",".join(ks) res, desc = db.query(sql) if res==0 and desc: for row in desc: gk = "" #按时汇总 if row['sumdelay'] == 2: clock = utils.timestamp(clock,'h') gk =row['gkey'] #按天汇总,当前时间0点 elif row['sumdelay'] == 1 and now.tm_hour==0: clock = utils.timestamp(0,'d') gk =row['gkey'] #按月汇总,当天是1号,时间是0点 elif row['sumdelay'] == 3 and (now.tm_mday==1 or now.tm_hour==0) : clock = utils.timestamp(0,'d') gk =row['gkey'] #按周汇总,当天是周1,时间是0点 elif row['sumdelay'] == 4 and (now.tm_wday!=1 or now.tm_hour!=0) : clock = utils.timestamp(0,'d') gk =row['gkey'] #gk =row['gkey'] if gk: #todo_gkeys.append(row) #print(stat_base.RD_ITM_HST_TMP_PRFX+row['gkey'],RD_TMP_PRF+row['gkey']) #rdb.rename(stat_base.RD_ITM_HST_TMP_PRFX+row['gkey'],RD_TMP_PRF+row['gkey']) key = stat_base.RD_ITM_HST_TMP_PRFX+gk r_item = rdb.hgetall(key) rdb.delete(key) tb = stat_base.get_hst_name(row['history_mrk']) cnt = 0 if r_item: logger.info("sum stat group %s from %s into %s" % (row['gkey'],key,tb)) ks = list(r_item.keys()) #每次写50条记录 for i in range(0,len(ks),50): value = ["('%s','%s','%s')" %(j,r_item[j],clock) for j in ks[i:i+50]] ins_sql = "insert into %s(itemid,val,clock)values%s" % (tb,','.join(value)) res,desc = db.query(ins_sql,1) if res == -1: logger.error(str(desc)) data_loge.info(ins_sql) else: cnt+=len(value) logger.info("[%s] count:[%s] ins:[%s] rows." %(gk,len(ks),cnt)) #汇总数据入库 """
def reg_group(gkey,info): """ 注册group 参数说明: gkey group的标识 info 组信息 返回值:[status,info] status: 0 成功 -100 数据库错误 """ if gkey == 'default': return [0, 1] pid = info.get('pid',1) name = info.get('name') items_mrk = info.get('items_mrk','') history_mrk = info.get('history_mrk',items_mrk) #检查group是否存在,如不存则插入 stat_db = mysqldb.get_db() res,desc = get_groups([gkey]) #print(res,desc) if res==0 and not desc: data = {'name':name,'pid':1,'gkey':gkey,'items_mrk':items_mrk,'history_mrk':history_mrk} rs, des = stat_db.insert('stat_item_group',data) gid = des else: gid = desc[0]['gid'] #检查项目表与记录表是否存在,不存在创建 itesm_tabse_ddl = """ Create Table stat_item_%s ( id int(11) NOT NULL AUTO_INCREMENT COMMENT '项目id号', name varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '项目名称', itemkey varchar(50) COLLATE utf8_bin NOT NULL COMMENT '项目的标识', gid int(11) NOT NULL DEFAULT '1' COMMENT '所属项目集的id号', sumdelay int(11) DEFAULT '1' COMMENT '间隔时间(秒)', PRIMARY KEY (id), UNIQUE KEY itemkey (itemkey) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='统计项目表'; """ % items_mrk if items_mrk: res,desc = stat_db.query("show tables like 'stat_item_%s'" % items_mrk ) if res==0 and not desc: rs,ds = stat_db.query(itesm_tabse_ddl,1) if rs== -1: return err_handle.db_err history_tabse_ddl = """ Create Table stat_history_%s ( itemid bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '项目id号', clock int(11) NOT NULL DEFAULT '0' COMMENT '数据产生时间', val float(16,4) NOT NULL DEFAULT '0.0000' COMMENT '值', gid int(11) DEFAULT '1' COMMENT '项目集id号', KEY history_1 (itemid,clock) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='存放统计数据'; """ % history_mrk if history_mrk: res,desc = stat_db.query("show tables like 'stat_history_%s'" % items_mrk ) if res==0 and not desc: rs,ds = stat_db.query(history_tabse_ddl,1) if rs ==-1: return err_handle.db_err return [0,gid]
def chang_flag(corp,flag,table): """更改原始公司表中的flag """ db = mysqlwrap.get_db() sql = "update base_%s set flag=%s where name='%s'" % (table,flag,corp) res,desc = db.query(sql,1)
'url_check':'http://gxqyxygs.gov.cn/checkCheckNo.jspx', 'url_list':'http://gxqyxygs.gov.cn/searchList.jspx'} }, 'ha':{'mod':'anhui','thread_count':50, 'urls':{'host':'http://222.143.24.157', 'url_home':'http://222.143.24.157/search.jspx', 'url_code':'http://222.143.24.157/validateCode.jspx?type=1&id=%s', 'url_check':'http://222.143.24.157/checkCheckNo.jspx', 'url_list':'http://222.143.24.157/searchList.jspx'} }, 'hl':{'mod':'anhui','thread_count':50, 'urls':{'host':'http://gsxt.hljaic.gov.cn', 'url_home':'http://gsxt.hljaic.gov.cn/search.jspx', 'url_code':'http://gsxt.hljaic.gov.cn/validateCode.jspx?type=1&id=%s', 'url_check':'http://gsxt.hljaic.gov.cn/checkCheckNo.jspx', 'url_list':'http://gsxt.hljaic.gov.cn/searchList.jspx'} }, } if __name__=="__main__": dbinfo ={'host':'192.168.10.126','port':3306,'dbname':'gov_corp', 'user':'******','passwd':'wbsp','charset':'utf8'} mysqlwrap.setup_db('default',dbinfo) mysqlwrap.get_db().connect() rediswrap.setup_redis('default','192.168.10.126',6380) for p in SF_DIST.keys(): sql = Data_tb % p mysqlwrap.get_db().query(sql,1) #print(get_proxy())