def savedata(data,table):
"""保存数据到mysql数据库
"""
db = mysqlwrap.get_db()
if type(data) != list:
data = [data]
for item in data:
fields = []
values = []
for k,v in item.items():
fields.append(k)
v=mysqlwrap.addslashes(v)
values.append('"%s"' % v)
sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values))
res,desc = db.query(sql,1)
#print(sql,res,desc)
#数据入库失败处理
if res == -1:
#print('insert_error:',desc)
if desc.args[0] in [1064,1062]:
return (0,())
item['table'] = 'data_%s'%table
datamodel.get_tmp_queue().push(item)
#插入dat_total总表
#print(item)
if 'reg_status' in item and item['reg_status'] in ['存续','在业'] :
fields.append('prov_tb')
values.append('"%s"' % table)
t_table = 'data_total'
sql = "insert into %s(%s)values(%s)" % (t_table,','.join(fields),','.join(values))
rs,ds = db.query(sql,1)
#print("insert total tb",rs,ds)
return (res,desc)
def output_data(self,queue,exit_flag):
"""输出结果到数据库
"""
kword_stat={}
kcount = 0
out_db = mysqlwrap.get_db(self._output['dbserver'])
#插入sql模版，如果记录存在则更新
tb = self._output['table']
insert_sql = "insert into " + tb + "(rowid,contents,tb_name,k_weight) values('%s','%s','" + self._input['data']['table'] + "','%s') on DUPLICATE KEY UPDATE contents='%s',k_weight='%s'"
while 1:
if exit_flag.is_set() and queue.qsize()==0:
import os
print("out_db exit...",os.getpid())
#sql = "insert into illegal_log(kcount,taskid,contents)values('%s','%s','%s')" % \
#(kcount,self.taskid,mysqlwrap.addslashes(json.dumps(kword_stat)))
#res,desc = out_db.query(sql,1)
get_result_queue(self.pid).push(kword_stat)
break
#print("im waiting for data.....",queue.qsize())
try:
try:
row = queue.get(timeout=5)
except:
continue
id = row.pop('id')
weight = row.pop('weight')
#小于或等设置的输出权重，刚放弃
if weight <= self._output['weight']:
continue
#统计关键匹配数量
for ks in row['content'].values():
for k in ks:
kcount+=1
if k in kword_stat:
kword_stat[k]+=1
else:
kword_stat[k]=1
content = mysqlwrap.addslashes(json.dumps(row.get('content')))
res, desc = out_db.query(insert_sql % (id, content, weight,content, weight))
except Exception as e:
logger.error(str(e))
def updata(data,table):
"""更新数据
"""
db = mysqlwrap.get_db()
if type(data) != list:
data = [data]
for item in data:
for k,v in item.items():
v=mysqlwrap.addslashes(v)
item[k]=v
where = "name='%s'" % item['name']
res,desc = db.update("data_%s" % table,item,where)
#数据入库失败处理
if 'reg_status' in item and item['reg_status'] in ['存续','在业','存续（在营、开业、在'] :
item['prov_tb']='"%s"' % table
t_table = 'data_total'
res,desc = db.update(t_table,item,where)
print("insert total tb",res,desc)
return (res,desc)