Beispiel #1
0
def savedata(data,table):
	"""保存数据到mysql数据库
	"""
	db = mysqlwrap.get_db()
	if type(data) != list:
		data = [data]
	for item in data:
		fields = []
		values = []
		for k,v in item.items():
			fields.append(k)
			v=mysqlwrap.addslashes(v)
			values.append('"%s"' % v)
		sql = "insert into data_%s(%s)values(%s)" % (table,','.join(fields),','.join(values))
		res,desc = db.query(sql,1)
		#print(sql,res,desc)
		#数据入库失败处理
		if res == -1:
			#print('insert_error:',desc)
			if desc.args[0] in [1064,1062]:
				return (0,())
			item['table'] = 'data_%s'%table
			datamodel.get_tmp_queue().push(item)
		#插入dat_total总表
		#print(item)
		if 'reg_status' in item and item['reg_status'] in ['存续','在业'] :
			fields.append('prov_tb')
			values.append('"%s"' % table)
			t_table = 'data_total'
			
			sql = "insert into %s(%s)values(%s)" % (t_table,','.join(fields),','.join(values))
			rs,ds = db.query(sql,1)
			#print("insert total tb",rs,ds)
	return (res,desc)
Beispiel #2
0
    def output_data(self,queue,exit_flag):
        """输出结果到数据库
        """
        kword_stat={}
        kcount = 0
        out_db = mysqlwrap.get_db(self._output['dbserver'])
        #插入sql模版,如果记录存在则更新
        tb = self._output['table']      
        insert_sql = "insert into " + tb + "(rowid,contents,tb_name,k_weight) values('%s','%s','" + self._input['data']['table'] + "','%s') on DUPLICATE KEY UPDATE contents='%s',k_weight='%s'"

        while 1:
            if exit_flag.is_set() and queue.qsize()==0:
                import os
                print("out_db exit...",os.getpid())
                #sql = "insert into illegal_log(kcount,taskid,contents)values('%s','%s','%s')" % \
                #(kcount,self.taskid,mysqlwrap.addslashes(json.dumps(kword_stat)))
                
                #res,desc = out_db.query(sql,1)
                get_result_queue(self.pid).push(kword_stat)
                break
            #print("im waiting for data.....",queue.qsize())
            try:
                try:
                    row = queue.get(timeout=5)
                except:
                    continue
                
                id = row.pop('id')
                weight = row.pop('weight')
                #小于或等设置的输出权重,刚放弃
                if weight <= self._output['weight']:
                    continue
                #统计关键匹配数量
                for ks in row['content'].values():
                    for k in ks:
                        kcount+=1
                        if k in kword_stat:
                            kword_stat[k]+=1
                        else:
                            kword_stat[k]=1
                            
                content = mysqlwrap.addslashes(json.dumps(row.get('content')))
                
                res, desc = out_db.query(insert_sql % (id, content, weight,content, weight))
                
            except Exception as e:
                
                logger.error(str(e))
Beispiel #3
0
def updata(data,table):
	"""更新数据
	"""
	db = mysqlwrap.get_db()
	if type(data) != list:
		data = [data]	
	for item in data:
		for k,v in item.items():
			v=mysqlwrap.addslashes(v)
			item[k]=v
		where = "name='%s'" % item['name']
		res,desc = db.update("data_%s" % table,item,where)
		#数据入库失败处理
		if 'reg_status' in item and item['reg_status'] in ['存续','在业','存续(在营、开业、在'] :
			item['prov_tb']='"%s"' % table
			t_table = 'data_total'
			res,desc = db.update(t_table,item,where)
			print("insert total tb",res,desc)
	return (res,desc)