def antilogN_func(line=None): '''外挂日志antilog采集''' if line == 'nothingData': senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'antilog', '') if not senderNodeId: return jsonresult = headJson + '''{}}''' return jsonresult % (senderNodeId, logTime, limitNum, 'p1-antilog', '4') if 'Account' not in line: return if 'TimeOut' in line: return modname = 'antilogNew' if 'zxy' in getareaname(): area = getareaname().replace('zxy', '', 1) else: area = getareaname() group = 'game' + hostname.split('game')[-1] try: acc = line.split(' ')[5].split('=')[1] lv = line.split(' ')[3].split('=')[1] ip = str2ip(line.split(' ')[4].split('=')[1]) st = line.split(' ')[7].split('=')[1] detime = ' '.join(line.split(' ')[0:2]) if int(line.split(' ')[0].replace('-', '')) < 140314: print 'datetime', detime return if st == 'deteched': app = line.split(' ')[8].split('=')[1] ver = line.split(' ')[9].split('=')[1] ret = 'NULL' # elif st == 'TimeOut': # app = 'TimeOut' # ver = 'NULL' # ret = 'NULL' # if 'Ret=' in line: # ret = line.split(' ')[8].split('=')[1].strip('\n') senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'antilog', '') jsonresult = '' if senderNodeId: jsonresult = headJson + '''{"zone":"%s","group":"%s","accountid":"","acc":"%s","lv":"%s","name":"","party":"","ip":"%s","detime":"%s","wg":"%s","flag":"","optime":"","ver":"%s","is_killed":"","ret":"%s"}}''' jsonresult = jsonresult % (senderNodeId, logTime, limitNum, 'p1-antilog', '1', area, group, acc, lv, ip, detime, app, ver, ret) # print jsonresult return jsonresult except: log(modname, 3, '日志分析错误: %s' % line) return
def line(socketconfig='/tools/config/nsconfig/collectNew.ini:line',port='2888[0-6]'): try: connectnum_json = '' connectnum = '1' logName = 'lineNum'+str(time.strftime('%Y%m%d',time.localtime(time.time()))) log = getLogger(logName) area,group,oid,ctime = gethostinfo() connectnum = getconnectnum(port) headJson = '' senderNodeId,logTime,limitNum,headJson = getNodeIdAndType('line','') if senderNodeId: connectnum_json = headJson+'''{"zone":"%s","group":"%s","time":"%s","num":"%s","oid":"%s"}}''' connectnum_json = connectnum_json %( str(senderNodeId),str(logTime),str(limitNum),'p1-line','1',str(area),str(group),str(ctime),str(connectnum),str(oid) ) #print connectnum_json,' dddd' # connectnum_json = {'zone':area,'group':group,'time':ctime,'num':connectnum,'oid':oid} lastLineNum = getiniConfig.getConfig('/home/config','monitorNewlinenumber.ini','lastNumConfig','lastLineNum') if str(lastLineNum) == '0': getiniConfig.write_config('/home/config','monitorNewlinenumber.ini','lastNumConfig','lastLineNum',str(connectnum)) else: zbxsend(connectnum) log.info('Newinfo :'+str(connectnum_json)) except Exception,e: print str(e) log.error("monitor lineNew error: %s, onlineNum is %s"%(str(e),str(connectnum))) try: connectnum = getconnectnum(port) zbxsend(connectnum) except Exception,e: log.error("monitor lineNew error1: %s"%str(e))
def submit(filepath, modname, jsonresultlist, funkey, p1, op='1'): '''socket发送给syslogtcp''' config = getconf(modname, filepath) ipsegment = '.'.join(getlanip().split('.')[0:3]) serverlist = eval(config.get('server')) socketserver = '' for server in serverlist.items(): if '.'.join(server[1].split('.')[0:3]) == ipsegment: socketserver = server[1] break if not socketserver: print '没有找到本网段的Flume服务器' return ping_ret = getstatusoutput('ping -c 2 %s' % socketserver) if ping_ret[0]: print 'ping不通服务器%s' % socketserver return socketport = config.get('port_shumen') if 'zxy' in gethostname(): socketport = config.get('port_zuixiaoyao') senderNodeId, logTime, limitNum, headJson = getNodeIdAndType(funkey, '') sockettcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sockettcp.settimeout(5) #设置socket超时时间 sockettcp.connect((socketserver, int(socketport))) except: try: time.sleep(2) sockettcp.connect((socketserver, int(socketport))) except: sockettcp.close() return False if op == '4': sockettcp.sendall('<134>%s\n' % jsonresultlist) sockettcp.close() print 'successfl!! ', op return True for json in jsonresultlist: if json: try: jsonRes = headJson + '''%s}''' jsonRes = jsonRes % (senderNodeId, logTime, limitNum, p1, '1', str(json)) #sockettcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #sockettcp.connect((socketserver,int(socketport))) sockettcp.sendall('<134>%s\n' % jsonRes) #sockettcp.close() #print 'successful!!' except: time.sleep(2) sockettcp.sendall('<134>%s\n' % jsonRes) try: sockettcp.connect((socketserver, int(socketport))) except: log(modname, 3, '发送采集数据失败。: %s %s' % (modname, json)) sockettcp.close() return True
def antifixedN_func(line=None): '''外挂日志antifixed采集''' if line == 'nothingData': senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'antifixed', '') if not senderNodeId: return jsonresult = headJson + '''{}}''' return jsonresult % (senderNodeId, logTime, limitNum, 'p1-antifixed', '4') if 'Acct' not in line: return modname = 'antifixedNew' if 'zxy' in getareaname(): area = getareaname().replace('zxy', '', 1) else: area = getareaname() group = 'game' + hostname.split('game')[-1] try: lvstr = line.replace('lv= ', 'lv=') ip = str2ip(lvstr.split(' ')[6].split('=')[1]) detime = lvstr.split(' ')[0][1:5] + '-' + line.split( ' ')[0][5:7] + '-' + line.split(' ')[0][7:9] + ' ' + line.split( ' ')[1][0:8] if int(lvstr.split(' ')[0][1:9]) < 20140314: print 'dateTime', detime return lv = lvstr.split(' ')[4].split('=')[1] acc = lvstr.split(' ')[8].split('=')[1] ret = lvstr.split(' ')[7].split('=')[1][1:-1] senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'antifixed', '') jsonresult = '' if senderNodeId: jsonresult = headJson + '''{"zone":"%s","group":"%s","accountid":"","acc":"%s","lv":"%s","name":"","party":"","wg":"","flag":"","optime":"","ver":"","is_killed":"","ret":"%s","ip":"%s","detime":"%s"}}''' jsonresult = jsonresult % (senderNodeId, logTime, limitNum, 'p1-antifixed', '1', area, group, acc, lv, ret, ip, detime) except: log(modname, 3, '日志分析错误: %s' % line) return return jsonresult
def macN_func(line=None, extra=None): '''实时登录数据采集''' print line, ' ddd ', extra if line == 'nothingData': print 'macN_funcmacN_func:', line senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'mac', 'login') if not senderNodeId: return jsonresult = headJson + '''{}}''' jsonresult = jsonresult % (senderNodeId, logTime, limitNum, 'p1-onlinemac', '4') print jsonresult return jsonresult if 'Acct' not in line: return modname = 'macNew' area = hostname.split('.')[0].split('login')[0] if 'zxy' in area: area = area.replace('zxy', '', 1) else: area = area jsonresult = '' try: senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'mac', 'login') if not senderNodeId: return mac = line.split(' ')[3].split('=')[1] ip = line.split(' ')[2].split('=')[1] name = line.split(' ')[4].split('=')[1].strip('\n') t = re.compile('\d* \d*:\d*:\d*').findall(line)[0] addtime = t[0:4] + '-' + t[4:6] + '-' + t[6:8] + t[8:] if senderNodeId: jsonresult = headJson + '''{"zone":"%s","mac":"%s","ip":"%s","name":"%s","addtime":"%s"}}''' jsonresult = jsonresult % ( str(senderNodeId), str(logTime), str(limitNum), 'p1-onlinemac', '1', str(area), str(mac), str(ip), str(name), str(addtime)) except: log(modname, 3, '日志分析错误: %s' % line) return print jsonresult return jsonresult
def diamondN_func(socketconfig='/tools/config/nsconfig/collectNew.ini:diamond', gameconf="/ss/dblog/db_server_user.ini:db_log"): '''福利点钻石数据采集''' filepath, modname = socketconfig.split(':') if 'collectNew' not in socketconfig: markfile = '/opt/mark_%s' % modname else: markfile = '/opt/mark_%s' % modname # if not os.path.exists('/opt/newmarks'): # os.mkdir('/opt/newmarks') # markfile = '/opt/newmarks/mark_%sNew'%modname mysqlconf = getmysqlconf(gameconf) if os.path.isfile(markfile): maxidx = int(file(markfile).read()) else: maxidx_shell = '''%s -e "select count(*) from t_bind_diamond_log;"|grep -v count''' % mysqlconf ret = getstatusoutput(maxidx_shell) if ret[0]: print 'mysql selected failed:%s' % str(ret) return if int(ret[1]) and int(ret[1]) > 1000: maxidx = int(ret[1]) - 1000 shell = '''%s -e "select * from t_bind_diamond_log where idx>%s;"|grep -v idx''' % ( mysqlconf, maxidx) ret = getstatusoutput(shell) if ret[0] and not ret[1]: senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'diamond', '') jsonresult = headJson + '''{}}''' jsonresult = jsonresult % (senderNodeId, logTime, limitNum, 'p1-diamondexchange', '4') result = submit(filepath, modname, jsonresult, 'diamond', 'p1-diamondexchange', '4') if not result: print 'socket submit error' return else: return True if ret[0] and ret[1]: print 'mysql selected failed:%s' % str(ret) return jsonresultlist, newmaxidx = analysis_diamond(ret[1]) result = submit(filepath, modname, jsonresultlist, 'diamond', 'p1-diamondexchange') if not result: print 'socket submit error' return f = file(markfile, 'w') f.write('%s' % (newmaxidx)) f.close() return True
def gmlogN_func(line=None, extra=None): '''GM聊天日志采集''' if line == 'nothingData': senderNodeId,logTime,limitNum,headJson = getNodeIdAndType('gmlog','') jsonresult = headJson+'''{}}''' #print 'op is 4' jsonresults = jsonresult%(senderNodeId,logTime,limitNum,'p1-gmlog','4') #print jsonresult return jsonresults if '[' not in line : return modname = 'gmlogNew' jsonresults = '' if 'zxy' in getareaname(): area = getareaname().replace('zxy', '', 1) else: area = getareaname() group = 'game' + hostname.split('game')[-1] try: sender = line.split(' ')[3].replace('[','') receiver = line.split(' ')[5].replace(']','') fc = line.split(' ')[2][1:2] num = line.split(' ')[2][2:] t = re.compile('\d* \d*:\d*:\d*').findall(line)[0] atime = t[0:4]+'-'+t[4:6]+'-'+t[6:8]+t[8:] text = line.split(']')[2].strip('\n').replace("'","\'") senderNodeId,logTime,limitNum,headJson = getNodeIdAndType('gmlog','') if senderNodeId: jsonresult = headJson+'''{"zone":"%s","group":"%s","sender":"%s","receiver":"%s","fc":"%s","num":"%s","atime":"%s","cmd":"%s"}}''' jsonresults = jsonresult%( senderNodeId,logTime,limitNum,'p1-gmlog','1',area,group,sender,receiver,fc,num,atime,text ) except: log(modname, 3, '日志分析错误: %s' %line) return return jsonresults
def welfareN_func(socketconfig='/tools/config/nsconfig/collectNew.ini:welfare', gameconf="/ss/dblog/db_server_user.ini:db_log"): '''福利点数据采集''' filepath, modname = socketconfig.split(':') if 'collectNew' not in socketconfig: markfile = '/opt/mark_%s' % modname else: markfile = '/opt/mark_%s' % modname # if not os.path.exists('/opt/newmarks'): # os.mkdir('/opt/newmarks') # markfile = '/opt/newmarks/mark_%sNew'%modname mysqlconf = getmysqlconf(gameconf) if os.path.isfile(markfile): lastmark = file(markfile).read() else: lastmark = '' tablelist = gettablelist(mysqlconf, lastmark) for table, lastuserlogid in tablelist: shell = '''%s -e "select * from %s where userlogtype=912 and userlogid>%s;"|grep -v userlogid''' % ( mysqlconf, table, lastuserlogid) ret = getstatusoutput(shell) if ret[0] and not ret[1]: senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'welfare', '') jsonresult = headJson + '''{}}''' jsonresult = jsonresult % (senderNodeId, logTime, limitNum, 'p1-diamondconsume', '4') result = submit(filepath, modname, jsonresult, 'welfare', 'p1-diamondconsume', '4') if not result: print 'socket submit error' return else: continue # return True if ret[0] and ret[1]: print 'mysql selected failed:%s' % str(ret) return jsonresultlist, maxuserlogid = analysis_welfare(ret[1]) result = submit(filepath, modname, jsonresultlist, 'welfare', 'p1-diamondconsume') if not result: print 'socket submit error' return f = file(markfile, 'w') f.write('%s:%s' % (table, maxuserlogid)) f.close() return True
def cpumen(socketconfig='/tools/config/nsconfig/collectNew.ini:cpumen',servername='game_server'): area,group,oid,ctime = gethostinfo() cpu,memory = getcpumem(servername) version_num = getversion() jsonresult = '' senderNodeId,logTime,limitNum,headJson = getNodeIdAndType('cpumen','game') if senderNodeId: #print headJson jsonresult = headJson+'''{"zone":"%s","group":"%s","time":"%s","memory":"%s","cpu":"%s","oid":"%s","version_num":"%s"}}''' #print jsonresult jsonresult = jsonresult%( str(senderNodeId),str(logTime),str(limitNum),'p1-mmon','1',str(area),str(group),str(ctime),str(memory),str(cpu),str(oid),str(version_num) ) # cpumem_json = {'zone':area,'group':group,'time':ctime,'memory':memory,'cpu':cpu,'oid':oid,'version_num':version_num} if jsonresult: result = submit(socketconfig,jsonresult) return result else: return False
def rankN_func(file=None, type=None, modname='rankNew'): """ rank日志数据采集 @param file: @param type: @param modname: @return: """ jsonresult = '''{"zone":"%s","group":"%s","type":"%s","msg":[%s]}''' resstr = '''{"id":"%s","ownername":"%s","lev":"%s","equipname":"%s","equipscore":"%s","other":"%s"},''' list = [] dics = { 'id': '', 'ownername': '', 'lev': '', 'equipname': '', 'equipscore': '', 'other': '' } num = 0 re = '' #结果集 try: f = open(file, 'r') # jsonresult['zone'] = zone # jsonresult['group'] = group # jsonresult['type'] = type for line in f: if not line.strip().strip('\r\n'): continue if num == 0: num += 1 continue line1 = line.decode('GBK').encode('UTF-8').strip().strip('\r\n') # print 'rank_func-File:%s %s\n' % (file,line1) tmp = line1.split(',') # print 'tmp:%s\n' %tmp if (len(tmp) == 5): dics['id'] = tmp[0] dics['ownername'] = tmp[1].replace("\"", "") dics['lev'] = tmp[2] dics['equipname'] = tmp[3] dics['equipscore'] = tmp[4].replace("\"", "") dics['other'] = '' else: dics['id'] = tmp[0] dics['ownername'] = tmp[1].replace("\"", "") dics['lev'] = tmp[2] dics['equipname'] = tmp[3] dics['equipscore'] = tmp[4].replace("\"", "") dics['other'] = tmp[5] res = resstr % (dics['id'], dics['ownername'], dics['lev'], dics['equipname'], dics['equipscore'], dics['other']) re += res # print '%s' %res # list.append('%s' %res) # list.append(dics) except: log(modname, 3, '日志分析错误: %s' % file) return # finally: f.close() # str = resstr % tmp[0],tmp[1],tmp[2],tmp[3],tmp[4] all = jsonresult % (zone, group, type, re.rstrip(',')) # print all senderNodeId, logTime, limitNum, headJson = getNodeIdAndType('rank', '') #print senderNodeId,logTime,limitNum jsonresults = '' if senderNodeId: jsonresults = headJson + all + '}' jsonresults = jsonresults % (senderNodeId, logTime, limitNum, 'p1-gametoplog', '1') #print jsonresults #loger = getLog('/var/log/runlog','rankNews') #loger.info(str(jsonresults)) return jsonresults
def actmod_func(line=None, f_time=None): '''行为日志采集''' # print '#Peng# 进入阶段一' server, zone, hs_all = gethostname() pid = 'null' if "shumenol" in hs_all: pid = 'sm' elif "zuixiaoyao" in hs_all: pid = 'zxy' if line == 'nothingData': senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'actmod', 'game') headJson_list = headJson.split(',') headJson_list.insert(-1, '"pid":"%s"') headJson = ','.join(headJson_list) headJson = str(headJson) jsonresult = headJson + '''{}}''' return jsonresult % (senderNodeId, logTime, limitNum, 'p1-misclog', '4', pid) if 'acctid,acct' in line: return modname = 'actmodNew' resultList = [] # print '进入阶段二' # if 'zxy' in getareaname(): # area = getareaname().replace('zxy', '', 1) # else: # area = getareaname() # group = 'game' + hostname.split('game')[-1] group = server[server.index('game'):] if os.path.exists('/opt/cur_file_mark_actmod'): f = open('/opt/cur_file_mark_actmod', 'r') file = f.readline().strip() f.close() timestamp = os.path.getmtime(file) ctime = datetime.datetime.fromtimestamp(timestamp) else: ctime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) try: # print '进入阶段三' # print '#Peng# 传入的文件名是:', f_time if f_time.count('_') > 2: f_name = f_time.split('_') file_time = f_name[1] + f_name[2] else: file_time = f_time.split('_', 1)[1].split('.')[0].replace('_', '') # server, zone, hs_all = gethostname() # pid = 'null' # if "shumenol" in hs_all: # pid = 'sm' # elif "zuixiaoyao" in hs_all: # pid = 'zxy' if 'zxy' in server: zone = server[server.index('zxy') + 3:server.index('game')] elif 'sm' in server: zone = server[server.index('sm') + 2:server.index('game')] else: zone = server[:server.index('game')] acctid = line.split(',')[0] acct = line.split(',')[1] actid = line.split(',')[2] act = line.split(',')[3] level = line.split(',')[4] # 没有关键字not_classic_sign_keyword,表示非经典服 if find_keyword_in_file(not_classic_sign_file, not_classic_sign_keyword): print '#Peng# 非经典服' godlev = line.split(',')[5] m = line.split(',')[6] ec = line.split(',')[7] km = line.split(',')[8] kgm = line.split(',')[9] ki = line.split(',')[13] gold = line.split(',')[14] ectype = line.split(',')[15] fbMsg = line.split(',')[16] kgt = line.split(',')[10] kgs = line.split(',')[11] kgi = line.split(',')[12] ectypetm = line.split(',')[18] killu = line.split(',')[19] bkillu = line.split(',')[20] bindmoney = line.split(',')[25] binddiamond = line.split(',')[26] gainbinddiamond = line.split(',')[27] tradeincnt = line.split(',')[28] tradeoutcnt = line.split(',')[29] tradeinmoney = line.split(',')[30] tradeoutmoney = line.split(',')[31] stallcnt = line.split(',')[32] stallmoney = line.split(',')[33] else: print '#Peng# 经典服' godlev = '' m = line.split(',')[5] ec = line.split(',')[6] km = line.split(',')[7] kgm = line.split(',')[8] ki = line.split(',')[12] gold = line.split(',')[13] ectype = line.split(',')[14] fbMsg = line.split(',')[15] kgt = line.split(',')[9] kgs = line.split(',')[10] kgi = line.split(',')[11] ectypetm = line.split(',')[17] killu = line.split(',')[18] bkillu = line.split(',')[19] bindmoney = line.split(',')[20] binddiamond = line.split(',')[21] gainbinddiamond = line.split(',')[22] tradeincnt = line.split(',')[23] tradeoutcnt = line.split(',')[24] tradeinmoney = line.split(',')[25] tradeoutmoney = line.split(',')[26] stallcnt = line.split(',')[27] stallmoney = line.split(',')[28] # ctime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime()) senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'actmod', 'game') headJson_list = headJson.split(',') headJson_list.insert(-1, '"pid":"%s"') headJson = ','.join(headJson_list) headJson = str(headJson) # print '#Peng# headJson数据是:',headJson jsonresult = headJson + '''{"gold": "%s", "tradeoutcnt": "%s", "bkillu": "%s", "binddiamond": "%s", "ec": "%s", "gainbinddiamond": "%s", "acctid": "%s", "acct": "%s", "ectype": "%s", "stallmoney": "%s", "tradeincnt": "%s", "zone": "%s", "ectypetm": "%s", "stallcnt": "%s", "actid": "%s", "killu": "%s", "kgs": "%s", "kgt": "%s", "time": "%s", "kgi": "%s", "kgm": "%s", "bindmoney": "%s", "level": "%s", "ki": "%s", "m": "%s", "tradeinmoney": "%s", "km": "%s", "server": "%s", "tradeoutmoney": "%s", "act": "%s","group":"%s"}}''' # if senderNodeId: jsonresults = jsonresult % ( senderNodeId, logTime, limitNum, 'p1-misclog', '1', pid, gold, tradeoutcnt, bkillu, binddiamond, ec, gainbinddiamond, acctid, acct, ectype, stallmoney, tradeincnt, zone, ectypetm, stallcnt, actid, killu, kgs, kgt, file_time, kgi, kgm, bindmoney, level, ki, m, tradeinmoney, km, server, tradeoutmoney, act, group) # print '经过处理后的数据:', jsonresults resultList.append(jsonresults) # commit(sockettcp,jsonresults,socketserver,socketport) # sockettcp.close() return resultList except: print '#Peng#:', traceback.format_exc() log(modname, 3, '日志内容分析错误: %s,错误原因:【%s】' % (line, traceback.format_exc())) return resultList
logData = str(jsonData) data = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + logData + "\n" logWrite(data) sockettcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: # status = sockettcp.connect_ex((socketserver,int(socketport))) status = 111 except Exception, e: error = "%s" % e data = time.strftime("%Y-%m-%d %H-%M-%S", time.localtime()) + error + "\n" logWrite(data) print "not connect ", e if (status != 113): senderNodeId, logTime, limitNum, headJson = getNodeIdAndType( 'loginerror', 'login') if not senderNodeId: return #print senderNodeId,logTime,limitNum if senderNodeId: jsonresult = '''{"senderNodeId":"%s","receiverNodeId":"nodereceive001","logTime":"%s","logSeqNo":"%s","logType":"p1-gamelogin","op":"1","params":%s}''' sockettcp.connect((socketserver, int(socketport))) sockettcp.settimeout(3) #print 'sssssssssssss: ',jsonData if not jsonData: jsonRes = headJson % (senderNodeId, logTime, limitNum, "p1-gamelogin", "4") + '''{}}''' try: print jsonRes, 'jsonData is null, op is 4' sockettcp.send('<134>%s\n' % jsonRes) sockettcp.close()