def apiHaodou(line): cols=line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: return method=cols[column.METHOD_CID] if method in StopMethods: return uid=column.uid(cols) if uid == None or uid == "": return None intTime=cols[column.TIME_CID] ip=cols[column.IP_CID] return "%s\t%s\t%s\thaodou"%(uid,ip,dayFromInt(intTime))
def count(): for line in sys.stdin: cols=line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue uuid=column.uuidOnly(cols[1:]) if uuid == None: continue time=cols[1] method=cols[column.METHOD_CID+1] uid=column.uid(cols[1:]) response="" if method in regMs and method in column2.FuncMap and len(cols) >= column.APP_LOG_COLUMNS4+1: response=column2.FuncMap[method](cols[-1]) if response == None: response="" #print method print "%s\t%s\t%s\t%s\t%s"%(uuid,time,uid,method,response)
def count(): for line in sys.stdin: cols = line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue uuid = column.uuidOnly(cols[1:]) if uuid == None: continue time = cols[1] method = cols[column.METHOD_CID + 1] uid = column.uid(cols[1:]) response = "" if method in regMs and method in column2.FuncMap and len( cols) >= column.APP_LOG_COLUMNS4 + 1: response = column2.FuncMap[method](cols[-1]) if response == None: response = "" #print method print "%s\t%s\t%s\t%s\t%s" % (uuid, time, uid, method, response)
def ips(): us = uids() for line in sys.stdin: cols = line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue uid = column.uid(cols) if uid == "": continue if uid not in us: continue ip = cols[column.IP_CID] if ip not in us[uid]: us[uid][ip] = [1, 0] else: us[uid][ip][0] += 1 wf = open("nutriUidIps.txt", "w") for uid in us: print uid + "\t" + str(us[uid]) wf.write(uid + "\t" + str(us[uid]) + "\n") wf.close()
def map(): us = {} for line in sys.stdin: cols = line.strip().split("\t") if len(cols) == Reduce_Cols and cols[1] in types: #过去的数据累加 print line.strip() continue if not column.valid(cols): continue uid = column.uid(cols) uuid = column.uuidOnly(cols) ip = cols[column.IP_CID] time = cols[column.TIME_CID] channel = cols[column.MEDIA_CID] if uid != None and len(uid) != 0: addItem2(U_Type, uuid, time, us, Uid_Fix + uid) addItem2(Channel_Type, channel, time, us, Uid_Fix + uid) addItem2(Ip_Type, ip, time, us, Uid_Fix + uid) if uuid != None and len(uuid) != 0: addItem2(U_Type, uid, time, us, Uuid_Fix + uuid) addItem2(Channel_Type, channel, time, us, Uuid_Fix + uuid) addItem2(Ip_Type, ip, time, us, Uuid_Fix + uuid) for u in us: output(u, us[u])
def distribute(): vs={} ms={} cs={} for line in sys.stdin: cols=line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue version=cols[column.VERSION_CID] channel=cols[column.MEDIA_CID] method=cols[column.METHOD_CID] uid=column.uid(cols) if uid == "": uid=None if method in regMs and uid == None: DictUtil.addOne(vs,version) DictUtil.addOne(ms,method) DictUtil.addOne(cs,channel) for v in vs: print "%s\t%d"%(v,vs[v]) for m in ms: print "%s\t%d"%(m,ms[m]) for c in cs: print "%s\t%d"%(c,cs[c])
def distribute(): vs = {} ms = {} cs = {} for line in sys.stdin: cols = line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue version = cols[column.VERSION_CID] channel = cols[column.MEDIA_CID] method = cols[column.METHOD_CID] uid = column.uid(cols) if uid == "": uid = None if method in regMs and uid == None: DictUtil.addOne(vs, version) DictUtil.addOne(ms, method) DictUtil.addOne(cs, channel) for v in vs: print "%s\t%d" % (v, vs[v]) for m in ms: print "%s\t%d" % (m, ms[m]) for c in cs: print "%s\t%d" % (c, cs[c])
def map(): us={} for line in sys.stdin: cols=line.strip().split("\t") if len(cols) == Reduce_Cols and cols[1] in types: #过去的数据累加 print line.strip() continue if not column.valid(cols): continue uid=column.uid(cols) uuid=column.uuidOnly(cols) ip=cols[column.IP_CID] time=cols[column.TIME_CID] channel=cols[column.MEDIA_CID] if uid != None and len(uid) !=0: addItem2(U_Type,uuid,time,us,Uid_Fix+uid) addItem2(Channel_Type,channel,time,us,Uid_Fix+uid) addItem2(Ip_Type,ip,time,us,Uid_Fix+uid) if uuid != None and len(uuid) !=0: addItem2(U_Type,uid,time,us,Uuid_Fix+uuid) addItem2(Channel_Type,channel,time,us,Uuid_Fix+uuid) addItem2(Ip_Type,ip,time,us,Uuid_Fix+uuid) for u in us: output(u,us[u])
def do(userFile): us={} vms={} for line in open(userFile): us[line.strip()]=1 for line in sys.stdin: cols=line.strip().split("\t") uid=column.uid(cols) if uid == None or uid == "": continue if uid not in us: continue v=cols[column.VERSION_CID] v=column.intVersion(v) method=cols[column.METHOD_CID] if v not in vms: vms[v]={} if method not in vms[v]: vms[v][method]=1 else: vms[v][method]+=1 for v in vms: for method in vms[v]: print "%d\t%s\t%d"%(v,method,vms[v][method])
def do(userFile): us = {} vms = {} for line in open(userFile): us[line.strip()] = 1 for line in sys.stdin: cols = line.strip().split("\t") uid = column.uid(cols) if uid == None or uid == "": continue if uid not in us: continue v = cols[column.VERSION_CID] v = column.intVersion(v) method = cols[column.METHOD_CID] if v not in vms: vms[v] = {} if method not in vms[v]: vms[v][method] = 1 else: vms[v][method] += 1 for v in vms: for method in vms[v]: print "%d\t%s\t%d" % (v, method, vms[v][method])
def count(): lastUuid="" regFirstDo="NO" hasUid="" ms=[] phone="" for line in sys.stdin: cols=line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue uuid=column.uuidOnly(cols[1:]) if uuid == None: continue uid=column.uid(cols[1:]) method=cols[column.METHOD_CID+1] resp="" if method in regMs and method in column2.FuncMap and len(cols) >= column.APP_LOG_COLUMNS4+1: resp=column2.FuncMap[method](cols[-1]) if resp == None: resp = "" if lastUuid == "": lastUuid=uuid if lastUuid != uuid: output(lastUuid,ms,regFirstDo,phone) lastUuid=uuid regFirstDo="NO" ms=[] hasUid="" phone="" if method in regMs and hasUid == "": if method == "passport.bindconnectstatus" and PositiveNumP.match(resp): hasHit=True if len(ms) >= 1: regFirstDo="OK" uid=resp #sys.stderr.write("bind uid:"+uid+"\n") elif method == "passport.reg" and PositiveNumP.match(resp): hasHit=True if len(ms) >= 1: regFirstDo="OK" uid=resp #sys.stderr.write("reg uid:"+uid+"\n") else: if resp != "": ms.append(method+"("+resp+")") else: ms.append(method) elif len(ms) >= 1: if (regFirstDo=="NO" or regFirstDo=="OK"): if method in regUserMs: regFirstDo=method elif uid != "": regFirstDo="OK" if uid != "": hasUid=uid if method == "common.sendcode": phone=column.getValue(cols[column.PARA_ID+1],"phone") #sys.stderr.write(method+"\t"+cols[column.PARA_ID+1]+"\t"+phone+"\n") if lastUuid != "": output(lastUuid,ms,regFirstDo,phone) for m in lms: if m.endswith("_NO"): continue v=lms[m] nv=0 if m+"_NO" in lms: nv=lms[m+"_NO"] print "%s\t%d\t%d\t%.4f"%(m,lms[m],nv,nv/(v+1e-32))
def count(): lastUuid = "" regFirstDo = "NO" hasUid = "" ms = [] phone = "" for line in sys.stdin: cols = line.strip().split("\t") if len(cols) < column.APP_LOG_COLUMNS: continue uuid = column.uuidOnly(cols[1:]) if uuid == None: continue uid = column.uid(cols[1:]) method = cols[column.METHOD_CID + 1] resp = "" if method in regMs and method in column2.FuncMap and len( cols) >= column.APP_LOG_COLUMNS4 + 1: resp = column2.FuncMap[method](cols[-1]) if resp == None: resp = "" if lastUuid == "": lastUuid = uuid if lastUuid != uuid: output(lastUuid, ms, regFirstDo, phone) lastUuid = uuid regFirstDo = "NO" ms = [] hasUid = "" phone = "" if method in regMs and hasUid == "": if method == "passport.bindconnectstatus" and PositiveNumP.match( resp): hasHit = True if len(ms) >= 1: regFirstDo = "OK" uid = resp #sys.stderr.write("bind uid:"+uid+"\n") elif method == "passport.reg" and PositiveNumP.match(resp): hasHit = True if len(ms) >= 1: regFirstDo = "OK" uid = resp #sys.stderr.write("reg uid:"+uid+"\n") else: if resp != "": ms.append(method + "(" + resp + ")") else: ms.append(method) elif len(ms) >= 1: if (regFirstDo == "NO" or regFirstDo == "OK"): if method in regUserMs: regFirstDo = method elif uid != "": regFirstDo = "OK" if uid != "": hasUid = uid if method == "common.sendcode": phone = column.getValue(cols[column.PARA_ID + 1], "phone") #sys.stderr.write(method+"\t"+cols[column.PARA_ID+1]+"\t"+phone+"\n") if lastUuid != "": output(lastUuid, ms, regFirstDo, phone) for m in lms: if m.endswith("_NO"): continue v = lms[m] nv = 0 if m + "_NO" in lms: nv = lms[m + "_NO"] print "%s\t%d\t%d\t%.4f" % (m, lms[m], nv, nv / (v + 1e-32))