def count():
orders={}
for line in sys.stdin:
cols=line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
method=cols[column.METHOD_CID]
if method != "search.getlistv3":
continue
para=cols[column.PARA_ID]
offset=column.getValue(para,"offset")
if offset != "0":
continue
order=column.getValue(para,"order")
if order not in orders:
orders[order]=1
else:
orders[order]+=1
ns={}
for order in orders:
n=order
if order in orderMap:
n=orderMap[order]
ns[n]=orders[order]
for n in ns:
print n+"\t"+str(ns[n])
def test():
cc=CardClick("人气")
print type(cc)
cc.posHit[0]=1
print cc
sys.path.append("../")
import column
sys.path.append("../abtest/")
import column2
cardBank=CardBank()
for line in sys.stdin:
cols=line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS+1:
continue
method=cols[column.METHOD_CID+1] #获得请求的方法
if method == "search.getlist" or method == "search.getsearchindex":
keyword=column.getValue(cols[column.PARA_ID+1],"keyword")
scene=column.getValue(cols[column.PARA_ID+1],"scene")
tagid=column.getValue(cols[column.PARA_ID+1],"tagid")
version=cols[column.VERSION_CID+1]
ip=cols[0]
if version == "":
continue
ret=column2.FuncMap[method](cols[-1]) #获得搜索返回的食谱列表
if (scene == "" or scene == "k1") and tagid == "" and keyword == "" and "food" in ret and ret["food"] != None and ret["food"] != 0:
print ret["food"]
print ip
#print line
continue
cardBank.addSearch(keyword,ret)
def getShareVersionItem(method,para,version): itemid=column.getValue(para,"itemid") type=column.getValue(para,"type") if version == None or version < 400: return getV3TypeId(itemid,type) else: return getV4TypeId(itemid,type)
def getShareVersionItem(method, para, version):
itemid = column.getValue(para, "itemid")
type = column.getValue(para, "type")
if version == None or version < 400:
return getV3TypeId(itemid, type)
else:
return getV4TypeId(itemid, type)
def count():
orders = {}
for line in sys.stdin:
cols = line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
method = cols[column.METHOD_CID]
if method != "search.getlistv3":
continue
para = cols[column.PARA_ID]
offset = column.getValue(para, "offset")
if offset != "0":
continue
order = column.getValue(para, "order")
if order not in orders:
orders[order] = 1
else:
orders[order] += 1
ns = {}
for order in orders:
n = order
if order in orderMap:
n = orderMap[order]
ns[n] = orders[order]
for n in ns:
print n + "\t" + str(ns[n])
def requestIdCheck(self, action):
rqid = column.getValue(action.para, "return_request_id")
qid = column.getValue(self.para, "request_id")
if rqid != "" and qid != "":
if rqid == qid:
return 1
else:
return -1
return 0
def getParentMethod(method, para):
offset = column.getValue(para, "offset")
if offset != "0" and offset != "":
return method
if method == "search.getlist":
scene = column.getValue(para, "scene")
if scene == "t1":
return "search.getcatelist"
elif scene == "t2":
return "recipe.getcollectlist"
elif scene == "k3":
return "info.getinfo"
return ""
def parseFeedLine(line,repeats={}):
cols=line.strip().split("\",\"")
if len(cols) < ColsNum:
return None
uuid=cols[1]
content=cols[4]
channel=column.getValue(cols[5],"channel_type")[3:-1]
if channel.startswith('"'):channel=channel[1:]
createTime=cols[7][0:-1]
version=column.getValue(cols[5],"soft_version")[3:-1]
if version.startswith('"'): version=version[1:]
feed=Feed(uuid,createTime,content,channel,version)
if uuid+"_"+content in repeats:
return None
repeats[uuid+"_"+content]=1
return feed
def getSearchName(method, para):
scene = column.getSearchType(method, para)
if scene == "t2":
tag = column.getValue(para, "tagid")
if tag != "":
scene += SplitSign + getActionItem.tagName(tag)
return scene
def getSearchName(method,para): scene=column.getSearchType(method,para) if scene == "t2": tag=column.getValue(para,"tagid") if tag != "": scene+=SplitSign+getActionItem.tagName(tag) return scene
def parseLine(line,type="appLog"):
cols=line.strip().split("\t")
if type == "userSort":
(action,user)=sequence.getActionUser(cols,0)
if action == None:
return (None,None)
if user == None:
return (None,None)
item=hitItemName.getPosName(action)
return (item,user.name())
elif type == "appLog":
if not column.valid(cols):
return (None,None)
method=cols[column.METHOD_CID]
para=cols[column.PARA_ID]
version=column.intVersion(cols[column.VERSION_CID])
item=getActionItem.getActionItem(method,para,version)
name=hitItemName.getMethodName(method,para)
offset=column.getValue(para,"offset")
if offset == "0":
name+=hitItemName.SplitSign+"0"
item=hitItemName.packItem(name,item)
uuid=column.uuid(cols)
return (item,uuid)
return (None,None)
def qid_m(f):
for line in f:
line = line.strip()
if line.startswith('{"status":'): #response日志
q = column.getQid2(line, "request_id")
time = line.strip().split('"time":')[-1][:-1] #获取response的时间
if q != None and q != "":
print q + "\t" + time + '\t' + line
elif line[0:4].find(".") > 0: #m_haodou_com日志
cols = line.strip().split("\01")
if len(cols) > 5:
ip = cols[0]
try:
time = str(toIntTime(cols[2].split()[0]))
print str(random.randint(
0,
99)) + "\t" + ip + "\t" + time + "\t" + line.strip()
except:
sys.stderr.write(line)
continue
else: #请求日志
cols = line.split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
sys.stderr.write(line)
continue
q = column.getValue(cols[column.PARA_ID], "request_id")
if q != None and q != "":
print q + "\t" + line
else:
print str(random.randint(0, 99)) + "\t" + line
def qid_m(f):
for line in f:
line=line.strip()
if line.startswith('{"status":'): #response日志
q=column.getQid2(line,"request_id")
time=line.strip().split('"time":')[-1][:-1] #获取response的时间
if q != None and q != "":
print q+"\t"+time+'\t'+line
elif line[0:4].find(".") > 0: #m_haodou_com日志
cols=line.strip().split("\01")
if len(cols) > 5:
ip=cols[0]
try:
time=str(toIntTime(cols[2].split()[0]))
print str(random.randint(0,99))+"\t"+ip+"\t"+time+"\t"+line.strip()
except:
sys.stderr.write(line)
continue
else: #请求日志
cols=line.split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
sys.stderr.write(line)
continue
q=column.getValue(cols[column.PARA_ID],"request_id")
if q != None and q != "":
print q+"\t"+line
else:
print str(random.randint(0,99))+"\t"+line
def albumItems(method, para, response):
ret = []
if "rids" in response:
for rid in response["rids"]:
ret.append("rid-" + rid)
ret.append("aid-" + column.getValue(para, "aid"))
return ret
def albumItems(method,para,response):
ret=[]
if "rids" in response:
for rid in response["rids"]:
ret.append("rid-"+rid)
ret.append("aid-"+column.getValue(para,"aid"))
return ret
def countSearch(file):
tags=readCates(file)
for line in sys.stdin:
if line.find("search.getlist") < 0:
continue
cols=line.strip().split("\t")
if not column.valid(cols):
continue
scene=column.getValue(cols[column.PARA_ID],"scene")
if scene == "t2":
continue
tagid=column.getValue(cols[column.PARA_ID],"tagid")
if tagid != None and tagid != "" and scene == "t1":
print tagid
continue
keyword=column.getValue(cols[column.PARA_ID],"keyword")
if keyword != None and keyword in tags:
print tags[keyword]
def getInfoItems(method, para, response):
ret = []
if "tags" in response:
for tag in response["tags"]:
ret.append("tag-" + tagName(tag))
ret.append("rid-" + column.getValue(para, "rid"))
if "pids" in response:
for pid in response["pids"]:
ret.append("pid-" + pid)
return ret
def getInfoItems(method,para,response):
ret=[]
if "tags" in response:
for tag in response["tags"]:
ret.append("tag-"+tagName(tag))
ret.append("rid-"+column.getValue(para,"rid"))
if "pids" in response:
for pid in response["pids"]:
ret.append("pid-"+pid)
return ret
def actionResultItems(method,para,response): if method in itemsPair: ret=itemsPair[method](method,para,response) offset=column.getValue(para,"offset") if offset == "0": item=getActionItem(method,para,400) if item != "#": ret.append(item) return ret else: return []
def actionResultItems(method, para, response):
if method in itemsPair:
ret = itemsPair[method](method, para, response)
offset = column.getValue(para, "offset")
if offset == "0":
item = getActionItem(method, para, 400)
if item != "#":
ret.append(item)
return ret
else:
return []
def testTag():
import column
for line in sys.stdin:
line = line.strip()
cols = line.split("\t")
if len(cols) < column.APP_LOG_COLUMNS + 1:
continue
method = cols[column.METHOD_CID]
if not method.startswith("search.getlist"):
continue
keyword = column.getValue(cols[column.PARA_ID], "keyword")
if keyword == None or keyword == "":
continue
tags = {}
kw = keyword
#sys.stderr.write(keyword+"\n")
try:
keyword = keyword.decode("utf-8")
except UnicodeDecodeError:
sys.stderr.write(keyword + "\n")
continue
offset = column.getValue(cols[column.PARA_ID], "offset")
if offset != "" and offset != "0":
continue
match(keyword, tags, 0, len(keyword), True)
uuid = column.uuidFirst(cols)
if uuid == None or uuid == "":
continue
d = {}
for sub in tags:
d[sub[1]] = 1
for tag in tags[sub]:
d[tag] = 1
if tag not in tagm:
continue
for t in tagm[tag]:
d[t] = 1
s = uuid + "\t" + kw
for t in d:
s += "\t" + t.encode("utf-8")
print s
def testTag():
import column
for line in sys.stdin:
line=line.strip()
cols=line.split("\t")
if len(cols) < column.APP_LOG_COLUMNS+1:
continue
method=cols[column.METHOD_CID]
if not method.startswith("search.getlist"):
continue
keyword=column.getValue(cols[column.PARA_ID],"keyword")
if keyword == None or keyword == "":
continue
tags={}
kw=keyword
#sys.stderr.write(keyword+"\n")
try:
keyword=keyword.decode("utf-8")
except UnicodeDecodeError:
sys.stderr.write(keyword+"\n")
continue
offset=column.getValue(cols[column.PARA_ID],"offset")
if offset != "" and offset != "0":
continue
match(keyword,tags,0,len(keyword),True)
uuid=column.uuidFirst(cols)
if uuid == None or uuid == "":
continue
d={}
for sub in tags:
d[sub[1]]=1
for tag in tags[sub]:
d[tag]=1
if tag not in tagm:
continue
for t in tagm[tag]:
d[t]=1
s=uuid+"\t"+kw
for t in d:
s+="\t"+t.encode("utf-8")
print s
def output(actions):
if len(actions) == 0:
return 0
version=actions[0].user.version
if version not in cs:
cs[version]={}
for action in actions:
offset=column.getValue(action.para,"offset")
if offset != "" and offset != "0": #下拉页不单独统计
continue
stype=hitItemName.getPosName(action)
if stype not in cs[version]:
cs[version][stype]=PosRet()
ret=cs[version][stype]
addAction(action,actions,ret)
def output(actions):
if len(actions) == 0:
return 0
version = actions[0].user.version
if version not in cs:
cs[version] = {}
for action in actions:
offset = column.getValue(action.para, "offset")
if offset != "" and offset != "0":
continue
stype = hitItemName.getName(action)
if stype not in cs[version]:
cs[version][stype] = HitRet()
ret = cs[version][stype]
addAction(action, actions, ret)
def qid(f):
lastQLine=""
for line in f:
line=line.strip()
if line.startswith('{"status":'):
q=column.getQid2(line,"request_id")
if q != None and q != "":
print q+"\t"+line
elif line.startswith('{"remote_addr":"'):
ip=mlog.getU(line)
time=mlog.value(line,'"log_time":')
if ip != "" and ip != None:
print str(random.randint(0,99))+"\t"+ip+"\t"+time+"\t"+line.strip()
else:
cols=line.split("\t")
if not column.valid(cols):
continue
q=column.getValue(cols[column.PARA_ID],"request_id")
if q != None and q != "":
print q+"\t"+line
else:
print str(random.randint(0,99))+"\t"+line
def appLogHit(aid):
channels={}
for line in sys.stdin:
cols=line.strip().split("\t")
if not column.valid(cols):
continue
method=cols[column.METHOD_CID]
if method != "info.getalbuminfo":
continue
version=column.intVersion(cols[column.VERSION_CID])
if version < 400:
continue
taid=column.getValue(cols[column.PARA_ID],"aid")
if taid != aid:
continue
appid=cols[column.APPID_CID]
if appid not in channels:
channels[appid]=1
else:
channels[appid]+=1
for appid in channels:
print "%s\t%d"%(appid,channels[appid])
def appLogHit(aid):
channels = {}
for line in sys.stdin:
cols = line.strip().split("\t")
if not column.valid(cols):
continue
method = cols[column.METHOD_CID]
if method != "info.getalbuminfo":
continue
version = column.intVersion(cols[column.VERSION_CID])
if version < 400:
continue
taid = column.getValue(cols[column.PARA_ID], "aid")
if taid != aid:
continue
appid = cols[column.APPID_CID]
if appid not in channels:
channels[appid] = 1
else:
channels[appid] += 1
for appid in channels:
print "%s\t%d" % (appid, channels[appid])
def qid(f):
lastQLine = ""
for line in f:
line = line.strip()
if line.startswith('{"status":'):
q = column.getQid2(line, "request_id")
if q != None and q != "":
print q + "\t" + line
elif line.startswith('{"remote_addr":"'):
ip = mlog.getU(line)
time = mlog.value(line, '"log_time":')
if ip != "" and ip != None:
print str(random.randint(
0, 99)) + "\t" + ip + "\t" + time + "\t" + line.strip()
else:
cols = line.split("\t")
if not column.valid(cols):
continue
q = column.getValue(cols[column.PARA_ID], "request_id")
if q != None and q != "":
print q + "\t" + line
else:
print str(random.randint(0, 99)) + "\t" + line
def getSearchKey(para):
scene=column.getValue(para,"scene")
if scene.startswith("k"):
kw="keyword-"+column.getValue(para,"keyword")
if scene == "k1":
tagid=column.getValue(para,"tagid")
return kw+"-tag-"+tagName(tagid)
else:
return kw
else:
tagid=column.getValue(para,"tagid")
if tagid == "":
tagid=column.getValue(para,"typeid")
if tagid != "" and tagid != "null":
return "tag-"+tagName(tagid)
elif scene.startswith("t"):
return "tag-"
return "keyword-"+column.getValue(para,"keyword")
def getSearchKey(para):
scene = column.getValue(para, "scene")
if scene.startswith("k"):
kw = "keyword-" + column.getValue(para, "keyword")
if scene == "k1":
tagid = column.getValue(para, "tagid")
return kw + "-tag-" + tagName(tagid)
else:
return kw
else:
tagid = column.getValue(para, "tagid")
if tagid == "":
tagid = column.getValue(para, "typeid")
if tagid != "" and tagid != "null":
return "tag-" + tagName(tagid)
elif scene.startswith("t"):
return "tag-"
return "keyword-" + column.getValue(para, "keyword")
def searchFollowMapper(f):
targetUsers={}
for line in open("uid.1211.txt"):
targetUsers[line.strip()]=1
lastU=""
lastIP=""
rids={}
mtid=""
ftid=""
mtnum=0
ftnum=0
fmtnum=0
oc=0
otherRids={}
searchRet={}
click=None
keywordClick={}
lastK=""
hc=0
hrc=0
hac=0
htc=0
keyword=""
cardBank=CardBank()
for line in f:
if True:
cols=line.strip().split("\t")
if len(cols) < 3:
continue
if line.find("m.haodou.com") > 0 and line.find("GET") > 0:
cols=cols[2].strip().split("\01")
if len(cols) < 5:
continue
p=cols[4].find(THead)
if cols[0] == lastIP and p > 0:
id=cols[4][p+THeadLen:]
if ftid == "" and id !="" and "topicId" in searchRet and searchRet["topicId"] == id:
#click.addTopicHit(id)
#oc+=1
#if "ttitle" in searchRet:
# title=searchRet["ttitle"]
# click.addTopicTitleHit(title,keyword)
#mtid=id
pass
lastIP=cols[0]
continue
if len(cols) < column.APP_LOG_COLUMNS+1:
lastIP=cols[0]
continue
lastIP=cols[0]
version=cols[column.VERSION_CID+1]
v=column.intVersion(version)
if v < 400:
continue
u=column.uuidFirst(cols[1:]) #获得uuid
if u not in targetUsers:
continue
if u == None:
continue
if u.find('{') >= 0:
continue
para=cols[column.PARA_ID+1] #得到请求参数
if lastU == "":
lastU=u
if lastU != u:
lastU=u
keyword=""
rids={}
otherRids={}
searchRet={}
click=None
cardBank.reset()
if mtid != "":
mtnum+=1
if ftid == mtid:
fmtnum+=1
if ftid != "":
ftnum+=1
mtid=""
amtid=""
ftid=""
method=cols[column.METHOD_CID+1] #获得请求的方法
hasSearch=False
if method == 'search.getsearchindex': #搜索方法
keyword=V45Fix+column.getValue(para,"keyword") #搜索的关键字
hasSearch=True
searchRet=column2.FuncMap[method](cols[-1]) #获得搜索返回的食谱列表
rids=getRidPos(searchRet)
if keyword not in keywordClick:
keywordClick[keyword]=Click()
click=keywordClick[keyword]
click.addSearchRet(searchRet,keyword)
cardBank.addSearch(keyword[len(V45Fix):],searchRet)
#避免出现时间序列不对
#if amtid != "" and "topicId" in searchRet and searchRet["topicId"] == mtid:
# click.addTopicHit(mtid)
# oc+=1
# if "ttitle" in searchRet:
# title=searchRet["ttitle"]
# click.addTopicTitleHit(title,keyword)
# mtid=amtid
otherRids={}
if "rids" in searchRet:
for i in range(FirstPageNum,len(searchRet["rids"]),1):
otherRids[searchRet["rids"][i]]=i
elif method == "search.getlist" and v < 450:
offset=column.getValue(para,"offset")
scene=column.getValue(para,"scene")
if scene != "k1":
continue
#keyword=column.getValue(para,"keyword") #搜索的关键字
searchRet=column2.FuncMap[method](cols[-1]) #
if offset != "0":
kw=column.getValue(para,"keyword")
if kw != keyword:
continue
if "rids" in searchRet:
for i in range(len(searchRet["rids"])):
otherRids[searchRet["rids"][i]]=i
continue
keyword=column.getValue(para,"keyword")
hasSearch=True
rids=getRidPos(searchRet)
otherRids={}
#searchRet=column2.FuncMap[method](cols[-1]) #获得搜索返回的食谱列表
if "rids" in searchRet:
for i in range(len(searchRet["rids"])):
if i >= FirstPageNum:
otherRids[searchRet["rids"][i]]=i
continue
rids[searchRet["rids"][i]]=i
if keyword not in keywordClick:
keywordClick[keyword]=Click()
click=keywordClick[keyword]
click.addSearchRet(searchRet,keyword)
cardBank.addSearch(keyword,searchRet,False)
if hasSearch:
if lastK != "":
keywordClick[lastK].addHitCount(hc)
if hrc > 0:
keywordClick[lastK].getMs().rb+=1
if hac > 0:
keywordClick[lastK].getMs().ab+=1
if htc > 0:
keywordClick[lastK].getMs().tb+=1
if hc + hrc+oc > 0:
keywordClick[lastK].addHasHit()
if mtid != "":
mtnum+=1
if mtid == ftid:
fmtnum+=1
if ftid != "":
ftnum+=1
mtid=""
ftid=""
hc=0
hrc=0
hac=0
htc=0
oc=0
lastK=keyword
if click == None:
continue
if method == "info.getinfo":
rid=column.getValue(para,"rid") #获得点击的食谱id
pos=-1
if rid in rids:
click.addRecipeHit(rid,rids[rid])
kw=keyword
if v >= 450:
kw=keyword[len(V45Fix):]
cardBank.addHit(kw,rid,rids[rid],(v >= 450))
hc+=1
pos=rids[rid]
elif rid in otherRids:
oc+=1
if pos >= 0:
ret=column2.FuncMap[method](cols[-1])
if ret == None:
pass
#sys.stderr.write(cols[-1])
elif "title" in ret:
title=ret["title"]
click.addTitleHit(title,pos,keyword)
elif method == "info.getfoodinfo":
fid=column.getValue(para,"foodid")
if fid != "" and "food" in searchRet and fid == searchRet["food"]:
click.addFoodHit(fid)
oc+=1
elif v >= 450:
if method == "info.getalbuminfo":
id=column.getValue(para,"aid") #获取点击的专辑id
if id !="" and "aid" in searchRet and searchRet["aid"]==id :
click.addAlbumHit(id)
oc+=1
if "atitle" in searchRet:
title=searchRet["atitle"]
click.addAlbumTitleHit(title,keyword)
elif method == "favorite.isfav":
id=column.getValue(para,"rid") #获取话题id
type=column.getValue(para,"type")
if type == "3" and mtid == "" and id != "" and "topicId" in searchRet and searchRet["topicId"] == id:
#print para
#print "response",cols[-1]
#print ""
click.addTopicHit(id)
oc+=1
if "ttitle" in searchRet:
title=searchRet["ttitle"]
click.addTopicTitleHit(title,keyword)
ftid=id
elif method == "search.getlist":
offset=column.getValue(para,"offset")
scene=column.getValue(para,"scene")
if offset != "0":
continue
if scene != "k1":
continue
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
tagid=column.getValue(para,"tagid")
if tagid == "" or tagid == "null":
click.getMs().r+=1
hrc+=1
else:
if tagid not in click.getMs().rt:
click.getMs().rt[tagid]=1
else:
click.getMs().rt[tagid]+=1
elif method == "search.getalbumlist":
offset=column.getValue(para,"offset")
if offset != "0":
continue
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
click.getMs().a+=1
hac+=1
oc+=1
elif method == "search.gettopiclist":
offset=column.getValue(para,"offset")
if offset != "0":
continue
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
click.getMs().t+=1
htc+=1
oc+=1
if lastK != "":
keywordClick[lastK].addHitCount(hc)
if hrc > 0:
keywordClick[lastK].getMs().rb+=1
if hac > 0:
keywordClick[lastK].getMs().ab+=1
if htc > 0:
keywordClick[lastK].getMs().tb+=1
if hc + hrc+oc > 0:
keywordClick[lastK].addHasHit()
if mtid != "":
mtnum+=1
if ftid == mtid:
fmtnum+=1
if ftid != "":
ftnum+=1
ck45=Click()
ck44=Click()
for kw in keywordClick:
if kw.startswith(V45Fix):
ck45.merge(keywordClick[kw])
else:
ck44.merge(keywordClick[kw])
print kw+"\t"+str(keywordClick[kw])
print "ck45_##total##"+"\t"+str(ck45)
print "ck44_##total##"+"\t"+str(ck44)
for card in cardBank.bank:
print CardFix+str(cardBank.bank[card])
sys.stderr.write("mtnum:%d\tftnum:%d\tfmtnum:%d\n"%(mtnum,ftnum,fmtnum))
def getCommentItem(method, para):
type = column.getValue(para, "type")
rid = column.getValue(para, "rid")
return getV4TypeId(rid, type)
def getSearchTagsItem(method, para):
return "keyword-" + column.getValue(para, "keyword")
def getPhotoviewItem(method,para): return "pid-"+column.getValue(para,"id")
def addHit(self, action):
#print "self-addHit",self
#print "action-addHit",action
if action.method in ons and not self.method in ons[action.method]:
#print "self",self
#print "action",action
#print ""
return False
if action.user.sameUserGivenIP(self.user) < 0:
return False
if self.method in ns and action.method in ns[self.method]:
#print "self",self
#print "action",action
#print ""
return False
if self.method == action.method:
offset = column.getValue(self.para,
"offset") #只有列表页可以有方法名与自己想同的子节点
offset1 = column.getValue(action.para, "offset")
if offset != "0" or offset1 == "" or offset1 == "0":
#print "self",self
#print "action",action
#print ""
return False
else:
self.pull.append(action.id)
action.parentId = self.id
action.parentMethod = self.method
return True
#else:
#if self.requestIdCheck(action) < 0:
#print "self",self
#print "action",action
#print ""
#return False
if action.method == "search.getlist":
scene = column.getValue(action.para, "scene")
if scene != "":
if self.method == "info.getinfo":
if scene != "k3":
#print "self",self
#print "action",action
#print ""
return False
elif self.method == "recipe.getcollectlist":
if scene != "t2":
return False
elif self.method == "search.getcatelist":
if scene != "t1":
return False
elif self.method == "recipe.getcollectrecomment":
if scene != "k2":
return False
#print "self-after-addHit",self
#print "action-after-addHit",action
if action.getItem() not in self.hit:
self.hit[action.getItem()] = 1
else:
self.hit[action.getItem()] += 1
self.sons.append(action.id)
action.parentId = self.id
action.parentMethod = self.method
return True
def getCommentItem(method,para): type=column.getValue(para,"type") rid=column.getValue(para,"rid") return getV4TypeId(rid,type)
def topicCheck(id, title):
idType = "album"
if id.startswith("topic"):
idType = "topic"
cs = {}
cvs = {}
cs["Android"] = {}
cs["Appstore"] = {}
cs["other"] = {}
pvs = {}
pvs["Android"] = {}
pvs["Appstore"] = {}
pvs["other"] = {}
us = {}
for line in sys.stdin:
if idType == "topic" and line.find("m.haodou.com") > 0:
cols = line.strip().split("\01")
us[cols[0]] = 1
if not line.find(id + ".") > 0:
continue
type = "other"
if line.find("Android") > 0:
type = "Android"
elif line.find("iPhone") > 0:
type = "Appstore"
ip = cols[0]
cs[type][ip] = 1
elif line.find(title) > 0:
#print line
type = "other"
if line.find('"appid":"2"') > 0:
type = "Android"
elif line.find('"appid":"4"') > 0:
type = "Appstore"
if idType == "album":
s = line.find('"uuid":"')
if s < 0:
continue
e = line.find('"', s + len('"uuid":"') + 1)
if e < 0:
continue
u = line[s + len('"uuid":"'):e]
else:
s = line.find('"user_ip":"')
if s < 0:
continue
e = line.find('"', s + len('"user_ip":"') + 1)
if e < 0:
continue
u = line[s + len('"user_ip":"'):e]
s = line.find('"vn":"')
if s < 0:
version = ""
else:
e = line.find('"', s + len('"vn":"') + 1)
if e < 0:
version = ""
else:
version = line[s + len('"vn":"'):e]
if version.startswith("v"):
version = version[1:]
if version not in pvs[type]:
pvs[type][version] = {}
pvs[type][version][u] = 1
else:
cols = line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
uuid = column.uuid(cols)
us[uuid] = 1
method = cols[column.METHOD_CID]
version = cols[column.VERSION_CID]
if version.startswith("v"):
version = version[1:]
if method != "info.getalbuminfo":
continue
aid = column.getValue(cols[column.PARA_ID], "aid")
if aid != id:
continue
appid = cols[column.APPID_CID]
type = "other"
if appid == "2":
type = "Android"
elif appid == "4":
type = "Appstore"
if uuid == None or uuid == "":
continue
cs[type][uuid] = 1
if type not in cvs:
cvs[type] = {}
if version not in cvs[type]:
cvs[type][version] = {}
cvs[type][version][uuid] = 1
#print cs
for type in pvs:
for version in pvs[type]:
viewNum = len(pvs[type][version])
innNum = 0
notNum = 0
for u in pvs[type][version]:
if u in cs[type]:
#print "In\t%s\t%s\t%s"%(type,u,version)
innNum += 1
elif u not in us:
#print "Not\t%s\t%s\t%s"%(type,u,version)
notNum += 1
print "%s\t%s\t%d\t%d\t%.4f\t%d\t%.4f" % (
type, version, viewNum, notNum, notNum /
(viewNum + 1e-12), innNum, innNum / (viewNum + 1e-12))
for type in cvs:
for version in cvs[type]:
viewNum = len(cvs[type][version])
notNum = 0
for uuid in cvs[type][version]:
if type not in pvs or version not in pvs[
type] or uuid not in pvs[type][version]:
notNum += 1
innNum = viewNum - notNum
print "appLog:%s\t%s\t%d\t%d\t%.4f\t%d\t%.4f" % (
type, version, viewNum, notNum, notNum /
(viewNum + 1e-12), innNum, innNum / (viewNum + 1e-12))
def paraName(para,name): return column.getValue(para,name)
def info():
lastU=""
lastM="_"
lastPara="_"
lastQid="_"
infoList=[]
infoM={}
for line in sys.stdin:
cols=line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
u=column.uuidFirst(cols[1:])
v=column.bigVersion(cols[column.VERSION_CID+1])
if v != "4":
continue
para=cols[column.PARA_ID+1]
qid=column.getValue(para,"request_id")
if qid == "" or u == None or u == "":
if u == None:
u=""
type="_"
if qid != "":
type+="q"
if u != "":
type+="u"
print "WW\t"+type+"\t"+u+"\t"+qid
continue
if lastU == "":
lastU=u
if lastU != u:
lastU=u
lastPara="_"
lastQid="_"
lastM="_"
infoList=[]
infoM={}
method=cols[column.METHOD_CID+1]
if method.startswith("info.getinfo") or method == "info.getlastestinfo":
rid=column.getValue(para,"rid")
if rid != None and rid != "":
if rid in infoM:
print infoM[rid][0]+"\t"+infoM[rid][1]+"\t"+u+"\t"+infoM[rid][2]+"\t"+rid
else:
print "VoidMethod_"+lastM+"\t"+lastPara+"\t"+u+"\t"+lastQid+"\t"+rid
if method in infoHead:
lastM=method
infoList=[]
lastPara="_"
lastQid="_"
ret=column2.FuncMap[method](cols[-1])
if ret == None:
sys.stderr.write(line)
continue
if method == "rank.getrankview":
infoList=ret["rids"]
lastPara=column.getValue(para,"id")
elif method == "info.getalbuminfo":
infoList=ret["rids"]
elif method == "recipeuser.getuserrecipelist":
infoList=ret
elif method == "userfeed.getfollowuserfeed":
for (id,type) in ret:
if type == "10":
infoList.append(id)
elif method.startswith("search.getlist"):
infoList=ret["rids"]
rqid=column.getValue(para,"return_request_id")
tagid=column.getValue(para,"tagid")
keyword=column.getValue(para,"keyword")
offset=column.getValue(para,"offset")
if rqid != None and rqid != "":
lastPara+="r"
if tagid != None and tagid != "":
lastPara+="t"
if keyword != None and keyword != "":
lastPara+="k"
if offset == "0":
lastPara+="0"
elif method == "recipephoto.getproducts":
for pid,rid in ret["prids"]:
if rid != "0":
infoList.append(rid)
elif method == "recipe.getcollectrecomment":
for rid,isLike in ret:
infoList.append(rid)
type=column.getValue(para,"type")
offset=column.getValue(para,"offset")
lastPara+=type+"_"+offset
print method+"\t"+lastPara+"\t"+u+"\t"+qid
print "VoidMethod_"+method+"\t"+lastPara+"\t"+u+"\t"+qid
lastQid=qid
for rid in infoList:
infoM[rid]=(method,lastPara,qid)
def wiki():
lastIp=""
v=""
selfIds={}
lastIp=""
allIds={}
ids={}
for line in sys.stdin:
cols=line.strip().split("\t")
ip=cols[0]
if lastIp == "":
lastIp=ip
if lastIp != ip:
lastIp=ip
selfIds={}
v=""
if len(cols) == 3:
topic=mlog.getTopic(cols[2])
if topic == None:
continue
print "all_"+v+"\t"+cols[0]+"\t"+topic
if topic in allIds:
print allIds[topic]+"_"+v+"\t"+cols[0]+"\t"+topic
if topic in selfIds:
print selfIds[topic]+"_"+v+"\t"+cols[0]+"\t"+topic
if topic in ids:
print ids[topic]+"_"+v+"\t"+cols[0]+"\t"+topic
else:
if len(cols) < column.APP_LOG_COLUMNS+1:
continue
method=cols[column.METHOD_CID+1]
v=cols[column.VERSION_CID+1]
if method == "wiki.getlistbytype":
if len(cols) < column.APP_LOG_COLUMNS4+2:
continue
ret=column2.wikiList(cols[-1])
if ret == None:
continue
type=column.getValue(cols[column.PARA_ID+1],"type")
if type == None or type =="":
continue
if type == "1":
for tid in ret:
ids[tid]="wikis"
elif type == "2":
for tid in ret:
ids[tid]="tables"
elif method == "recipe.getcollectlist":
if len(cols) == column.APP_LOG_COLUMNS4+2:
ret=column2.getRecoms(cols[-1])
if ret == None:
continue
if "wiki" in ret:
for id,title in ret["wiki"]:
allIds[id]="wiki"
ids[id]="wikis"
selfIds[id]="self_wiki"
if "table" in ret:
for id,title in ret["table"]:
allIds[id]="table"
ids[id]="tables"
selfIds[id]="self_table"
def getVideoListItem(method, para):
return "info.getvideolist-" + column.getValue(para, "cate_id")
def getFollowItem(method, para):
return "uid-" + column.getValue(para, "fid")
def getUserByNameItem(method, para):
return "name-" + column.getValue(para, "name")
def getRecomListItem(method,para): type=column.getValue(para,"type") return "recom-"+type
def getSearchTagsItems(method, para, response):
ret = []
keyword = column.getValue(para, "keyword")
for id in response:
ret.append("keyword-" + keyword + "-tag-" + tagName(id))
return ret
def topicCheck(id,title):
idType="album"
if id.startswith("topic"):
idType="topic"
cs={}
cvs={}
cs["Android"]={}
cs["Appstore"]={}
cs["other"]={}
pvs={}
pvs["Android"]={}
pvs["Appstore"]={}
pvs["other"]={}
us={}
for line in sys.stdin:
if idType == "topic" and line.find("m.haodou.com") > 0:
cols=line.strip().split("\01")
us[cols[0]]=1
if not line.find(id+".") > 0:
continue
type="other"
if line.find("Android") > 0:
type="Android"
elif line.find("iPhone") >0:
type="Appstore"
ip=cols[0]
cs[type][ip]=1
elif line.find(title) > 0:
#print line
type="other"
if line.find('"appid":"2"') > 0:
type="Android"
elif line.find('"appid":"4"') > 0:
type="Appstore"
if idType == "album":
s=line.find('"uuid":"')
if s < 0:
continue
e=line.find('"',s+len('"uuid":"')+1)
if e < 0:
continue
u=line[s+len('"uuid":"'):e]
else:
s=line.find('"user_ip":"')
if s < 0:
continue
e=line.find('"',s+len('"user_ip":"')+1)
if e < 0:
continue
u=line[s+len('"user_ip":"'):e]
s=line.find('"vn":"')
if s < 0:
version=""
else:
e=line.find('"',s+len('"vn":"')+1)
if e < 0:
version=""
else:
version=line[s+len('"vn":"'):e]
if version.startswith("v"):
version=version[1:]
if version not in pvs[type]:
pvs[type][version]={}
pvs[type][version][u]=1
else:
cols=line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
uuid=column.uuid(cols)
us[uuid]=1
method=cols[column.METHOD_CID]
version=cols[column.VERSION_CID]
if version.startswith("v"):
version=version[1:]
if method != "info.getalbuminfo":
continue
aid=column.getValue(cols[column.PARA_ID],"aid")
if aid != id:
continue
appid=cols[column.APPID_CID]
type="other"
if appid == "2":
type="Android"
elif appid == "4":
type="Appstore"
if uuid == None or uuid == "":
continue
cs[type][uuid]=1
if type not in cvs:
cvs[type]={}
if version not in cvs[type]:
cvs[type][version]={}
cvs[type][version][uuid]=1
#print cs
for type in pvs:
for version in pvs[type]:
viewNum=len(pvs[type][version])
innNum=0
notNum=0
for u in pvs[type][version]:
if u in cs[type]:
#print "In\t%s\t%s\t%s"%(type,u,version)
innNum+=1
elif u not in us:
#print "Not\t%s\t%s\t%s"%(type,u,version)
notNum+=1
print "%s\t%s\t%d\t%d\t%.4f\t%d\t%.4f"%(type,version,viewNum,notNum,notNum/(viewNum+1e-12),innNum,innNum/(viewNum+1e-12))
for type in cvs:
for version in cvs[type]:
viewNum=len(cvs[type][version])
notNum=0
for uuid in cvs[type][version]:
if type not in pvs or version not in pvs[type] or uuid not in pvs[type][version]:
notNum+=1
innNum=viewNum-notNum
print "appLog:%s\t%s\t%d\t%d\t%.4f\t%d\t%.4f"%(type,version,viewNum,notNum,notNum/(viewNum+1e-12),innNum,innNum/(viewNum+1e-12))
def searchFollow(ers):
for m in searchHead:
if m not in ers:
ers[m]={}
lastIp=""
lastWds={}
lastTids={}
liveMethods={}
for line in sys.stdin:
cols=line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
ip=cols[0]
if ip == "":
continue
u=column.ActionUser(cols[1:])
if u != None and u != "":
ip+="_"+u
if lastIp == "":
lastIp=ip
if lastIp != ip:
lastWds={}
lastTids={}
liveQids={}
liveMethods={}
lastIp=ip
method=cols[column.METHOD_CID+1]
v=column.bigVersion(cols[column.VERSION_CID+1])
if method.startswith("search.getlist"):
para=cols[column.PARA_ID+1]
offset=column.getValue(para,"offset")
if offset != "0":
#sys.stderr.write("offset:"+offset+"\n")
continue
tid=column.getValue(para,"tagid")
if tid != None and tid != "" and tid != "null":
r=column.getValue(para,"return_request_id")
if r != None and r != "":
print "recipe.getcollectlist_return_"+method+"_"+v+"\t"+ip+"\ttid_"+tid
continue
if tid in lastTids and len(lastTids[tid]) >= 1:
print lastTids[tid][-1]+"_"+method+"_"+v+"\t"+ip+"\ttid_"+tid
else:
print "VoidMethod_tid_"+method+"_"+v+"\t"+ip+"\ttid_"+tid
else:
kw=column.getValue(para,"keyword")
if kw in lastWds and len(lastWds[kw]) >=1:
print lastWds[kw][-1]+"_"+method+"_"+v+"\t"+ip+"\tkw_"+kw
else:
print "VoidMethod_kw_"+method+"_"+v+"\t"+ip+"\tkw_"+kw
elif method in searchHead:
tmp={}
for m in liveMethods:
if method in ers[m]:
tmp[m]=1
for m in tmp:
for tid in lastTids:
ms=lastTids[tid]
for i in range(len(ms)-1,-1,-1):
if ms[i] == m:
del ms[i]
for kw in lastWds:
ms=lastWds[kw]
for i in range(len(ms)-1,-1,-1):
if ms[i] == m:
del ms[i]
#sys.stderr.write("delete "+m+" because "+method+"\n")
del liveMethods[m]
liveMethods[method]=1
kwret=None
tidret=None
if method == "search.gethotsearch" or method == "search.getsuggestion":
ret=column2.FuncMap[method](cols[-1])
if ret != None:
kwret=[]
for w in ret:
kwret.append(column2.escapeUnicode(w).encode("utf-8"))
elif method == "search.getcatelist":
tidret=column2.FuncMap[method](cols[-1])
elif method == "recipe.getcollectlist":
ret=column2.FuncMap[method](cols[-1])
if ret != None and "tag" in ret:
tidret=[]
for id,title in ret["tag"]:
tidret.append(id)
elif method == "info.getinfo":
ret=column2.FuncMap[method](cols[-1])
if ret != None and "tags" in ret:
tidret=ret["tags"]
if kwret != None:
for kw in kwret:
if kw not in lastWds:
lastWds[kw]=[]
lastWds[kw].append(method)
if tidret != None:
for tid in tidret:
if tid not in lastTids:
lastTids[tid]=[]
lastTids[tid].append(method)
def searchFollowMapper(f):
lastU=""
lastM="_" #最后调用的获取菜谱列表的方法
KeyWordClick={} #点击次数
RidList=set([])#搜索返回的食谱列表
#for line in sys.stdin:
for line in f:
try:
cols=line.strip().split("\t")
#print line
if len(cols) < column.APP_LOG_COLUMNS:
continue
u=column.uuidFirst(cols[1:]) #获得uuid
para=cols[column.PARA_ID+1] #得到请求参数
if lastU == "":#第一个用户
lastU=u
if lastU != u: #下一个用户
lastU=u
lastM="_"
RidList=set([])
method=cols[column.METHOD_CID+1] #获得请求的方法
if method in getRecipeMethod:
lastM = method #最后获得菜谱列表的方法
RidList=set([])
if method in searchHead: #搜索方法
keyword=column.getValue(para,"keyword") #搜索的关键字
#print keyword
offset=column.getValue(para,"offset")
#print offset
if len(keyword) == 0: keyword ="_"
if len(offset) == 0: offset ="0"
if keyword not in KeyWordClick: #还没统计过的新关键词
KeyWordClick[keyword]=[0,0,0,0,{}] #第一个用于统计没下拉的搜索的次数，第二个统计下拉搜索的次数，第三个统计所有可能的点击， 第三个用于统计匹配到菜谱的点击，第五个字典用于菜谱点击统计
if offset != '0': #搜索下拉
KeyWordClick[keyword][searchPullDown] += 1 #统计搜索下拉
KeyWordClick[keyword][searchCnt] += 1 #统计第一次搜索
else: #第一次搜索
KeyWordClick[keyword][searchCnt] += 1 #统计第一次搜索
ret=column2.FuncMap[method](cols[-1]) #获得搜索返回的食谱列表
RidList=RidList | set(ret['rids']) #获得搜索返回的食谱列表,搜索过程会连续翻页，产生多次搜索调用，因此合并返回的列表
#print offset
#产生点击
if (method.startswith("info.getinfo") or method == "info.getlastestinfo"):
rid=column.getValue(para,"rid") #获得点击的食谱id
#print rid
#搜索次数统计
if rid in RidList: #点击的菜谱ID在搜索的返回列表中
KeyWordClick[keyword][clikAll] += 1 #可能的点击+1
KeyWordClick[keyword][clikRid] += 1 #匹配的点击+1
#菜谱点击统计
if rid not in KeyWordClick[keyword][clikRidList]: #如果该菜谱从来没有被统计过
KeyWordClick[keyword][clikRidList][rid] = 1 ;
else:
KeyWordClick[keyword][clikRidList][rid] += 1 ; #菜谱点击次数+1
elif lastM in searchHead : #如果没有匹配到菜谱，但最后调用的是搜索方法，仍然统计可能的点击次数
KeyWordClick[keyword][clikAll] += 1 #可能的点击+1
#菜谱点击统计
if rid not in KeyWordClick[keyword][clikRidList]: #如果该菜谱从来没有被统计过
KeyWordClick[keyword][clikRidList][rid] = 1 ;
else:
KeyWordClick[keyword][clikRidList][rid] += 1 ; #菜谱点击次数+1
except Exception,ex:
continue
def searchFollowMapper(f):
lastIP=""
rids={}
oc=0
otherRids={}
otherAids={}
otherTids={}
searchRet={}
click=None
keywordClick={}
lastK=""
hc=0
keyword=""
cardBank=CardBank()
for line in f:
if True:
cols=line.strip().split("\t")
if len(cols) < 3:
continue
ip=cols[0]
if lastIP == "":
lastIP=ip
if lastIP != ip:
lastIP=ip
keyword=""
rids={}
uc={}
otherRids={}
otherAids={}
otherTids={}
searchRet={}
click=None
lastSearchRet={}
lastClick=None
cardBank.reset()
if line.find("m.haodou.com") > 0 and line.find("GET") > 0:
cols=cols[2].strip().split("\01")
if len(cols) < 5:
continue
p=cols[4].find(THead)
if p > 0:
end=cols[4].find("&",p+THeadLen)
if end < 0:
end=len(cols[4])
id=cols[4][p+THeadLen:end]
if id !="":
if "topicId" in searchRet and searchRet["topicId"] == id:
click.addTopicHit(id)
oc+=1
if "ttitle" in searchRet:
title=searchRet["ttitle"]
click.addTopicTitleHit(title,keyword)
elif id in otherTids:
oc+=1
click.getMs().tb+=1
else:
if cols[4].find("uuid") > 0:
sys.stderr.write(line)
mtid=id
continue
if len(cols) < column.APP_LOG_COLUMNS+1:
continue
version=cols[column.VERSION_CID+1]
v=column.intVersion(version)
if v < 400:
continue
u=column.uuidFirst(cols[1:]) #获得uuid
if u == None or u.find("{") >= 0:
u=""
para=cols[column.PARA_ID+1] #得到请求参数
method=cols[column.METHOD_CID+1] #获得请求的方法
hasSearch=False
if method == 'search.getsearchindex': #搜索方法
keyword=V45Fix+column.getValue(para,"keyword") #搜索的关键字
hasSearch=True
searchRet=column2.FuncMap[method](cols[-1]) #获得搜索返回的食谱列表
rids=getRidPos(searchRet)
if keyword not in keywordClick:
keywordClick[keyword]=Click()
click=keywordClick[keyword]
click.addSearchRet(searchRet,keyword)
cardBank.addSearch(keyword[len(V45Fix):],searchRet)
otherRids={}
otherAids={}
otherTids={}
if "rids" in searchRet:
for i in range(FirstPageNum,len(searchRet["rids"]),1):
otherRids[searchRet["rids"][i]]=i
elif method == "search.getlist" and (v < 450 or v >= 480):
offset=column.getValue(para,"offset")
scene=column.getValue(para,"scene")
if scene != "k1":
continue
#keyword=column.getValue(para,"keyword") #搜索的关键字
searchRet=column2.FuncMap[method](cols[-1]) #
if offset != "0":
kw=column.getValue(para,"keyword")
if kw != keyword:
continue
if "rids" in searchRet:
for i in range(len(searchRet["rids"])):
otherRids[searchRet["rids"][i]]=i
continue
keyword=column.getValue(para,"keyword")
hasSearch=True
rids=getRidPos(searchRet)
otherRids={}
#otherAids={}
#otherTids={}
if "rids" in searchRet:
for i in range(len(searchRet["rids"])):
if i >= FirstPageNum:
otherRids[searchRet["rids"][i]]=i
continue
rids[searchRet["rids"][i]]=i
if keyword not in keywordClick:
keywordClick[keyword]=Click()
click=keywordClick[keyword]
click.addSearchRet(searchRet,keyword)
cardBank.addSearch(keyword,searchRet,False)
if hasSearch:
if lastK != "":
keywordClick[lastK].addHitCount(hc)
if hc + oc > 0:
keywordClick[lastK].addHasHit()
hc=0
oc=0
lastK=keyword
if click == None:
continue
if method == "info.getinfo":
rid=column.getValue(para,"rid") #获得点击的食谱id
pos=-1
if rid in rids:
click.addRecipeHit(rid,rids[rid])
kw=keyword
if v >= 450:
kw=keyword[len(V45Fix):]
cardBank.addHit(kw,rid,rids[rid],(v >= 450))
hc+=1
pos=rids[rid]
elif rid in otherRids:
oc+=1
click.getMs().rb+=1
if pos >= 0:
ret=column2.FuncMap[method](cols[-1])
if ret == None:
pass
#sys.stderr.write(cols[-1])
elif "title" in ret:
title=ret["title"]
click.addTitleHit(title,pos,keyword)
elif method == "info.getfoodinfo":
fid=column.getValue(para,"foodid")
if fid != "" and "food" in searchRet and fid == searchRet["food"]:
click.addFoodHit(fid)
oc+=1
elif v >= 450:
if method == "info.getalbuminfo":
id=column.getValue(para,"aid") #获取点击的专辑id
if id !="":
if "aid" in searchRet and searchRet["aid"]==id :
click.addAlbumHit(id)
oc+=1
if "atitle" in searchRet:
title=searchRet["atitle"]
click.addAlbumTitleHit(title,keyword)
elif id in otherAids:
oc+=1
click.getMs().ab+=1
elif method == "search.gettags":
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
tags=column2.FuncMap[method](cols[-1])
for tagid in tags:
if tagid not in click.getMs().rTagShow:
click.getMs().rTagShow[tagid]=1
else:
click.getMs().rTagShow[tagid]+=1
elif method == "search.getlist":
offset=column.getValue(para,"offset")
scene=column.getValue(para,"scene")
if scene != "k1":
continue
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
searchRet=column2.FuncMap[method](cols[-1]) #
if "rids" in searchRet:
for i in range(len(searchRet["rids"])):
otherRids[searchRet["rids"][i]]=i
if offset != "0":
continue
tagid=column.getValue(para,"tagid")
if tagid == "" or tagid == "null":
click.getMs().r+=1
else:
if tagid not in click.getMs().rt:
click.getMs().rt[tagid]=1
else:
click.getMs().rt[tagid]+=1
click.getMs().rtn=1
elif method == "search.getalbumlist":
offset=column.getValue(para,"offset")
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
ret=column2.getList(cols[-1],"AlbumId")
for aid in ret:
otherAids[aid]=1
if offset == "0":
click.getMs().a+=1
elif method == "search.gettopiclist":
offset=column.getValue(para,"offset")
kw=V45Fix+column.getValue(para,"keyword") #搜索的关键字
if kw != keyword:
continue
ret=column2.getList(cols[-1],"TopicId","int")
for tid in ret:
#sys.stderr.write("tid:"+tid+"\n")
otherTids[tid]=1
if offset == "0":
click.getMs().t+=1
if lastK != "":
keywordClick[lastK].addHitCount(hc)
if hc + oc > 0:
keywordClick[lastK].addHasHit()
ck45=Click()
ck44=Click()
for kw in keywordClick:
if kw.startswith(V45Fix):
ck45.merge(keywordClick[kw])
else:
ck44.merge(keywordClick[kw])
print kw+"\t"+str(keywordClick[kw])
print "ck45_##total##"+"\t"+str(ck45)
print "ck44_##total##"+"\t"+str(ck44)
for card in cardBank.bank:
print CardFix+str(cardBank.bank[card])
def getFavoriteItem(method,para): type=column.getValue(para,"type") rid=column.getValue(para,"rid") return getV3TypeId(rid,type)
def getShareItem(method, para):
type = column.getValue(para, "type")
itemid = column.getValue(para, "itemid")
return getV4TypeId(itemid, type)
def getShareItem(method,para): type=column.getValue(para,"type") itemid=column.getValue(para,"itemid") return getV4TypeId(itemid,type)
def getFavoriteItem(method, para):
type = column.getValue(para, "type")
rid = column.getValue(para, "rid")
return getV3TypeId(rid, type)
def count():
lastUuid=""
regFirstDo="NO"
hasUid=""
ms=[]
phone=""
for line in sys.stdin:
cols=line.strip().split("\t")
if len(cols) < column.APP_LOG_COLUMNS:
continue
uuid=column.uuidOnly(cols[1:])
if uuid == None:
continue
uid=column.uid(cols[1:])
method=cols[column.METHOD_CID+1]
resp=""
if method in regMs and method in column2.FuncMap and len(cols) >= column.APP_LOG_COLUMNS4+1:
resp=column2.FuncMap[method](cols[-1])
if resp == None:
resp = ""
if lastUuid == "":
lastUuid=uuid
if lastUuid != uuid:
output(lastUuid,ms,regFirstDo,phone)
lastUuid=uuid
regFirstDo="NO"
ms=[]
hasUid=""
phone=""
if method in regMs and hasUid == "":
if method == "passport.bindconnectstatus" and PositiveNumP.match(resp):
hasHit=True
if len(ms) >= 1:
regFirstDo="OK"
uid=resp
#sys.stderr.write("bind uid:"+uid+"\n")
elif method == "passport.reg" and PositiveNumP.match(resp):
hasHit=True
if len(ms) >= 1:
regFirstDo="OK"
uid=resp
#sys.stderr.write("reg uid:"+uid+"\n")
else:
if resp != "":
ms.append(method+"("+resp+")")
else:
ms.append(method)
elif len(ms) >= 1:
if (regFirstDo=="NO" or regFirstDo=="OK"):
if method in regUserMs:
regFirstDo=method
elif uid != "":
regFirstDo="OK"
if uid != "":
hasUid=uid
if method == "common.sendcode":
phone=column.getValue(cols[column.PARA_ID+1],"phone")
#sys.stderr.write(method+"\t"+cols[column.PARA_ID+1]+"\t"+phone+"\n")
if lastUuid != "":
output(lastUuid,ms,regFirstDo,phone)
for m in lms:
if m.endswith("_NO"):
continue
v=lms[m]
nv=0
if m+"_NO" in lms:
nv=lms[m+"_NO"]
print "%s\t%d\t%d\t%.4f"%(m,lms[m],nv,nv/(v+1e-32))
def getRecipeidItem(method, para):
return "rid-" + column.getValue(para, "recipeid")
def getAidItem(method, para):
return "aid-" + column.getValue(para, "aid")
def getPhotoviewItem(method, para):
return "pid-" + column.getValue(para, "id")
