def mapaction(self, data):
try:
datatype = find_value_fromdict_singlekey(data, "jhd_datatype")
if datatype in ["biqu", "BIQU_ANDROID", "biqu_all"]:
eventid = find_value_fromdict_singlekey(data, "jhd_eventId")
usermap = find_value_fromdict_singlekey(data, "jhd_map")
# 搜索区分单程、往返机票
if eventid == "ac36":
# et: 返程时间
et = usermap.get("et", None) if isinstance(usermap, dict) else None
# print(et, type(usermap), usermap)
if et:
for acid in ["ac36", "jhf_ac36_et1"]:
data["jhd_eventId"] = acid
yield data
else:
for acid in ["ac36", "jhf_ac36_et0"]:
data["jhd_eventId"] = acid
yield data
# 预订区分单程、往返机票
elif eventid == "ac49":
try:
wf = int(usermap["wf"])
except:
wf = -1
if wf == 1:
for acid in ["ac49", "jhf_ac49_wf1"]:
data["jhd_eventId"] = acid
yield data
else:
for acid in ["ac49", "jhf_ac49_wf0"]:
data["jhd_eventId"] = acid
yield data
# 付款区分单程、往返机票
elif eventid == "ac53":
fst = usermap.get("fst", None) if isinstance(usermap, dict) else None
if fst:
for acid in ["ac53", "jhf_ac53_fst1"]:
data["jhd_eventId"] = acid
yield data
else:
for acid in ["ac53", "jhf_ac53_fst0"]:
data["jhd_eventId"] = acid
yield data
else:
yield data
else:
yield data
except:
import traceback
print(traceback.print_exc())
def parse(self, dataDict, userDict, mode=UserProfileBuilder):
try:
userProfile = mode()
userkey = find_value_fromdict_singlekey(dataDict, "jhd_userkey")
userkey = userkey.strip()
ver = find_value_fromdict_singlekey(dataDict, "jhd_vr")
userProfile.setUid(userkey)
opTime = find_value_fromdict_singlekey(
dataDict, "jhd_opTime").replace("-", "").replace(":", "")
# 以用户实际操作时间作为 LastLoginTime
if len(opTime) == 14:
# opTime = "".join([self.yyyymmdd, self.hhmm])
try:
opTime = time.strftime(
"%Y%m%d%H%M%S",
time.strptime("".join([self.yyyymmdd, opTime[8:]]),
"%Y%m%d%H%M%S"))
except:
opTime = "".join([self.yyyymmdd, self.hhmm])
opTime = time.strftime(
"%Y%m%d%H%M%S", time.strptime(opTime, "%Y%m%d%H%M%S"))
else:
opTime = "".join([self.yyyymmdd, "000001"])
userProfile.setPub(
find_value_fromdict_singlekey(dataDict, "jhd_pb"))
# userProfile.setPlat(find_value_fromdict_singlekey(dataDict, "jhd_os").split("_")[0])
userProfile.setPlat(
find_value_fromdict_singlekey(dataDict, "jhd_os").split("_")[0]
if find_value_fromdict_singlekey(dataDict, "jhd_os") else "#")
userProfile.setOS(find_value_fromdict_singlekey(
dataDict, "jhd_os"))
userProfile.setPushid(
find_value_fromdict_singlekey(dataDict, "jhd_pushid"))
userProfile.setUA(find_value_fromdict_singlekey(
dataDict, "jhd_ua"))
userProfile.setFirstLoginTime(opTime)
userProfile.setLastLoginTime(opTime)
userProfile.setVer(ver)
userProfile.setProperties(dataDict.get("jhd_properties", {}))
if (userDict.has_key(userkey)):
userDict[userkey] = \
self.mergeUserProfile(userProfile.build(), userDict[userkey])
else:
userDict[userkey] = userProfile.build()
except:
import traceback
print(traceback.print_exc())
return userProfile
def parse(self, data, entities, mode=UserMapMetaBuilder):
try:
optype = find_value_fromdict_singlekey(data, "type")
if optype != "ac":
return
eventid = find_value_fromdict_singlekey(data, "event")
entity = self.userDict[eventid] if eventid in self.userDict else mode(eventid)
jhmap = find_value_fromdict_singlekey(data, "usermap")
if not isinstance(jhmap, dict):
return
for key in jhmap:
entity.addField(key, jhmap[key], self.enum_config.get(self.datatype, None))
if eventid not in self.userDict:
entities.append(entity.build())
self.userDict.setdefault(eventid, entity)
return entity
except:
import traceback
print(traceback.print_exc())
def parse(self, data, userDict, mode=UserActiveUpdateBuilder):
userData = []
try:
userkey = find_value_fromdict_singlekey(data, "jhd_userkey")
user_active = mode() if userkey not in self.userDict else self.userDict[userkey]
user_active.setUserkey(userkey)
self.userDict.setdefault(userkey, user_active)
return user_active
except:
import traceback
print(traceback.print_exc())
def parse(self, data, userDict, mode=UserProfileBuilderH5):
try:
userProfile = super(UserProfileH5, self).parse(data,
userDict,
mode=mode)
if not userProfile:
return
device = find_value_fromdict_singlekey(
data, self.comparison_table["device"])
browser = find_value_fromdict_singlekey(
data, self.comparison_table["browser"])
system = find_value_fromdict_singlekey(
data, self.comparison_table["system"])
support = find_value_fromdict_singlekey(
data, self.comparison_table["support"])
userProfile.setSupport(support)
userProfile.setDevice(device)
userProfile.setBrowser(browser)
userProfile.setSystem(system)
return userProfile
except:
import traceback
print(traceback.print_exc())
def parse(self, data, eventEntities, mode=UserEventBuilderH5):
try:
eventEntity = super(UserEventH5,
self).parse(data,
eventEntities,
mode=UserEventBuilderH5)
if not eventEntity:
return
device = find_value_fromdict_singlekey(
data, self.comparison_table["device"])
browser = find_value_fromdict_singlekey(
data, self.comparison_table["browser"])
system = find_value_fromdict_singlekey(
data, self.comparison_table["system"])
support = find_value_fromdict_singlekey(
data, self.comparison_table["support"])
eventEntity.setSupport(support)
eventEntity.setDevice(device)
eventEntity.setBrowser(browser)
eventEntity.setSystem(system)
return eventEntity
except:
import traceback
print(traceback.print_exc())
def transform(self, line):
data = json.loads(line)
# singlekey
datatype = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_datatype"])
userkey = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_userkey"])
userkey = userkey.strip()
# appendkey
vr = find_value_fromdict_singlekey(data,
self.comparison_table["jhd_vr"])
ip = find_value_fromdict_singlekey(data,
self.comparison_table["jhd_ip"])
ip = ip.strip("/").strip('''"''')
pushid = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_pushid"])
ua = find_value_fromdict_singlekey(data,
self.comparison_table["jhd_ua"])
try:
device_name = find_value_fromdict_singlekey(
data, self.comparison_table["device"]).get("name", None)
except:
device_name = None
if device_name:
tmp = device_name.lower()
if "android" in tmp:
m = self.pattern_ua.search(ua)
if m:
ua = m.group(1)
ua = ua.lower()
else:
ua = tmp
try:
system_name = find_value_fromdict_singlekey(
data, self.comparison_table["system"]).get("name", None)
except:
system_name = None
useros = system_name
if system_name:
tmp = system_name.lower()
if "android" in tmp:
useros = "android"
elif "iphone" in tmp:
useros = "ios"
usermap = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_map"])
if usermap == "" or usermap == "null" or (not usermap):
usermap = {}
# countkey
# wei.wei
usermap["browser_ua"] = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_ua"])
# 添加日志接收时间（戳），2017-04-13日添加
_jh_recv_ts = find_value_fromdict_singlekey(
data, self.comparison_table["_jh_recv_ts"])
if _jh_recv_ts:
usermap["_jh_recv_ts"] = _jh_recv_ts
else:
usermap["_jh_recv_ts"] = time.time()
# other
ts = find_value_fromdict_singlekey(data,
self.comparison_table["jhd_ts"])
if bool(ts) == False:
ts = usermap["_jh_recv_ts"]
try:
optime = time.strftime("%Y%m%d%H%M%S",
time.localtime(int(ts / float(1000))))
except:
optime = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
opa = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_opType"])
opa = self.comparison_key.get(opa, opa)
eventid = find_value_fromdict_singlekey(
data,
self.comparison_table["jhd_eventId"]) if opa == "action" else opa
pagename = "jhd_page" if opa == "page" else None
try:
uri = find_value_fromdict_singlekey(
data, self.comparison_table["jhd_pageName"])
if opa == "action":
value = find_value_fromdict_singlekey(
data, self.comparison_table["value"])
usermap["uri"] = uri
usermap["value"] = value
elif opa == "page":
ref = find_value_fromdict_singlekey(
data, self.comparison_table["ref"])
usermap["uri"] = uri
usermap["ref"] = ref
elif opa == "end":
value = find_value_fromdict_singlekey(
data, self.comparison_table["value"])
usermap["uri"] = uri
usermap["value"] = value
except:
import traceback
print(traceback.print_exc())
return {
"jhd_auth": "off",
"jhd_sdk_type": "js",
"jhd_ua": ua,
"jhd_map": usermap,
"jhd_loc": None,
"jhd_pushid": pushid,
"jhd_opTime": optime,
"jhd_eventId": eventid if eventid != "page" else pagename,
"jhd_ip": ip,
"jhd_pb": None,
"jhd_userkey": userkey,
"jhd_os": useros,
"jhd_opType": opa,
"jhd_sdk_version": vr,
"jhd_netType": None,
"jhd_vr": vr,
"jhd_ts": ts,
"jhd_interval": usermap["value"] if opa == "end" else None,
"jhd_datatype": datatype
}
def parse_item(self, data, userDict, mode=UserCrumbsBuilder):
try:
# singlekey
datatype = find_value_fromdict_singlekey(data, "jhd_datatype")
userkey = find_value_fromdict_singlekey(data, "jhd_userkey")
userkey = userkey.strip()
pushid = find_value_fromdict_singlekey(data, "jhd_pushid")
# appendkey
os = find_value_fromdict_singlekey(data, "jhd_os")
pub = find_value_fromdict_singlekey(data, "jhd_pb")
net = find_value_fromdict_singlekey(data, "jhd_netType")
vr = find_value_fromdict_singlekey(data, "jhd_vr")
ua = find_value_fromdict_singlekey(data, "jhd_ua")
ip = find_value_fromdict_singlekey(data, "jhd_ip")
# countkey
opa = find_value_fromdict_singlekey(data, "jhd_opType")
eventid = find_value_fromdict_singlekey(data, "jhd_eventId")
eventid = opa if opa in set(["in", "end"]) else eventid
pagename = find_value_fromdict_singlekey(data, "jhd_pageName")
# addkey
intervalStr = find_value_fromdict_singlekey(data, "jhd_interval")
try:
interval = float(intervalStr)
except:
interval = None
# other
usermap = find_value_fromdict_singlekey(data, "jhd_map")
if usermap == "" or usermap == "null" or (not usermap):
usermap = -1
opa_tm = find_value_fromdict_singlekey(data, "jhd_opTime")
opatm = opa_tm.replace("+", "").replace("-", "").replace("-", "")
hh_mm_ss = opa_tm.split("+")[1] if "+" in opa_tm else ":".join([opa_tm[8:10], opa_tm[10:12], opa_tm[12:14]])
userData = mode() if userkey not in userDict else userDict[userkey]
userData.setDatatype(datatype)
userData.setUserkey(userkey)
userData.setPushId(pushid)
userData.setTM(self.yyyy_mm_dd)
userData.setOS(os)
userData.setPub(pub)
userData.setNet(net)
userData.setVr(vr)
userData.setUA(ua)
userData.setIP(ip)
userData.setItemCount(eventid, opa, hh_mm_ss, usermap, interval=interval)
userData.setItemCount(pagename, opa, hh_mm_ss, usermap, interval=interval, isadd=False)
userData.setLastOpaTime(opatm)
userDict.setdefault(userkey, userData)
return userData
except:
import traceback
print(traceback.print_exc())
def parse(self, data, eventEntities, mode=UserEventBuilder):
try:
eventEntity = mode()
optype = find_value_fromdict_singlekey(data, "jhd_opType")
eventid = find_value_fromdict_singlekey(data, "jhd_eventId")
userkey = find_value_fromdict_singlekey(data, "jhd_userkey")
userkey = userkey.strip() if userkey else userkey
pushid = find_value_fromdict_singlekey(data, "jhd_pushid")
pushid = pushid.strip() if pushid else pushid
ts = find_value_fromdict_singlekey(data, "jhd_ts")
jhd_opTime = find_value_fromdict_singlekey(data, "jhd_opTime")
if jhd_opTime is None and ts:
jhd_opTime = time.strftime("%Y%m%d%H%M%S", time.localtime(ts))
if ts is None and jhd_opTime:
ts = time.mktime(time.strptime(jhd_opTime,
"%Y%m%d%H%M%S")) * 1000
ip = find_value_fromdict_singlekey(data, "jhd_ip")
sdk_version = find_value_fromdict_singlekey(
data, "jhd_sdk_version")
pub = find_value_fromdict_singlekey(data, "jhd_pb")
os = find_value_fromdict_singlekey(data, "jhd_os")
net = find_value_fromdict_singlekey(data, "jhd_netType")
ua = find_value_fromdict_singlekey(data, "jhd_ua")
vr = find_value_fromdict_singlekey(data, "jhd_vr")
jhmap = find_value_fromdict_singlekey(data, "jhd_map")
# Event
eventEntity.setOpType(optype)
eventEntity.setEventId(eventid)
# who
eventEntity.setUid(userkey)
eventEntity.setPushid(pushid)
# when
eventEntity.setTS(ts)
# where
eventEntity.setIP(ip)
# how
eventEntity.setPub(pub)
eventEntity.setOS(os)
eventEntity.setNet(net)
eventEntity.setUA(ua)
eventEntity.setVR(vr)
eventEntity.setMap(jhmap)
# what
eventEntity.setSdkVr(sdk_version)
eventEntities.append(eventEntity.build())
return eventEntity
except:
import traceback
print(traceback.print_exc())