def paths(self, **kwargs):
logtype = kwargs["logtype"]
if logtype == "logfile":
_paths = get_file_path(**kwargs)
return [
PathProperty(_path=_path, pathtype="logfile")
for _path in _paths
]
elif logtype == "uvfile":
num = kwargs["num"]
datatype = kwargs["datatype"]
iszip = kwargs.get("iszip", True)
_paths = [get_uvfile_path(num, datatype, iszip=iszip)]
return [
PathProperty(_path=_path, pathtype="uvfile")
for _path in _paths
]
# 所有类型日志路径
if isinstance(logtype, list):
paths = []
for _logtype in logtype:
kwargs = dict(kwargs, **{"logtype": _logtype})
paths += self.paths(**kwargs)
return paths
def __init__(self, datatype, yyyy_mm_dd, hhmm="2359", last=1440):
self.datatype = datatype
self.yyyy_mm_dd = yyyy_mm_dd
self.yyyymmdd = yyyy_mm_dd.replace("-", "")
self.paths = get_file_path(datatype=self.datatype,
yyyymmdd=self.yyyymmdd,
hhmm=hhmm,
last=last)
self.userDict = {}
def __init__(self, datatype, yyyy_mm_dd, hhmm="2359", last=1440):
self.datatype = datatype
self.yyyy_mm_dd = yyyy_mm_dd
self.yyyymmdd = yyyy_mm_dd.replace("-", "")
self.paths = get_file_path(datatype=self.datatype, yyyymmdd=self.yyyymmdd, hhmm=hhmm, last=last)
self.userDict = {}
# 配置需要收集的mapkey
self.enum_config = {
# "biqu": [], # format
}
def __init__(self, datatype, yyyy_mm_dd, hhmm="2359", last=1):
self.datatype = datatype
self.yyyy_mm_dd = yyyy_mm_dd
self.yyyymmdd = yyyy_mm_dd.replace("-", "")
self.hhmm = hhmm
self.paths = get_file_path(datatype=datatype,
yyyymmdd=self.yyyymmdd,
hhmm=hhmm,
last=last)
self.paths.sort()
self.data = {}
def __init__(self, datatype, yyyy_mm_dd, hhmm="2359", last=1440):
self.datatype = datatype
self.yyyy_mm_dd = yyyy_mm_dd
self.yyyymmdd = yyyy_mm_dd.replace("-", "")
self.paths = get_file_path(datatype=self.datatype, yyyymmdd=self.yyyymmdd, hhmm=hhmm, last=last)
self.userDict = {}
self.singlekey = ["jhd_datatype", "jhd_userkey", "jhd_pushid"]
self.appendkey = ["jhd_pb", "jhd_vr", "jhd_os", "jhd_netType", "jhd_ua", "jhd_ip"]
self.countkey = {"item_count": ["jhd_eventId", "jhd_opType", "jhd_pageName"]}
self.eventtype = {"jhd_eventId": "action", "jhd_pageName": "page"}
self.addkey = {"item_add": ["jhd_interval"]}
self.statkeys = list(
itertools.chain(self.singlekey, self.appendkey, self.countkey["item_count"], self.addkey["item_add"]))
self.uvkeys = list(
itertools.chain(self.singlekey, self.appendkey, \
self.countkey.keys(), \
self.eventtype.keys(), \
self.addkey.keys()))
def mapCollector(self, datatype, num, ruleFunction):
pass
# dayStr = time.strftime("%Y-%m-%d", time.localtime(time.time() - 86400 * num))
yyyymmddhhmm = time.strftime("%Y%m%d%H%M",
time.localtime(time.time() - 86400 * num))
paths = get_file_path(datatype=datatype,
yyyymmdd=yyyymmddhhmm[:8],
hhmm="2359",
last=1440)
result = {}
for path in paths:
for data in self.pipline(path):
try:
ruleFunction.rules(result, data)
except:
import traceback
print(traceback.print_exc())
return result
def getData(self, datatype, num):
curDay = time.strftime("%Y%m%d",
time.localtime(time.time() - 86400 * num))
paths = self.daily_paths if self.daily_paths else get_file_path(
datatype=datatype, yyyymmdd=curDay, hhmm="2359", last=1440)
result = {}
for path in paths:
for line in JHOpen().readLines(path):
if not line:
continue
try:
data = json.loads(line)
optype = data["type"].strip()
if optype != "ac":
continue
eventid = data["event"].strip()
result.setdefault(optype, set()).add(eventid)
except:
import traceback
print(traceback.print_exc())
print(line)
return result