def init_logfilter(conf, source):
kwargs = dict(conf["filter"])
kwargs["pre_count"] = conf.getint("filter", "pre_count")
kwargs["pre_term"] = config.getdur(conf, "filter", "pre_term")
kwargs["fourier_sample_rule"] = [
tuple(config.str2dur(s) for s in dt_cond.split("_"))
for dt_cond in config.gettuple(conf, "filter",
"fourier_sample_rule")]
kwargs["fourier_th_spec"] = conf.getfloat("filter", "fourier_th_spec")
kwargs["fourier_th_eval"] = conf.getfloat("filter", "fourier_th_eval")
kwargs["fourier_th_restore"] = conf.getfloat("filter",
"fourier_th_restore")
kwargs["fourier_peak_order"] = conf.getint("filter", "fourier_peak_order")
kwargs["corr_sample_rule"] = [
tuple(config.str2dur(s) for s in dt_cond.split("_"))
for dt_cond in config.gettuple(conf, "filter", "corr_sample_rule")]
kwargs["corr_th"] = conf.getfloat("filter", "corr_th")
kwargs["corr_diff"] = [config.str2dur(diffstr) for diffstr
in config.gettuple(conf, "filter", "corr_diff")]
kwargs["linear_sample_rule"] = [
tuple(config.str2dur(s) for s in dt_cond.split("_"))
for dt_cond in config.gettuple(conf, "filter",
"linear_sample_rule")]
kwargs["linear_count"] = conf.getint("filter", "linear_count")
kwargs["linear_th"] = conf.getfloat("filter", "linear_th")
return LogFilter(source, **kwargs)
def _add_event(dt, host, event_name):
info = {}
if self.conf.has_option(section, "info"):
for i in config.gettuple(self.conf, section, "info"):
if i == "ifname":
info[i] = random.choice(
config.gettuple(self.conf, section, "ifname"))
elif i == "user":
info[i] = random.choice(
config.gettuple(self.conf, section, "user"))
self.l_event.append((dt, host, event_name, info))
_recur(dt, host, event_name)
def _generate_log(self, event):
dt = event[0]
host = event[1]
event_name = event[2]
info = event[3]
for log_name in config.gettuple(self.conf, "event_" + event_name,
"logs"):
section = "log_" + log_name
mode = self.conf.get(section, "mode")
form = self.conf.get(section, "format")
mes = form
while True:
match = self._var_re.search(mes)
if match is None:
break
var_type = match.group().strip("$")
if var_type in info.keys():
var_string = info[var_type]
elif var_type == "pid":
var_string = str(random.randint(1, 65535))
elif var_type == "host":
var_string = host
else:
raise ValueError
mes = "".join(
(mes[:match.start()] + var_string + mes[match.end():]))
if mode == "each":
self.l_log.append((dt, host, mes))
elif mode == "delay_rand":
delay_min = config.getdur(self.conf, section, "delay_min")
delay_max = config.getdur(self.conf, section, "delay_max")
log_dt = self._dt_delta_rand(dt, delay_min, delay_max)
self.l_log.append((log_dt, host, mes))
elif mode == "drop_rand":
drop_p = self.conf.getfloat(section, "drop_p")
if random.random() > drop_p:
self.l_log.append((dt, host, mes))
elif mode == "other_host_rand":
l_host = []
for t_group in config.gettuple(self.conf, section, "groups"):
for t_host in self.d_host[t_group]:
if not t_host == host:
l_host.append(t_host)
self.l_log.append((dt, random.choice(l_host), mes))
def _generate_event(self, event_name):
section = "event_" + event_name
def _recur(dt, host, event_name):
if not self.conf.has_option(section, "recurrence"):
return
if self.conf.getboolean(section, "recurrence"):
if random.random() < self.conf.getfloat(section, "recur_p"):
durmin = config.getdur(self.conf, section, "recur_dur_min")
durmax = config.getdur(self.conf, section, "recur_dur_max")
new_dt = self._dt_delta_rand(dt, durmin, durmax)
_add_event(new_dt, host, event_name)
def _add_event(dt, host, event_name):
info = {}
if self.conf.has_option(section, "info"):
for i in config.gettuple(self.conf, section, "info"):
if i == "ifname":
info[i] = random.choice(
config.gettuple(self.conf, section, "ifname"))
elif i == "user":
info[i] = random.choice(
config.gettuple(self.conf, section, "user"))
self.l_event.append((dt, host, event_name, info))
_recur(dt, host, event_name)
for group in config.gettuple(self.conf, section, "groups"):
for host in self.d_host[group]:
occ = self.conf.get(section, "occurrence")
if occ == "random_uniform":
freq = self.conf.getfloat(section, "frequency")
for dt in self.rand_uniform(self.top_dt, self.end_dt,
freq):
_add_event(dt, host, event_name)
elif occ in ("random", "random_exp"):
freq = self.conf.getfloat(section, "frequency")
for dt in self._rand_exp(self.top_dt, self.end_dt, freq):
_add_event(dt, host, event_name)
elif occ == "hourly":
dursec = 60 * 60
first_dt = self.top_dt + datetime.timedelta(
seconds=random.randint(0, dursec - 1))
now_dt = first_dt
while now_dt < self.end_dt:
_add_event(now_dt, host, event_name)
now_dt += datetime.timedelta(seconds=dursec)
elif occ == "daily":
dursec = 24 * 60 * 60
first_dt = self.top_dt + datetime.timedelta(
seconds=random.randint(0, dursec - 1))
now_dt = first_dt
while now_dt < self.end_dt:
_add_event(now_dt, host, event_name)
now_dt += datetime.timedelta(seconds=dursec)
def __init__(self, conf):
self._arearule = conf["dag"]["area"]
self._areadict = config.GroupDef(conf["dag"]["area_def"])
if self._arearule == "all":
self._testfunc = self._test_all
elif self._arearule == "each":
self._testfunc = self._test_each
else:
self.areas = config.gettuple(conf, "dag", "area")
self._testfunc = self._test_ingroup
def __init__(self, conf_fn, seed=None):
if seed is None:
random.seed()
else:
random.seed(seed)
self.conf = configparser.ConfigParser()
self.conf.read(conf_fn)
self.term = config.getterm(self.conf, "main", "term")
self.top_dt, self.end_dt = self.term
self.d_host = {}
for group in config.gettuple(self.conf, "main", "host_groups"):
for host in config.gettuple(self.conf, "main", "group_" + group):
self.d_host.setdefault(group, []).append(host)
self.l_event = []
for event_name in config.gettuple(self.conf, "main", "events"):
self._generate_event(event_name)
self.l_log = []
for event in self.l_event:
self._generate_log(event)
def filter_periodic(conf, ld, l_dt, dt_range, evdef, method):
"""Return True and the interval if a_cnt is periodic."""
ret_false = False, None, None
gid_name = conf.get("dag", "event_gid")
p_cnt = conf.getint("filter", "pre_count")
p_term = config.getdur(conf, "filter", "pre_term")
# preliminary test
if len(l_dt) < p_cnt:
_logger.debug("time-series count too small, skip")
return ret_false
elif max(l_dt) - min(l_dt) < p_term:
_logger.debug("time-series range too small, skip")
return ret_false
# periodicity test
for dt_cond in config.gettuple(conf, "filter", "sample_rule"):
dt_length, binsize = [config.str2dur(s) for s in dt_cond.split("_")]
if (dt_range[1] - dt_range[0]) == dt_length:
temp_l_dt = l_dt
else:
temp_l_dt = reload_ts(ld, evdef, dt_length, dt_range, gid_name)
a_cnt = dtutil.discretize_sequential(temp_l_dt,
dt_range,
binsize,
binarize=False)
remain_dt = None
if method == "remove":
flag, interval = period.fourier_remove(conf, a_cnt, binsize)
elif method == "replace":
flag, remain_array, interval = period.fourier_replace(
conf, a_cnt, binsize)
if remain_array is not None:
remain_dt = revert_event(remain_array, dt_range, binsize)
elif method == "corr":
flag, interval = period.periodic_corr(conf, a_cnt, binsize)
else:
raise NotImplementedError
if flag:
return flag, remain_dt, interval
return ret_false