# Default Data Set Parameters
a = { 'N' : 50,
'T' : 1000,
'periods' : [15, 40, 70, 90, 120],
'L' : 10,
'L2' : 200,
'M' : 3,
'pA' : 0.1,
'noise_sig' : 0.0 }
anomalies_list = []
data_list = []
#Frahst_alg = FRAHST('F-7.A-recS.R-static', p)
Frahst_alg = FRAHST('F-3.A-eng.R-eng', p)
for i in xrange(initial_conditions):
D = gen_funcs[anomaly_type](**a) # so tidy!
#data = load_ts_data('isp_routers', 'full')
data = D['data']
#data = zscore_win(data, 100)
z_iter = iter(data)
numStreams = data.shape[1]
'''Initialise'''
Frahst_alg.re_init(numStreams)
print 'data set ', i
a = {
'N': 50,
'T': 1000,
'periods': [15, 40, 70, 90, 120],
'L': 10,
'L2': 200,
'M': 3,
'pA': 0.1,
'noise_sig': 0.0
}
anomalies_list = []
data_list = []
#Frahst_alg = FRAHST('F-7.A-recS.R-static', p)
Frahst_alg = FRAHST('F-3.A-eng.R-eng', p)
for i in xrange(initial_conditions):
D = gen_funcs[anomaly_type](**a) # so tidy!
#data = load_ts_data('isp_routers', 'full')
data = D['data']
#data = zscore_win(data, 100)
z_iter = iter(data)
numStreams = data.shape[1]
'''Initialise'''
Frahst_alg.re_init(numStreams)
print 'data set ', i
'''Begin Frahst'''
# Main iterative loop.
def __init__(self, a, dat_changes, p, alg_changes, alg_versions, path,
init_c):
#def gen_res_str(self, a, dat_changes, p, alg_changes, alg_versions):
""" generate structured array to hold results
i x j x k = Algorithm version x alg_params x data_parmas
Also generates FRAHST dictionary
"""
self.k = dat_changes
self.j = alg_changes
self.i = alg_versions
# Data type for Results structure
dt = ([
('key', 'a5'), # Key to FRAHST instance
(
'met',
[
('TP', 'i4'), # Metrics
('FP', 'i4'),
('TN', 'i4'),
('FN', 'i4'),
('precision', 'f4'),
('recall', 'f4'),
('F05', 'f4'),
('F1', 'f4'),
('F2', 'f4'),
('ACC', 'f4'),
('FDR', 'f4'),
('FPR', 'f4')
]),
(
'params',
[
(
'alg',
[
('alpha', 'f4'), # Alg Parameters
('init_r', 'i2'),
('holdOffTime', 'i2'),
('EWMA_filter_alpha', 'f4'),
('residual_thresh', 'f4'),
('ht_AR_win', 'i2'),
('AR_order', 'i1'),
('x_thresh', 'f4'),
('sample_N', 'i4'),
('dependency_lag', 'i4'),
('t_thresh', 'f4'),
('FP_rate', 'f4'),
('F_min', 'f4'),
('epsilon', 'f4'),
('e_low', 'f4'),
('e_high', 'f4'),
('r_upper_bound', 'i4'),
('fix_init_Q', np.bool),
('small_value', 'f4'),
('ignoreUp2', 'i2'),
('z_win', 'i2')
]),
(
'dat',
[
('N', 'i4'), # Data Parameters
('T', 'i4'),
('periods', np.object),
('L', 'i2'),
('L2', 'i2'),
('M', 'i2'),
('pA', 'f4'),
('noise_sig', 'f4')
])
])
])
# Big results Structure
self.R = np.zeros(
(
len(alg_versions), # i
len(alg_changes.values()[0]), # j
len(dat_changes.values()[0])),
dtype=dt) # k
# FRahst Dictionary
self.F_dict = {}
# For each Change to Algorith Parameters - j
alg_var = alg_changes.keys()[0]
alg_values = alg_changes.values()[0]
# For each Change to Dataset Parameters - k
dat_var = dat_changes.keys()[0]
dat_values = dat_changes.values()[0]
for i in xrange(self.R.shape[0]): # alg version
for j in xrange(self.R.shape[1]): # alg parameters
# update p
p[alg_var] = alg_values[j]
if alg_var == 'FP_rate':
# update threshold
p['t_thresh'] = sp.stats.t.isf(1.0 * p['FP_rate'],
p['sample_N'])
for k in xrange(self.R.shape[2]): # data parameters
# update a
a[dat_var] = dat_values[k]
key = 'F' + str(i) + str(j) + str(k)
self.F_dict[key] = FRAHST(alg_versions[i], p.copy(),
a['N'])
# Fill in rest of values
self.R[i, j, k]['key'] = key
self.R[i, j, k]['params']['alg']['alpha'] = p['alpha']
self.R[i, j, k]['params']['alg']['init_r'] = p['init_r']
self.R[
i, j,
k]['params']['alg']['holdOffTime'] = p['holdOffTime']
self.R[i, j, k]['params']['alg']['EWMA_filter_alpha'] = p[
'EWMA_filter_alpha']
self.R[i, j, k]['params']['alg']['residual_thresh'] = p[
'residual_thresh']
self.R[i, j,
k]['params']['alg']['ht_AR_win'] = p['ht_AR_win']
self.R[i, j,
k]['params']['alg']['AR_order'] = p['AR_order']
self.R[i, j,
k]['params']['alg']['x_thresh'] = p['x_thresh']
self.R[i, j,
k]['params']['alg']['sample_N'] = p['sample_N']
self.R[i, j, k]['params']['alg']['dependency_lag'] = p[
'dependency_lag']
self.R[i, j,
k]['params']['alg']['t_thresh'] = p['t_thresh']
self.R[i, j, k]['params']['alg']['FP_rate'] = p['FP_rate']
self.R[i, j, k]['params']['alg']['F_min'] = p['F_min']
self.R[i, j, k]['params']['alg']['epsilon'] = p['epsilon']
self.R[i, j, k]['params']['alg']['e_low'] = p['e_low']
self.R[i, j, k]['params']['alg']['e_high'] = p['e_high']
self.R[i, j, k]['params']['alg']['r_upper_bound'] = p[
'r_upper_bound']
self.R[i, j,
k]['params']['alg']['fix_init_Q'] = p['fix_init_Q']
self.R[
i, j,
k]['params']['alg']['small_value'] = p['small_value']
self.R[i, j,
k]['params']['alg']['ignoreUp2'] = p['ignoreUp2']
self.R[i, j, k]['params']['alg']['z_win'] = p['z_win']
self.R[i, j, k]['params']['dat']['N'] = a['N']
self.R[i, j, k]['params']['dat']['T'] = a['T']
self.R[i, j, k]['params']['dat']['periods'] = a['periods']
self.R[i, j, k]['params']['dat']['L'] = a['L']
self.R[i, j, k]['params']['dat']['L2'] = a['L2']
self.R[i, j, k]['params']['dat']['M'] = a['M']
self.R[i, j, k]['params']['dat']['pA'] = a['pA']
self.R[i, j,
k]['params']['dat']['noise_sig'] = a['noise_sig']
# Aditional Experimental variables worth storing
self.alg_ver = alg_versions
self.alg_par = alg_changes
self.dat_par = dat_changes
self.path = path
self.init_c = init_c
# set time sample buffer
time_sample_list = np.array([0.0] * initial_conditions)
for i in range(initial_conditions):
'''Generate Data Set'''
a['N'] = n
D = gen_funcs[anomaly_type](**a)
data = D['data']
''' Mean Centering '''
data = zscore_win(data, 100)
#data = zscore(data)
data = np.nan_to_num(data)
z_iter = iter(data)
numStreams = data.shape[1]
# Initialise Algorithm
F = FRAHST('F-7.A-recS.R-static.S-none', p, numStreams)
# Start time Profiling
start = time.time()
'''Begin Frahst'''
# Main iterative loop.
for zt in z_iter:
zt = zt.reshape(zt.shape[0], 1) # Convert to a column Vector
if np.any(F.st['anomaly']):
F.st['anomaly'][:] = False # reset anomaly var
'''Frahst Version '''
F.run(zt)
'''Anomaly Detection method'''
F.detect_anom(zt)
# set time sample buffer
time_sample_list = np.array([0.0]*initial_conditions)
for i in range(initial_conditions):
'''Generate Data Set'''
a['N'] = n
D = gen_funcs[anomaly_type](**a)
data = D['data']
''' Mean Centering '''
data = zscore_win(data, 100)
#data = zscore(data)
data = np.nan_to_num(data)
z_iter = iter(data)
numStreams = data.shape[1]
# Initialise Algorithm
F = FRAHST('F-7.A-recS.R-static.S-none', p, numStreams)
# Start time Profiling
start = time.time()
'''Begin Frahst'''
# Main iterative loop.
for zt in z_iter:
zt = zt.reshape(zt.shape[0],1) # Convert to a column Vector
if np.any(F.st['anomaly']):
F.st['anomaly'][:] = False # reset anomaly var
'''Frahst Version '''
F.run(zt)