# For Profiling start = time.time() for n in num_streams: for snr in SNRs: for l in anomaly_lengths: for m in anomaly_magnitudes: A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0 = Tseries(0) s1 = Tseries(0) s0.makeSeries( [1, 3, 1], [100, l, 200 - l], [baseLine, baseLine, baseLine + m], gradient=float(m) / float(l), noise_type="none", ) s1.makeSeries( [1, 4, 1], [200, l, 100 - l], [baseLine, baseLine, baseLine - m], gradient=float(m) / float(l), noise_type="none", )
# -*- coding: utf-8 -*- """ Created on Sun Nov 21 13:05:27 2010 @author: musselle """ from ControlCharts import Tseries # import numpy as np # from matplotlib.pyplot import plot, figure import scipy # Initialise stream series1 = Tseries(0) series2 = Tseries(0) series3 = Tseries(0) series1.makeSeries([3],[5],[2], gradient = 10, noise = 0.000000001) series2.makeSeries([3],[4],[2], gradient = 10, noise = 0.000000001) series3.makeSeries([3],[8],[2], gradient = 5, noise = 0.000000001) series1.makeSeries([2,1,2],[95, 100, 100],[50, 50, 50], amp = 10, noise = 0.00000001) series2.makeSeries([2],[296],[40], amp = 10, noise = 0.000000001, period = 10, phase = 5) series3.makeSeries([2,4,2],[142, 10, 140],[40, 40, 20], gradient = 2, amp = 10, noise = 0.00000001)
# For Profiling start = time.time() for n in num_streams: for snr in SNRs: for l in anomaly_lengths: for m in anomaly_magnitudes: A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0lin = Tseries(0) # linear component s0sin = Tseries(0) # Sine component s1lin = Tseries(0) s1sin = Tseries(0) if anomaly_type == 'peak': s0lin.makeSeries([1,3,4,1], [interval_length, l/2, l/2, 2 * interval_length - l], [baseLine, baseLine, baseLine + m, baseLine], gradient = float(m)/float(l/2), noise_type ='none') s0sin.makeSeries([2], [3 * interval_length], [0.0], amp = amp, period = period, noise_type ='none') # sum sin and linear components to get data stream s0 = np.array(s0lin) + np.array(s0sin)
import time from PedrosFrahst import frahst_pedro #=============================================================================== # Runscript #=============================================================================== #============ # Initialise #============ 'Create time series string' start = time.time() # Initialise stream series1 = Tseries(0) series2 = Tseries(0) series3 = Tseries(0) #series4 = Tseries(0) series1.makeSeries([1],[300],[5], noise = 1) series2.makeSeries([1,4,1],[200, 10, 90],[6, 6, -4], gradient = 1, noise = 1) series3.makeSeries([1,3,1],[100,10,190],[2,2,12], gradient = 1, noise = 1) #series4.makeSeries([2],[300],[5], period = 10, amp = 1, noise = 1) streams = scipy.c_[series1, series2, series3] # ,series4] # Run SPIRIT and Frahst # Parameters
def genDataMatrix(anomaly_type, n, snr, l, m, initial_conditions, period, amp, baseLine=0.0, baseLine_MA_window=15): '''Generate dataset matrix with given parameters Matrix is timesteps x num_strems x initial conditions ''' A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0lin = Tseries(0) # linear component s0sin = Tseries(0) # Sine component s1lin = Tseries(0) s1sin = Tseries(0) if anomaly_type == 'peak': s0lin.makeSeries([1, 3, 4, 1], [100, l / 2, l / 2, 200 - l], [baseLine, baseLine, baseLine + m, baseLine], gradient=float(m) / float(l / 2), noise_type='none') s0sin.makeSeries([2], [300], [0.0], amp=amp, period=period, noise_type='none') # sum sin and linear components to get data stream s0 = np.array(s0lin) + np.array(s0sin) s1lin.makeSeries([1, 4, 3, 1], [200, l / 2, l / 2, 100 - l], [baseLine, baseLine, baseLine - m, baseLine], gradient=float(m) / float(l / 2), noise_type='none') s1sin.makeSeries([2], [300], [0.0], amp=amp, period=period, noise_type='none') # sum sin and linear components to get data stream s1 = np.array(s1lin) + np.array(s1sin) elif anomaly_type == 'shift': s0lin.makeSeries([1, 3, 1], [100, l, 200 - l], [baseLine, baseLine, baseLine + m], gradient=float(m) / float(l), noise_type='none') s0sin.makeSeries([2], [300], [0.0], amp=amp, period=period, noise_type='none') # sum sin and linear components to get data stream s0 = np.array(s0lin) + np.array(s0sin) s1lin.makeSeries([1, 4, 1], [200, l, 100 - l], [baseLine, baseLine, baseLine - m], gradient=float(m) / float(l), noise_type='none') s1sin.makeSeries([2], [300], [0.0], amp=amp, period=period, noise_type='none') # sum sin and linear components to get data stream s1 = np.array(s1lin) + np.array(s1sin) # The rest of the ts for k in range(2, n): name = 's' + str(k) vars()[name] = Tseries(0) vars()[name].makeSeries([2], [300], [baseLine], amp=amp, period=period, noise_type='none') # Concat into one matrix S = scipy.c_[s0] for k in range(1, n): S = scipy.c_[S, vars()['s' + str(k)]] # Concatonate to 3d array, timesteps x streams x initial condition if type(A) == int: A = S # first loop only else: A = np.dstack((A, S))
import matplotlib.pyplot as plt from Frahst_v3_1 import FRAHST_V3_1 from SPIRIT import SPIRIT from utils import analysis, QRsolveA, pltSummary # =============================================================================== # Runscript # =============================================================================== # ============ # Initialise # ============ "Create time series string" series_1 = Tseries(0) series_2 = Tseries(0) series_3 = Tseries(0) series_4 = Tseries(0) series_5 = Tseries(0) # S 1 # CYCLIC - Normal - Cyclic series_1.cyclicEt(100, base=6, noise=0.1, amp=2, period=5, phase=0, noise_type="gauss") series_1.normalEt(100, base=6, noise=0.00000000000000001, noise_type="gauss") series_1.cyclicEt(100, base=6, noise=0.1, amp=2, period=5, phase=0, noise_type="gauss") # S 2 # Cyclic series_2.cyclicEt(300, base=6, noise=0.1, amp=2, period=5, phase=1.5, noise_type="gauss") # S 3 # Cyclic
return data if __name__ == '__main__' : first = 1 if first: #s1 = Tseries(0) s2 = Tseries(0) #s1.makeSeries([2,1,2], [300, 300, 300], noise = 0.5, period = 50, amp = 5) #s2.makeSeries([2], [900], noise = 0.5, period = 50, amp = 5) #data = sp.r_['1,2,0', s1, s2] s0lin = Tseries(0) s0sin = Tseries(0) s2lin = Tseries(0) s2sin = Tseries(0) interval_length = 300 l = 10 m = 10 baseLine = 0 amp = 5 period = 50 s0lin.makeSeries([1,3,4,1], [interval_length, l/2, l/2, 2 * interval_length - l], [baseLine, baseLine, baseLine + m, baseLine], gradient = float(m)/float(l/2), noise = 0.5) s0sin.makeSeries([2], [3 * interval_length], [0.0], amp = amp, period = period, noise = 0.5)
def fractional_decay_MA(data, alpha): """ MA based on incremetal time fractions """ aved_data = np.zeros_like(data) u = np.zeros(data.shape[1]) for i in range(data.shape[0]): t = i + 1 new_data_vec = data[i] u = ((t - 1) / float(t)) * alpha * u + (1 / float(t)) * new_data_vec aved_data[i] = u return aved_data if __name__ == '__main__': # Create Series s0 = Tseries(0) s1 = Tseries(0) s2 = Tseries(0) s0.makeSeries([1, 3, 1], [100, 10, 190], base=[0, 0, 2], gradient=0.2, noise=0.5) s1.makeSeries([1, 4, 1], [200, 10, 90], base=[0, 0, -2], gradient=0.2, noise=0.5) s2.makeSeries([1], [300], noise=0.5) streams = scipy.c_[s0, s1, s2]
# -*- coding: utf-8 -*- """ Created on Sat Mar 05 23:19:11 2011 THIS IS THE VERSION TO RUN ON MAC, NOT WINDOWS @author: musselle """ from numpy import eye, zeros, dot, sqrt, log10, trace, arccos, nan, arange import numpy as np import numpy.random as npr import scipy as sp from numpy.random import rand from numpy.linalg import qr, eig, norm, solve from matplotlib.pyplot import plot, figure, title, step from artSigs import genCosSignals_no_rand , genCosSignals import scipy.io as sio from utils import analysis, QRsolveA, pltSummary2 from PedrosFrahst import frahst_pedro_original from Frahst_v3_1 import FRAHST_V3_1 from load_syn_ping_data import load_n_store def FRAHST_V3_3(data, r=1, alpha=0.96, e_low=0.96, e_high=0.98, fix_init_Q = 0, holdOffTime=0, evalMetrics = 'F', static_r = 0, r_upper_bound = None, ignoreUp2 = 0): """ Fast Rank Adaptive Householder Subspace Tracking Algorithm (FRAHST) Version 3.3 - Add decay of S and in the event of vanishing inputs - Make sure rank of S does not drop (and work out what that means!) - stops S going singular Version 3.2 - Added ability to fix r to a static value., and also give it an upper bound. If undefined, defaults to num of data streams.
def genDataMatrix(anomaly_type, n, snr, l, m, initial_conditions, period, amp, baseLine = 0.0, baseLine_MA_window = 15): '''Generate dataset matrix with given parameters Matrix is timesteps x num_strems x initial conditions ''' A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0lin = Tseries(0) # linear component s0sin = Tseries(0) # Sine component s1lin = Tseries(0) s1sin = Tseries(0) if anomaly_type == 'peak': s0lin.makeSeries([1,3,4,1], [100, l/2, l/2, 200 - l], [baseLine, baseLine, baseLine + m, baseLine], gradient = float(m)/float(l/2), noise_type ='none') s0sin.makeSeries([2], [300], [0.0], amp = amp , period = period, noise_type ='none') # sum sin and linear components to get data stream s0 = np.array(s0lin) + np.array(s0sin) s1lin.makeSeries([1,4,3,1],[200, l/2, l/2, 100 - l],[baseLine, baseLine, baseLine - m, baseLine], gradient = float(m)/float(l/2), noise_type ='none') s1sin.makeSeries([2], [300], [0.0], amp = amp , period = period, noise_type ='none') # sum sin and linear components to get data stream s1 = np.array(s1lin) + np.array(s1sin) elif anomaly_type == 'shift': s0lin.makeSeries([1,3,1], [100, l, 200 - l], [baseLine, baseLine, baseLine + m], gradient = float(m)/float(l), noise_type ='none') s0sin.makeSeries([2], [300], [0.0], amp = amp, period = period, noise_type ='none') # sum sin and linear components to get data stream s0 = np.array(s0lin) + np.array(s0sin) s1lin.makeSeries([1,4,1],[200, l, 100 - l],[baseLine, baseLine, baseLine - m], gradient = float(m)/float(l), noise_type ='none') s1sin.makeSeries([2], [300], [0.0], amp = amp , period = period, noise_type ='none') # sum sin and linear components to get data stream s1 = np.array(s1lin) + np.array(s1sin) # The rest of the ts for k in range(2, n) : name = 's'+ str(k) vars()[name] = Tseries(0) vars()[name].makeSeries([2], [300], [baseLine], amp = amp , period = period, noise_type ='none') # Concat into one matrix S = scipy.c_[s0] for k in range(1, n) : S = scipy.c_[ S, vars()['s'+ str(k)] ] # Concatonate to 3d array, timesteps x streams x initial condition if type(A) == int: A = S # first loop only else: A = np.dstack((A, S))
# Author: C Musselle --<> # Purpose: Generate Synthetic data streams to test for anomaly detection and monitouring applications # Created: 11/18/11 import numpy as np import scipy as sp import matplotlib.pyplot as plt import sys import os from ControlCharts import Tseries """ Code Description: . """ s0 = Tseries(0) s1 = Tseries(0) s2 = Tseries(0) s3 = Tseries(0) s4 = Tseries(0) s5 = Tseries(0) s6 = Tseries(0) s7 = Tseries(0) s8 = Tseries(0) s9_lin = Tseries(0) s9_sin = Tseries(0) s0.makeSeries([2], [1000], amp=1, noise=0.25) s1.makeSeries([2], [1000], amp=1, noise=0.25) s2.makeSeries([2], [1000], amp=1, noise=0.25) s3.makeSeries([2], [1000], amp=1, noise=0.25)
def gen_a_peak_dip(N, T, L, M, pA, k = 5, interval = [10,121], seed = None, noise_sig = 0.1, L2 = None, periods = None, periods2 = None): """ Adds short peak or dip to data interval used is (low (Inclusive), High (exclusive)] """ if seed: print 'Setting seed to %i' % (seed) npr.seed(seed) if periods is None: periods = [] num_left = float(interval[1] - interval[0]) num_needed = float(k) for i in xrange(interval[0], interval[1]+1): # probability of selection = (number needed)/(number left) p = num_needed / num_left if npr.rand() <= p: periods.append(i) num_needed -= 1 else: num_left -=1 #if seed is not None: ## Code to make random sins combination #A, sins = sin_rand_combo(N, T, periods, seed = seed, noise_scale = noise_sig) #else: ## Code to make random sins combination # Seed already set at start of function A, sins = sin_rand_combo(N, T, periods, noise_scale = noise_sig) # Anomaly will occur (and finish) Between time points 50 and T - 10 start_point = npr.randint(50, T - L - 10) # Code to make linear Anomalous trend baseLine = 0 # Randomly choose peak or dip if npr.rand() < 0.5: anom_type = 'Peak' s0lin = Tseries(0) s0lin.makeSeries([1,3,4,1], [start_point, L/2, L/2, T - start_point - L], [baseLine, baseLine, baseLine + M, baseLine], gradient = float(M)/float(L/2), noise_type ='none') else: anom_type = 'Dip' s0lin = Tseries(0) s0lin.makeSeries([1,4,3,1], [start_point, L/2, L/2, T - start_point - L], [baseLine, baseLine, baseLine - M, baseLine], gradient = float(M)/float(L/2), noise_type ='none') # Select stream(s) to be anomalous if type(pA) == int: num_anom = pA elif pA < 1.0: num_anom = np.floor(pA * N) if num_anom > 1: anoms = [] num_left = float(N) num_needed = float(num_anom) for i in xrange(N): # probability of selection = (number needed)/(number left) p = num_needed / num_left if npr.rand() <= p: anoms.append(i) num_needed -= 1 num_left -= 1 else: num_left -= 1 A[:,anoms] = A[:,anoms] + np.atleast_2d(s0lin).T else: anoms = npr.randint(N) A[:,anoms] = A[:,anoms] + np.array(s0lin) ''' Grount truth Table ''' if anoms.__class__ == int: anoms = [anoms] gt_table = np.zeros(num_anom, dtype = [('start','i4'),('loc','i4'),('len','i4'),('mag','i4'),('type','a10')]) for i, a in enumerate(anoms): gt_table[i] = (start_point, a, L, M, anom_type) output = dict(data = A, gt = gt_table, trends = sins, periods = periods) return output
def gen_a_grad_persist(N, T, L, M, pA, k = 5, interval = [10,101], seed = None, noise_sig = 0.1, L2 = None, periods = None, periods2 = None): """ Adds longer persisted anomaly. gradient up/down to it and vice versa after L steps """ if seed: print 'Setting seed to %i' % (seed) npr.seed(seed) if periods is None: periods = [] num_left = float(interval[1] - interval[0]) num_needed = float(k) for i in xrange(interval[0], interval[1]+1): # probability of selection = (number needed)/(number left) p = num_needed / num_left if npr.rand() <= p: periods.append(i) num_needed -= 1 else: num_left -=1 # Seed already set at start of function A, sins = sin_rand_combo(N, T, periods, noise_scale = noise_sig) # Anomaly will occur (and finish) Between time points 50 and T - 10 start_point = npr.randint(50, T - L - L2 - 10) # Code to make linear Anomalous trend baseLine = 0 # Randomly choose peak or dip if npr.rand() < 0.5: anom_type = 'Grad Up--Down' s0lin = Tseries(0) s0lin.makeSeries([1,3,1,4,1], [start_point, L/2, L2, L/2, T - start_point - L - L2], [baseLine, baseLine, baseLine + M, baseLine + M, baseLine], gradient = float(M)/float(L/2), noise_type ='none') else: anom_type = 'Grad Down--Up' s0lin = Tseries(0) s0lin.makeSeries([1,4,1,3,1], [start_point, L/2, L2, L/2, T - start_point - L - L2], [baseLine, baseLine, baseLine - M, baseLine - M, baseLine], gradient = float(M)/float(L/2), noise_type ='none') # Select stream(s) to be anomalous if type(pA) == int: num_anom = pA elif pA < 1.0: num_anom = np.floor(pA * N) if num_anom > 1: anoms = [] num_left = float(N) num_needed = float(num_anom) for i in xrange(N): # probability of selection = (number needed)/(number left) p = num_needed / num_left if npr.rand() <= p: anoms.append(i) num_needed -= 1 num_left -=1 else: num_left -=1 A[:,anoms] = A[:,anoms] + np.atleast_2d(s0lin).T else: anoms = npr.randint(N) A[:,anoms] = A[:,anoms] + np.array(s0lin) ''' Ground Truth Table ''' if anoms.__class__ == int: anoms = [anoms] gt_table = np.zeros(num_anom*2, dtype = [('start','i4'),('loc','i4'),('len','i4'),('mag','i4'),('type','a10')]) count = 0 for i, a in enumerate(anoms): a1_type = anom_type[5:].split('--')[0] a2_type = anom_type[5:].split('--')[1] gt_table[i+count] = (start_point, a, L, M, 'Grad ' + a1_type) gt_table[i+count+1] = (start_point+L2+(L/2.0), a, L, M, 'Grad ' + a2_type) count += 1 output = dict(data = A, gt = gt_table, trends = sins, periods = periods) return output
# -*- coding: utf-8 -*- """ Created on Sat Mar 05 23:19:11 2011 THIS IS THE VERSION TO RUN ON MAC, NOT WINDOWS @author: musselle """ from numpy import eye, zeros, dot, sqrt, log10, trace, arccos, nan, arange import numpy as np import numpy.random as npr import scipy as sp from numpy.random import rand from numpy.linalg import qr, eig, norm, solve from matplotlib.pyplot import plot, figure, title, step from artSigs import genCosSignals_no_rand, genCosSignals import scipy.io as sio from utils import analysis, QRsolveA, pltSummary2 from PedrosFrahst import frahst_pedro_original from Frahst_v3_1 import FRAHST_V3_1 from load_syn_ping_data import load_n_store def FRAHST_V3_3(data, r=1, alpha=0.96, e_low=0.96, e_high=0.98, fix_init_Q=0, holdOffTime=0, evalMetrics='F', static_r=0, r_upper_bound=None,
z22 = np.sin(2 * np.pi * t / p22) + npr.randn(t.shape[0]) * noise_scale z33 = np.sin(2 * np.pi * t / p33) + npr.randn(t.shape[0]) * noise_scale data = sp.r_['1,2,0', sp.r_[z1, z11], sp.r_[z2, z22], sp.r_[z3, z33]] return data if __name__ == '__main__': first = 1 if first: #s1 = Tseries(0) s2 = Tseries(0) #s1.makeSeries([2,1,2], [300, 300, 300], noise = 0.5, period = 50, amp = 5) #s2.makeSeries([2], [900], noise = 0.5, period = 50, amp = 5) #data = sp.r_['1,2,0', s1, s2] s0lin = Tseries(0) s0sin = Tseries(0) s2lin = Tseries(0) s2sin = Tseries(0) s3 = Tseries(0) s4 = Tseries(0) interval_length = 300 l = 10 m = 10 baseLine = 0
from ControlCharts import Tseries from CUSUM import cusum import numpy as np import matplotlib.pyplot as plt #=============================================================================== # Runscript #=============================================================================== #============ # Initialise #============ 'Create time series string' series_1 = Tseries(0) # NORMAL - normalEt(self, size, base=0, noise, noise_type = 'gauss') series_1.normalEt(50, 0, 1,'gauss') series_1.normalEt(50, 3, 1,'gauss') # CYCLIC - cyclicEt(self, size, base=0, noise=2, amp=10, period=25, \ # noise_type = 'gauss'): series_1.cyclicEt(100, 3, 1, 5, 50, 'gauss') # NORMAL - series_1.normalEt(50, 3, 1,'gauss') series_1.normalEt(50, -3, 1,'gauss') # UP - upEt(self,size,base=0,noise=1,gradient=0.2, noise_type = 'gauss') series_1.upEt(100, -3, 1, 0.2, 'gauss')
# -*- coding: utf-8 -*- """ Created on Sat Mar 05 23:19:11 2011 THIS IS THE VERSION TO RUN ON MAC, NOT WINDOWS @author: musselle """ from numpy import eye, zeros, dot, sqrt, log10, trace, arccos, nan, arange, ones import numpy as np import scipy as sp import numpy.random as npr from numpy.linalg import qr, eig, norm, solve from matplotlib.pyplot import plot, figure, title, step, ylim from artSigs import genCosSignals_no_rand, genCosSignals import scipy.io as sio from utils import analysis, QRsolveA, pltSummary2 from PedrosFrahst import frahst_pedro_original from Frahst_v3_1 import FRAHST_V3_1 from Frahst_v3_3 import FRAHST_V3_3 from Frahst_v3_4 import FRAHST_V3_4 from Frahst_v4_0 import FRAHST_V4_0 from load_syn_ping_data import load_n_store from QR_eig_solve import QRsolve_eigV from create_Abilene_links_data import create_abilene_links_data from MAfunctions import MA_over_window from ControlCharts import Tseries def FRAHST_V6_3(data, r=1, alpha=0.96, L=1,
from ControlCharts import Tseries from CUSUM import cusum import numpy as np import matplotlib.pyplot as plt #=============================================================================== # Runscript #=============================================================================== #============ # Initialise #============ 'Create time series string' series_1 = Tseries(0) # NORMAL - normalEt(self, size, base=0, noise, noise_type = 'gauss') series_1.normalEt(50, 0, 1, 'gauss') series_1.normalEt(50, 3, 1, 'gauss') # CYCLIC - cyclicEt(self, size, base=0, noise=2, amp=10, period=25, \ # phase = 0, noise_type = 'gauss'): series_1.cyclicEt(100, 3, 1, 5, 50) # NORMAL - series_1.normalEt(50, 3, 1, 'gauss') series_1.normalEt(50, -3, 1, 'gauss') # UP - upEt(self,size,base=0,noise=1,gradient=0.2, noise_type = 'gauss') series_1.upEt(100, -3, 1, 0.2, 'gauss')
count = 1 total = len(num_streams) * len(SNR) * len(anomaly_length) * len(anomaly_magnitude) for a in num_streams: for b in SNR: for c in anomaly_length: for d in anomaly_magnitude: A = 0 for e in range(initial_conditions): # Seed random number generator np.random.seed(e) # Two ts that have anomalous shift s0 = Tseries(0) s1 = Tseries(0) s0.makeSeries([1,3,1],[100, c, 200 - c],[baseLine, baseLine, baseLine + d], gradient = d/c, noise_type ='none') s1.makeSeries([1,4,1],[200, c, 100 - c],[baseLine, baseLine, baseLine - d], gradient = d/c, noise_type ='none') # The rest of the ts for i in range(2, a) : name = 's'+ str(i) vars()[name] = Tseries(0) vars()[name].makeSeries([1],[300],[5], noise_type ='none') # concat into one matrix streams = scipy.c_[s0] for i in range(1, a) : streams = scipy.c_[streams, vars()[name] ]
total = len(num_streams) * len(SNR) * len(anomaly_length) * len( anomaly_magnitude) for a in num_streams: for b in SNR: for c in anomaly_length: for d in anomaly_magnitude: A = 0 for e in range(initial_conditions): # Seed random number generator np.random.seed(e) # Two ts that have anomalous shift s0 = Tseries(0) s1 = Tseries(0) s0.makeSeries([1, 3, 1], [100, c, 200 - c], [baseLine, baseLine, baseLine + d], gradient=d / c, noise_type='none') s1.makeSeries([1, 4, 1], [200, c, 100 - c], [baseLine, baseLine, baseLine - d], gradient=d / c, noise_type='none') # The rest of the ts for i in range(2, a): name = 's' + str(i) vars()[name] = Tseries(0) vars()[name].makeSeries([1], [300], [5], noise_type='none')
score = [] # reset score tagMetric.append(tag) # list of cumulative sums of Smax tag = np.zeros((len(stream))) # reset tag for next win size # Plot results x = range(1, len(stream)) plt.figure() plt.subplot(311) plt.plot(stream) plt.title('Input Stream') plt.subplot(312) for i in range(len(win_sizes)): # i = 1:size(win_sizes,2) plt.plot(sMaxScore[i]) #'Color', ColOrd(i,:)) plt.title('Smax Score') plt.subplot(313) for i in range(len(win_sizes)): plt.plot(tagMetric[i]) plt.title('Tag') if __name__ == '__main__': series = Tseries(0) series.makeSeries([1,2,3,4] , [100,100,100,100], noise = 4) cusum_alg(series, [30, 60, 90])
def singleShiftRerun(parameter_string): params = parameter_string.split(', ') # Data Sets baseLine = 0.0 num_streams = [int(params[0].split()[-1])] # n SNRs = [int(params[1].split()[-1])] # snr anomaly_lengths = [int(params[2].split()[-1])] # l anomaly_magnitudes = [int(params[3].split()[-1])] # m initial_conditions = 20 # i # Algorithm Parameters thresholds = params[4].split()[-1].split(',') e_highs = [float(thresholds[1][0:-1])] # eh e_lows = [float(thresholds[0][1:])] # el alphas = [float(params[5].split()[-1])] # a holdOffs = [int(params[6].split()[-1])] # h # Algorithm flags run_spirit = 0 run_frahst = 1 run_frahst_pedro = 0 #=============================================================================== # Initialise Data sets #========-======================================================================= # For Profiling start = time.time() for n in num_streams: for snr in SNRs: for l in anomaly_lengths: for m in anomaly_magnitudes: A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0 = Tseries(0) s1 = Tseries(0) s0.makeSeries([1, 3, 1], [100, l, 200 - l], [baseLine, baseLine, baseLine + m], gradient=float(m) / float(l), noise_type='none') s1.makeSeries([1, 4, 1], [200, l, 100 - l], [baseLine, baseLine, baseLine - m], gradient=float(m) / float(l), noise_type='none') # The rest of the ts for k in range(2, n): name = 's' + str(k) vars()[name] = Tseries(0) vars()[name].makeSeries([1], [300], [baseLine], noise_type='none') # Concat into one matrix S = scipy.c_[s0] for k in range(1, n): S = scipy.c_[S, vars()['s' + str(k)]] # Concatonate to 3d array, timesteps x streams x initial condition if type(A) == int: A = S else: A = np.dstack((A, S)) # Calculate the noise Ps = np.sum(A[:, :, 0]**2) Pn = Ps / (10.**(snr / 10.)) scale = Pn / (n * 300.) noise = np.random.randn(A.shape[0], A.shape[1], A.shape[2]) * np.sqrt(scale) A = A + noise #=============================================================================== # Ground Truths #=============================================================================== # # time step | length ground_truths = np.array([[100, l], [200, l]]) #============================================================================== # Run Algorithm #============================================================================== alg_count = 1 for eh in e_highs: for el in e_lows: for a in alphas: for h in holdOffs: print 'Running Algorithm(s) with:\nE_Thresh = (' + str(el) + ',' + str(eh) + ')\n' + \ 'alpha = ' + str(a) + '\nHoldOff = ' + str(h) SPIRIT_metricList = [] FRAHST_metricList = [] PEDRO_FRAHST_metricList = [] for i in range(initial_conditions): # Load Data streams = A[:, :, i] if run_spirit == 1: # SPIRIT res_sp = SPIRIT(streams, a, [el, eh], evalMetrics='F', reorthog=False, holdOffTime=h) res_sp['Alg'] = 'SPIRIT: alpha = ' + str(a) + \ ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' pltSummary2( res_sp, streams, (el, eh)) SPIRIT_metricList.append( analysis( res_sp, ground_truths, 300)) # data = res_sp # Title = 'SPIRIT alpha = ' + str(a) + ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' # # plot_4x1(streams, data['hidden'], data['e_ratio'], data['orthog_error'], # ['Input Data','Hidden\nVariables', # 'Energy Ratio', 'Orthogonality\nError (dB)'] , 'Time Steps', Title) # plot_4x1(streams, data['hidden'], data['orthog_error'], data['subspace_error'], # ['Input Data','Hidden\nVariables', 'Orthogonality\nError (dB)','Subspace\nError (dB)'], # 'Time Steps', Title) # if run_frahst == 1: # My version of Frahst res_fr = FRAHST_V3_1( streams, alpha=a, e_low=el, e_high=eh, holdOffTime=h, fix_init_Q=1, r=1, evalMetrics='F') res_fr[ 'Alg'] = 'MyFrahst: alpha = ' + str( a ) + ' ,E_Thresh = (' + str( el) + ',' + str(eh) + ')' FRAHST_metricList.append( analysis( res_fr, ground_truths, 300)) pltSummary2( res_fr, streams, (el, eh)) # # data = res_fr # Title = 'My Frahst with alpha = ' + str(a) + ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' # plot_4x1(streams, data['hidden'], data['orthog_error'], data['subspace_error'], # ['Input Data','Hidden\nVariables', 'Orthogonality\nError (dB)','Subspace\nError (dB)'], # 'Time Steps', Title) if run_frahst_pedro == 1: # Pedros version of Frahst res_frped = frahst_pedro( streams, alpha=a, e_low=el, e_high=eh, holdOffTime=h, r=1, evalMetrics='F') res_frped['Alg'] = 'Pedros Frahst: alpha = ' + str(a) + \ ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' PEDRO_FRAHST_metricList.append( analysis( res_frped, ground_truths, 300)) pltSummary2( res_frped, streams, (el, eh)) # # data = res_frped # Title = 'Pedros Frahst: alpha = ' + str(a) + \ # ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' # # plot_4x1(streams, data['hidden'], data['orthog_error'], data['subspace_error'], # ['Input Data','Hidden\nVariables', 'Orthogonality\nError (dB)','Subspace\nError (dB)'], # 'Time Steps', Title) finish = time.time() - start print 'Runtime = ' + str(finish) + 'seconds\n' print 'In H:M:S = ' + GetInHMS(finish)
def singleShiftRerun(parameter_string): params = parameter_string.split(', ') # Data Sets baseLine = 0.0 num_streams = [int(params[0].split()[-1])] # n SNRs = [int(params[1].split()[-1])] # snr anomaly_lengths = [int(params[2].split()[-1])] # l anomaly_magnitudes = [int(params[3].split()[-1])] # m initial_conditions = 20 # i # Algorithm Parameters thresholds = params[4].split()[-1].split(',') e_highs = [float(thresholds[1][0:-1])] # eh e_lows = [float(thresholds[0][1:])] # el alphas = [float(params[5].split()[-1])] # a holdOffs = [int(params[6].split()[-1])] # h # Algorithm flags run_spirit = 0 run_frahst = 1 run_frahst_pedro = 0 #=============================================================================== # Initialise Data sets #========-======================================================================= # For Profiling start = time.time() for n in num_streams: for snr in SNRs: for l in anomaly_lengths: for m in anomaly_magnitudes: A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0 = Tseries(0) s1 = Tseries(0) s0.makeSeries([1,3,1],[100, l, 200 - l],[baseLine, baseLine, baseLine + m], gradient = float(m)/float(l), noise_type ='none') s1.makeSeries([1,4,1],[200, l, 100 - l],[baseLine, baseLine, baseLine - m], gradient = float(m)/float(l), noise_type ='none') # The rest of the ts for k in range(2, n) : name = 's'+ str(k) vars()[name] = Tseries(0) vars()[name].makeSeries([1], [300], [baseLine], noise_type ='none') # Concat into one matrix S = scipy.c_[s0] for k in range(1, n) : S = scipy.c_[ S, vars()['s'+ str(k)] ] # Concatonate to 3d array, timesteps x streams x initial condition if type(A) == int: A = S else: A = np.dstack((A, S)) # Calculate the noise Ps = np.sum(A[:,:,0] ** 2) Pn = Ps / (10. ** (snr/10.)) scale = Pn / (n * 300.) noise = np.random.randn(A.shape[0], A.shape[1], A.shape[2]) * np.sqrt(scale) A = A + noise #=============================================================================== # Ground Truths #=============================================================================== # # time step | length ground_truths = np.array([[100, l], [200, l]]) #============================================================================== # Run Algorithm #============================================================================== alg_count = 1 for eh in e_highs : for el in e_lows : for a in alphas : for h in holdOffs : print 'Running Algorithm(s) with:\nE_Thresh = (' + str(el) + ',' + str(eh) + ')\n' + \ 'alpha = ' + str(a) + '\nHoldOff = ' + str(h) SPIRIT_metricList = [] FRAHST_metricList = [] PEDRO_FRAHST_metricList = [] for i in range(initial_conditions): # Load Data streams = A[:,:,i] if run_spirit == 1: # SPIRIT res_sp = SPIRIT(streams, a, [el, eh], evalMetrics = 'F', reorthog = False, holdOffTime = h) res_sp['Alg'] = 'SPIRIT: alpha = ' + str(a) + \ ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' pltSummary2(res_sp, streams, (el, eh)) SPIRIT_metricList.append(analysis(res_sp, ground_truths, 300 )) # data = res_sp # Title = 'SPIRIT alpha = ' + str(a) + ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' # # plot_4x1(streams, data['hidden'], data['e_ratio'], data['orthog_error'], # ['Input Data','Hidden\nVariables', # 'Energy Ratio', 'Orthogonality\nError (dB)'] , 'Time Steps', Title) # plot_4x1(streams, data['hidden'], data['orthog_error'], data['subspace_error'], # ['Input Data','Hidden\nVariables', 'Orthogonality\nError (dB)','Subspace\nError (dB)'], # 'Time Steps', Title) # if run_frahst == 1: # My version of Frahst res_fr = FRAHST_V3_1(streams, alpha=a, e_low=el, e_high=eh, holdOffTime=h, fix_init_Q = 1, r = 1, evalMetrics = 'F') res_fr['Alg'] = 'MyFrahst: alpha = ' + str(a) + ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' FRAHST_metricList.append(analysis(res_fr, ground_truths, 300 )) pltSummary2(res_fr, streams, (el, eh)) # # data = res_fr # Title = 'My Frahst with alpha = ' + str(a) + ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' # plot_4x1(streams, data['hidden'], data['orthog_error'], data['subspace_error'], # ['Input Data','Hidden\nVariables', 'Orthogonality\nError (dB)','Subspace\nError (dB)'], # 'Time Steps', Title) if run_frahst_pedro == 1: # Pedros version of Frahst res_frped = frahst_pedro(streams, alpha=a, e_low=el, e_high=eh, holdOffTime=h, r = 1, evalMetrics = 'F') res_frped['Alg'] = 'Pedros Frahst: alpha = ' + str(a) + \ ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' PEDRO_FRAHST_metricList.append(analysis(res_frped, ground_truths, 300 )) pltSummary2(res_frped, streams, (el, eh)) # # data = res_frped # Title = 'Pedros Frahst: alpha = ' + str(a) + \ # ' ,E_Thresh = (' + str(el) + ',' + str(eh) + ')' # # plot_4x1(streams, data['hidden'], data['orthog_error'], data['subspace_error'], # ['Input Data','Hidden\nVariables', 'Orthogonality\nError (dB)','Subspace\nError (dB)'], # 'Time Steps', Title) finish = time.time() - start print 'Runtime = ' + str(finish) + 'seconds\n' print 'In H:M:S = ' + GetInHMS(finish)
# For Profiling start = time.time() for n in num_streams: for snr in SNRs: for l in anomaly_lengths: for m in anomaly_magnitudes: A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0lin = Tseries(0) # linear component s0sin = Tseries(0) # Sine component s1lin = Tseries(0) s1sin = Tseries(0) if anomaly_type == 'peak': s0lin.makeSeries([1, 3, 4, 1], [ interval_length, l / 2, l / 2, 2 * interval_length - l ], [baseLine, baseLine, baseLine + m, baseLine], gradient=float(m) / float(l / 2), noise_type='none') s0sin.makeSeries([2], [3 * interval_length], [0.0], amp=amp, period=period,
# -*- coding: utf-8 -*- """ Created on Wed May 18 11:23:36 2011 Test SPIRIT and Frahst on Contol Time Series: Cyclic @author: - Musselle """ from ControlCharts import Tseries from CUSUM import cusum import numpy as np import matplotlib.pyplot as plt from Frahst_v3_1 import FRAHST_V3_1 from SPIRIT import SPIRIT from utils import analysis, QRsolveA, pltSummary #=============================================================================== # Runscript #=============================================================================== #============ # Initialise #============ 'Create time series string' series_1 = Tseries(0) series_2 = Tseries(0) series_3 = Tseries(0) series_4 = Tseries(0) series_5 = Tseries(0) # S 1 # CYCLIC - Normal - Cyclic series_1.cyclicEt(100, base=6, noise=0.1, amp=2, period=5, phase=0, \ noise_type = 'gauss') series_1.normalEt(100, base=6, noise=0.00000000000000001, noise_type='gauss')
# -*- coding: utf-8 -*- """ Created on Wed May 18 11:23:36 2011 Test SPIRIT and Frahst on Contol Time Series: Cyclic @author: - Musselle """ from ControlCharts import Tseries from CUSUM import cusum import numpy as np import matplotlib.pyplot as plt from Frahst_v3_1 import FRAHST_V3_1 from SPIRIT import SPIRIT from utils import analysis, QRsolveA, pltSummary #=============================================================================== # Runscript #=============================================================================== #============ # Initialise #============ 'Create time series string' series_1 = Tseries(0) series_2 = Tseries(0) series_3 = Tseries(0) series_4 = Tseries(0) series_5 = Tseries(0) # S 1 # CYCLIC - Normal - Cyclic series_1.cyclicEt(100, base=6, noise=0.1, amp=2, period=5, phase=0, \ noise_type = 'gauss') series_1.normalEt(100, base=6, noise = 0.00000000000000001, noise_type ='gauss')
# -*- coding: utf-8 -*- """ Created on Sat Mar 05 23:19:11 2011 THIS IS THE VERSION TO RUN ON MAC, NOT WINDOWS @author: musselle """ from numpy import eye, zeros, dot, sqrt, log10, trace, arccos, nan, arange, ones import numpy as np import scipy as sp import numpy.random as npr from numpy.linalg import qr, eig, norm, solve from matplotlib.pyplot import plot, figure, title, step, ylim from artSigs import genCosSignals_no_rand , genCosSignals import scipy.io as sio from utils import analysis, QRsolveA, pltSummary2 from PedrosFrahst import frahst_pedro_original from Frahst_v3_1 import FRAHST_V3_1 from Frahst_v3_3 import FRAHST_V3_3 from Frahst_v3_4 import FRAHST_V3_4 from Frahst_v4_0 import FRAHST_V4_0 from load_syn_ping_data import load_n_store from QR_eig_solve import QRsolve_eigV from create_Abilene_links_data import create_abilene_links_data from MAfunctions import MA_over_window from ControlCharts import Tseries def FRAHST_V6_3(data, r=1, alpha=0.96, L = 1, holdOffTime=0, evalMetrics = 'F', EW_mean_alpha = 0.1, EWMA_filter_alpha = 0.3, residual_thresh = 0.1, F_min = 0.9, epsilon = 0.05, static_r = 0, r_upper_bound = None,
import scipy import time from PedrosFrahst import frahst_pedro #=============================================================================== # Runscript #=============================================================================== #============ # Initialise #============ 'Create time series string' start = time.time() # Initialise stream series1 = Tseries(0) series2 = Tseries(0) series3 = Tseries(0) #series4 = Tseries(0) series1.makeSeries([1], [300], [5], noise=1) series2.makeSeries([1, 4, 1], [200, 10, 90], [6, 6, -4], gradient=1, noise=1) series3.makeSeries([1, 3, 1], [100, 10, 190], [2, 2, 12], gradient=1, noise=1) #series4.makeSeries([2],[300],[5], period = 10, amp = 1, noise = 1) streams = scipy.c_[series1, series2, series3] # ,series4] # Run SPIRIT and Frahst # Parameters
return aved_data def fractional_decay_MA(data, alpha): """ MA based on incremetal time fractions """ aved_data = np.zeros_like(data) u = np.zeros(data.shape[1]) for i in range(data.shape[0]): t = i + 1 new_data_vec = data[i] u = ((t - 1)/float(t)) * alpha * u + (1/float(t)) * new_data_vec aved_data[i] = u return aved_data if __name__ == '__main__': # Create Series s0 = Tseries(0) s1 = Tseries(0) s2 = Tseries(0) s0.makeSeries([1,3,1], [100,10,190], base = [0, 0, 2], gradient = 0.2, noise = 0.5) s1.makeSeries([1,4,1], [200,10,90], base = [0, 0, -2], gradient = 0.2, noise = 0.5) s2.makeSeries([1], [300], noise = 0.5) streams = scipy.c_[s0, s1, s2] N = 25 # Test Moving Averages MA1 = MA_over_window1(streams, N) MA2 = MA_over_window2(streams, N) MA3 = MA_over_window3(streams, N)
# Purpose: Generate Synthetic data streams to test for anomaly detection and monitouring applications # Created: 11/18/11 import numpy as np import scipy as sp import matplotlib.pyplot as plt import sys import os from ControlCharts import Tseries """ Code Description: . """ s0 = Tseries(0) s1 = Tseries(0) s2 = Tseries(0) s3 = Tseries(0) s4 = Tseries(0) s5 = Tseries(0) s6 = Tseries(0) s7 = Tseries(0) s8 = Tseries(0) s9_lin = Tseries(0) s9_sin = Tseries(0) s0.makeSeries([2], [1000], amp = 1, noise = 0.25) s1.makeSeries([2], [1000], amp = 1, noise = 0.25) s2.makeSeries([2], [1000], amp = 1, noise = 0.25) s3.makeSeries([2], [1000], amp = 1, noise = 0.25)
# For Profiling start = time.time() for n in num_streams: for snr in SNRs: for l in anomaly_lengths: for m in anomaly_magnitudes: A = 0 for i in range(initial_conditions): # Seed random number generator np.random.seed(i) # Two ts that have anomalous shift s0lin = Tseries(0) # linear component s0sin = Tseries(0) # Sine component s1lin = Tseries(0) s1sin = Tseries(0) if anomaly_type == 'peak': s0lin.makeSeries([1,3,4,1], [100, l/2, l/2, 200 - l], [baseLine, baseLine, baseLine + m, baseLine], gradient = float(m)/float(l/2), noise_type ='none') s0sin.makeSeries([2], [300], [0.0], amp = amp , period = period, noise_type ='none') # sum sin and linear components to get data stream s0 = np.array(s0lin) + np.array(s0sin) s1lin.makeSeries([1,4,3,1],[200, l/2, l/2, 100 - l],[baseLine, baseLine, baseLine - m, baseLine],
else: # Repeat last entries count = count + 1 Q_t.append(Q_t[-1]) S_t.append(S_t[-1]) rr.append(rr[-1]) hiddenV[t-1,:len(hh[0])] = hh[0] return Q_t, S_t, rr, E_t, E_dash_t, hiddenV, count if __name__ == '__main__': # Initialise stream series1 = Tseries(0) series2 = Tseries(0) series3 = Tseries(0) series1.makeSeries([3],[5],[2], gradient = 10, noise = 0.000000001) series2.makeSeries([3],[4],[2], gradient = 10, noise = 0.000000001) series3.makeSeries([3],[8],[2], gradient = 5, noise = 0.000000001) series1.makeSeries([2,1,2],[95, 100, 100],[50, 50, 50], amp = 10, noise = 0.00000001) series2.makeSeries([2],[296],[40], amp = 10, noise = 0.000000001, period = 10, phase = 5) series3.makeSeries([2,4,2],[142, 10, 140],[40, 40, 2], gradient = 20,
count = 1 total = len(num_streams) * len(SNR) * len(anomaly_length) * len(anomaly_magnitude) for n in num_streams: for snr in SNR: for l in anomaly_length: for m in anomaly_magnitude: A = 0 for e in range(initial_conditions): # Seed random number generator np.random.seed(e) # Two ts that have anomalous shift s0 = Tseries(0) s1 = Tseries(0) if anomaly_type == 'peak': s0.makeSeries([1,3,4,1], [100, l/2, l/2, 200 - l], [baseLine, baseLine, baseLine + m, baseLine], gradient = float(m)/float(l/2), noise_type ='none') s1.makeSeries([1,4,3,1],[200, l/2, l/2, 100 - l],[baseLine, baseLine, baseLine - m, baseLine], gradient = float(m)/float(l/2), noise_type ='none') elif anomaly_type == 'shift': s0.makeSeries([1,3,1],[100, l, 200 - l],[baseLine, baseLine, baseLine + m], gradient = float(m)/float(l), noise_type ='none') s1.makeSeries([1,4,1],[200, l, 100 - l],[baseLine, baseLine, baseLine - m], gradient = float(m)/float(l), noise_type ='none') # The rest of the ts for i in range(2, n) : name = 's'+ str(i) vars()[name] = Tseries(0)
score = [] # reset score tagMetric.append(tag) # list of cumulative sums of Smax tag = np.zeros((len(stream))) # reset tag for next win size # Plot results x = range(1, len(stream)) plt.figure() plt.subplot(311) plt.plot(stream) plt.title('Input Stream') plt.subplot(312) for i in range(len(win_sizes)): # i = 1:size(win_sizes,2) plt.plot(sMaxScore[i]) #'Color', ColOrd(i,:)) plt.title('Smax Score') plt.subplot(313) for i in range(len(win_sizes)): plt.plot(tagMetric[i]) plt.title('Tag') if __name__ == '__main__': series = Tseries(0) series.makeSeries([1, 2, 3, 4], [100, 100, 100, 100], noise=4) cusum_alg(series, [30, 60, 90])