def adaptive_radius(self, node, alpha = 1.): cnn = self.graph.connected_components() sub_g = [ sub_g for sub_g in cnn if node in sub_g ][0] edges = [ n.get_edges() for n in sub_g ] edges = Util.flatList(edges) if len(edges) < 2: return self.r edges = list(set(edges)) dists = [ distance.euclidean(e.head.data.pos, e.tail.data.pos) for e in edges ] # print ">>>", np.mean(dists) + alpha * np.std(dists), self.r return np.mean(dists) + alpha * np.std(dists)
def getPValue_V1( all_buses, id_bus, i, alpha_m, anomalyMethod = "KNN", h = None ): alphas = [] for id_bus_j in range( len(all_buses) ): if id_bus_j == id_bus: continue; own = all_buses[id_bus_j] fleet = all_buses[:id_bus_j] + all_buses[id_bus_j+1 :] own_ = Util.shrink(i, own, TH1) fleet_ = [ Util.shrink(i, bus, TH1) for bus in fleet ] flat_fleet_ = Util.flatList( fleet_ ) model = AnomalyModel(flat_fleet_, anomalyMethod, h) for his_j in own_: alpha = model.getAnomalyScore(his_j) alphas += [alpha] return len( [d for d in alphas if d >= alpha_m] )*1. / len(alphas)
def vizualize_buses( all_buses, dates_all_buses, dim = 2, path = "buses_viz/", m = '.' ): Util.mkdir(path) # ''' viz0 = Visualize(); viz1 = Visualize(); viz2 = Visualize(); viz3 = Visualize() c = Visualize.colors( len(all_buses) ) D = Util.flatList(all_buses) viz1.PCA_Plot( zip(*D), dim = dim, fig=path+"_Buses_All.png", color='b', marker = m ) # DD = Util.shuffle_list(D) # for i in xrange(0, len(DD)-1, 1000): viz1.MDS_Plot( zip(*DD[i:i+1000]), dim = dim, fig=path+"_"+str(i)+"_Buses_All.png", color='b', marker = m ) X = viz1.PCA_Transform( zip(*D), dim = dim ) all_buses_transformed = [] for ib in range( len(all_buses) ): print ib+1, Xb = [ x for i,x in enumerate(X) if D[i] in all_buses[ib] ] all_buses_transformed.append( Xb ) viz0.do_plot( zip(*Xb), color = c[ib], marker = m ) viz1.plot( zip(*Xb), fig=path+"Bus"+str(ib)+".png", color = c[ib], marker = m ) viz0.end_plot(fig=path+"_Buses_All_c.png") # ''' window = 30; step = 10; window_t = datetime.timedelta(days = window); step_t = datetime.timedelta(days = step) t = datetime.datetime(year=2011, month=6, day=1) while t <= datetime.datetime(year=2015, month=9, day=1): viz2.do_plot( [[-0.39, 0.39], [-0.39, 0.39]], color='w' ) for ib, bus in enumerate(all_buses_transformed): # bus_tt = [x for ix,x in enumerate(bus) if ix < len(dates_all_buses[ib]) and dates_all_buses[ib][ix] > t and dates_all_buses[ib][ix] <= t+window_t] bus_tt = [x for ix,x in enumerate(bus) if ix < len(dates_all_buses[ib]) and dates_all_buses[ib][ix] <= t+window_t] if len( bus_tt ) > 0: viz2.do_plot( zip(* bus_tt ), color = c[ib], marker = m ) viz3.do_plot( [[-0.39, 0.39], [-0.39, 0.39]], color='w' ); viz3.do_plot( zip(* bus_tt ), color = c[ib], marker = m ); viz3.end_plot(fig=path+"Bus"+str(ib)+"_"+Util.date2str(t+window_t)+".png") viz2.end_plot(fig=path+"_Buses_"+Util.date2str(t+window_t)+".png") t += step_t '''
from sklearn.neighbors import NearestNeighbors
#-----------------------------------
if __name__ == "__main__":
random.seed( 12345 )
# como_extract.computeHistogramsAllBuses(); exit(0)
(all_buses, periods_all_buses) = Util.pickleLoad(DATA_FILE_NAME+"_"+SIGNAL_CODE+".txt")
dates_all_buses = [ [ Util.getDate(tm) for tm in times ] for times in periods_all_buses ]
# como_ploting.Ploting.vizualize_buses(all_buses, dates_all_buses, m = '.'); exit(0)
#-----------------------------------
for id_bus in range( len(all_buses) ): # for each test bus
h = IGNG( radius = 2*PARAMS["R"] ); h.train( [Util.centroid( Util.flatList(all_buses) )] ); dir_imgs = "cosmo_IGNG_deriv_mature/"
# h = GNG(period = 200); h.train( [Util.centroid( Util.flatList(all_buses) )] ); dir_imgs = "cosmo_GNG/"
own_test = all_buses[id_bus]
fleet_test = all_buses[:id_bus] + all_buses[id_bus+1 :]
filename = DBFILES[id_bus]
busname = filename.split("_")[0]
dates = dates_all_buses[id_bus]
Z1 = []; Z2 = []; S1 = []; S2 = []
#--------------------------
for i, his_test in enumerate( own_test ): # for each day
sys.stdout.write( "\r%s" % "---------------------------- progress = " + str(i*100./len(own_test)) + " " + DBFILES[id_bus] + " " ); sys.stdout.flush()
S2 = []
# --------------------------
for i, his_test in enumerate(own_test): # for each day
sys.stdout.write(
"\r%s" % "---------------------------- progress = "
+ str(i * 100.0 / len(own_test))
+ " "
+ DBFILES[id_bus]
+ " "
)
sys.stdout.flush()
own_test_ = Util.shrink(i, own_test, TH1)
fleet_test_ = [Util.shrink(i, bus, TH1) for bus in fleet_test]
flat_fleet_test_ = Util.flatList(fleet_test_)
# ------
center = Util.centroid(own_test_)
score1 = Util.dist(his_test, center)
pvalue1 = 0.5
# pvalue1, score1 = como_anomaly.normalityProba_V3( "centroid", own_test_, his_test, all_buses, id_bus, i )
# pvalue1, score1 = como_anomaly.normalityProba_V3( "medoid", own_test_, his_test, all_buses, id_bus, i )
# ------
# pvalue1, score1 = como_anomaly.normalityProba_V1( "online", flat_fleet_test_, his_test, all_buses, id_bus, i, h )
# pvalue1, score1 = como_anomaly.normalityProba_V1( "IGNG", flat_fleet_test_, his_test, all_buses, id_bus, i )
# pvalue1, score1 = como_anomaly.normalityProba_V1( "GNG", flat_fleet_test_, his_test, all_buses, id_bus, i )
# pvalue1, score1 = como_anomaly.normalityProba_V1( "KNN", flat_fleet_test_, his_test, all_buses, id_bus, i )
# pvalue2, score2 = como_anomaly.normalityProba_V2( "RNN", own_test_, his_test, all_buses, id_bus, i )
pvalue2, score2 = 0.5, 0.5