def __init__(self, params, graph, dir_path, gpu, device):
self._params = params
self._graph = graph
self._dir_path = dir_path
self._mp = MotifProbability(self._params["vertices"],
self._params["probability"],
self._params["clique_size"],
self._params["directed"])
self._motif_mat = None
self._motif_features = {
"motif3":
FeatureMeta(nth_nodes_motif(3, gpu=gpu, device=device), {"m3"}),
"motif4":
FeatureMeta(nth_nodes_motif(4, gpu=gpu, device=device), {"m4"})
}
def _calc_motif3(self, gpu, device):
if self._dir_path != "":
if os.path.exists(os.path.join(self._dir_path, "motif3.pkl")):
pkl3 = pickle.load(
open(os.path.join(self._dir_path, "motif3.pkl"), "rb"))
if type(pkl3) == dict:
return pkl3
elif type(pkl3) == list:
motif3 = {v: pkl3[v] for v in range(len(pkl3))}
return motif3
else:
motif3 = pkl3._features
motif3dict = {v: motif3[v] for v in range(len(motif3))}
return motif3dict
(graph, vertices_dict) = (self._graph, {v: v for v in self._graph.nodes()}) if not \
sorted(list(self._graph.nodes()))[-1] != len(self._graph) - 1 else self._relabel_graph()
raw_ftr = GraphFeatures(graph, {
"motif3":
FeatureMeta(nth_nodes_motif(3, gpu=gpu, device=device), {"m3"})
},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True if self._dir_path != "" else False)
motif3 = raw_ftr['motif3']._features
motif3dict = {
vertices_dict[v]: motif3[v]
for v in range(len(vertices_dict))
}
return motif3dict
def _calc_betweenness(self):
raw_ftr = GraphFeatures(self._graph,
{"betweenness": FeatureMeta(BetweennessCentralityCalculator, {"betweenness"})},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True)
feature_dict = raw_ftr["betweenness"]._features
feature_mx = np.zeros((len(feature_dict), 1))
for i in feature_dict.keys():
feature_mx[i] = feature_dict[i]
return self._log_norm(feature_mx)
def _calc_bfs(self):
raw_ftr = GraphFeatures(
self._graph,
{"bfs_moments": FeatureMeta(BfsMomentsCalculator, {"bfs"})},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True)
feature_dict = raw_ftr["bfs_moments"]._features
feature_mx = np.zeros(
(len(feature_dict), len(list(feature_dict.values())[0][0])))
for i in feature_dict.keys():
for j in range(len(feature_dict[i][0])):
feature_mx[i, j] = feature_dict[i][0][j]
return self._log_norm(feature_mx)
def _calc_motif3(self):
raw_ftr = GraphFeatures(self._graph, {
"motif3":
FeatureMeta(nth_nodes_motif(3, gpu=self._gpu, device=self._device),
{"m3"})
},
dir_path=self._dir_path)
raw_ftr.build(should_dump=self._dump)
feature = raw_ftr['motif3']._features
if type(feature) == dict:
motif_matrix = self._to_matrix(feature)
else:
motif_matrix = feature
return self._log_norm(motif_matrix)
def _calc_motif4(self):
# FOR NOW, NO GPU FOR US
if os.path.exists(os.path.join(self._dir_path, "motif4.pkl")):
pkl4 = pickle.load(
open(os.path.join(self._dir_path, "motif4.pkl"), "rb"))
if type(pkl4) == dict:
motif4 = self._to_matrix(pkl4)
elif type(pkl4) == MotifsNodeCalculator:
motif4 = np.array(pkl4._features)
else:
motif4 = np.array(pkl4)
if self._motif_choice == "All_Motifs":
mp = MotifProbability(self._params['vertices'],
self._params['probability'],
self._params['clique_size'],
self._params['directed'])
motif3_count = 1 + mp.get_3_clique_motifs(3)[
-1] # The full 3 clique is the last motif 3.
clique_motifs = [
m - motif3_count for m in mp.get_3_clique_motifs(4)
]
return motif4[:, clique_motifs]
else:
return motif4
raw_ftr = GraphFeatures(self._graph, {
"motif4":
FeatureMeta(nth_nodes_motif(4, gpu=self._gpu, device=self._device),
{"m4"})
},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True)
feature = raw_ftr['motif4']._features
if type(feature) == dict:
motif_matrix = self._to_matrix(feature)
else:
motif_matrix = feature
normed_matrix = self._log_norm(motif_matrix)
if self._motif_choice == "All_Motifs":
mp = MotifProbability(self._params['vertices'],
self._params['probability'],
self._params['clique_size'],
self._params['directed'])
motif3_count = 1 + mp.get_3_clique_motifs(3)[
-1] # The full 3 clique is the last motif 3.
clique_motifs = [
m - motif3_count for m in mp.get_3_clique_motifs(4)
]
return normed_matrix[:, clique_motifs]
else:
return normed_matrix
def _calc_bfs(self):
raw_ftr = GraphFeatures(
self._graph,
{"bfs_moments": FeatureMeta(BfsMomentsCalculator, {"bfs"})},
dir_path="")
raw_ftr.build(should_dump=False)
feat = raw_ftr["bfs_moments"]._features
if type(feat) == list:
feature_mx = np.array(feat)
else:
feature_mx = np.zeros((len(feat), len(list(feat.values())[0][0])))
for i in feat.keys():
for j in range(len(feat[i][0])):
feature_mx[i, j] = feat[i][0][j]
return self._log_norm(feature_mx)
def _calc_motif4(self):
raw_ftr = GraphFeatures(self._graph, {
"motif4":
FeatureMeta(nth_nodes_motif(4, gpu=self._gpu, device=self._device),
{"m4"})
},
dir_path="")
raw_ftr.build(should_dump=False)
feature = raw_ftr['motif4']._features
if type(feature) == dict:
motif_matrix = self._to_matrix(feature)
else:
motif_matrix = feature
normed_matrix = self._log_norm(motif_matrix)
return normed_matrix
def _calc_motif3(self):
raw_ftr = GraphFeatures(self._graph,
{"motif3": FeatureMeta(nth_nodes_motif(3, gpu=self._gpu, device=self._device), {"m3"})},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True)
feature = raw_ftr['motif3']._features
if type(feature) == dict:
motif_matrix = self._to_matrix(feature)
else:
motif_matrix = feature
normed_matrix = self._log_norm(motif_matrix)
if self._motif_choice == "All_Motifs":
mp = MotifProbability(self._params['vertices'], self._params['probability'],
self._params['subgraph_size'], self._params['directed'])
clique_motifs = mp.get_3_clique_motifs(3)
return normed_matrix[:, clique_motifs]
else:
return normed_matrix
def _calc_motif4(self):
raw_ftr = GraphFeatures(self._graph,
{"motif4": FeatureMeta(nth_nodes_motif(4, gpu=self._gpu, device=self._device), {"m4"})},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True)
feature = raw_ftr['motif4']._features
if type(feature) == dict:
motif_matrix = self._to_matrix(feature)
else:
motif_matrix = feature
normed_matrix = self._log_norm(motif_matrix)
if self._motif_choice == "All_Motifs":
mp = MotifProbability(self._params['vertices'], self._params['probability'],
self._params['subgraph_size'], self._params['directed'])
motif3_count = 1 + mp.get_3_clique_motifs(3)[-1] # The full 3 clique is the last motif 3.
clique_motifs = [m - motif3_count for m in mp.get_3_clique_motifs(4)]
return normed_matrix[:, clique_motifs]
else:
return normed_matrix
def _calc_motif3(self):
# FOR NOW, NO GPU FOR US
if os.path.exists(os.path.join(self._dir_path, "motif3.pkl")):
pkl3 = pickle.load(
open(os.path.join(self._dir_path, "motif3.pkl"), "rb"))
if type(pkl3) == dict:
motif3 = self._to_matrix(pkl3)
elif type(pkl3) == MotifsNodeCalculator:
motif3 = np.array(pkl3._features)
else:
motif3 = np.array(pkl3)
if self._motif_choice == "All_Motifs":
mp = MotifProbability(self._params['vertices'],
self._params['probability'],
self._params['clique_size'],
self._params['directed'])
clique_motifs = mp.get_3_clique_motifs(3)
return motif3[:, clique_motifs]
else:
return motif3
raw_ftr = GraphFeatures(self._graph, {
"motif3":
FeatureMeta(nth_nodes_motif(3, gpu=self._gpu, device=self._device),
{"m3"})
},
dir_path=self._dir_path)
raw_ftr.build(should_dump=True)
feature = raw_ftr['motif3']._features
if type(feature) == dict:
motif_matrix = self._to_matrix(feature)
else:
motif_matrix = feature
normed_matrix = self._log_norm(motif_matrix)
if self._motif_choice == "All_Motifs":
mp = MotifProbability(self._params['vertices'],
self._params['probability'],
self._params['clique_size'],
self._params['directed'])
clique_motifs = mp.get_3_clique_motifs(3)
return normed_matrix[:, clique_motifs]
else:
return normed_matrix
from vertices.bfs_moments import BfsMomentsCalculator
from vertices.closeness_centrality import ClosenessCentralityCalculator
from vertices.communicability_betweenness_centrality import CommunicabilityBetweennessCentralityCalculator
from vertices.eccentricity import EccentricityCalculator
from vertices.fiedler_vector import FiedlerVectorCalculator
from vertices.flow import FlowCalculator
from vertices.general import GeneralCalculator
from vertices.hierarchy_energy import HierarchyEnergyCalculator
from vertices.k_core import KCoreCalculator
from vertices.load_centrality import LoadCentralityCalculator
from vertices.louvain import LouvainCalculator
from vertices.motifs import nth_nodes_motif
ANOMALY_DETECTION_FEATURES = {
"attractor_basin":
FeatureMeta(AttractorBasinCalculator, {"ab"}), # directed only
"average_neighbor_degree":
FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}),
"betweenness_centrality":
FeatureMeta(BetweennessCentralityCalculator, {"betweenness"}),
"bfs_moments":
FeatureMeta(BfsMomentsCalculator, {"bfs"}),
"closeness_centrality":
FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
# "communicability_betweenness_centrality": FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
# {"communicability"}),
"eccentricity":
FeatureMeta(EccentricityCalculator, {"ecc"}),
"fiedler_vector":
FeatureMeta(FiedlerVectorCalculator, {"fv"}),
# "flow": FeatureMeta(FlowCalculator, {}),
from vertices.bfs_moments import BfsMomentsCalculator
from vertices.closeness_centrality import ClosenessCentralityCalculator
from vertices.communicability_betweenness_centrality import CommunicabilityBetweennessCentralityCalculator
from vertices.eccentricity import EccentricityCalculator
from vertices.fiedler_vector import FiedlerVectorCalculator
from vertices.general import GeneralCalculator
from vertices.hierarchy_energy import HierarchyEnergyCalculator
from vertices.k_core import KCoreCalculator
from vertices.load_centrality import LoadCentralityCalculator
from vertices.louvain import LouvainCalculator
from vertices.motifs import nth_nodes_motif
from vertices.page_rank import PageRankCalculator
ANOMALY_DETECTION_FEATURES = {
"attractor_basin":
FeatureMeta(AttractorBasinCalculator, {"ab"}), # directed only
"average_neighbor_degree":
FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}),
"betweenness_centrality":
FeatureMeta(BetweennessCentralityCalculator, {"betweenness"}),
# "bfs_moments": FeatureMeta(BfsMomentsCalculator, {"bfs"}),
# "closeness_centrality": FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
# "communicability_betweenness_centrality": FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
# {"communicability"}),
"eccentricity":
FeatureMeta(EccentricityCalculator, {"ecc"}),
"fiedler_vector":
FeatureMeta(FiedlerVectorCalculator, {"fv"}),
# "flow": FeatureMeta(FlowCalculator, {}),
"general":
FeatureMeta(GeneralCalculator, {"gen"}),
import os
from betweenness_centrality import BetweennessCentralityCalculator
from bfs_moments import BfsMomentsCalculator
from feature_calculators import FeatureMeta
CODE_DIR = "code"
DATA_INPUT_DIR = "dataset_input"
PKL_DIR = "pkl"
FEATURES_PKL_DIR = os.path.join(PKL_DIR, "features")
NORM_REDUCED = "_REDUCED_"
NORM_REDUCED_SYMMETRIC = "_REDUCED_SYMMETRIC_"
DEG = "_DEGREE_"
IN_DEG = "_IN_DEGREE_"
OUT_DEG = "_OUT_DEGREE_"
CENTRALITY = ["betweenness_centrality", FeatureMeta(BetweennessCentralityCalculator, {"betweenness"})]
BFS = ["bfs_moments", FeatureMeta(BfsMomentsCalculator, {"bfs"})]
class FactorLoss:
def __init__(self):
self._begin_low_limit = 0 # 0 .. 1
self._end_low_limit = 0.3 # 0.5 .. 1
self._interval = 1e-4
self._curr_start = self._begin_low_limit
self._curr_epoch = 0
def factor_loss(self, output, target, jump=False):
# scale = 1 - self._curr_start
# shift = 1 - scale
# "average_neighbor_degree": FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}),
# "betweenness_centrality": FeatureMeta(BetweennessCentralityCalculator, {"betweenness"}),
# "bfs_moments": FeatureMeta(BfsMomentsCalculator, {"bfs"}),
# "closeness_centrality": FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
# "communicability_betweenness_centrality": FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
# {"communicability"}),
# "eccentricity": FeatureMeta(EccentricityCalculator, {"ecc"}),
# "fiedler_vector": FeatureMeta(FiedlerVectorCalculator, {"fv"}),
# "flow": FeatureMeta(FlowCalculator, {}),
# "general": FeatureMeta(GeneralCalculator, {"gen"}),
# Isn't OK - also in previous version
# "hierarchy_energy": FeatureMeta(HierarchyEnergyCalculator, {"hierarchy"}),
# "k_core": FeatureMeta(KCoreCalculator, {"kc"}),
# "load_centrality": FeatureMeta(LoadCentralityCalculator, {"load_c"}),
# "louvain": FeatureMeta(LouvainCalculator, {"lov"}),
"motif3": FeatureMeta(nth_nodes_motif(3), {"m3"}),
# "multi_dimensional_scaling": FeatureMeta(MultiDimensionalScalingCalculator, {"mds"}),
# "page_rank": FeatureMeta(PageRankCalculator, {"pr"}),
"motif4": FeatureMeta(nth_nodes_motif(4), {"m4"}),
# "first_neighbor_histogram": FeatureMeta(nth_neighbor_calculator(1), {"fnh", "first_neighbor"}),
# "second_neighbor_histogram": FeatureMeta(nth_neighbor_calculator(2), {"snh", "second_neighbor"}),
}
SOURCE = 'SourceID'
DEST = 'DestinationID'
DURATION = 'Duration'
TIME = 'StartTime'
COMMUNITY = 'Community'
TARGET = 'target'
ALL_BETA_PATH = "all_times_beta"
For more information, please read the README in the git.
"""
CODE_DIR = "code"
DATA_INPUT_DIR = "dataset_input"
PKL_DIR = "pkl"
FEATURES_PKL_DIR = os.path.join(PKL_DIR, "features")
NORM_REDUCED = "_REDUCED_"
NORM_REDUCED_SYMMETRIC = "_REDUCED_SYMMETRIC_"
DEG = "_DEGREE_"
IN_DEG = "_IN_DEGREE_"
OUT_DEG = "_OUT_DEGREE_"
CENTRALITY = [
"betweenness_centrality",
FeatureMeta(BetweennessCentralityCalculator, {"betweenness"})
]
BFS = ["bfs_moments", FeatureMeta(BfsMomentsCalculator, {"bfs"})]
class FactorLoss:
def __init__(self):
self._begin_low_limit = 0 # 0 .. 1
self._end_low_limit = 0.3 # 0.5 .. 1
self._interval = 1e-4
self._curr_start = self._begin_low_limit
self._curr_epoch = 0
def factor_loss(self, output, target, jump=False):
# scale = 1 - self._curr_start
from vertices.closeness_centrality import ClosenessCentralityCalculator
from vertices.communicability_betweenness_centrality import CommunicabilityBetweennessCentralityCalculator
from vertices.eccentricity import EccentricityCalculator
from vertices.fiedler_vector import FiedlerVectorCalculator
from vertices.flow import FlowCalculator
from vertices.general import GeneralCalculator
from vertices.hierarchy_energy import HierarchyEnergyCalculator
from vertices.k_core import KCoreCalculator
from vertices.load_centrality import LoadCentralityCalculator
from vertices.louvain import LouvainCalculator
from vertices.motifs import nth_nodes_motif
from vertices.multi_dimensional_scaling import MultiDimensionalScalingCalculator
from vertices.page_rank import PageRankCalculator
ANOMALY_DETECTION_FEATURES = {
"attractor_basin": FeatureMeta(AttractorBasinCalculator, {"ab"}), # directed only
"average_neighbor_degree": FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}),
"betweenness_centrality": FeatureMeta(BetweennessCentralityCalculator, {"betweenness"}),
"bfs_moments": FeatureMeta(BfsMomentsCalculator, {"bfs"}),
"closeness_centrality": FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
# "communicability_betweenness_centrality": FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
# {"communicability"}),
"eccentricity": FeatureMeta(EccentricityCalculator, {"ecc"}),
"fiedler_vector": FeatureMeta(FiedlerVectorCalculator, {"fv"}),
# "flow": FeatureMeta(FlowCalculator, {}),
"general": FeatureMeta(GeneralCalculator, {"gen"}),
# Isn't OK - also in previous version
# "hierarchy_energy": FeatureMeta(HierarchyEnergyCalculator, {"hierarchy"}),
"k_core": FeatureMeta(KCoreCalculator, {"kc"}),
"load_centrality": FeatureMeta(LoadCentralityCalculator, {"load_c"}),
"louvain": FeatureMeta(LouvainCalculator, {"lov"}),
def __init__(self, gpu=False, device=0):
self.NODE_LEVEL = {
"attractor_basin":
FeatureMeta(AttractorBasinCalculator, {"ab"}), # Directed
"average_neighbor_degree":
FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}), # Any
"betweenness_centrality":
FeatureMeta(BetweennessCentralityCalculator,
{"betweenness"}), # Any
"bfs_moments":
FeatureMeta(BfsMomentsCalculator, {"bfs"}), # Any
"closeness_centrality":
FeatureMeta(ClosenessCentralityCalculator, {"closeness"}), # Any
"communicability_betweenness_centrality":
FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
{"communicability"}), # Undirected
"eccentricity":
FeatureMeta(EccentricityCalculator, {"ecc"}), # Any
"fiedler_vector":
FeatureMeta(FiedlerVectorCalculator,
{"fv"}), # Undirected (due to a code limitation)
"flow":
FeatureMeta(FlowCalculator, {}), # Directed
# General - calculating degrees. Directed will get (in_deg, out_deg) and undirected will get degree only per vertex.
"general":
FeatureMeta(GeneralCalculator, {"gen"}), # Any
"hierarchy_energy":
FeatureMeta(HierarchyEnergyCalculator,
{"hierarchy"}), # Directed (but works for any)
"k_core":
FeatureMeta(KCoreCalculator, {"kc"}), # Any
"load_centrality":
FeatureMeta(LoadCentralityCalculator, {"load_c"}), # Any
"louvain":
FeatureMeta(LouvainCalculator, {"lov"}), # Undirected
"motif3":
FeatureMeta(nth_nodes_motif(3, gpu, device), {"m3"}), # Any
"page_rank":
FeatureMeta(PageRankCalculator,
{"pr"}), # Directed (but works for any)
"motif4":
FeatureMeta(nth_nodes_motif(4, gpu, device), {"m4"}), # Any
}
self.MOTIFS = {
"motif3": FeatureMeta(nth_nodes_motif(3, gpu, device), {"m3"}),
"motif4": FeatureMeta(nth_nodes_motif(4, gpu, device), {"m4"})
}
"""
def __init__(self, gpu=False):
self.NODE_LEVEL = {
"attractor_basin":
FeatureMeta(AttractorBasinCalculator, {"ab"}),
"average_neighbor_degree":
FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}),
"betweenness_centrality":
FeatureMeta(BetweennessCentralityCalculator, {"betweenness"}),
"bfs_moments":
FeatureMeta(BfsMomentsCalculator, {"bfs"}),
"closeness_centrality":
FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
"communicability_betweenness_centrality":
FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
{"communicability"}),
"eccentricity":
FeatureMeta(EccentricityCalculator, {"ecc"}),
"fiedler_vector":
FeatureMeta(FiedlerVectorCalculator, {"fv"}),
"flow":
FeatureMeta(FlowCalculator, {}),
"general":
FeatureMeta(GeneralCalculator, {"gen"}),
"hierarchy_energy":
FeatureMeta(HierarchyEnergyCalculator, {"hierarchy"}),
"k_core":
FeatureMeta(KCoreCalculator, {"kc"}),
"load_centrality":
FeatureMeta(LoadCentralityCalculator, {"load_c"}),
"louvain":
FeatureMeta(LouvainCalculator, {"lov"}),
"motif3":
FeatureMeta(nth_nodes_motif(3, gpu), {"m3"}),
"page_rank":
FeatureMeta(PageRankCalculator, {"pr"}),
"motif4":
FeatureMeta(nth_nodes_motif(4, gpu), {"m4"}),
}
self.MOTIFS = {
"motif3": FeatureMeta(nth_nodes_motif(3, gpu), {"m3"}),
"motif4": FeatureMeta(nth_nodes_motif(4, gpu), {"m4"})
}
from vertices.closeness_centrality import ClosenessCentralityCalculator
from vertices.communicability_betweenness_centrality import CommunicabilityBetweennessCentralityCalculator
from vertices.eccentricity import EccentricityCalculator
from vertices.fiedler_vector import FiedlerVectorCalculator
from vertices.flow import FlowCalculator
from vertices.general import GeneralCalculator
from vertices.k_core import KCoreCalculator
from vertices.load_centrality import LoadCentralityCalculator
from vertices.louvain import LouvainCalculator
from vertices.motifs import nth_nodes_motif
from vertices.multi_dimensional_scaling import MultiDimensionalScalingCalculator
from vertices.page_rank import PageRankCalculator
ANOMALY_DETECTION_FEATURES = {
"attractor_basin":
FeatureMeta(AttractorBasinCalculator, {"ab"}),
"average_neighbor_degree":
FeatureMeta(AverageNeighborDegreeCalculator, {"avg_nd"}),
"betweenness_centrality":
FeatureMeta(BetweennessCentralityCalculator, {"betweenness"}),
"bfs_moments":
FeatureMeta(BfsMomentsCalculator, {"bfs"}),
"closeness_centrality":
FeatureMeta(ClosenessCentralityCalculator, {"closeness"}),
# "communicability_betweenness_centrality": FeatureMeta(CommunicabilityBetweennessCentralityCalculator,
# {"communicability"}),
"eccentricity":
FeatureMeta(EccentricityCalculator, {"ecc"}),
"fiedler_vector":
FeatureMeta(FiedlerVectorCalculator, {"fv"}),
"flow":