def plot_all_roc(root_path, general_models, special_models, anomaly_name):
mark_style = ['o', '^', 's', 'v']
colors = ['purple', 'red', 'blue', 'green']
ModelClass_dic = get_classes()
f = plt.figure()
for i,model_name in enumerate(special_models):
model = ModelClass_dic[model_name](model_root=root_path)
score_name = list(model.get_score_methods().keys())[0]
d = load_object(os.path.join(model.model_path,'eval', 'ROC', anomaly_name, 'roc_score_' + score_name + '.pkl'))
plt.plot(d['dBs'], d['aucs'], color=colors[-1], marker=mark_style[i], label=model_name+', score method - '+score_name)
for i,model_name in enumerate(general_models):
model = ModelClass_dic[model_name](model_root=root_path)
score_methods_dic = model.get_score_methods()
for j,score_name in enumerate(score_methods_dic.keys()):
d = load_object(os.path.join(model.model_path,'eval', 'ROC', anomaly_name, 'roc_score_' + score_name + '.pkl'))
plt.plot(d['dBs'], d['aucs'], color=colors[i], marker=mark_style[j], label=model_name+', score method - '+score_name)
plt.ylim([0, 1])
plt.xlabel('ISR in dB of anomaly', fontsize=18)
plt.ylabel('AUC score', fontsize=18)
plt.title(anomaly_name, fontsize=20)
plt.legend()
plt.gca().grid(True)
f.set_size_inches(8, 6.5, forward=True)
fig_path = os.path.join(root_path, '0_all_ROC', 'dB_vs_AUC_'+anomaly_name)
save_fig(f, fig_path)
plt.close()
def load_model(self):
max_path = os.path.join(self.model_path, "cepstrum_max.pkl")
scaler_path = os.path.join(self.model_path, "train_scaler.pkl")
if use_scaling:
self.scaler = load_object(scaler_path)
self.cepstrum_max = load_object(max_path)
self.loaded = True
def load_model(self):
scaler_path = os.path.join(self.model_path, "train_scaler.pkl")
params_path = os.path.join(self.model_path, "model_params.pkl")
if use_scaling:
self.scaler = load_object(scaler_path)
params_dic = load_object(params_path)
self.freqs = params_dic['freqs']
self.means = params_dic['means']
self.stds = params_dic['stds']
self.gaussians = params_dic['gaussians']
self.loaded = True
def load_model(self):
params_path = os.path.join(self.model_path, "model_params.pkl")
params_dic = load_object(params_path)
self.freqs = params_dic['freqs']
self.means = params_dic['means']
self.stds = params_dic['stds']
self.gaussians = params_dic['gaussians']
self.loaded = True
def load_model(self):
if use_whitening:
self.whiten_path = os.path.join(self.model_path, "zca_scaler.pkl")
self.scaler = load_object(
os.path.join(self.model_path, 'train_scaler.pkl'))
deep_model_input_shape = (block_shape[0], block_shape[1], 1)
self.build_model(deep_model_input_shape, opt, loss_fn)
self.load_weights()
self.loaded = True
def predict_by_ae(data_iq, sample_rate, model_weights_dir):
gpus = conf['gpus']
train_params = conf['learning']['ae']
batch_size = conf['learning']['ae']['batch_size']
rbw_set = conf['preprocessing']['ae']['rbw_set']
found_anomaly_per_rbw = []
for rbw in rbw_set:
print('loading data and geting spectrogram...')
freqs, time, data_spectro = compute_fft_test_data(
data_iq, sample_rate, rbw, model_weights_dir)
print('spliting to block and predicting AutoEncoders errors...')
block_shape = load_object(
os.path.join(model_weights_dir, 'block_shape.pkl'))
block_indices, data_blocks = reshape_to_blocks(data_spectro,
block_shape)
conv_model = AeDeepModel(train_params, model_weights_dir, gpus)
conv_model.build_model(data_blocks.shape[1:])
conv_model.load_weights()
data_ae_errors = predict_ae_error_vectors(data_blocks, data_blocks,
conv_model, batch_size)
data_ae_errors = np.reshape(
data_ae_errors, (block_indices.shape[0], block_indices.shape[1]))
print('declaring anomaly block...')
error_median, error_std = load_val_stat(model_weights_dir)
anomalies_indices = detect_reconstruction_anomalies_median(
data_ae_errors, error_median, error_std)
print('voting between blocks...')
has_anomaly = voting_anomalies(anomalies_indices, block_indices, time,
freqs)
found_anomaly_per_rbw.append(has_anomaly)
print('finished!')
return any(found_anomaly_per_rbw)
def save_roc_plot(iq_normal,
sample_rate,
dBs,
num_ROC_samples=500,
score_method=None,
score_name='normal'):
fprs, tprs, aucs = [], [], []
normal_plot_saved = 0
for ind, dB in enumerate(dBs):
gc.collect()
print('creating anomaly with {}dB...'.format(dB))
sweep_params['dB'] = dB
basic_len = get_basic_block_len(sample_rate)
roc_start_indices_path = os.path.join(model_root,
'roc_start_indices.pkl')
num_samples = num_ROC_samples
if os.path.isfile(roc_start_indices_path):
a_starts, c_starts = load_object(roc_start_indices_path)
if len(a_starts) != num_samples:
print(
'wrong length of roc_start_indices.pkl...\nchanging to roc_start_indices.pkl len'
)
num_samples = len(a_starts)
else:
a_starts = np.random.randint(0, iq_normal.shape[0] - basic_len,
(num_samples, ))
c_starts = np.random.randint(0, iq_normal.shape[0] - basic_len,
(num_samples, ))
persist_object([a_starts, c_starts], roc_start_indices_path)
tot_starts = np.concatenate([a_starts, c_starts])
y_true = np.concatenate(
[np.ones((num_samples, )),
np.zeros((num_samples, ))]).astype(bool)
y_score = np.zeros((2 * num_samples, ))
for i in range(2 * num_samples):
if y_true[i]:
basic_iq = trim_iq_basic_block(iq_normal,
sample_rate,
start=tot_starts[i])
basic_iq = CW(basic_iq, sample_rate, anomal_freq, dB)
if i < 2:
model.plot_prediction(basic_iq, sample_rate)
f = plt.gcf()
f.suptitle('using model "' + model.name +
'" on sweep with ISR: ' + str(dB) + 'dB')
f.set_size_inches(12, 8, forward=True)
fig_path = os.path.join(
plots_path,
'{0:02d}_ISR_dB_{1}_sample_{2}'.format(ind, dB, i))
save_fig(f, fig_path)
plt.close()
else:
basic_iq = trim_iq_basic_block(iq_normal,
sample_rate,
start=tot_starts[i])
if normal_plot_saved < 2:
model.plot_prediction(basic_iq, sample_rate)
f = plt.gcf()
f.suptitle('using model "' + model.name +
'" on normal data')
f.set_size_inches(12, 8, forward=True)
fig_path = os.path.join(
plots_path,
'normal_sample_{0}'.format(normal_plot_saved))
save_fig(f, fig_path)
plt.close()
normal_plot_saved += 1
if score_method:
y_score[i] = score_method(basic_iq, sample_rate)
else:
y_score[i] = model.predict_basic_block_score(
basic_iq, sample_rate)
fpr, tpr, thresholds = roc_curve(y_true, y_score)
fprs.append(fpr)
tprs.append(tpr)
aucs.append(roc_auc_score(y_true, y_score))
# ploting
f = plt.figure(0)
for j in range(len(dBs) - 1, -1, -1):
plt.plot(fprs[j], tprs[j])
plt.legend([
'anomaly in {}dB, auc: {:.3f}'.format(dBs[j], aucs[j])
for j in range(len(dBs) - 1, -1, -1)
])
plt.xlabel('False Positive Rate', fontsize=18)
plt.ylabel('True Positive Rate', fontsize=18)
plt.title('ROC for anomalies with different ISR [dB]', fontsize=20)
plt.gca().grid(True)
f.set_size_inches(8, 6.5, forward=True)
fig_path = os.path.join(plots_path, 'All_ROCs_' + score_name)
save_fig(f, fig_path)
plt.close()
persist_object(
{
'dBs': dBs,
'aucs': aucs,
'name': model.name + '_score_' + score_name
}, os.path.join(plots_path, 'roc_score_' + score_name + '.pkl'))
f = plt.figure(1)
plt.plot(dBs, aucs, '-o')
plt.ylim([0, 1])
plt.xlabel('ISR in dB of anomaly', fontsize=18)
plt.ylabel('AUC score', fontsize=18)
plt.title(
'Sweep anomaly\nArea Under Curve as function of\nInterference Signal Ratio',
fontsize=20)
plt.gca().grid(True)
f.set_size_inches(8, 6.5, forward=True)
fig_path = os.path.join(plots_path, 'dB_vs_AUC_' + score_name)
save_fig(f, fig_path)
plt.close()
test_data = series_to_supervised(test_data, n_in=seq_input_length, n_out=seq_output_length)
# # Trim the data to fit the sequence length (SEQ_LENGTH)
test_data = trim_by_seq_length(test_data, seq_input_length)
(X_test, Y_test) = get_X_and_Y_columns(test_data)
X_test = reshape_to_seq(X_test, seq_input_length)
Y_test = reshape_to_seq(Y_test, seq_output_length)
# Pad input/output sequences
if use_padding:
X_test = pad_sequences(X_test, maxlen=seq_pad_length, dtype='float32', padding=input_padding)
inp_shape = X_test.shape
model_obj.load_weights()
test_errors = predict_rnn_error_vectors(X_test, Y_test, model_obj,batch_size)
scaled_train_errors = load_object(train_errors_path)
scaled_test_errors = scale_test_vectors(test_errors, error_scaler_path)
gmm = load_object(gmm_save_path)
test_scores = (gmm.score_samples(scaled_test_errors))
try:
train_scores = load_object(train_scores_path)
except:
raise Exception('No train scores are found, please train to obtain them')
# ## Anomaly detection phase - EMD
# Dataset-wise
# for now, just return the EMD between the train and test scores
emd_dists=compute_emd_distributions(train_scores,test_scores)