def compute_average_gradient_norm(model:Model, dataset_id, dataset_fold_id=None, batch_size=128,
cutoff=None, normalize_timeseries=False, learning_rate=1e-3):
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in ['pre', 'post'], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice, dataset_id, sequence_length)
y_train = to_categorical(y_train, len(np.unique(y_train)))
optm = Adam(lr=learning_rate)
model.compile(optimizer=optm, loss='categorical_crossentropy', metrics=['accuracy'])
average_gradient = _average_gradient_norm(model, X_train, y_train, batch_size)
print("Average gradient norm : ", average_gradient)
def evaluate_model(model: Model,
dataset_id,
dataset_prefix,
dataset_fold_id=None,
batch_size=128,
test_data_subset=None,
cutoff=None,
normalize_timeseries=False):
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_test)
if max_nb_variables != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, max_nb_variables)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
_, X_test = cutoff_sequence(None, X_test, choice, dataset_id,
max_nb_variables)
if not is_timeseries:
X_test = pad_sequences(X_test,
maxlen=MAX_NB_VARIABLES[dataset_id],
padding='post',
truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
if dataset_fold_id is None:
# weight_fn = "./weights/%s_weights.h5" % dataset_prefix
weight_fn = "./model_weights.h5"
else:
# weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix, dataset_fold_id)
weight_fn = "./model_weights.h5"
model.load_weights(weight_fn)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nEvaluating : ")
loss, accuracy = model.evaluate(X_test, y_test, batch_size=batch_size)
print()
print("Final Accuracy : ", accuracy)
return accuracy, loss
def train_model(model:Model, dataset_id, dataset_prefix, dataset_fold_id=None, epochs=50, batch_size=128, val_subset=None,
cutoff=None, normalize_timeseries=False, learning_rate=1e-3, monitor='val_acc', optimization_mode='max', compile_model=True):
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_train)
if max_nb_variables != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, max_nb_variables)
else:
assert cutoff in ['pre', 'post'], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice, dataset_id, max_nb_variables)
classes = np.unique(y_train)
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
print("Class weights : ", class_weight)
y_train = to_categorical(y_train, len(np.unique(y_train)))
y_test = to_categorical(y_test, len(np.unique(y_test)))
if is_timeseries:
factor = 1. / np.cbrt(2)
else:
factor = 1. / np.sqrt(2)
if dataset_fold_id is None:
weight_fn = "./weights/%s_weights.h5" % dataset_prefix
else:
weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix, dataset_fold_id)
model_checkpoint = ModelCheckpoint(weight_fn, verbose=1, mode=optimization_mode,
monitor=monitor, save_best_only=True, save_weights_only=True)
reduce_lr = ReduceLROnPlateau(monitor=monitor, patience=100, mode=optimization_mode,
factor=factor, cooldown=0, min_lr=1e-4, verbose=2)
callback_list = [model_checkpoint, reduce_lr]
optm = Adam(lr=learning_rate)
if compile_model:
model.compile(optimizer=optm, loss='categorical_crossentropy', metrics=['accuracy'])
if val_subset is not None:
X_test = X_test[:val_subset]
y_test = y_test[:val_subset]
model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, callbacks=callback_list,
class_weight=class_weight, verbose=2, validation_data=(X_test, y_test))
def evaluate_model(model: Model,
dataset_id,
dataset_prefix,
batch_size=128,
test_data_subset=None,
cutoff=None,
normalize_timeseries=False):
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
max_nb_words, sequence_length = calculate_dataset_metrics(X_test)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
_, X_test = cutoff_sequence(None, X_test, choice, dataset_id,
sequence_length)
if not is_timeseries:
#X_test is the testing data, y_test is the labels
X_test = pad_sequences(X_test,
maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id],
padding='post',
truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy', f1_score]) #f1_score,
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nEvaluating : ")
loss, accuracy, f = model.evaluate(X_test, y_test, batch_size=batch_size)
print()
print("Final Accuracy : ", accuracy)
print("Final Error: ", (1 - accuracy))
print("loss = ", loss)
print("f_score = ", f)
# print("specificity", r)
# print("precision_value = ",p)
# print("recall_value = ",r)
return accuracy, loss, f #,precision_val, recall_val
def hyperparameter_search_over_model(model_gen,
dataset_id,
param_grid,
cutoff=None,
normalize_timeseries=False):
X_train, y_train, _, _, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
max_nb_words, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, _ = cutoff_sequence(X_train, None, choice, dataset_id,
sequence_length)
if not is_timeseries:
print(
"Model hyper parameters can only be searched for time series models"
)
return
classes = np.unique(y_train)
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
y_train = to_categorical(y_train, len(np.unique(y_train)))
clf = KerasClassifier(build_fn=model_gen,
epochs=50,
class_weight=class_weight,
verbose=0)
grid = GridSearchCV(clf, param_grid=param_grid, n_jobs=1, verbose=10, cv=3)
result = grid.fit(X_train, y_train)
print("Best: %f using %s" % (result.best_score_, result.best_params_))
means = result.cv_results_['mean_test_score']
stds = result.cv_results_['std_test_score']
params = result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
def evaluate_model(model, dataset_id, dataset_prefix, batch_size=128, test_data_subset=None,
cutoff=None, normalize_timeseries=False):
_, _, X_test, y_test, is_timeseries = load_dataset_at(dataset_id,
normalize_timeseries=normalize_timeseries)
max_nb_words, sequence_length = calculate_dataset_metrics(X_test)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in ['pre', 'post'], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
_, X_test = cutoff_sequence(None, X_test, choice, dataset_id, sequence_length)
if not is_timeseries:
X_test = pad_sequences(X_test, maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id], padding='post', truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm, loss='categorical_crossentropy', metrics=['accuracy'])
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nEvaluating : ")
loss, accuracy = model.evaluate(X_test, y_test, batch_size=batch_size)
from evaluate_metrics import plot_confusion_matrix
y_pred = model.predict(X_test, batch_size = batch_size)
y_pred = (np.argmax(y_pred, axis=1)).tolist()
y_test = (np.argmax(y_test, axis=1)).tolist()
plot_confusion_matrix.run(y_test, y_pred, ['zero', 'two', 'three', 'four'])
print()
print("Final Accuracy : ", accuracy)
return accuracy
def visualize_cam(model: Model,
dataset_id,
dataset_prefix,
class_id,
cutoff=None,
normalize_timeseries=False):
X_train, y_train, _, _, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, _ = cutoff_sequence(X_train, _, choice, dataset_id,
sequence_length)
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
class_weights = model.layers[-1].get_weights()[0]
conv_layers = [
layer for layer in model.layers if layer.__class__.__name__ == 'Conv1D'
]
final_conv = conv_layers[-1].name
final_softmax = model.layers[-1].name
out_names = [final_conv, final_softmax]
if class_id > 0:
class_id = class_id - 1
y_train_ids = np.where(y_train[:, 0] == class_id)
sequence_input = X_train[y_train_ids[0], ...]
choice = np.random.choice(range(len(sequence_input)), 1)
sequence_input = sequence_input[choice, :, :]
eval_functions = build_function(model, out_names)
conv_out, predictions = get_outputs(model, sequence_input, eval_functions)
conv_out = conv_out[0, :, :]
conv_out = conv_out.transpose((1, 0))
conv_channels = conv_out.shape[0]
conv_cam = np.zeros(sequence_length, dtype=np.float32)
for i, w in enumerate(class_weights[conv_channels:, class_id]):
conv_cam += w * conv_out[i, :]
conv_cam /= conv_cam.max()
sequence_input = sequence_input.reshape((-1, 1))
conv_cam = conv_cam.reshape((-1, 1))
sequence_df = pd.DataFrame(sequence_input,
index=range(sequence_input.shape[0]),
columns=range(sequence_input.shape[1]))
conv_cam_df = pd.DataFrame(conv_cam,
index=range(conv_cam.shape[0]),
columns=[1])
fig, axs = plt.subplots(2, 1, squeeze=False, figsize=(6, 6))
class_label = class_id + 1
sequence_df.plot(title='Sequence (class = %d)' % (class_label),
subplots=False,
legend=None,
ax=axs[0][0])
conv_cam_df.plot(title='Convolution Class Activation Map (class = %d)' %
(class_label),
subplots=False,
legend=None,
ax=axs[1][0])
plt.show()
def visualize_context_vector(model: Model,
dataset_id,
dataset_prefix,
cutoff=None,
limit=None,
normalize_timeseries=False,
visualize_sequence=True,
visualize_classwise=False):
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, sequence_length)
attn_lstm_layer = [(i, layer) for (i, layer) in enumerate(model.layers)
if layer.__class__.__name__ == 'AttentionLSTM']
if len(attn_lstm_layer) == 0:
raise AttributeError('Provided model does not have an Attention layer')
else:
i, attn_lstm_layer = attn_lstm_layer[
0] # use first attention lstm layer only
attn_lstm_layer.return_attention = True
model.layers[i] = attn_lstm_layer
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
attention_output = model.layers[i].call(model.input)
eval_functions = build_function(model,
attn_lstm_layer.name,
outputs=[attention_output])
attention_vectors = []
for i in range(X_train.shape[0]):
activations = get_outputs(model,
X_train[i, :, :][np.newaxis, ...],
eval_functions,
verbose=False)[0]
attention_vector = np.sum(activations, axis=1).squeeze()
attention_vectors.append(attention_vector)
attention_vectors = np.array(attention_vectors)
attention_vector_final = np.mean(attention_vectors, axis=0)
if visualize_sequence:
# plot input sequence part that is paid attention too in detail
attention_vector_final = attention_vector_final.reshape(
(1, attention_vector_final.shape[0]))
X_train_attention = np.zeros_like(X_train)
X_test_attention = np.zeros_like(X_test)
for i in range(X_train.shape[0]):
X_train_attention[
i, :, :] = attention_vector_final * X_train[i, :, :]
for i in range(X_test.shape[0]):
X_test_attention[
i, :, :] = attention_vector_final * X_test[i, :, :]
plot_dataset(dataset_id,
seed=1,
limit=limit,
cutoff=cutoff,
normalize_timeseries=normalize_timeseries,
plot_data=(X_train, y_train, X_test, y_test,
X_train_attention, X_test_attention),
type='Context',
plot_classwise=visualize_classwise)
else:
# plot only attention chart
train_df = pd.DataFrame({'attention (%)': attention_vector_final},
index=range(attention_vector_final.shape[0]))
train_df.plot(kind='bar',
title='Attention Mechanism (Train) as '
'a function of input'
' dimensions.')
plt.show()
def write_cam(model: Model,
dataset_id,
dataset_prefix,
cutoff=None,
normalize_timeseries=False):
""" Same as visualize_cam, but writes the result data to a file. """
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_test)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, _ = cutoff_sequence(_, X_test, choice, dataset_id,
sequence_length)
print("Weights path : ", "./weights/%s_weights.h5" % dataset_prefix)
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
class_weights = model.layers[-1].get_weights()[0]
conv_layers = [
layer for layer in model.layers if layer.__class__.__name__ == 'Conv1D'
]
final_conv = conv_layers[-1].name
final_softmax = model.layers[-1].name
out_names = [final_conv, final_softmax]
eval_functions = build_function(model, out_names)
parts = os.path.split(dataset_prefix)
if len(parts) > 1:
basepath = os.path.join('cam_features', os.pathsep.join(parts[:-1]))
dataset_name = parts[-1]
else:
basepath = 'cam_features/'
dataset_name = dataset_prefix
if not os.path.exists(basepath):
os.makedirs(basepath)
cam_features = []
for i in range(X_test.shape[0]):
print("Sample %d running" % (i + 1))
y_id = y_test[i, 0]
sequence_input = X_test[[i], :, :]
conv_out, predictions = get_outputs(model, sequence_input,
eval_functions)
conv_out = conv_out[0, :, :] # (T, C)
conv_out = conv_out.transpose((1, 0)) # (C, T)
conv_channels = conv_out.shape[0]
conv_cam = class_weights[:conv_channels, [int(y_id)]] * conv_out
conv_cam = np.mean(conv_cam, axis=0)
conv_cam = conv_cam.reshape((-1, 1))
cam_features.append(conv_cam)
cam_features = np.concatenate(cam_features, -1).transpose()
print("Num features = ", cam_features.shape)
conv_cam_df = pd.DataFrame(cam_features)
conv_cam_df.to_csv(basepath +
'/%s_features_mean_unnormalized.csv' % dataset_name,
header=False,
index=False)
def write_context_vector(model: Model,
dataset_id,
dataset_prefix,
cutoff=None,
limit=None,
normalize_timeseries=False,
visualize_sequence=True,
visualize_classwise=False):
""" Same as visualize_context_vector, but writes the context vectors to a file. Unused. """
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, sequence_length)
attn_lstm_layer = [(i, layer) for (i, layer) in enumerate(model.layers)
if layer.__class__.__name__ == 'AttentionLSTM']
if len(attn_lstm_layer) == 0:
raise AttributeError('Provided model does not have an Attention layer')
else:
i, attn_lstm_layer = attn_lstm_layer[
0] # use first attention lstm layer only
attn_lstm_layer.return_attention = True
model.layers[i] = attn_lstm_layer
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
attention_output = model.layers[i].call(model.input)
eval_functions = build_function(model,
attn_lstm_layer.name,
outputs=[attention_output])
train_attention_vectors = []
test_attention_vectors = []
if not os.path.exists('lstm_features/'):
os.makedirs('lstm_features/')
output_shape = [X_train.shape[-1], 1, 1]
for i in range(X_train.shape[0]):
activations = get_outputs(model,
X_train[i, :, :][np.newaxis, ...],
eval_functions,
verbose=False)[0]
# print("activations", activations.shape)
attention_vector = activations.reshape((-1, 1, 1))
attention_vector = (attention_vector - attention_vector.min()) / (
attention_vector.max() - attention_vector.min())
attention_vector = (attention_vector * 2.) - 1.
attention_vector = resize(attention_vector,
output_shape,
mode='reflect',
anti_aliasing=True)
attention_vector = attention_vector.reshape([1, -1])
train_attention_vectors.append(attention_vector)
for i in range(X_test.shape[0]):
activations = get_outputs(model,
X_test[i, :, :][np.newaxis, ...],
eval_functions,
verbose=False)[0]
# print("activations", activations.shape)
attention_vector = activations.reshape((-1, 1, 1))
attention_vector = (attention_vector - attention_vector.min()) / (
attention_vector.max() - attention_vector.min())
attention_vector = (attention_vector * 2.) - 1.
attention_vector = resize(attention_vector,
output_shape,
mode='reflect',
anti_aliasing=True)
attention_vector = attention_vector.reshape([1, -1])
test_attention_vectors.append(attention_vector)
train_attention_vectors = np.array(train_attention_vectors)
test_attention_vectors = np.array(test_attention_vectors)
print("Train Attention Vectors Shape :", train_attention_vectors.shape)
print("Test Attentin Vectors Shape :", test_attention_vectors.shape)
if visualize_sequence:
# plot input sequence part that is paid attention too in detail
X_train_attention = train_attention_vectors * X_train
X_test_attention = test_attention_vectors * X_test
X_train_attention = X_train_attention.squeeze(1)
X_test_attention = X_test_attention.squeeze(1)
df = pd.DataFrame(X_test_attention)
df['label'] = y_test[:, 0]
df.to_csv('lstm_features/features.csv')
else:
# plot only attention chart
choice = np.random.randint(0, train_attention_vectors.shape[0])
train_df = pd.DataFrame(
{'attention (%)': train_attention_vectors[choice, 0]},
index=range(train_attention_vectors.shape[-1]))
train_df.plot(kind='bar',
title='Attention Mechanism (Train) as '
'a function of input'
' dimensions. Class = %d' % (y_train[choice]))
plt.show()
def extract_features(model: Model,
dataset_id,
dataset_prefix,
layer_name,
cutoff=None,
normalize_timeseries=False):
""" Same as visualize_features, but saves them to a file instead. """
layer_name = layer_name.lower()
assert layer_name in ['cnn', 'lstm', 'lstmfcn']
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, sequence_length)
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
conv_layers = [
layer for layer in model.layers if layer.__class__.__name__ == 'Conv1D'
]
lstm_layers = [
layer for layer in model.layers if layer.__class__.__name__ == 'LSTM'
or layer.__class__.__name__ == 'AttentionLSTM'
]
lstmfcn_layer = model.layers[-2]
if layer_name == 'cnn':
feature_layer = conv_layers[-1]
elif layer_name == 'lstm':
feature_layer = lstm_layers[-1]
else:
feature_layer = lstmfcn_layer
dataset_name = os.path.split(dataset_prefix)[-1]
if dataset_name is None or len(dataset_name) == 0:
dataset_name = dataset_prefix
save_dir = 'layer_features/%s/' % dataset_name
if not os.path.exists(save_dir):
os.makedirs(save_dir)
extraction_model = Model(model.input, feature_layer.output)
train_features = extraction_model.predict(X_train, batch_size=128)
test_features = extraction_model.predict(X_test, batch_size=128)
shape = train_features.shape
if len(shape) > 2:
train_features = train_features.reshape(
(shape[0], shape[1] * shape[2]))
shape = test_features.shape
if len(shape) > 2:
test_features = test_features.reshape((shape[0], shape[1] * shape[2]))
print("Train feature shape : ", train_features.shape, "Classes : ",
len(np.unique(y_train)))
print("Test features shape : ", test_features.shape, "Classes : ",
len(np.unique(y_test)))
np.save(save_dir + '%s_%s_train_features.npy' % (layer_name, dataset_name),
train_features)
np.save(save_dir + '%s_%s_train_labels.npy' % (layer_name, dataset_name),
y_train)
np.save(save_dir + '%s_%s_test_features.npy' % (layer_name, dataset_name),
test_features)
np.save(save_dir + '%s_%s_test_labels.npy' % (layer_name, dataset_name),
y_test)
print("Saved train feature vectors at %s" %
(save_dir + '%s_%s_train_features.npy' % (layer_name, dataset_name)))
print("Saved test feature vectors at %s" %
(save_dir + '%s_%s_test_features.npy' % (layer_name, dataset_name)))
print()
def train_model(model: Model,
dataset_id,
dataset_prefix,
dataset_fold_id=None,
epochs=50,
batch_size=128,
val_subset=None,
cutoff=None,
normalize_timeseries=False,
learning_rate=1e-3,
monitor='val_loss',
optimization_mode='auto',
compile_model=True):
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_train)
print("x_train shape", np.shape(X_train))
print("x_train type", type(X_train))
print("y_train shape", np.shape(y_train))
print("y_train type", type(y_train))
if max_nb_variables != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, max_nb_variables)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, max_nb_variables)
classes = np.unique(y_train)
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
print("Class weights : ", class_weight)
y_train = to_categorical(y_train, len(np.unique(y_train)))
y_test = to_categorical(y_test, len(np.unique(y_test)))
if is_timeseries:
factor = 1. / np.cbrt(2)
else:
factor = 1. / np.sqrt(2)
if dataset_fold_id is None:
weight_fn = "./weights/%s_weights.h5" % dataset_prefix
else:
weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix,
dataset_fold_id)
model_checkpoint = ModelCheckpoint(weight_fn,
verbose=1,
mode=optimization_mode,
monitor=monitor,
save_best_only=True,
save_weights_only=True)
reduce_lr = ReduceLROnPlateau(monitor=monitor,
patience=100,
mode=optimization_mode,
factor=factor,
cooldown=0,
min_lr=1e-4,
verbose=2)
early_stopping = EarlyStopping(monitor=monitor,
min_delta=1e-3,
patience=25,
verbose=1,
mode='auto',
baseline=None,
restore_best_weights=False)
callback_list = [model_checkpoint, reduce_lr, early_stopping]
# callback_list = [model_checkpoint, reduce_lr]
optm = Adam(lr=learning_rate)
if compile_model:
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
if val_subset is not None:
X_test = X_test[:val_subset]
y_test = y_test[:val_subset]
history = model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=callback_list,
class_weight=class_weight,
verbose=2,
validation_data=(X_test, y_test))
training_acc_list = list(history.history['acc'])
val_acc_list = list(history.history['val_acc'])
training_loss_list = list(history.history['loss'])
val_loss_list = list(history.history['val_loss'])
length = len(training_acc_list)
data = np.concatenate((np.array(training_acc_list).reshape(
(length, 1)), np.array(val_acc_list).reshape(
(length, 1)), np.array(training_loss_list).reshape(
(length, 1)), np.array(val_loss_list).reshape((length, 1))),
axis=1)
df = pandas.DataFrame(data,
columns=[
'training_accuracy', 'validation_accuracy',
'training_loss', 'validation_loss'
])
df.to_excel("training_record.xlsx", index=False)
def visualize_cam(model: Model,
dataset_id,
dataset_prefix,
class_id,
cutoff=None,
normalize_timeseries=False,
seed=0):
"""
Used to visualize the Class Activation Maps of the Keras Model.
Args:
model: A Keras Model.
dataset_id: Integer id representing the dataset index containd in
`utils/constants.py`.
dataset_prefix: Name of the dataset. Used for weight saving.
class_id: Index of the class whose activation is to be visualized.
cutoff: Optional integer which slices of the first `cutoff` timesteps
from the input signal.
normalize_timeseries: Bool / Integer. Determines whether to normalize
the timeseries.
If False, does not normalize the time series.
If True / int not equal to 2, performs standard sample-wise
z-normalization.
If 2: Performs full dataset z-normalization.
seed: Random seed number for Numpy.
"""
np.random.seed(seed)
X_train, y_train, _, _, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, _ = cutoff_sequence(X_train, _, choice, dataset_id,
sequence_length)
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
class_weights = model.layers[-1].get_weights()[0]
conv_layers = [
layer for layer in model.layers if layer.__class__.__name__ == 'Conv1D'
]
final_conv = conv_layers[-1].name
final_softmax = model.layers[-1].name
out_names = [final_conv, final_softmax]
if class_id > 0:
class_id = class_id - 1
y_train_ids = np.where(y_train[:, 0] == class_id)
sequence_input = X_train[y_train_ids[0], ...]
choice = np.random.choice(range(len(sequence_input)), 1)
sequence_input = sequence_input[choice, :, :]
eval_functions = build_function(model, out_names)
conv_out, predictions = get_outputs(model, sequence_input, eval_functions)
conv_out = conv_out[0, :, :] # (T, C)
conv_out = (conv_out - conv_out.min(axis=0, keepdims=True)) / \
(conv_out.max(axis=0, keepdims=True) - conv_out.min(axis=0, keepdims=True))
conv_out = (conv_out * 2.) - 1.
conv_out = conv_out.transpose((1, 0)) # (C, T)
conv_channels = conv_out.shape[0]
conv_cam = class_weights[:conv_channels, [class_id]] * conv_out
conv_cam = np.sum(conv_cam, axis=0)
conv_cam /= conv_cam.max()
sequence_input = sequence_input.reshape((-1, 1))
conv_cam = conv_cam.reshape((-1, 1))
sequence_df = pd.DataFrame(sequence_input,
index=range(sequence_input.shape[0]),
columns=range(sequence_input.shape[1]))
conv_cam_df = pd.DataFrame(conv_cam,
index=range(conv_cam.shape[0]),
columns=[1])
fig, axs = plt.subplots(2, 1, squeeze=False, figsize=(6, 6))
class_label = class_id + 1
sequence_df.plot(title='Sequence (class = %d)' % (class_label),
subplots=False,
legend=None,
ax=axs[0][0])
conv_cam_df.plot(title='Convolution Class Activation Map (class = %d)' %
(class_label),
subplots=False,
legend=None,
ax=axs[1][0])
plt.show()
def train_model(model: Model,
dataset_id,
dataset_prefix,
epochs=50,
batch_size=128,
val_subset=None,
cutoff=None,
normalize_timeseries=False,
learning_rate=1e-3):
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
max_nb_words, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, sequence_length)
if not is_timeseries:
X_train = pad_sequences(X_train,
maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id],
padding='post',
truncating='post')
X_test = pad_sequences(X_test,
maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id],
padding='post',
truncating='post')
classes = np.unique(y_train)
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
print("Class weights : ", class_weight)
y_train = to_categorical(y_train, len(np.unique(y_train)))
y_test = to_categorical(y_test, len(np.unique(y_test)))
if is_timeseries:
factor = 1. / np.cbrt(2)
else:
factor = 1. / np.sqrt(2)
model_checkpoint = ModelCheckpoint("./weights/%s_weights.h5" %
dataset_prefix,
verbose=1,
monitor='loss',
save_best_only=True,
save_weights_only=True)
reduce_lr = ReduceLROnPlateau(monitor='loss',
patience=100,
mode='auto',
factor=factor,
cooldown=0,
min_lr=1e-4,
verbose=2)
callback_list = [model_checkpoint, reduce_lr]
optm = Adam(lr=learning_rate)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
if val_subset is not None:
X_test = X_test[:val_subset]
y_test = y_test[:val_subset]
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=callback_list,
class_weight=class_weight,
verbose=2,
validation_data=(X_test, y_test))
def train_model(model: Model,
dataset_id,
dataset_prefix,
epochs=50,
batch_size=128,
val_subset=None,
cutoff=None,
normalize_timeseries=False,
learning_rate=1e-3):
"""
Trains a provided Model, given a dataset id.
Args:
model: A Keras Model.
dataset_id: Integer id representing the dataset index containd in
`utils/constants.py`.
dataset_prefix: Name of the dataset. Used for weight saving.
epochs: Number of epochs to train.
batch_size: Size of each batch for training.
val_subset: Optional integer id to subset the test set. To be used if
the test set evaluation time significantly surpasses training time
per epoch.
cutoff: Optional integer which slices of the first `cutoff` timesteps
from the input signal.
normalize_timeseries: Bool / Integer. Determines whether to normalize
the timeseries.
If False, does not normalize the time series.
If True / int not equal to 2, performs standard sample-wise
z-normalization.
If 2: Performs full dataset z-normalization.
learning_rate: Initial learning rate.
"""
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
max_nb_words, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, sequence_length)
if not is_timeseries:
X_train = pad_sequences(X_train,
maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id],
padding='post',
truncating='post')
X_test = pad_sequences(X_test,
maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id],
padding='post',
truncating='post')
classes = np.unique(y_train)
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
print("Class weights : ", class_weight)
y_train = to_categorical(y_train, len(np.unique(y_train)))
y_test = to_categorical(y_test, len(np.unique(y_test)))
if is_timeseries:
factor = 1. / np.cbrt(2)
else:
factor = 1. / np.sqrt(2)
path_splits = os.path.split(dataset_prefix)
if len(path_splits) > 1:
base_path = os.path.join('weights', *path_splits)
if not os.path.exists(base_path):
os.makedirs(base_path)
base_path = os.path.join(base_path, path_splits[-1])
else:
all_weights_path = os.path.join('weights', dataset_prefix)
if not os.path.exists(all_weights_path):
os.makedirs(all_weights_path)
model_checkpoint = ModelCheckpoint("./weights/%s_weights.h5" %
dataset_prefix,
verbose=1,
monitor='loss',
save_best_only=True,
save_weights_only=True)
reduce_lr = ReduceLROnPlateau(monitor='loss',
patience=100,
mode='auto',
factor=factor,
cooldown=0,
min_lr=1e-4,
verbose=2)
callback_list = [model_checkpoint, reduce_lr]
optm = Adam(lr=learning_rate)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
if val_subset is not None:
X_test = X_test[:val_subset]
y_test = y_test[:val_subset]
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=callback_list,
class_weight=class_weight,
verbose=2,
validation_data=(X_test, y_test))
def evaluate_model(model: Model,
dataset_id,
dataset_prefix,
batch_size=128,
test_data_subset=None,
cutoff=None,
normalize_timeseries=False):
"""
Evaluates a given Keras Model on the provided dataset.
Args:
model: A Keras Model.
dataset_id: Integer id representing the dataset index containd in
`utils/constants.py`.
dataset_prefix: Name of the dataset. Used for weight saving.
batch_size: Size of each batch for evaluation.
test_data_subset: Optional integer id to subset the test set. To be used if
the test set evaluation time is significantly.
cutoff: Optional integer which slices of the first `cutoff` timesteps
from the input signal.
normalize_timeseries: Bool / Integer. Determines whether to normalize
the timeseries.
If False, does not normalize the time series.
If True / int not equal to 2, performs standard sample-wise
z-normalization.
If 2: Performs full dataset z-normalization.
Returns:
The test set accuracy of the model.
"""
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
max_nb_words, sequence_length = calculate_dataset_metrics(X_test)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
_, X_test = cutoff_sequence(None, X_test, choice, dataset_id,
sequence_length)
if not is_timeseries:
X_test = pad_sequences(X_test,
maxlen=MAX_SEQUENCE_LENGTH_LIST[dataset_id],
padding='post',
truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
print("Weights loaded from ", "./weights/%s_weights.h5" % dataset_prefix)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nEvaluating : ")
loss, accuracy = model.evaluate(X_test, y_test, batch_size=batch_size)
print()
print("Final Accuracy : ", accuracy)
return accuracy
def evaluate_model(model,
dataset_id,
dataset_prefix,
dataset_fold_id=None,
batch_size=128,
test_data_subset=None,
cutoff=None,
normalize_timeseries=False):
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_test)
if max_nb_variables != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, max_nb_variables)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
_, X_test = cutoff_sequence(None, X_test, choice, dataset_id,
max_nb_variables)
if not is_timeseries:
X_test = pad_sequences(X_test,
maxlen=MAX_NB_VARIABLES[dataset_id],
padding='post',
truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy'])
if dataset_fold_id is None:
weight_fn = "./weights/%s_weights.h5" % dataset_prefix
else:
weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix,
dataset_fold_id)
model.load_weights(weight_fn)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nEvaluating : ")
loss, accuracy = model.evaluate(X_test, y_test, batch_size=batch_size)
'''
print "\nPredicting"
for i in range(X_test.shape[0]):
pred = model.predict(X_test[i][np.newaxis,...], batch_size=batch_size)
indices = np.argmax(pred)
print indices
'''
from evaluate_metrics import plot_confusion_matrix
from sklearn import metrics
y_pred = model.predict(X_test, batch_size=batch_size)
y_pred = (np.argmax(y_pred, axis=1)).tolist()
y_test = (np.argmax(y_test, axis=1)).tolist()
plot_confusion_matrix.run(y_test,
y_pred,
class_names=NB_CLASSES_NAMES[dataset_id])
cfmatrix = metrics.confusion_matrix(y_test, y_pred)
report = metrics.classification_report(y_test, y_pred)
averagef_mahi = np.mean(classifaction_report_csv(report).iloc[:, 1])
microf = metrics.f1_score(y_test, y_pred, average='micro')
averagef = metrics.f1_score(y_test, y_pred, average='macro')
weightedf_mahi = sum((classifaction_report_csv(report).iloc[:, 1]) *
(cfmatrix.sum(axis=1) / sum(cfmatrix.sum(axis=1))))
weightedf = metrics.f1_score(y_test, y_pred, average='weighted')
print()
print("Binary F-Score : ", microf)
print("Final F-Score : ", averagef)
print("Weighted F-Score : ", weightedf)
print("Mahis F-Score : ", averagef_mahi)
print("Mahis W F-Score : ", weightedf_mahi)
print()
print("Final Accuracy : ", accuracy)
return accuracy, loss
def train_model(model: Model,
dataset_id,
dataset_prefix,
dataset_fold_id=None,
epochs=50,
batch_size=128,
val_subset=None,
cutoff=None,
normalize_timeseries=False,
learning_rate=1e-3,
monitor='loss',
optimization_mode='auto',
compile_model=True):
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_train)
if max_nb_variables != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, max_nb_variables)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, max_nb_variables)
classes = np.unique(y_train)
le = LabelEncoder()
y_ind = le.fit_transform(y_train.ravel())
recip_freq = len(y_train) / (len(le.classes_) *
np.bincount(y_ind).astype(np.float64))
class_weight = recip_freq[le.transform(classes)]
print("Class weights : ", class_weight)
y_train = to_categorical(y_train, len(np.unique(y_train)))
y_test = to_categorical(y_test, len(np.unique(y_test)))
if is_timeseries:
factor = 1. / np.cbrt(2)
else:
factor = 1. / np.sqrt(2)
if dataset_fold_id is None:
weight_fn = "./weights/%s_weights.h5" % dataset_prefix
else:
weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix,
dataset_fold_id)
model_checkpoint = ModelCheckpoint(weight_fn,
verbose=1,
mode=optimization_mode,
monitor=monitor,
save_best_only=True,
save_weights_only=True)
reduce_lr = ReduceLROnPlateau(monitor=monitor,
patience=100,
mode=optimization_mode,
factor=factor,
cooldown=0,
min_lr=1e-4,
verbose=2)
time_callback = TimeHistory()
callback_list = [model_checkpoint, reduce_lr, time_callback]
optm = Adam(lr=learning_rate)
if compile_model:
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=[
'accuracy', f1_m, precision_m, recall_m, tp_m, fp_m,
tn_m, fn_m
])
if val_subset is not None:
X_test = X_test[:val_subset]
y_test = y_test[:val_subset]
modelHistory = model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
callbacks=callback_list,
class_weight=class_weight,
verbose=2,
validation_data=(X_test, y_test))
modelTimes = time_callback.times
historyF1 = modelHistory.history['val_f1_m']
historyPrecision = modelHistory.history['val_precision_m']
historyRecall = modelHistory.history['val_recall_m']
historyAccuracy = modelHistory.history['val_acc']
historyTP = modelHistory.history['val_tp_m']
historyFP = modelHistory.history['val_fp_m']
historyTN = modelHistory.history['val_tn_m']
historyFN = modelHistory.history['val_fn_m']
convergenceEpochs = str(historyAccuracy.index(max(historyAccuracy)))
totalTrainingTime = np.sum(
np.array(modelTimes[int(convergenceEpochs)]).astype(np.float))
print('type(totalTrainingTime): ' + str(type(totalTrainingTime)))
print('type(convergenceEpochs): ' + str(type(convergenceEpochs)))
meanTimePerEpoch = totalTrainingTime / int(convergenceEpochs)
with open(resultsFilename, "a") as text_file:
print(f"Training complete for fold ...\n", file=text_file)
print(f"Epoch Times: {str(modelTimes[0:int(convergenceEpochs)])}\n",
file=text_file)
print(f"Number of Epochs: {str(convergenceEpochs)}\n", file=text_file)
print(
f"Mean Epoch Time: {float(sum(modelTimes[0:int(convergenceEpochs)]))/int(convergenceEpochs)}\n",
file=text_file)
print(f"Fold Test Metrics:\n", file=text_file)
print(
f"Fold TP: {str(historyTP)}\nFold TN: {str(historyTN)}\nFold FP: {str(historyFP)}\nFold FN: {str(historyFN)}\n",
file=text_file)
print(
f"Fold Accuracy: {str(historyAccuracy)}\nFold Precision: {str(historyPrecision)}\nFold Recall: {str(historyRecall)}\nFold F1: {str(historyF1)}\n",
file=text_file)
print(f"Accuracy Maxes at Epoch: {str(convergenceEpochs)}\n",
file=text_file)
print(
f"Training Epochs until max accuracy: {str(convergenceEpochs)}\n",
file=text_file)
print(f"Total Training Time: {str(totalTrainingTime)}\n",
file=text_file)
print(f"Mean Time per Epoch: {str(meanTimePerEpoch)}\n",
file=text_file)
def predict_model(model,
dataset_id,
dataset_prefix,
dataset_fold_id=None,
batch_size=128,
test_data_subset=None,
cutoff=None,
normalize_timeseries=False):
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_test)
if not is_timeseries:
X_test = pad_sequences(X_test,
maxlen=MAX_NB_VARIABLES[dataset_id],
padding='post',
truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=['accuracy', f1_score])
if dataset_fold_id is None:
weight_fn = "./weights/%s_weights.h5" % dataset_prefix
else:
weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix,
dataset_fold_id)
model.load_weights(weight_fn)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nPredicting : ")
# metrics = model.evaluate(X_test, y_test, batch_size=batch_size)
# print()
# print("Final metrics %s: " % model.metrics_names, metrics)
y_pred = model.predict(X_test, batch_size=batch_size)
y_test = (np.argmax(y_test, axis=1)).tolist()
y_pred = (np.argmax(y_pred, axis=1)).tolist()
cfmatrix = metrics.confusion_matrix(y_test, y_pred)
report = metrics.classification_report(y_test, y_pred)
averagef_mahi = np.mean(classifaction_report_csv(report).iloc[:, 1])
microf = metrics.f1_score(y_test, y_pred, average='micro')
averagef = metrics.f1_score(y_test, y_pred, average='macro')
weightedf_mahi = sum((classifaction_report_csv(report).iloc[:, 1]) *
(cfmatrix.sum(axis=1) / sum(cfmatrix.sum(axis=1))))
weightedf = metrics.f1_score(y_test, y_pred, average='weighted')
print()
print("Binary F-Score : ", microf)
print("Final F-Score : ", averagef)
print("Weighted F-Score : ", weightedf)
print("Mahis F-Score : ", averagef_mahi)
print("Mahis W F-Score : ", weightedf_mahi)
return averagef
def visualize_filters(model: Model,
dataset_id,
dataset_prefix,
conv_id=0,
filter_id=0,
seed=0,
cutoff=None,
normalize_timeseries=False):
"""
Used to visualize the output filters of a particular convolution layer.
Args:
model: A Keras Model.
dataset_id: Integer id representing the dataset index containd in
`utils/constants.py`.
dataset_prefix: Name of the dataset. Used for weight saving.
conv_id: Convolution layer ID. Can be 0, 1 or 2 for LSTMFCN and
its univariate variants (as it uses 3 Conv blocks).
filter_id: ID of the filter that is under observation.
seed: Numpy random seed.
cutoff: Optional integer which slices of the first `cutoff` timesteps
from the input signal.
normalize_timeseries: Bool / Integer. Determines whether to normalize
the timeseries.
If False, does not normalize the time series.
If True / int not equal to 2, performs standard sample-wise
z-normalization.
If 2: Performs full dataset z-normalization.
"""
np.random.seed(seed)
assert conv_id >= 0 and conv_id < 3, "Convolution layer ID must be between 0 and 2"
X_train, y_train, _, _, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, _ = cutoff_sequence(X_train, _, choice, dataset_id,
sequence_length)
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
conv_layers = [
layer for layer in model.layers if layer.__class__.__name__ == 'Conv1D'
]
conv_layer = conv_layers[conv_id]
conv_layer_name = conv_layer.name
eval_functions = build_function(model, [conv_layer_name])
save_dir = os.path.split(dataset_prefix)[0]
if not os.path.exists('cnn_filters/%s' % (save_dir)):
os.makedirs('cnn_filters/%s' % (save_dir))
dataset_name = os.path.split(dataset_prefix)[-1]
if dataset_name is None or len(dataset_name) == 0:
dataset_name = dataset_prefix
# Select single datapoint
sample_index = np.random.randint(0, X_train.shape[0])
y_id = y_train[sample_index, 0]
sequence_input = X_train[[sample_index], :, :]
# Get features of the cnn layer out
conv_out = get_outputs(model, sequence_input, eval_functions)[0]
conv_out = conv_out[0, :, :] # [T, C]
# select single filter
assert filter_id > 0 and filter_id < conv_out.shape[-1]
channel = conv_out[:, filter_id]
channel = channel.reshape((-1, 1))
conv_filters = pd.DataFrame(channel)
conv_filters.to_csv('cnn_filters/%s_features.csv' % (dataset_prefix),
header=None,
index=False)
sequence_input = sequence_input[0, :, :].transpose()
sequence_df = pd.DataFrame(sequence_input,
index=range(sequence_input.shape[0]))
conv_cam_df = pd.DataFrame(conv_filters,
index=range(conv_filters.shape[0]))
fig, axs = plt.subplots(2, 1, squeeze=False, figsize=(6, 6))
class_label = y_id + 1
plt.rcParams.update({'font.size': 24})
sequence_df.plot(title='Dataset %s : Sequence ID = %d (class = %d)' %
(dataset_name, sample_index + 1, class_label),
subplots=False,
legend=None,
ax=axs[0][0])
conv_cam_df.plot(title='Convolution Layer %d Filter ID %d (class = %d)' %
(conv_id + 1, filter_id + 1, class_label),
subplots=False,
legend=None,
ax=axs[1][0])
# Formatting
plt.xlabel('Timesteps', axes=axs[0][0])
axs[0][0].set_ylabel('Value')
axs[1][0].set_ylabel('Value')
def mjrFormatter(x, pos):
return '{:.2f}'.format(x)
plt.gca().yaxis.set_major_formatter(FuncFormatter(mjrFormatter))
plt.show()
def visualize_context_vector(model: Model,
dataset_id,
dataset_prefix,
cutoff=None,
limit=None,
normalize_timeseries=False,
visualize_sequence=True,
visualize_classwise=False):
"""
Visualize the Context Vector of the Attention LSTM.
Args:
model: an Attention LSTM-FCN Model.
dataset_id: Integer id representing the dataset index containd in
`utils/constants.py`.
dataset_prefix: Name of the dataset. Used for weight saving.
batch_size: Size of each batch for evaluation.
test_data_subset: Optional integer id to subset the test set. To be used if
the test set evaluation time is significantly.
cutoff: Optional integer which slices of the first `cutoff` timesteps
from the input signal.
limit: Number of samples to be visualized in one plot.
normalize_timeseries: Bool / Integer. Determines whether to normalize
the timeseries.
If False, does not normalize the time series.
If True / int not equal to 2, performs standard sample-wise
z-normalization.
If 2: Performs full dataset z-normalization.
visualize_sequence: Bool flag, whetehr to visualize the sequence attended to
by the Context Vector or just the Context Vector itself.
visualize_classwise: Bool flag. Wheter to visualize the samples
seperated by class. When doing so, `limit` is multiplied by
the number of classes so it is better to set `limit` to 1 in
such cases.
"""
X_train, y_train, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id, normalize_timeseries=normalize_timeseries)
_, sequence_length = calculate_dataset_metrics(X_train)
if sequence_length != MAX_SEQUENCE_LENGTH_LIST[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, sequence_length)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
X_train, X_test = cutoff_sequence(X_train, X_test, choice,
dataset_id, sequence_length)
attn_lstm_layer = [(i, layer) for (i, layer) in enumerate(model.layers)
if layer.__class__.__name__ == 'AttentionLSTM']
if len(attn_lstm_layer) == 0:
raise AttributeError('Provided model does not have an Attention layer')
else:
i, attn_lstm_layer = attn_lstm_layer[
0] # use first attention lstm layer only
attn_lstm_layer.return_attention = True
model.layers[i] = attn_lstm_layer
model.load_weights("./weights/%s_weights.h5" % dataset_prefix)
attention_output = model.layers[i].call(model.input)
eval_functions = build_function(model,
attn_lstm_layer.name,
outputs=[attention_output])
train_attention_vectors = []
test_attention_vectors = []
output_shape = [X_train.shape[-1], 1, 1]
for i in range(X_train.shape[0]):
activations = get_outputs(model,
X_train[i, :, :][np.newaxis, ...],
eval_functions,
verbose=False)[0]
# print("activations", activations.shape)
attention_vector = activations.reshape((-1, 1, 1))
attention_vector = (attention_vector - attention_vector.min()) / (
attention_vector.max() - attention_vector.min())
attention_vector = (attention_vector * 2.) - 1.
attention_vector = resize(attention_vector,
output_shape,
mode='reflect',
anti_aliasing=True)
attention_vector = attention_vector.reshape([1, -1])
train_attention_vectors.append(attention_vector)
for i in range(X_test.shape[0]):
activations = get_outputs(model,
X_test[i, :, :][np.newaxis, ...],
eval_functions,
verbose=False)[0]
# print("activations", activations.shape)
attention_vector = activations.reshape((-1, 1, 1))
attention_vector = (attention_vector - attention_vector.min()) / (
attention_vector.max() - attention_vector.min())
attention_vector = (attention_vector * 2.) - 1.
attention_vector = resize(attention_vector,
output_shape,
mode='reflect',
anti_aliasing=True)
attention_vector = attention_vector.reshape([1, -1])
test_attention_vectors.append(attention_vector)
train_attention_vectors = np.array(train_attention_vectors)
test_attention_vectors = np.array(test_attention_vectors)
print("Train Attention Vectors Shape :", train_attention_vectors.shape)
print("Test Attentin Vectors Shape :", test_attention_vectors.shape)
if visualize_sequence:
# plot input sequence part that is paid attention too in detail
X_train_attention = train_attention_vectors * X_train
X_test_attention = test_attention_vectors * X_test
plot_dataset(dataset_id,
seed=1,
limit=limit,
cutoff=cutoff,
normalize_timeseries=normalize_timeseries,
plot_data=(X_train, y_train, X_test, y_test,
X_train_attention, X_test_attention),
type='Context',
plot_classwise=visualize_classwise)
else:
# plot only attention chart
choice = np.random.randint(0, train_attention_vectors.shape[0])
train_df = pd.DataFrame(
{'attention (%)': train_attention_vectors[choice, 0]},
index=range(train_attention_vectors.shape[-1]))
train_df.plot(kind='bar',
title='Attention Mechanism (Train) as '
'a function of input'
' dimensions. Class = %d' % (y_train[choice]))
plt.show()
def evaluate_model(model: Model,
dataset_id,
dataset_prefix,
dataset_fold_id=None,
batch_size=128,
test_data_subset=None,
cutoff=None,
normalize_timeseries=False):
_, _, X_test, y_test, is_timeseries = load_dataset_at(
dataset_id,
fold_index=dataset_fold_id,
normalize_timeseries=normalize_timeseries)
max_timesteps, max_nb_variables = calculate_dataset_metrics(X_test)
if max_nb_variables != MAX_NB_VARIABLES[dataset_id]:
if cutoff is None:
choice = cutoff_choice(dataset_id, max_nb_variables)
else:
assert cutoff in [
'pre', 'post'
], 'Cutoff parameter value must be either "pre" or "post"'
choice = cutoff
if choice not in ['pre', 'post']:
return
else:
_, X_test = cutoff_sequence(None, X_test, choice, dataset_id,
max_nb_variables)
if not is_timeseries:
X_test = pad_sequences(X_test,
maxlen=MAX_NB_VARIABLES[dataset_id],
padding='post',
truncating='post')
y_test = to_categorical(y_test, len(np.unique(y_test)))
optm = Adam(lr=1e-3)
model.compile(optimizer=optm,
loss='categorical_crossentropy',
metrics=[
'accuracy', f1_m, precision_m, recall_m, tp_m, fp_m,
tn_m, fn_m
])
if dataset_fold_id is None:
weight_fn = "./weights/%s_weights.h5" % dataset_prefix
else:
weight_fn = "./weights/%s_fold_%d_weights.h5" % (dataset_prefix,
dataset_fold_id)
model.load_weights(weight_fn)
if test_data_subset is not None:
X_test = X_test[:test_data_subset]
y_test = y_test[:test_data_subset]
print("\nEvaluating : ")
loss, accuracy, f1_score, precision, recall, tp, fp, tn, fn = model.evaluate(
X_test, y_test, batch_size=batch_size)
predictions = (model.predict(X_test) > 0.5)
truelabels = y_test == 1
print('predictions: ' + str(predictions[:, 0]))
print('truelabels: ' + str(truelabels[:, 0]))
predictions = predictions[:, 0]
truelabels = truelabels[:, 0]
tp = np.sum(np.logical_and(predictions, truelabels))
tn = np.sum(np.logical_and(np.invert(predictions), np.invert(truelabels)))
fp = np.sum(np.logical_and(predictions, np.invert(truelabels)))
fn = np.sum(np.logical_and(np.invert(predictions), (truelabels)))
precision = tp / (tp + fp)
recall = tp / (tp + fn)
accuracy = (tp + tn) / (tp + tn + fp + fn)
f1_score = (2 * precision * recall) / (precision + recall)
print(f"TP: {str(tp)}\nTN: {str(tn)}\nFP: {str(fp)}\nFN: {str(fn)}\n")
print(
f"Accuracy: {str(accuracy)}\nPrecision: {str(precision)}\nRecall: {str(recall)}\nF1: {str(f1_score)}\n"
)
with open(resultsFilename, "a") as text_file:
print(f"Final Test Metrics for ...:\n", file=text_file)
print(f"TP: {str(tp)}\nTN: {str(tn)}\nFP: {str(fp)}\nFN: {str(fn)}\n",
file=text_file)
print(
f"Accuracy: {str(accuracy)}\nPrecision: {str(precision)}\nRecall: {str(recall)}\nF1: {str(f1_score)}\n",
file=text_file)
return accuracy, loss
