def getDECNetworkResults(dec, enc):
#Load test dataset
test_data = LoadDataset("dataset/kaggle_original_train/", 0)
test_data, test_label, val, val_label = test_data.load_data()
big_data = LoadDataset("dataset/kaggle_augmented_train_new/", 0)
big_data, _, _, _ = big_data.load_data()
# make save directory
os.makedirs(os.path.join("dec"), exist_ok=True)
os.chdir("dec")
encoded = enc.predict(test_data)
q, _ = dec.predict(test_data, verbose=0)
y_pred = q.argmax(1)
print(y_pred)
confusion_matrix(test_label.astype(np.int64), y_pred)
#Take prediction time
for i in range(20):
iterate = 5000 * (i + 1)
data = big_data[0:iterate, :]
print(data.shape)
print("KMEAN")
start = time.time()
q, _ = dec.predict(data, verbose=0)
y_pred = q.argmax(1)
end = time.time()
print(end - start)
train_x = np.reshape(test_data, (3720, 64, 64))
TSNE = TSNEAlgo()
TSNE.tsne_fit(encoded, perplexity=35)
TSNE.tsne_plot(train_x,
y_pred.astype(int),
save_name="Pred",
save_data_dir="dec")
TSNE.tsne_plot(train_x,
test_label.astype(int),
save_name="True",
save_data_dir="dec")
def anomaly():
"""
Testing models ability to find data anomalies
!!! not working in current version
"""
#Load anomaly dataset
anomaly_data = LoadDataset("dataset/kaggle_anomalies/", 0)
anomaly_data, anomaly_label, val, val_label = anomaly_data.load_data()
for i in range(len(anomaly_label)):
anomaly_label[i] = anomaly_label[i] + 5
#Concatinate test and anomaly
test_anomaly_data = np.vstack((test_data, anomaly_data))
test_anomaly_label = np.hstack((test_label, anomaly_label))
"""# Get k-means cluster distance
tf.keras.backend.clear_session()
model = "auto"
if model == "auto":
model_type = "COAPNET"
latent_vector = "globalAverage"
model = buildNetwork(model_type, latent_vector,latent_dim = 64, epochs = 50,train = True,noisy = False)
auto, enc, pre = model.getModel()
getNetworkResults(model,latent_vector)
if model == "dec":
train_data = LoadDataset("dataset/kaggle_original_train/",0)
train_data, train_label, val, val_label = train_data.load_data()
encoded_x= np.reshape(train_data,(3720, 64 *64))
model_type = "COAPNET"
latent_vector = "globalAverage"
model = buildNetwork(model_type, latent_vector,latent_dim = 64, epochs = 5,train = True,noisy = False)
auto, enc, pre = model.getModel()
from DEC import DeepEmbeddedClustering
from results import getDECNetworkResults
dec = DeepEmbeddedClustering(auto,enc,train_data,train_label,5)
dec.buildModel()
dec.trainModel()
dec,enc = dec.getModel()
def getNetworkResults(model, model_type):
"""
Main result function
"""
#Specify wanted results
plot_input_output = False
plot_data = False
plot_class_activation_map = False
plot_activation_map = False
plot_kernel_inspection = False
hierarchical_clustering = False
kmean_cluster = True
spectral_cluster = True
pred_time = False
#Load test dataset
test_data = LoadDataset("dataset/kaggle_original_train/", 0)
test_data, test_label, val, val_label = test_data.load_data()
# make save directory
os.makedirs(os.path.join(model_type), exist_ok=True)
os.chdir(model_type)
# get autoencoder, encoder and decoder
AUTO, ENC, DEC = model.getModel()
#Predict encoder output
encoded = ENC.predict(test_data)
#Fit the tsne algorithm
TSNE = TSNEAlgo()
TSNE.tsne_fit(encoded, perplexity=35)
if plot_input_output == True:
# Visualize input and output from autoencoder
visualize_input_output(AUTO, test_data, model_type)
if model_type == "globalAverage" and plot_class_activation_map == True:
visualize_class_activation_map(model, test_data)
if plot_activation_map == True:
visualize_activation_map(model, test_data)
if plot_data == True:
#Predict results using TSNE
TSNE.tsne_plot(test_data, test_label, model_type, model_type)
if plot_kernel_inspection == True:
kernel_inspection(model, test_data)
if hierarchical_clustering == True:
hierarchical(ENC,
TSNE,
test_data,
test_label,
save_name="hierarchical.png")
if kmean_cluster == True:
kmean(ENC, TSNE, test_data, test_label)
if spectral_cluster == True:
spectral(ENC, TSNE, test_data, test_label)
if pred_time == True:
os.chdir("..")
data = LoadDataset("dataset/kaggle_augmented_train_new/", 0)
data, test_label, val, val_label = data.load_data()
os.chdir(model_type)
predictionTime(ENC, data)
def loadData(self):
train_data = LoadDataset(self.load_data_dir, 0.1)
self.train_data, self.train_label, self.validation_data, self.validation_label = train_data.load_data(
)
return