def getDECNetworkResults(dec, enc): #Load test dataset test_data = LoadDataset("dataset/kaggle_original_train/", 0) test_data, test_label, val, val_label = test_data.load_data() big_data = LoadDataset("dataset/kaggle_augmented_train_new/", 0) big_data, _, _, _ = big_data.load_data() # make save directory os.makedirs(os.path.join("dec"), exist_ok=True) os.chdir("dec") encoded = enc.predict(test_data) q, _ = dec.predict(test_data, verbose=0) y_pred = q.argmax(1) print(y_pred) confusion_matrix(test_label.astype(np.int64), y_pred) #Take prediction time for i in range(20): iterate = 5000 * (i + 1) data = big_data[0:iterate, :] print(data.shape) print("KMEAN") start = time.time() q, _ = dec.predict(data, verbose=0) y_pred = q.argmax(1) end = time.time() print(end - start) train_x = np.reshape(test_data, (3720, 64, 64)) TSNE = TSNEAlgo() TSNE.tsne_fit(encoded, perplexity=35) TSNE.tsne_plot(train_x, y_pred.astype(int), save_name="Pred", save_data_dir="dec") TSNE.tsne_plot(train_x, test_label.astype(int), save_name="True", save_data_dir="dec")
def anomaly(): """ Testing models ability to find data anomalies !!! not working in current version """ #Load anomaly dataset anomaly_data = LoadDataset("dataset/kaggle_anomalies/", 0) anomaly_data, anomaly_label, val, val_label = anomaly_data.load_data() for i in range(len(anomaly_label)): anomaly_label[i] = anomaly_label[i] + 5 #Concatinate test and anomaly test_anomaly_data = np.vstack((test_data, anomaly_data)) test_anomaly_label = np.hstack((test_label, anomaly_label)) """# Get k-means cluster distance
tf.keras.backend.clear_session() model = "auto" if model == "auto": model_type = "COAPNET" latent_vector = "globalAverage" model = buildNetwork(model_type, latent_vector,latent_dim = 64, epochs = 50,train = True,noisy = False) auto, enc, pre = model.getModel() getNetworkResults(model,latent_vector) if model == "dec": train_data = LoadDataset("dataset/kaggle_original_train/",0) train_data, train_label, val, val_label = train_data.load_data() encoded_x= np.reshape(train_data,(3720, 64 *64)) model_type = "COAPNET" latent_vector = "globalAverage" model = buildNetwork(model_type, latent_vector,latent_dim = 64, epochs = 5,train = True,noisy = False) auto, enc, pre = model.getModel() from DEC import DeepEmbeddedClustering from results import getDECNetworkResults dec = DeepEmbeddedClustering(auto,enc,train_data,train_label,5) dec.buildModel() dec.trainModel() dec,enc = dec.getModel()
def getNetworkResults(model, model_type): """ Main result function """ #Specify wanted results plot_input_output = False plot_data = False plot_class_activation_map = False plot_activation_map = False plot_kernel_inspection = False hierarchical_clustering = False kmean_cluster = True spectral_cluster = True pred_time = False #Load test dataset test_data = LoadDataset("dataset/kaggle_original_train/", 0) test_data, test_label, val, val_label = test_data.load_data() # make save directory os.makedirs(os.path.join(model_type), exist_ok=True) os.chdir(model_type) # get autoencoder, encoder and decoder AUTO, ENC, DEC = model.getModel() #Predict encoder output encoded = ENC.predict(test_data) #Fit the tsne algorithm TSNE = TSNEAlgo() TSNE.tsne_fit(encoded, perplexity=35) if plot_input_output == True: # Visualize input and output from autoencoder visualize_input_output(AUTO, test_data, model_type) if model_type == "globalAverage" and plot_class_activation_map == True: visualize_class_activation_map(model, test_data) if plot_activation_map == True: visualize_activation_map(model, test_data) if plot_data == True: #Predict results using TSNE TSNE.tsne_plot(test_data, test_label, model_type, model_type) if plot_kernel_inspection == True: kernel_inspection(model, test_data) if hierarchical_clustering == True: hierarchical(ENC, TSNE, test_data, test_label, save_name="hierarchical.png") if kmean_cluster == True: kmean(ENC, TSNE, test_data, test_label) if spectral_cluster == True: spectral(ENC, TSNE, test_data, test_label) if pred_time == True: os.chdir("..") data = LoadDataset("dataset/kaggle_augmented_train_new/", 0) data, test_label, val, val_label = data.load_data() os.chdir(model_type) predictionTime(ENC, data)
def loadData(self): train_data = LoadDataset(self.load_data_dir, 0.1) self.train_data, self.train_label, self.validation_data, self.validation_label = train_data.load_data( ) return