def fit(self, input_vector_database, classes): # Init GSOM Parameters gsom_params = Params.GSOMParameters( self.SF, self.learning_itr, self.smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN, temporal_context_count=self.temporal_contexts, forget_itr_count=self.forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Process the input files self.output_loc, output_loc_images = self.generate_output_config( self.SF, self.forget_threshold) # Setup the age threshold based on the input vector length generalise_params.setup_age_threshold(input_vector_database.shape[0]) self.params = generalise_params # Process the clustering algorithm result_dict = self.run(input_vector_database, self.plot_for_itr, classes, output_loc_images) return result_dict, classes
def run(data_filename): SF = 0.83 forget_threshold = 60 # To include forgetting, threshold should be < learning iterations. temporal_contexts = 1 # If stationary data - keep this at 1 learning_itr = 100 smoothing_irt = 50 plot_for_itr = 4 # Unused parameter - just for visualization. Keep this as it is. # File Config dataset = 'anomaly' # data_filename = "../../data/NSLKDD-3.csv".replace('\\', '/') experiment_id = 'Exp-new-gsom-' + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S') output_save_location = join('output/', experiment_id) # Init GSOM Parameters gsom_params = Params.GSOMParameters(SF, learning_itr, smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN, temporal_context_count=temporal_contexts, forget_itr_count=forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Process the input files input_vector_database, labels, classes,X_test, y_test = Parser.InputParser.parse_input_zoo_data(data_filename) output_loc, output_loc_images = generate_output_config(dataset, SF, forget_threshold, temporal_contexts, output_save_location) # Setup the age threshold based on the input vector length generalise_params.setup_age_threshold(input_vector_database[0].shape[0]) # Process the clustering algorithm algorithm controller = Core.Controller(generalise_params) controller_start = time.time() result_dict, y_pred = controller.run(input_vector_database,X_test, plot_for_itr, classes, output_loc_images) return y_test, y_pred
def GSOM_model(SF,forget_threshold,temporal_contexts,learning_itr,smoothing_irt,plot_for_itr,data_filename,output_save_location,name): # Init GSOM Parameters gsom_params = Params.GSOMParameters(SF, learning_itr, smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN, temporal_context_count=temporal_contexts, forget_itr_count=forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Process the input files input_vector_database, labels, classes = Parser.InputParser.parse_input_train_data(data_filename, None) # Setup the age threshold based on the input vector length generalise_params.setup_age_threshold(input_vector_database[0].shape[0]) # Process the clustering algorithm controller = Core.Controller(generalise_params) controller_start = time.time() result_dict = controller.run(input_vector_database, plot_for_itr, classes) print('Algorithms completed in', round(time.time() - controller_start, 2), '(s)') gsom_nodemap = result_dict[0]['gsom'] # Saving gsom node map saved_gsom_nodemap_for_0_7 = joblib.dump(gsom_nodemap, output_save_location+'gsom_nodemap_{}.joblib'.format(name)) # Display display = Display_Utils.Display(result_dict[0]['gsom'], None) display.setup_labels_for_gsom_nodemap(classes, 2, 'Latent Space of cnn_5100_input_file_to_gsom : SF=0.7',output_save_location+'latent_space_{}_hitvalues'.format(name)) print('Completed.')
def generate_params(self, dim): # Init GSOM Parameters gsom_params = Params.GSOMParameters(self.SF, self.learning_itr, self.smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN, temporal_context_count=self.temporal_contexts, forget_itr_count=self.forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Setup the age threshold based on the input vector length generalise_params.setup_age_threshold(dim) return generalise_params
os.makedirs(output_loc_images) return output_loc, output_loc_images if __name__ == "__main__": SF = 0.83 forget_threshold = 60 temporal_contexts = 1 learning_itr = 100 smoothing_irt = 50 plot_for_itr = 4 # Init GSOM Parameters gsom_params = Params.GSOMParameters(SF, learning_itr, smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN, temporal_context_count=temporal_contexts, forget_itr_count=forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Setup the age threshold based on the input vector length generalise_params.setup_age_threshold(Lock.INPUT_SIZE) # Process the input files output_loc, output_loc_images = generate_output_config(SF, forget_threshold) X_train_emotion = Lock.emotion_feature X_train_behaviour = Lock.behaviour_feature y_train_emotion = Lock.emotion_label y_train_behaviour = Lock.behaviour_label result_dict = []
def generate_latent_map(visualize=False): # File config dataset_location = config.Configuration.dataset_location experiment_id = 'Exp-' + datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d-%H-%M-%S') output_save_location = join('output/', experiment_id) ID_column_name = config.Configuration.ID_column_name is_normalized = config.Configuration.normalize_dataset # Grid search config sheet_names = config.Configuration.excel_sheet_names # GSOM Config SF_values = config.Configuration.SF_values learning_itr = config.Configuration.learning_itr smoothing_irt = config.Configuration.smoothing_irt forget_threshold_values = [ int(learning_itr * (1 - tp)) for tp in config.Configuration.transience_percentages ] temporal_contexts = 1 plot_for_itr = 4 for dataset in sheet_names: # Process the input files input_vector_database, labels, selected_vars = Parser.InputParser.parse_data( dataset_location, dataset, ID_column_name, is_normalized) print('Latent Spaces for {}, using: \n{}'.format( dataset, list(selected_vars))) for SF, forget_threshold in itertools.product(SF_values, forget_threshold_values): print('\nProcessing with SF={} with Forgetting={}'.format( SF, forget_threshold)) forget_threshold_label = Utils.Utilities.forget_thresh_label( learning_itr, forget_threshold) output_loc, output_loc_images = generate_output_config( dataset, SF, forget_threshold, temporal_contexts, output_save_location) # Init GSOM Parameters gsom_params = Params.GSOMParameters( SF, learning_itr, smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN, temporal_context_count=temporal_contexts, forget_itr_count=forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Setup the age threshold based on the input vector length generalise_params.setup_age_threshold( input_vector_database[0].shape[0]) # Process the clustering algorithm algorithm controller = Core.Controller(generalise_params) result_dict = controller.run(input_vector_database, plot_for_itr, labels, output_loc_images) Utils.Utilities.save_object( result_dict, join( output_loc, 'gsom_nodemap_SF-{}_F-{}'.format(SF, forget_threshold_label))) gsom_nodemap = result_dict[0]['gsom'] print('Latent Space generated with {} neurons.'.format( len(gsom_nodemap))) # Visualizations display = Display_Utils.Display(gsom_nodemap, None) display.setup_labels_for_gsom_nodemap( gsom_nodemap, labels, 'Latent Space of {} : SF={} with Forget={}'.format( dataset, SF, forget_threshold_label), join( output_loc, '{}_latent_space_SF-{}_F-{}_hit_count'.format( dataset, SF, forget_threshold_label)), selected_vars=selected_vars) # Plot profile images profiler.Profiler.plot_profile_images(dataset_location, dataset, ID_column_name, output_loc, SF, forget_threshold_label, is_normalized) # Plot interactive profile visualization if visualize: int_display.InteractiveDisplay.plot_interactive( dataset, output_loc, SF, forget_threshold_label)
if __name__ == '__main__': if mode == 1: print('Start running IKASL algorithm.') # Init GSOM Parameters """ GSOMParameters(spread_factor, learning_itr, smooth_itr, max_neighbourhood_radius=4, start_learning_rate=0.3, smooth_neighbourhood_radius_factor=0.5, smooth_learning_factor=0.5, distance='EUC', fd=0.1, alpha=0.9, r=0.95) IKASLParameters(self, gsom_parameters, aggregate_proximity=2, hit_threshold_fraction=0.05, aggregate_function='AVG', aggregate_inside_hitnode_proximity=True) """ gsom_params = Params.GSOMParameters( 0.83, 100, 100, distance=Elements.DistanceFunction.COSINE) ikasl_params = Params.IKASLParameters( gsom_params, aggregate_proximity=2, hit_threshold_fraction=0.05, aggregate_function=Elements.AggregateFunction.PROXIMITY_AVERAGE, aggregate_inside_hitnode_proximity=False) construct_output_title(ikasl_params) # Process the input files input_vector_database, labels = Parser.InputParser.parse_input( input_folder_path) # Process the IKASL algorithm print('Starting IKASL ...')
output_save_location = 'output/' if not exists(output_save_location): makedirs(output_save_location) SF = 0.7 forget_threshold = 1000 plot_output_name = output_save_location + output_save_filename + str( SF) + '_mage_' + str(forget_threshold) + 'itr' if __name__ == '__main__': print('Start GSOM algorithm.') gsom_params = Params.GSOMParameters( SF, 50, 10, distance=Params.DistanceFunction.EUCLIDEAN, forget_itr_count=forget_threshold) generalise_params = Params.GeneraliseParameters(gsom_params) # Process the input files input_vector_database, labels, classes = Parser.InputParser.parse_input_zoo_data( input_filename, None) # Process the clustering algorithm algorithm controller = Core.Controller(generalise_params) controller_start = time.time() result_dict = controller.run(input_vector_database) print('Algorithms completed in', round(time.time() - controller_start, 2), '(s)') Utils.Utilities.save_object(result_dict,