def fit(self, input_vector_database, classes):
# Init GSOM Parameters
gsom_params = Params.GSOMParameters(
self.SF,
self.learning_itr,
self.smoothing_irt,
distance=Params.DistanceFunction.EUCLIDEAN,
temporal_context_count=self.temporal_contexts,
forget_itr_count=self.forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Process the input files
self.output_loc, output_loc_images = self.generate_output_config(
self.SF, self.forget_threshold)
# Setup the age threshold based on the input vector length
generalise_params.setup_age_threshold(input_vector_database.shape[0])
self.params = generalise_params
# Process the clustering algorithm
result_dict = self.run(input_vector_database, self.plot_for_itr,
classes, output_loc_images)
return result_dict, classes
def run(data_filename):
SF = 0.83
forget_threshold = 60 # To include forgetting, threshold should be < learning iterations.
temporal_contexts = 1 # If stationary data - keep this at 1
learning_itr = 100
smoothing_irt = 50
plot_for_itr = 4 # Unused parameter - just for visualization. Keep this as it is.
# File Config
dataset = 'anomaly'
# data_filename = "../../data/NSLKDD-3.csv".replace('\\', '/')
experiment_id = 'Exp-new-gsom-' + datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
output_save_location = join('output/', experiment_id)
# Init GSOM Parameters
gsom_params = Params.GSOMParameters(SF, learning_itr, smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN,
temporal_context_count=temporal_contexts, forget_itr_count=forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Process the input files
input_vector_database, labels, classes,X_test, y_test = Parser.InputParser.parse_input_zoo_data(data_filename)
output_loc, output_loc_images = generate_output_config(dataset, SF, forget_threshold, temporal_contexts,
output_save_location)
# Setup the age threshold based on the input vector length
generalise_params.setup_age_threshold(input_vector_database[0].shape[0])
# Process the clustering algorithm algorithm
controller = Core.Controller(generalise_params)
controller_start = time.time()
result_dict, y_pred = controller.run(input_vector_database,X_test, plot_for_itr, classes, output_loc_images)
return y_test, y_pred
def GSOM_model(SF,forget_threshold,temporal_contexts,learning_itr,smoothing_irt,plot_for_itr,data_filename,output_save_location,name):
# Init GSOM Parameters
gsom_params = Params.GSOMParameters(SF, learning_itr, smoothing_irt, distance=Params.DistanceFunction.EUCLIDEAN,
temporal_context_count=temporal_contexts, forget_itr_count=forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Process the input files
input_vector_database, labels, classes = Parser.InputParser.parse_input_train_data(data_filename, None)
# Setup the age threshold based on the input vector length
generalise_params.setup_age_threshold(input_vector_database[0].shape[0])
# Process the clustering algorithm
controller = Core.Controller(generalise_params)
controller_start = time.time()
result_dict = controller.run(input_vector_database, plot_for_itr, classes)
print('Algorithms completed in', round(time.time() - controller_start, 2), '(s)')
gsom_nodemap = result_dict[0]['gsom']
# Saving gsom node map
saved_gsom_nodemap_for_0_7 = joblib.dump(gsom_nodemap, output_save_location+'gsom_nodemap_{}.joblib'.format(name))
# Display
display = Display_Utils.Display(result_dict[0]['gsom'], None)
display.setup_labels_for_gsom_nodemap(classes, 2, 'Latent Space of cnn_5100_input_file_to_gsom : SF=0.7',output_save_location+'latent_space_{}_hitvalues'.format(name))
print('Completed.')
def generate_params(self, dim):
# Init GSOM Parameters
gsom_params = Params.GSOMParameters(self.SF, self.learning_itr, self.smoothing_irt,
distance=Params.DistanceFunction.EUCLIDEAN,
temporal_context_count=self.temporal_contexts,
forget_itr_count=self.forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Setup the age threshold based on the input vector length
generalise_params.setup_age_threshold(dim)
return generalise_params
os.makedirs(output_loc_images)
return output_loc, output_loc_images
if __name__ == "__main__":
SF = 0.83
forget_threshold = 60
temporal_contexts = 1
learning_itr = 100
smoothing_irt = 50
plot_for_itr = 4
# Init GSOM Parameters
gsom_params = Params.GSOMParameters(SF, learning_itr, smoothing_irt,
distance=Params.DistanceFunction.EUCLIDEAN,
temporal_context_count=temporal_contexts,
forget_itr_count=forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Setup the age threshold based on the input vector length
generalise_params.setup_age_threshold(Lock.INPUT_SIZE)
# Process the input files
output_loc, output_loc_images = generate_output_config(SF, forget_threshold)
X_train_emotion = Lock.emotion_feature
X_train_behaviour = Lock.behaviour_feature
y_train_emotion = Lock.emotion_label
y_train_behaviour = Lock.behaviour_label
result_dict = []
def generate_latent_map(visualize=False):
# File config
dataset_location = config.Configuration.dataset_location
experiment_id = 'Exp-' + datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H-%M-%S')
output_save_location = join('output/', experiment_id)
ID_column_name = config.Configuration.ID_column_name
is_normalized = config.Configuration.normalize_dataset
# Grid search config
sheet_names = config.Configuration.excel_sheet_names
# GSOM Config
SF_values = config.Configuration.SF_values
learning_itr = config.Configuration.learning_itr
smoothing_irt = config.Configuration.smoothing_irt
forget_threshold_values = [
int(learning_itr * (1 - tp))
for tp in config.Configuration.transience_percentages
]
temporal_contexts = 1
plot_for_itr = 4
for dataset in sheet_names:
# Process the input files
input_vector_database, labels, selected_vars = Parser.InputParser.parse_data(
dataset_location, dataset, ID_column_name, is_normalized)
print('Latent Spaces for {}, using: \n{}'.format(
dataset, list(selected_vars)))
for SF, forget_threshold in itertools.product(SF_values,
forget_threshold_values):
print('\nProcessing with SF={} with Forgetting={}'.format(
SF, forget_threshold))
forget_threshold_label = Utils.Utilities.forget_thresh_label(
learning_itr, forget_threshold)
output_loc, output_loc_images = generate_output_config(
dataset, SF, forget_threshold, temporal_contexts,
output_save_location)
# Init GSOM Parameters
gsom_params = Params.GSOMParameters(
SF,
learning_itr,
smoothing_irt,
distance=Params.DistanceFunction.EUCLIDEAN,
temporal_context_count=temporal_contexts,
forget_itr_count=forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Setup the age threshold based on the input vector length
generalise_params.setup_age_threshold(
input_vector_database[0].shape[0])
# Process the clustering algorithm algorithm
controller = Core.Controller(generalise_params)
result_dict = controller.run(input_vector_database, plot_for_itr,
labels, output_loc_images)
Utils.Utilities.save_object(
result_dict,
join(
output_loc,
'gsom_nodemap_SF-{}_F-{}'.format(SF,
forget_threshold_label)))
gsom_nodemap = result_dict[0]['gsom']
print('Latent Space generated with {} neurons.'.format(
len(gsom_nodemap)))
# Visualizations
display = Display_Utils.Display(gsom_nodemap, None)
display.setup_labels_for_gsom_nodemap(
gsom_nodemap,
labels,
'Latent Space of {} : SF={} with Forget={}'.format(
dataset, SF, forget_threshold_label),
join(
output_loc, '{}_latent_space_SF-{}_F-{}_hit_count'.format(
dataset, SF, forget_threshold_label)),
selected_vars=selected_vars)
# Plot profile images
profiler.Profiler.plot_profile_images(dataset_location, dataset,
ID_column_name, output_loc,
SF, forget_threshold_label,
is_normalized)
# Plot interactive profile visualization
if visualize:
int_display.InteractiveDisplay.plot_interactive(
dataset, output_loc, SF, forget_threshold_label)
if __name__ == '__main__':
if mode == 1:
print('Start running IKASL algorithm.')
# Init GSOM Parameters
"""
GSOMParameters(spread_factor, learning_itr, smooth_itr, max_neighbourhood_radius=4, start_learning_rate=0.3,
smooth_neighbourhood_radius_factor=0.5, smooth_learning_factor=0.5, distance='EUC', fd=0.1,
alpha=0.9, r=0.95)
IKASLParameters(self, gsom_parameters, aggregate_proximity=2, hit_threshold_fraction=0.05,
aggregate_function='AVG', aggregate_inside_hitnode_proximity=True)
"""
gsom_params = Params.GSOMParameters(
0.83, 100, 100, distance=Elements.DistanceFunction.COSINE)
ikasl_params = Params.IKASLParameters(
gsom_params,
aggregate_proximity=2,
hit_threshold_fraction=0.05,
aggregate_function=Elements.AggregateFunction.PROXIMITY_AVERAGE,
aggregate_inside_hitnode_proximity=False)
construct_output_title(ikasl_params)
# Process the input files
input_vector_database, labels = Parser.InputParser.parse_input(
input_folder_path)
# Process the IKASL algorithm
print('Starting IKASL ...')
output_save_location = 'output/'
if not exists(output_save_location):
makedirs(output_save_location)
SF = 0.7
forget_threshold = 1000
plot_output_name = output_save_location + output_save_filename + str(
SF) + '_mage_' + str(forget_threshold) + 'itr'
if __name__ == '__main__':
print('Start GSOM algorithm.')
gsom_params = Params.GSOMParameters(
SF,
50,
10,
distance=Params.DistanceFunction.EUCLIDEAN,
forget_itr_count=forget_threshold)
generalise_params = Params.GeneraliseParameters(gsom_params)
# Process the input files
input_vector_database, labels, classes = Parser.InputParser.parse_input_zoo_data(
input_filename, None)
# Process the clustering algorithm algorithm
controller = Core.Controller(generalise_params)
controller_start = time.time()
result_dict = controller.run(input_vector_database)
print('Algorithms completed in', round(time.time() - controller_start, 2),
'(s)')
Utils.Utilities.save_object(result_dict,
