def create_new_distance(self, amount):
""" Create a new distance. We must select the trained model and the unknown model. The amount is the max amount of letters to compare. """
# Generate the new id for this distance
try:
new_id = self.main_dict[list(self.main_dict.keys())[-1]].get_id() + 1
except (KeyError, IndexError):
new_id = 1
# Create the new object
new_distance = Detection(new_id)
# Structures to ignore
exceptions = ['models', 'database', 'datasets', 'notes', 'connections', 'experiments', 'template_example_module', 'labels']
# Get the training module
# 1- List all the structures in the db, so we can pick our type of module
structures = __database__.get_structures()
print_info('From which structure you want to pick up the trained model?:')
for structure in structures:
if structure not in exceptions:
print_info('\t'+structure)
training_structure_name = raw_input('Name:')
training_structure_name = training_structure_name.strip()
# 2- Verify is there
try:
selected_training_structure = structures[training_structure_name]
except KeyError:
print_error('No such structure available.')
return False
# 3- Get the main dict and list the 'objects'
print_info('Select the training module to use:')
for object in selected_training_structure:
print '\t',
print_info(selected_training_structure[object])
model_training_id = raw_input('Id:')
print
# Get the testing module
# 1- List all the structures in the db, so we can pick our type of module
structures = __database__.get_structures()
print_info('From which structure you want to pick up the testing model?:')
for structure in structures:
if structure not in exceptions:
print_info('\t'+structure)
testing_structure_name = raw_input('Name:')
testing_structure_name = testing_structure_name.strip()
# 2- Verify is there
try:
selected_testing_structure = structures[testing_structure_name]
except KeyError:
print_error('No such structure available.')
return False
# 3- Get the main dict and list the 'objects'
print_info('Select the testing module to use:')
for object in selected_testing_structure:
print '\t',
print_info(selected_testing_structure[object])
model_testing_id = raw_input('Id:')
# Run the distance rutine
if new_distance.detect(training_structure_name, selected_training_structure, model_training_id, training_structure_name, selected_testing_structure, model_testing_id, amount):
# Store on DB the new distance only if the comparison was successful.
self.main_dict[new_id] = new_distance
print_info('New distance created with id {}'.format(new_id))
def compare_all(self, structure_name, amount, verbose):
""" Compare all the models between themselves in the specified structure. Do not repeat the comparison if it already exists """
structures = __database__.get_structures()
try:
structure = structures[structure_name]
except KeyError:
print_error('No such structure available.')
return False
for train_object in structure:
train_model_id = int(structure[train_object].get_id())
# Run this model against the rest
for test_object in structure:
test_model_id = int(structure[test_object].get_id())
if not self.has_distance(train_model_id, test_model_id, structure_name, structure_name):
print
print('Training model:'),
print_info(structure[train_object])
print('\tTesting model: '),
print_info(structure[test_object])
# Generate the new id for this distance
try:
new_id = self.main_dict[list(self.main_dict.keys())[-1]].get_id() + 1
except (KeyError, IndexError):
new_id = 1
# Create the new object
new_distance = Detection(new_id)
# Run the distance rutine
if new_distance.detect(structure_name, structure, train_model_id, structure_name, structure, test_model_id, amount, verbose):
# Store on DB the new distance only if the comparison was successful
self.main_dict[new_id] = new_distance
print_info('\tNew distance created with id {}'.format(new_id))
def regenerate(self):
""" Regenerate """
print_info('Regenerating distance {}'.format(self.get_id()))
structures = __database__.get_structures()
structure_training = structures[self.training_structure_name]
structure_testing = structures[self.testing_structure_name]
self.dict_of_distances = []
self.detect(self.training_structure_name, structure_training, self.model_training_id, self.testing_structure_name, structure_testing, self.model_testing_id, self.get_amount())
def check_need_for_regeneration(self):
""" Check if the training or testing of this comparison changed since we use them """
structures = __database__.get_structures()
try:
current_training_model_len = len(structures[self.training_structure_name][int(self.model_training_id)].get_state())
except KeyError:
#print_warning('Warning! In distance id {}, the training model was deleted. However, this distance can still be used.'.format(self.get_id()))
return False
try:
current_testing_model_len = len(structures[self.testing_structure_name][int(self.model_testing_id)].get_state())
except KeyError:
#print_warning('Warning! In distance id {}, the testing model was deleted. However, this distance can still be used.'.format(self.get_id()))
return False
if len(self.training_states) != current_training_model_len or len(self.testing_states) != current_testing_model_len:
return True
else:
return False
def create_new_distance(self, amount, train_id, temp_test_id, verbose):
""" Create a new distance. We must select the trained model and the unknown model. The amount is the max amount of letters to compare. """
train_id = train_id.split(',')
train_id.sort()
# For each train model passed
for train_id in train_id:
test_id = temp_test_id.split(',')
test_id.sort()
# For each test model passed
distances_ids = []
for tid in test_id:
# Generate the new id for this distance
try:
new_id = self.main_dict[list(self.main_dict.keys())[-1]].get_id() + 1
except (KeyError, IndexError):
new_id = 1
# Create the new object
new_distance = Detection(new_id)
# Remember the ids of the distances in this execution
structures = __database__.get_structures()
# Now the structures are fixed
model_training_id = train_id
# Suppose the markov_models_1 structure and don't ask
training_structure_name = "markov_models_1"
selected_training_structure = structures[training_structure_name]
# Suppose the markov_models_1 structure and don't ask
testing_structure_name = "markov_models_1"
selected_testing_structure = structures[testing_structure_name]
# Run the distance rutine
if new_distance.detect(training_structure_name, selected_training_structure, model_training_id, testing_structure_name, selected_testing_structure, tid, amount, verbose):
# Store on DB the new distance only if the comparison was successful.
distances_ids.append(new_id)
self.main_dict[new_id] = new_distance
if verbose > 1:
print
print_info('New distance created with id {}'.format(new_id))
# Finish for
# Compute the performance metrics based on the errors of the comparison of each training model with its testing model.
total_errors = {}
total_errors['TP'] = 0
total_errors['TN'] = 0
total_errors['FN'] = 0
total_errors['FP'] = 0
for did in distances_ids:
distance = self.get_distance(did)
test_id = distance.get_model_testing_id()
error = distance.get_current_error_type()
#if verbose:
# print_info('Test model id {}, error {}'.format(test_id, error))
if error == 'TP':
total_errors['TP'] += 1
elif error == 'TN':
total_errors['TN'] += 1
elif error == 'FN':
total_errors['FN'] += 1
elif error == 'FP':
total_errors['FP'] += 1
error_string_1 = ""
for error in total_errors:
error_string_1 += '{}:{} '.format(error, total_errors[error])
if verbose:
print_info(error_string_1)
# Call the performance metrics
performance_metrics = self.compute_total_performance_metrics(total_errors)
error_string_2 = ""
for pmetric in performance_metrics:
error_string_2 += '{}:{} '.format(pmetric, performance_metrics[pmetric])
if verbose:
print_info(error_string_2)
# Forget the distances ids for this execution
del distances_ids
return error_string_1 + ',' + error_string_2