class TestingController():
def __init__(self, config, experiment_name, model, new_testing):
test_config = config['testing']
model_config = config['model']
self.model = model
self.reads = test_config['reads']
self.batch_size = test_config['batch_size']
self.use_assembler = test_config['signal_window_stride'] < model_config['signal_window_size']
self.save_predictions = test_config['save_predictions']
self.inference_controller = InferenceController()
self.file_controller = FileController(experiment_name)
self.results = [] if new_testing else self.file_controller.load_testing()
def pretty_print_progress(self, start, end, total):
try:
progress_str = '['
for i in range(0, total, max(total//50, 1)):
if i >= start and i < end:
progress_str += 'x'
else:
progress_str += '-'
progress_str += ']'
return progress_str
except:
return '[ Failed to get progress string ]'
def get_assembly(self, y_pred, iteration, read_id, bacteria):
if self.use_assembler == False:
return ''.join(y_pred)
assembly_path = self.file_controller.get_assembly_filepath(iteration, read_id, bacteria)
return assemble_and_output(assembly_path, y_pred)
def get_result(self, assembly, aligner, read_id, bacteria):
try:
besthit = next(aligner.map(assembly))
cigacc = 1-(besthit.NM/besthit.blen)
return self.get_result_dict(read_id, bacteria, besthit.ctg, besthit.r_st, besthit.r_en, besthit.NM, besthit.blen, besthit.cigar_str, cigacc)
except:
return self.get_result_dict(read_id, bacteria, 0, 0, 0, 0, 0, 0, 0)
def save_prediction(self, prediction_str, bacteria, read_id, iteration):
if self.save_predictions == False:
return
self.file_controller.save_prediction(prediction_str, bacteria, read_id, iteration)
def get_result_dict(self, read_id, bacteria, ctg, r_st, r_en, nm, blen, cig, cigacc):
return {
'read_id':read_id,
'bacteria':bacteria,
'ctg': ctg,
'r_st': r_st,
'r_en': r_en,
'NM': nm,
'blen': blen,
'cig': cig,
'cigacc': cigacc
}
def test(self, bacteria, generator, aligner):
print(f' - Testing {bacteria}.')
for i in range(self.reads):
try:
x, read_id = next(generator.get_batched_read())
start_time = time.time()
y_pred = []
for b in range(0, len(x), self.batch_size):
x_batch = x[b:b+self.batch_size]
progress_bar = self.pretty_print_progress(b, b+len(x_batch), len(x))
print(f"{i+1:02d}/{self.reads:02d} Predicting batch {progress_bar} {b:04d}-{b+len(x_batch):04d}/{len(x):04d}", end="\r")
y_batch_pred = self.inference_controller.predict_batch(x_batch, self.model)
y_batch_pred_strings = convert_to_base_strings(y_batch_pred)
y_pred.extend(y_batch_pred_strings)
assembly = self.get_assembly(y_pred, i, read_id, bacteria)
self.save_prediction(assembly, bacteria, read_id, i)
result = self.get_result(assembly, aligner, read_id, bacteria)
result['time'] = time.time() - start_time
self.results.append(result)
print(f"{i:02d}/{self.reads} Done | CIG ACC: {result['cigacc']}"+" "*70) # 70 blanks to overwrite the previous print
self.file_controller.save_testing(self.results)
except Exception as e:
print(e)
traceback.print_exc()
class EvaluationController():
def __init__(self, experiment_name):
self.file_controller = FileController(experiment_name)
def count_unmatched_reads(self):
data = self.get_accuracy_list(include_unmatched=True)
count = 0
for accuracy in data:
if accuracy == 0:
count += 1
return count
def count_unmatched_reads_per_bacteria(self):
data = self.get_accuracy_list_per_bacteria(include_unmatched=True)
counts = {}
for bacteria in data.keys():
count = 0
for accuracy in data[bacteria]:
if accuracy == 0:
count += 1
counts[bacteria] = count
return counts
def count_reads(self):
data = self.file_controller.load_testing()
return len(data)
def count_reads_per_bacteria(self):
data = self.get_accuracy_list_per_bacteria(include_unmatched=True)
counts = {}
for bacteria in data.keys():
counts[bacteria] = len(data[bacteria])
return counts
def get_accuracy_list(self, include_unmatched=True):
data = self.file_controller.load_testing()
accuracy = []
for measurement in data:
if include_unmatched == False and measurement['cigacc'] == 0:
continue
accuracy.append(measurement['cigacc'] * 100)
return accuracy
def get_accuracy_list_per_bacteria(self, include_unmatched=True):
data = self.file_controller.load_testing()
accuracies = {}
for measurement in data:
key = measurement['bacteria']
if key not in accuracies.keys():
accuracies[key] = []
accuracies[key].append(measurement['cigacc'] * 100)
return accuracies
def get_accuracy_mean(self, include_unmatched=True):
data = self.get_accuracy_list(include_unmatched)
data = np.array(data)
return data.mean()
def get_accuracy_mean_per_bacteria(self, include_unmatched=True):
data = self.get_accuracy_list_per_bacteria(include_unmatched)
means = {}
for bacteria in data.keys():
bacteria_data = data[bacteria]
bacteria_data = np.array(bacteria_data)
means[bacteria] = bacteria_data.mean()
return means
def get_total_testing_time(self):
total_time_seconds = 0
data = self.file_controller.load_testing()
for measurement in data:
total_time_seconds += measurement['time']
total_time_seconds = math.floor(total_time_seconds)
total_time = str(datetime.timedelta(seconds=total_time_seconds))
return total_time
def get_total_training_time(self):
data = self.file_controller.load_training()
data = np.array(data)
training_times = data[:, 3]
_, training_stop_idx = self.get_best_validation_loss()
total_time_seconds = training_times[
training_stop_idx] - training_times[0]
total_time_seconds = math.floor(total_time_seconds)
total_time = str(datetime.timedelta(seconds=total_time_seconds))
return total_time
def get_best_validation_loss(self):
data = self.file_controller.load_training()
data = np.array(data)
validation_losses = data[:, 2]
min_validation_idx = -1
min_validation_loss = 1e10
for i, validation_loss in enumerate(validation_losses):
if validation_loss < 0:
continue
if validation_loss < min_validation_loss:
min_validation_loss = validation_loss
min_validation_idx = i
return min_validation_loss, min_validation_idx
def get_SMDI(self):
data = self.file_controller.load_testing()
smdi_dict = {"S": 0, "M": 0, "D": 0, "I": 0}
total_length = 0
for measurement in data:
if measurement['cigacc'] == 0:
continue
total_length += measurement['blen']
cigar_string = measurement['cig']
result = re.findall(r'[\d]+[SMDI]', cigar_string) #[6M, 5D, ...]
for r in result:
amount = int(r[:-1]) # 6
key = r[-1] # M
smdi_dict[key] += amount
for key in 'SMDI':
smdi_dict[key] /= total_length
return smdi_dict
def get_SMDI_per_bacteria(self):
data = self.file_controller.load_testing()
smdi_dicts = {}
lenghts = {}
for measurement in data:
if measurement['cigacc'] == 0:
continue
bacteria = measurement['bacteria']
if bacteria not in smdi_dicts.keys():
smdi_dicts[bacteria] = {"S": 0, "M": 0, "D": 0, "I": 0}
lenghts[bacteria] = 0
lenghts[bacteria] += measurement['blen']
cigar_string = measurement['cig']
result = re.findall(r'[\d]+[SMDI]', cigar_string)
for r in result:
amount = int(r[:-1])
key = r[-1]
smdi_dicts[bacteria][key] += amount
for bacteria in smdi_dicts.keys():
for key in 'SMDI':
smdi_dicts[bacteria][key] /= lenghts[bacteria]
return smdi_dicts