def evaluate(net, data_loader): correct = 0 total = 0 net.reset() target_scores = [] non_target_scores = [] for data in tqdm(data_loader): inputs, output = data[0], data[2] mask, scores = gate_activation_ce(net, inputs) selected_score = scores[mask] if selected_score.size == 0: scores = 1 / 7 * np.ones(7) else: xo = np.sum(selected_score, axis=0) / selected_score.size scores = softmax(xo) total += 1 correct += (scores.argmax() == output) if output == 0: target_scores.append(scores[0]) else: non_target_scores.append(scores[0]) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return float(correct) / total, eer
def evaluate(net, data_loader): """ compute the eer equal error rate :param net: network :param data_loader: test dataset, contains audio files in a numpy array format :return eer """ target_scores = [] non_target_scores = [] for data in tqdm(data_loader): net.reset() sample_input, output = data[0], data[1] xo = gate_lfcc(net, sample_input) if output == 1: target_scores.append(xo) else: non_target_scores.append(xo) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return eer
def evaluate_acc_eer(net, data_loader): """ compute the eer equal error rate and the accuracy :param net: network :param data_loader: test dataset, contains audio files in a numpy array format :return eer and accuracy """ correct = 0 total = 0 net.reset() target_scores = [] non_target_scores = [] for data in tqdm(data_loader): sample_input, output = data[0], data[1] sample_input = whiten(sample_input) mask, score = gate_activation(net, sample_input) selected_score = score[mask] if selected_score.size == 0: xo = 0.5 else: xo = np.sum(selected_score) / selected_score.size total += 1 correct += ((xo > 0.5) == output) if output == 1: target_scores.append(xo) else: non_target_scores.append(xo) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return float(correct) / total, eer
def evaluate_eer_acc(g, conf, data, backprop=False, use_gate=True, loading_bar=True): """ returns the equal error rate and the accuracy """ predictions, targets = feed_and_predict(data, g, conf, backprop, use_gate=use_gate, loading_bar=loading_bar) accuracy = np.mean(np.abs(predictions - targets) < 0.5) target_scores = predictions[targets == 1] non_target_scores = predictions[targets == 0] target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return eer, accuracy
def eval_eer_gc(g, config, batch): jitter = 1e-8 net = neat_local.nn.RecurrentNet.create(g, config, device="cpu", dtype=torch.float32) net.reset() input, output = batch # input: batch x t x BIN; output: batch input = input.transpose(0, 1) # input: t x batch x BIN batch_size = output.shape[0] norm = torch.zeros(batch_size) contribution = torch.zeros(batch_size) for input_t in input: xo = net.activate(input_t) # batch x 2 score = xo[:, 1] # batch confidence = xo[:, 0] # batch contribution += score * confidence # batch norm += confidence # batch predictions = contribution / (norm + jitter) # batch target_scores = predictions[output == 1] non_target_scores = predictions[output == 0] target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return eer
def eval_genomes(genomes, config_): """ Most important part of NEAT since it is here that we adapt NEAT to our problem. We tell what is the phenotype of a genome and how to calculate its fitness (same idea than a loss) :param config_: config from the config file :param genomes: list of all the genomes to get evaluated """ for _, genome in tqdm(genomes): net = neat.nn.RecurrentNetwork.create(genome, config_) target_scores = [] non_target_scores = [] for data in trainloader: inputs, output = data[0], data[1] net.reset() mask, score = gate_activation(net, inputs) selected_score = score[mask] if selected_score.size == 0: xo = 0.5 else: xo = np.sum(selected_score) / selected_score.size if output == 1: target_scores.append(xo) else: non_target_scores.append(xo) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) genome.fitness = 2 * (.5 - eer)
def evaluate(net, data_loader): correct = 0 total = 0 net.reset() target_scores = [] non_target_scores = [] for data in tqdm(data_loader): inputs, output = data[0], data[1] """ score_weight, score = gate_activation(net, inputs) selected_score = score[mask] selected_score = gate_activation(net, inputs) if selected_score.size == 0: xo = 0.5 else: xo = np.sum(selected_score) / selected_score.size """ selected_score = gate_average(net, inputs) xo = np.sum(selected_score) / selected_score.size total += 1 correct += ((xo > 0.5) == output) if output == 1: target_scores.append(xo) else: non_target_scores.append(xo) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return target_scores, non_target_scores, float(correct) / total, eer
def compute_eer(target_scores, non_target_scores): """ Return the equal error rate from the scores :param target_scores: numpy array scores of the bonafide files :param non_target_scores: numpy array scores of the spoofed files :return: eer equal error rate """ target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return eer
def evaluate_genome(g, conf, batches): tgs = [] ntgs = [] for batch in batches: tg, ntg = evaluate_batch(g, conf, batch) tgs.append(tg) ntgs.append(ntg) target_scores = np.concatenate(tgs) non_target_scores = np.concatenate(ntgs) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return eer
def eval_genome_eer(g, conf, batch, backprop=False, use_gate=True): """ Same than eval_genomes() but for 1 genome. This function is used for parallel evaluation. The input is already preprocessed with shape batch_size x t x bins t: index of the windows used for the pre-processing bins: number of features extracted --> corresponds to the number of input neurons of the recurrent net Here the fitness function is the equal error rate """ # inputs: batch_size x t x bins # outputs: batch_size inputs, targets = batch # inputs: t x batch_size x bins inputs = inputs.transpose(0, 1) net = neat_local.nn.RecurrentNet.create(g, conf, device="cpu", dtype=torch.float32) assert not backprop net.reset(len(targets)) contribution = torch.zeros(len(targets)) norm = torch.zeros(len(targets)) for input_t in inputs: # input_t: batch_size x bins xo = net.activate(input_t) # batch_size x 2 score = xo[:, 1] confidence = xo[:, 0] if use_gate else torch.ones_like(score) contribution += score * confidence # batch_size norm += confidence # batch_size jitter = 1e-8 prediction = contribution / (norm + jitter) # batch_size target_scores = prediction[targets == 1].numpy() # select with mask when target == 1 non_target_scores = prediction[ targets == 0].numpy() # select with mask when target == 0 pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return 2 * (.5 - eer)
def eval_genome(genome, config, batch_data): """ Most important part of NEAT since it is here that we adapt NEAT to our problem. We tell what is the phenotype of a genome and how to calculate its fitness (same idea than a loss) :param config: config from the config file :param genome: one genome to get evaluated :param batch_data: data to use to evaluate the genomes :return fitness: returns the fitness of the genome this version is intented to use ParallelEvaluator and should be much faster """ net = neat.nn.RecurrentNetwork.create(genome, config) target_scores = [] non_target_scores = [] for data in batch_data: inputs, output = data[0], data[1] inputs = whiten(inputs) net.reset() """ score_weight, score = gate_activation(net, inputs) selected_score = score[mask] selected_score = gate_activation(net, inputs) if selected_score.size == 0: xo = 0.5 else: xo = np.sum(selected_score) / selected_score.size """ xo = gate_mfcc(net, inputs) if output == 1: target_scores.append(xo) else: non_target_scores.append(xo) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return 2*(.5 - eer)
def evaluate(g, conf, data): """ returns the equal error rate, the accuracy (both multi class and binary class) and the confusion matrix """ predictions, targets = feed_and_predict_ce(data, g, conf) print(predictions) target_scores = predictions[targets == 0] non_target_scores = predictions[targets > 0] pmiss, pfa = rocch(target_scores[:, 0], non_target_scores[:, 0]) eer = rocch2eer(pmiss, pfa) predictions = np.argmax(predictions, axis=1) accuracy = (predictions == targets).sum() / len(data) c_matrix = confusion_matrix(targets, predictions, normalize="pred") predictions[predictions > 0] = 1 targets[targets > 0] = 1 anti_spoofing_accuracy = (predictions == targets).sum() / len(data) return eer, accuracy, anti_spoofing_accuracy, c_matrix
def eer_gc(genome, config, validation_set): """ function to use for selecting the grand xhampion of each generation :param genome: genome one genome to get evaluated :param config: file configuration file :param validation_set: ASVDataset data use for validation :return: """ net = neat.nn.RecurrentNetwork.create(genome, config) target_scores = [] non_target_scores = [] for data in tqdm(validation_set): inputs, output = data[0], data[1] inputs = whiten(inputs) net.reset() mask, score = gate_activation(net, inputs) selected_score = score[mask] if selected_score.size == 0: xo = 0.5 else: xo = np.sum(selected_score) / selected_score.size if output == 1: target_scores.append(xo) else: non_target_scores.append(xo) target_scores = np.array(target_scores) non_target_scores = np.array(non_target_scores) pmiss, pfa = rocch(target_scores, non_target_scores) eer = rocch2eer(pmiss, pfa) return eer