def evaluate(model, generator, data_type, devices, max_iteration, cuda): """Evaluate Args: model: object. generator: object. data_type: 'train' | 'validate'. devices: list of devices, e.g. ['a'] | ['a', 'b', 'c'] max_iteration: int, maximum iteration for validation cuda: bool. Returns: accuracy: float """ # Generate function generate_func = generator.generate_validate(data_type=data_type, devices=devices, shuffle=True, max_iteration=max_iteration) # Forward dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs_device = dict['output_device'] # (audios_num, classes_num) devices = dict['device'] # (audios_num, classes_num) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) # (audios_num,) predictions_device = np.argmax(outputs_device, axis=-1) # (audios_num,) # Evaluate classes_num = outputs.shape[-1] devices_num = outputs_device.shape[-1] loss = F.nll_loss(Variable(torch.Tensor(outputs)), Variable(torch.LongTensor(targets))).data.numpy() loss = float(loss) loss_device = F.nll_loss(Variable(torch.Tensor(outputs_device)), Variable(torch.LongTensor(devices))).data.numpy() loss_device = float(loss_device) confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) accuracy = calculate_accuracy(targets, predictions, classes_num, average='macro') accuracy_device = calculate_accuracy(devices, predictions_device, devices_num, average='macro') return accuracy, loss, accuracy_device, loss_device
def model_validate(classifier, class_wise_accuracy=False, plot_confusion_matrix=False): x_val, y_val = prepare_data(datatype='validate') if class_wise_accuracy: predict = classifier.predict(x_val) if plot_confusion_matrix: cm = calculate_confusion_matrix(y_val, predict, 10) plot_confusion_matrix2(cm, "svm", cfg.labels) class_wise_accuracy = calculate_accuracy(y_val, predict, 10) print_accuracy(class_wise_accuracy, cfg.labels) score = np.mean(class_wise_accuracy) else: score = classifier.score(x_val, y_val) print('The accuracy of validation: {:.4f}'.format(score)) return score
def evaluate(model, generator, data_type, devices, max_iteration): """Evaluate Args: model: object. generator: object. data_type: 'train' | 'validate'. devices: list of devices, e.g. ['a'] | ['a', 'b', 'c'] max_iteration: int, maximum iteration for validation Returns: accuracy: float """ # Generate function generate_func = generator.generate_validate(data_type=data_type, devices=devices, shuffle=True, max_iteration=max_iteration) # Forward dict = forward(model=model, generate_func=generate_func, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) # (audios_num,) # Evaluate classes_num = outputs.shape[-1] loss = K.mean( keras.metrics.sparse_categorical_crossentropy(K.constant(targets), K.constant(outputs))) loss = K.eval(loss) confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) accuracy = calculate_accuracy(targets, predictions, classes_num, average='macro') return accuracy, loss
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir subdir = args.subdir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename cuda = args.cuda labels = config.labels if 'mobile' in subdir: devices = ['a', 'b', 'c'] else: devices = ['a'] validation = True classes_num = len(labels) # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir, 'development.h5') dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold1_train.txt') dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_evaluate.txt'.format(holdout_fold)) model_path = os.path.join(workspace, 'models', subdir, filename, 'holdout_fold={}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) # Load model model = Model(classes_num) checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Predict & evaluate for device in devices: print('Device: {}'.format(device)) # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, dev_train_csv=dev_train_csv, dev_validate_csv=dev_validate_csv) generate_func = generator.generate_validate(data_type='validate', devices=device, shuffle=False) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) classes_num = outputs.shape[-1] # Evaluate confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) class_wise_accuracy = calculate_accuracy(targets, predictions, classes_num) # Print print_accuracy(class_wise_accuracy, labels) print('confusion_matrix: \n', confusion_matrix) logging.info('confusion_matrix: \n', confusion_matrix)
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir subdir = args.subdir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename # data_type = args.data_type labels = config.labels if 'mobile' in subdir: devices = ['a', 'b', 'c'] else: devices = ['a'] # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir, 'development_hpss_lrad.h5') dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_train.txt'.format(holdout_fold)) dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_evaluate.txt'.format(holdout_fold)) # model_path = os.path.join(workspace, 'models', subdir, filename, # 'holdout_fold={}'.format(holdout_fold), # 'md_{}_iters_max_attention2_2019-05-31 10:33:46.h5'.format(iteration)) # model_path = os.path.join(workspace, 'appendixes', # 'md_{}_iters_max_attention2_2019-05-31 00:48:09.h5'.format(iteration)) model_path = '/home/r506/Downloads/dcase2018_task1-master/models/' \ 'TUT-urban-acoustic-scenes-2018-development/main_keras/' \ 'new/DAN-DFF/md_9700_iters_max_attention2_78.1.h5' model = keras.models.load_model(model_path) # Predict & evaluate for device in devices: print('Device: {}'.format(device)) # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, dev_train_csv=dev_train_csv, dev_validate_csv=dev_validate_csv) generate_func = generator.generate_validate(data_type='validate', devices=device, shuffle=False) # Inference dict = forward(model=model, generate_func=generate_func, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) # 多分类交叉熵 predictions = np.argmax(outputs, axis=-1) classes_num = outputs.shape[-1] # Evaluate targets = np.argmax(targets, axis=-1) confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) class_wise_accuracy = calculate_accuracy(targets, predictions, classes_num) # Print print_accuracy(class_wise_accuracy, labels) print('confusion_matrix: \n', confusion_matrix) # Plot confusion matrix # plot_confusion_matrix( # confusion_matrix, # title='Device {}'.format(device.upper()), # labels=labels, # values=class_wise_accuracy) np.save('data5', confusion_matrix) plot_confusion_matrix2( confusion_matrix, title='The best performance of the proposed DAN-DFF method', labels=labels, )
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir subdir = args.subdir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename cuda = args.cuda labels = config.labels if 'mobile' in subdir: devices = ['a', 'b', 'c'] else: devices = ['a'] validation = True classes_num = len(labels) devices_num = len(devices) # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir, 'development.h5') dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold1_train.csv') dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_evaluate.csv'.format(holdout_fold)) model_path = os.path.join(workspace, 'models', subdir, filename, 'holdout_fold={}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) # Load model model = Model(classes_num, devices_num, cond_layer) checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Predict & evaluate for device in devices: print('Device: {}'.format(device)) # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, dev_train_csv=dev_train_csv, dev_validate_csv=dev_validate_csv) generate_func = generator.generate_validate(data_type='validate', devices=device, shuffle=False) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) # (outputs, targets, audio_names, outputs_heatmap) = forward_heatmap(model=model,################### # generate_func=generate_func, # cuda=cuda, # has_target=True) predictions = np.argmax(outputs, axis=-1) classes_num = outputs.shape[-1] ########################################################################## # heatmaps = [] # classes = [] # for i in range(0, len(predictions)): # pred_num = predictions[i] # if pred_num == targets[i]: # if not (pred_num in classes): # classes.append(pred_num) # print 'classes:' # print classes # logging.info('\n') # logging.info(outputs_heatmap[i][pred_num]) # logging.info('class num: ') # logging.info(pred_num) # heatmaps.append(outputs_heatmap[i][pred_num]) ############################################################################ # Evaluate confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) class_wise_accuracy = calculate_accuracy(targets, predictions, classes_num) # save # np.save(os.path.join(workspace, 'logs', 'main_pytorch', str(device)+"heatmap.npy"),heatmaps)############## # np.save(os.path.join(workspace, 'logs', 'main_pytorch', str(device)+"confusionMat.npy"),confusion_matrix)######### # Print print_accuracy(class_wise_accuracy, labels) print('confusion_matrix: \n', confusion_matrix) logging.info('confusion_matrix: \n', confusion_matrix)