def full_evaluation(images, root_output_path, dataset, labels): log = section_logger(1) evaluation_name = strftime("%Y%m%d%H%M", gmtime()) output_path = os.path.join(root_output_path, evaluation_name) os.makedirs(output_path, exist_ok=True) log('Save original images and predictions') save_originals(output_path, images, labels) log('Compute semantic similarity T-SNE visualizations') compute_semantic_similarity(images, output_path, dataset) log('Compute LRP metric') lrp_metric = compute_lrp_metric(images) log('Compute Uncertainty metrics') uncertainty, _ = compute_uncertainty_metrics(images, output_path, labels) log('Compute mAP') map = compute_map(images) log('Compute semantic similarity metrics') ssm = compute_semantic_similarity_metric(images) log('Save results') save_full_analysis(images, lrp_metric, uncertainty, map, ssm)
def extract_distributions(data): log = section_logger(1) log('Extracting distributions ') raw_data = data.annotations.join(data.images.set_index("image_id"), on="image_id")[[ 'file_name', 'name', 'class' ]] raw_data.columns = ['image_id', 'name', 'class'] frequencies = raw_data.pivot_table(index='image_id', columns='name', values='class', aggfunc='count', fill_value=0.0) frequencies['sum'] = frequencies.sum(1) frequencies[frequencies.columns.difference(['image_id', 'sum'])] = \ frequencies[frequencies.columns.difference(['image_id', 'sum'])].div(frequencies["sum"], axis=0) img_objs = extract_global_objs(frequencies) obj_df = extract_objects(data.annotations) return img_objs, obj_df
def extract_objects_df(idx, image, log_each=25000): log = section_logger(1) obj_data = { 'image_id': [], 'obj_id': [], 'label': [], 'x': [], 'y': [], 'w': [], 'h': [] } if idx % log_each == 0: log('Logging image [{}]'.format(idx)) for obj in image['objects']: if len(obj['synsets']) != 1: continue obj_data['image_id'].append(image['image_id']) obj_data['obj_id'].append(obj['object_id']) obj_data['x'].append(obj['x']) obj_data['y'].append(obj['y']) obj_data['w'].append(obj['w']) obj_data['h'].append(obj['h']) obj_data['label'].append(obj['synsets'][0]) image_df = pd.DataFrame(data=obj_data) return image_df
def generate_analysis(coco_path, voc_path, vg_path, output_path): log = section_logger() log('Loading definitions for COCO') coco_definitions = COCOSet(coco_path) log('Converting COCO data to dataframes') coco_df = extract_coco_object_dataframe(coco_definitions) log('Loading definitions for VOC') voc_definitions = load_VOC(voc_path) log('Converting VOC data to dataframes') voc_df = extract_voc_object_dataframe(voc_definitions, limit=1e8) log('Loading definitions for Visual Genome') vg_definitions = load_visual_genome(vg_path) log('Converting VG data to dataframes') vg_df = extract_object_dataframe(vg_definitions, limit=1e8) log('Saving dataframes') voc_df.to_csv(os.path.join(output_path, 'voc_df.csv')) vg_df.to_csv(os.path.join(output_path, 'vg_df.csv')) coco_df.to_csv(os.path.join(output_path, 'coco_df.csv'))
def generate_rp_from_vg(output_path, input_path, perc): vg.set_base(input_path) section = section_logger() section('Loading Visual Genome') data = vg.load_visual_genome(os.path.join(input_path, conf.VG_DATA)) section('Extracting Image Region Information') regions = extract_region_information(data)
def complete_evaluation(input_path, output_path, max_results=20): log = section_logger(0) log('Initializing data') device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') log('Load dataset') dataset = AD20kFasterRCNN( '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktrain.csv', '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/training', transforms=get_transform(train=True)) labels = dataset.labels dataset_test = AD20kFasterRCNN( '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktest.csv', '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/validation', transforms=get_transform(train=False), labels=labels, is_test=True) log('Initializing model') num_classes = len(dataset.labels) model = load_frcnn(input_path, num_classes, device) images = [] log('Compute model results') current_results = 0 for id, objects in enumerate(dataset_test): image = objects[0].to(device) predictions = model([image]) clean_data = lambda data: { k: data[k].tolist() for k in data.keys() if k in ['boxes', 'labels', 'scores'] } images.append({ 'image': image.cpu().detach().numpy(), 'groundTruth': clean_data(objects[1]), 'predictions': clean_data(predictions[0]) }) del image del predictions if current_results > max_results: break else: current_results += 1 log('Perform full evaluation') full_evaluation(images, output_path, dataset, labels)
def semantic_similarity_evaluation(root_output_path, max_labels_hierarchy=800): log = section_logger(0) output_path = os.path.join(root_output_path, 'semantic_similarity') os.makedirs(output_path, exist_ok=True) log('Loading the dataset') dataset = AD20kFasterRCNN( '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktrain.csv', '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/training', transforms=get_transform(train=True)) nlp = spacy.load("en_core_web_md") labels = dataset.labels log('Creating Wordnet tree') wordnet_tree, synsetmap = create_wordnet_tree(nlp, labels) log('Drawing Wordnet data') save_wordnet_data(output_path, wordnet_tree, synsetmap, examples=200, max_combinations=2000) draw_full_hierarchy(output_path, wordnet_tree) log('Generating tuples') label_tuples = generate_tuples(labels, 10000) log('Computing NLP similarities') label_tuples['nlp_cosine'] = label_tuples.apply( lambda row: nlp_similarity_fn(nlp, str(row['l1']), str(row['l2'])), axis=1) log('Computing Wordnet similarities') label_tuples['wordnet_wup'] = label_tuples.apply( lambda row: wordnet_similarity(str(row['l1']), str(row['l2']))['wup'], axis=1) label_tuples['wordnet_path'] = label_tuples.apply( lambda row: wordnet_similarity(str(row['l1']), str(row['l2']))['path'], axis=1) label_tuples['path_to_ancestor'] = label_tuples.apply( compute_path_to_ancestor(nlp), axis=1) label_tuples.to_csv(os.path.join(output_path, 'distances.csv')) log('Computing tSNE Clustering') embeddings = np.array([nlp(label).vector for label in labels]) for perplexity in [50, 100, 150, 200]: tsne_embedding = TSNE(n_components=2, perplexity=perplexity).fit_transform(embeddings) show_tsne(tsne_embedding, output_path, "ADE20k_{}".format(perplexity), labels)
def extract_distributions(data, max_imgs=conf.MAX_LOADED_IMAGES): log = section_logger(1) log('Extracting distributions ') global_objects = ms.Multiset() occurrences = ms.Multiset() images = [] for i, image in enumerate(data['objects']): if i > max_imgs: break if not check_image_exists(image['image_id']): print('Skipping unexistent image: {}.jpg'.format( image['image_id'])) if i % 25 == 0: log("Processing image {}".format(i)) image_objs = {} image_objs['id'] = image['image_id'] objs = ms.Multiset() objs_pd = {} total = 0 appeared = set() for j, obj in enumerate(image['objects']): if len(obj['names']) > 0: obj_name = obj['names'][0] obj_name = simplify(obj_name) if len(obj_name.strip()) == 0: continue total += 1 appeared.add(obj_name) global_objects.add(obj_name) objs.add(obj_name) for obj in appeared: occurrences.add(obj) for key in objs.distinct_elements(): objs_pd[key] = objs.get(key, 0) / total image_objs['pds'] = objs_pd images.append(image_objs) return global_objects, occurrences, images
def extract_data_analytics(): section = section_logger() section('Extracting Visual Genome') vg = load_visual_genome() section('Generating DataFrames from the object files') odf = extract_object_dataframe(vg, int(1e6)) section('Generating DataFrames from the relationship files') rdf = extract_relationship_dataframe(vg, int(1e6)) section('Saving CSVs') odf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_objects.csv')) rdf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_relationships.csv'))
def compute_knowledge_graph(): section = section_logger() section('Loading JSON files from dataset') vg = load_visual_genome() section('Generating Knowledge Graph') vi_graph, synset_ids = extract_knowledge_graph(vg, limit=1e10) section('Annotating with Wordnet synsets') wn_graph, synset_ids = add_wordnet_synsets(vi_graph, synset_ids) section('Saving graph') save_graph_info(wn_graph, synset_ids) return wn_graph
def add_wordnet_synsets(graph, synset_ids, report=2e5): old_synset_ids = dict(synset_ids) section = section_logger(1) counter = 0 for syn_name, syn_id in old_synset_ids.items(): counter += 1 if counter % report == 0: section("Loaded relationships in {} images".format(counter)) wordnet_syn = wordnet.synset(syn_name) extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hypernyms', 'generalize.v.01') extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hyponyms', 'specialize.v.01') return graph, synset_ids
def create_restricted_genome(input_path, output_path, restrictions): vg.set_base(input_path) section = section_logger() section('Loading Visual Genome') data = vg.load_visual_genome(os.path.join(input_path, conf.VG_DATA)) section('Creating image dataframes') image_df = create_images_df(data) section('Creating object dataframes') objects_df = get_objects_df(data, image_df) section('Filter objects') filtered_df = filter_objects(objects_df, restrictions) section('Save results') save_results(output_path, filtered_df)
def generate_opd_from_ad20k(input_path, output_path, top_objects, perc): section = section_logger() section('Loading AD20k Dataset') ad20k_definitions = load_or_dataset(input_path) section('Creating distributions') image_df, obj_df = extract_distributions(ad20k_definitions) section('Saving Raw DataFrame') save_raw_data(output_path, obj_df, image_df) section('Filtering objects') data_df = filter_top_objects(image_df, obj_df, top_objects) splits = split_distributions(data_df, perc) section('Saving final distribution') save_distributions(output_path, splits)
def generate_or_dataset_from_ad20k(dataset_path, output_path): section = section_logger() section('Loading Global Mat file') global_info = load_mat_file(dataset_path) labels = global_info.iloc[:, 1] section('Loading Training data') training_df = load_df_from(os.path.join(dataset_path, TRAINING_PATH), labels) section('Loading Test data') test_df = load_df_from(os.path.join(dataset_path, TEST_PATH), labels) section('Saving data to CSV') save_df(output_path, training_df, test_df, global_info) return None
def generate_vgopd_from_coco(output_path, input_path, top_objects, perc): section = section_logger() section('Loading VOC Dataset') voc_definitions = PascalVOCOR(input_path) section('Creating distributions') image_df, obj_df = extract_distributions(voc_definitions) section('Saving Raw DataFrame') save_raw_data(output_path, obj_df, image_df) section('Filtering objects') data_df = filter_top_objects(image_df, obj_df, top_objects) splits = split_distributions(data_df, perc) section('Saving final distribution') save_distributions(output_path, splits)
def generate_vgopd_from_restrictedvg(output_path, input_path, top_objects, perc): section = section_logger() section('Loading RestrictedVG Dataset') restricted_definitions = load_dataframe(input_path) section('Creating distributions') image_df, obj_df = extract_distributions(restricted_definitions) section('Saving Raw DataFrame') save_raw_data(output_path, obj_df, image_df) section('Filtering objects') data_df = filter_top_objects(image_df, obj_df, top_objects) splits = split_distributions(data_df, perc) section('Saving final distribution') save_distributions(output_path, splits)
def extract_voc_object_dataframe(voc, limit=10, report=2e5): section = section_logger(1) data = [] counter = 0 for img in voc[0]: if counter > limit: break else: counter += 1 if counter % report == 0: section("Loaded objects in {} images".format(counter)) rows = map(extract_voc_object_data(img['filename']), img['objects']) data.extend(rows) odf = pd.DataFrame( data, columns=['object_id', 'synsets', 'names', 'img', 'x', 'y', 'h', 'w']) return odf
def extract_distributions(raw_data): log = section_logger(1) log('Extracting distributions ') data = raw_data.loc[:, ['image_id', 'name', 'class']] frequencies = data.pivot_table(index='image_id', columns='name', values='class', aggfunc='count', fill_value=0.0) frequencies['sum'] = frequencies.sum(1) frequencies[frequencies.columns.difference(['image_id', 'sum'])] = \ frequencies[frequencies.columns.difference(['image_id', 'sum'])].div(frequencies["sum"], axis=0) img_objs = extract_global_objs(frequencies) obj_df = extract_objects(raw_data) return img_objs, obj_df
def generate_vgopd_from_vg(output_path, input_path, top_objects, perc): vg.set_base(input_path) section = section_logger() section('Loading Visual Genome') data = vg.load_visual_genome(os.path.join(input_path, conf.VG_DATA)) section('Creating distributions') global_objects, occurrences, data_pd = extract_distributions(data) section('Converting to DataFrame') obj_df, image_df = convert_to_dataframe(data_pd, global_objects, occurrences) save_raw_data(output_path, obj_df, image_df) section('Filtering objects') data_df = filter_top_objects(image_df, obj_df, top_objects) splits = split_distributions(data_df, perc) section('Saving final distribution') save_distributions(output_path, splits)
def extract_relationship_dataframe(vg, limit=10, report=2e5): section = section_logger(1) data = [] counter = 0 for img in vg['relationships']: if counter > limit: break else: counter += 1 if counter % report == 0: section("Loaded relationships in {} images".format(counter)) rows = map(extract_relationship_data(img['image_id']), img['relationships']) data.extend(list(rows)) rdf = pd.DataFrame(data, columns=['img', 'object', 'subject', 'predicate', 'synsets']) return rdf
def extract_object_dataframe(vg, limit=10, report=2e5): section = section_logger(1) data = [] counter = 0 for img in vg['objects']: if counter > limit: break else: counter += 1 if counter % report == 0: section("Loaded objects in {} images".format(counter)) current_image = vg['imageIdMap'][img['image_id']] assert current_image['image_id'] == img['image_id'], 'Position in the list {} does not correspond to ID {}'.format(img['image_id'], current_image['image_id']) rows = map(extract_object_data(img['image_id'], current_image['width'], current_image['height']), img['objects']) data.extend(rows) odf = pd.DataFrame(data, columns=['object_id', 'synsets', 'names', 'img', 'x', 'y', 'h', 'w']) return odf
def extract_knowledge_graph(vg, limit=10, report=2e5): section = section_logger(1) counter = 0 synset_ids = {} knowledge_graph = nx.DiGraph() for img in vg['relationships']: if counter > limit: break else: counter += 1 if counter % report == 0: section("Loaded relationships in {} images".format(counter)) for r in img['relationships']: object = r['object']['synsets'] subject = r['subject']['synsets'] predicate = r['synsets'] for syn_obj in object: for syn_sub in subject: id_obj = add_key_to_dict(syn_obj, synset_ids) id_sub = add_key_to_dict(syn_sub, synset_ids) if len(predicate) == 1: predicate_str = predicate[0] else: predicate_str = 'relation.n.01' knowledge_graph.add_edge(id_obj, id_sub, predicate=predicate_str) return knowledge_graph, synset_ids
def load_df_from(input_path, labels, max_images=1e6): index_folders = os.listdir(input_path) section = section_logger(1) num_images = 0 result_df = None for index in index_folders: index_folder = os.path.join(input_path, index) for category in os.listdir(index_folder): category_folder = os.path.join(index_folder, category) category_dfs = [] section('Processing category [{}]'.format(category)) for attribute_file_path in glob( os.path.join(category_folder, '*.txt')): attribute_file_name = attribute_file_path[attribute_file_path. rindex('/') + 1:] image_id = attribute_file_name[:attribute_file_name.index('.')] image_id = image_id[:image_id.rindex('_')] box_df = generate_box_information(category_folder, category, image_id, labels) category_dfs.append(box_df) num_images += 1 if result_df is not None: category_dfs.append(result_df) result_df = pd.concat(category_dfs) if num_images > max_images: return result_df return result_df
def train_model(model, name, output_path, lr, epoch_blocks, num_epochs, dataloaders, optimizer, scheduler, loss_criterion, device, epoch_loss_history, epoch_size=conf.STEPS_PER_EPOCH, batch_log_freq=conf.BATCH_LOG_FREQUENCY, skip_eval=False): log = section_logger(1) step = section_logger(2) best_model_wts = copy.deepcopy(model.state_dict()) best_eae = float("inf") for epoch_block in range(epoch_blocks): for epoch in range(num_epochs): log('Processing epoch {}'.format(epoch)) for phase in ['train', 'eval']: log('Starting [{}] phase'.format(phase)) if phase == 'train': model.train() elif skip_eval: continue else: model.eval() running_loss = 0.0 batch_n = 0 log('Starting optimization') for inputs, labels in dataloaders[phase]: inputs = inputs.to(device) labels = labels.to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): outputs = model(inputs) loss = loss_criterion(outputs, labels) if phase == 'train': loss.backward() optimizer.step() running_loss += loss.item() if batch_n % batch_log_freq == 0: step('[{}]: {}'.format(batch_n, loss.item())) if batch_n > epoch_size: break batch_n += 1 log('Calculating epoch stats') epoch_loss = running_loss / batch_n log('Saving best model') if phase == 'val' and epoch_loss < best_eae: best_eae = epoch_loss best_model_wts = copy.deepcopy(model.state_dict()) epoch_loss_history[phase].append(epoch_loss) scheduler.step(epoch) save_results(name, output_path, model, epoch_loss_history, lr) log('Finishing epochs block') model.load_state_dict(best_model_wts) return model
def alternative_evaluation(input_path, output_path, n, log_step=1): device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') log = section_logger(0) dataset = AD20kFasterRCNN( '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktrain.csv', '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/training', transforms=get_transform(train=True)) dataset_test = AD20kFasterRCNN( '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktest.csv', '/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/validation', transforms=get_transform(train=False), labels=dataset.labels, is_test=True) nlp = spacy.load("en_core_web_md") num_classes = len(dataset.labels) model = load_frcnn(input_path, num_classes, device) num_examples = 0 results = [] metaparameters = generate_metaparameters(10, METAPARAMETER_DEF, static=False) for id, objects in enumerate(dataset_test): image = objects[0].to(device) predictions = model([image]) if id % log_step == 0: log("Processing image [{}] with {} predictions and {} boxes". format(id, len(predictions[0]['boxes']), len(objects[1]['boxes']))) for meta_id in range(len(metaparameters['iou_threshold'])): iou_threshold = metaparameters['iou_threshold'][meta_id] nlp_threshold = metaparameters['iou_threshold'][meta_id] wnt_threshold = metaparameters['iou_threshold'][meta_id] partial_results = nlp_evaluation(id, objects, predictions[0], dataset.labels, nlp, iou_threshold=iou_threshold, nlp_threshold=nlp_threshold, wnt_threshold=wnt_threshold) results.append(partial_results) if num_examples < n: num_examples += 1 else: break results_df = pd.concat(results) results_df.to_csv(os.path.join(output_path, 'FRCNN_ad20k_evaluations.csv'))
def nlp_evaluation(id, objects, predictions, labels, nlp, iou_threshold=0.5, nlp_threshold=0.7, wnt_threshold=0.5): _, width, height = objects[0].shape log = section_logger(2) map_stats = { "positives": 0, "negatives": 0, "background": 0, "confidencesPos": [], "confidencesNeg": [] } nlp_stats = { "positives": 0, "negatives": 0, "background": 0, "confidencesPos": [], "confidencesNeg": [], "embedding_similarities": [] } wordnet_stats = { "positives": 0, "negatives": 0, "background": 0, "confidencesPos": [], "confidencesNeg": [], "path_similarities": [], "wup_similarities": [] } for pred_id, pred_box in enumerate(predictions['boxes']): matched_map = False matched_nlp = False matched_wordnet = False confidence = float( predictions['scores'][pred_id].cpu().detach().numpy()) for gt_id, gt_box in enumerate(objects[1]['boxes']): adjusted_box = adjust_box(pred_box, width, height) gt_box = adjust_box(gt_box, width, height) pred_label_id = predictions['labels'][pred_id] gt_label_id = objects[1]['labels'][gt_id] pred_label = labels[pred_label_id] gt_label = labels[gt_label_id] iou_pred = IoU(adjusted_box, gt_box) if iou_pred > iou_threshold and pred_label == gt_label: map_stats["positives"] += 1 map_stats["confidencesPos"].append(confidence) else: map_stats["negatives"] += 1 map_stats["confidencesNeg"].append(confidence) nlp_similarity = nlp_similarity_fn(nlp, gt_label, pred_label) nlp_stats["embedding_similarities"].append(nlp_similarity) if iou_pred > iou_threshold and nlp_similarity > nlp_threshold: nlp_stats["positives"] += 1 nlp_stats["confidencesPos"].append(confidence) else: nlp_stats["confidencesNeg"].append(confidence) nlp_stats["negatives"] += 1 wnt_similarity = wordnet_similarity(pred_label, gt_label) wordnet_stats["wup_similarities"].append(wnt_similarity['wup']) if iou_pred > iou_threshold and wnt_similarity[ 'path'] > wnt_threshold: wordnet_stats["positives"] += 1 wordnet_stats["confidencesPos"].append(confidence) else: wordnet_stats["confidencesNeg"].append(confidence) wordnet_stats["negatives"] += 1 if iou_pred > iou_threshold: matched_map = True matched_nlp = True matched_wordnet = True if not matched_map: map_stats["background"] += 1 if not matched_nlp: nlp_stats["background"] += 1 if not matched_wordnet: wordnet_stats["background"] += 1 map_stats = clean_stats_for_numpy(map_stats) wordnet_stats = clean_stats_for_numpy(wordnet_stats) nlp_stats = clean_stats_for_numpy(nlp_stats) image_stats = [ id, len(predictions['boxes']), len(objects[1]['boxes']), np.mean(wordnet_stats['wup_similarities']), np.std(wordnet_stats['wup_similarities']), np.mean(wordnet_stats['path_similarities']), np.std(wordnet_stats['path_similarities']), np.mean(nlp_stats['embedding_similarities']), np.std(nlp_stats['embedding_similarities']), np.mean(wordnet_stats['confidencesPos']), np.std(wordnet_stats['confidencesPos']), np.mean(wordnet_stats['confidencesNeg']), np.std(wordnet_stats['confidencesNeg']), np.mean(nlp_stats['confidencesPos']), np.std(nlp_stats['confidencesPos']), np.mean(nlp_stats['confidencesNeg']), np.std(nlp_stats['confidencesNeg']), np.mean(map_stats['confidencesPos']), np.std(map_stats['confidencesPos']), np.mean(map_stats['confidencesNeg']), np.std(map_stats['confidencesNeg']), iou_threshold, nlp_threshold, wnt_threshold, map_stats['positives'], map_stats['negatives'], map_stats['background'], nlp_stats['positives'], nlp_stats['negatives'], nlp_stats['background'], wordnet_stats['positives'], wordnet_stats['negatives'], wordnet_stats['background'] ] return pd.DataFrame( [image_stats], columns=[ 'id', 'predictions', 'objects', 'wordnet.mean_wpu', 'wordnet.sd_wpu', 'wordnet.mean_path', 'wordnet.sd_path', 'nlp.mean_embbeding', 'nlp.sd_embedding', 'wordnet.mean_confidence_pos', 'wordnet.sd_confidence_pos', 'wordnet.mean_confidence_neg', 'wordnet.sd_confidence_neg', 'nlp.mean_confidence_pos', 'nlp.sd_confidence_pos', 'nlp.mean_confidence_neg', 'nlp.sd_confidence_neg', 'map.mean_confidence_pos', 'map.sd_confidence_pos', 'map.mean_confidence_neg', 'map.sd_confidence_neg', 'iou_threshold', 'nlp_threshold', 'wnt_threshold', 'map_pos', 'map_neg', 'map_back', 'nlp_pos', 'nlp_neg', 'nlp_back', 'wordnet_pos', 'wordnet_neg', 'wordnet_back' ])
def compute_uncertainty_metrics(predictions_list, output_path, labels, epsilon=1e-2): # TODOlog = section_logger(0): add logs to detect the slow step log = section_logger(2) accumulator_q = 0 accumulator_quot = 0 for pred_id, predictions in enumerate(predictions_list): log('Processing uncertainty for image {}'.format(pred_id)) _, height, width = predictions['image'].shape pairs = [] pairs_spatial = [] ground_truth_list = predictions['groundTruth'] detections_list = predictions['predictions'] log('Computing spatial and label Qs') for gt_id in range(len(ground_truth_list['labels'])): pair_row = [] pair_spatial_row = [] gt = extract_boxes(ground_truth_list, gt_id, width, height, epsilon) for det_id in range(len(detections_list['labels'])): prediction = extract_boxes(detections_list, det_id, width, height, epsilon) spatial_q = compute_spatial_quality(gt, prediction) label_q = compute_label_quality(gt, prediction) pPDQ = np.sqrt(spatial_q * label_q) pair_row.append(pPDQ) pair_spatial_row.append(spatial_q) pairs.append(pair_row) pairs_spatial.append(pair_spatial_row) log('Finding optimal assignment') optimal_assignment = find_optimal_assignment(pairs) optimal_spatial_assignment = find_optimal_assignment(pairs_spatial) optimally_assigned_predictions = { key: [ predictions['predictions'][key][assignment[1]] for assignment in optimal_assignment ] for key in ['scores', 'labels', 'boxes'] } optimally_assigned_spatial_predictions = { key: [ predictions['predictions'][key][assignment[1]] for assignment in optimal_spatial_assignment ] for key in ['scores', 'labels', 'boxes'] } log('Computing confusion metrics') confusion_metrics = compute_confusion_metrics(predictions, optimal_assignment) log('Save image with the assignment') postfix = "_".join([str(metric) for metric in confusion_metrics]) integral_image = torch.tensor(predictions['image'] * 255).numpy().astype(np.int32) integral_image = np.transpose(integral_image, (1, 2, 0)) log('Saving images') show_objects_in_image(output_path, integral_image, predictions['groundTruth'], postfix, "{}_uncertainty_both".format(pred_id), labels, predictions=optimally_assigned_predictions, prediction_classes=labels) show_objects_in_image( output_path, integral_image, predictions['groundTruth'], postfix, "{}_uncertainty_spatial".format(pred_id), labels, predictions=optimally_assigned_spatial_predictions, prediction_classes=labels) # Add to accumulators accumulator_quot += sum(confusion_metrics) accumulator_q += 1 # Compute global PDQ return accumulator_q / accumulator_quot, optimal_assignment