def full_evaluation(images, root_output_path, dataset, labels):
log = section_logger(1)
evaluation_name = strftime("%Y%m%d%H%M", gmtime())
output_path = os.path.join(root_output_path, evaluation_name)
os.makedirs(output_path, exist_ok=True)
log('Save original images and predictions')
save_originals(output_path, images, labels)
log('Compute semantic similarity T-SNE visualizations')
compute_semantic_similarity(images, output_path, dataset)
log('Compute LRP metric')
lrp_metric = compute_lrp_metric(images)
log('Compute Uncertainty metrics')
uncertainty, _ = compute_uncertainty_metrics(images, output_path, labels)
log('Compute mAP')
map = compute_map(images)
log('Compute semantic similarity metrics')
ssm = compute_semantic_similarity_metric(images)
log('Save results')
save_full_analysis(images, lrp_metric, uncertainty, map, ssm)
def extract_distributions(data):
log = section_logger(1)
log('Extracting distributions ')
raw_data = data.annotations.join(data.images.set_index("image_id"),
on="image_id")[[
'file_name', 'name', 'class'
]]
raw_data.columns = ['image_id', 'name', 'class']
frequencies = raw_data.pivot_table(index='image_id',
columns='name',
values='class',
aggfunc='count',
fill_value=0.0)
frequencies['sum'] = frequencies.sum(1)
frequencies[frequencies.columns.difference(['image_id', 'sum'])] = \
frequencies[frequencies.columns.difference(['image_id', 'sum'])].div(frequencies["sum"], axis=0)
img_objs = extract_global_objs(frequencies)
obj_df = extract_objects(data.annotations)
return img_objs, obj_df
def extract_objects_df(idx, image, log_each=25000):
log = section_logger(1)
obj_data = {
'image_id': [],
'obj_id': [],
'label': [],
'x': [],
'y': [],
'w': [],
'h': []
}
if idx % log_each == 0:
log('Logging image [{}]'.format(idx))
for obj in image['objects']:
if len(obj['synsets']) != 1:
continue
obj_data['image_id'].append(image['image_id'])
obj_data['obj_id'].append(obj['object_id'])
obj_data['x'].append(obj['x'])
obj_data['y'].append(obj['y'])
obj_data['w'].append(obj['w'])
obj_data['h'].append(obj['h'])
obj_data['label'].append(obj['synsets'][0])
image_df = pd.DataFrame(data=obj_data)
return image_df
def generate_analysis(coco_path, voc_path, vg_path, output_path):
log = section_logger()
log('Loading definitions for COCO')
coco_definitions = COCOSet(coco_path)
log('Converting COCO data to dataframes')
coco_df = extract_coco_object_dataframe(coco_definitions)
log('Loading definitions for VOC')
voc_definitions = load_VOC(voc_path)
log('Converting VOC data to dataframes')
voc_df = extract_voc_object_dataframe(voc_definitions, limit=1e8)
log('Loading definitions for Visual Genome')
vg_definitions = load_visual_genome(vg_path)
log('Converting VG data to dataframes')
vg_df = extract_object_dataframe(vg_definitions, limit=1e8)
log('Saving dataframes')
voc_df.to_csv(os.path.join(output_path, 'voc_df.csv'))
vg_df.to_csv(os.path.join(output_path, 'vg_df.csv'))
coco_df.to_csv(os.path.join(output_path, 'coco_df.csv'))
def generate_rp_from_vg(output_path, input_path, perc):
vg.set_base(input_path)
section = section_logger()
section('Loading Visual Genome')
data = vg.load_visual_genome(os.path.join(input_path, conf.VG_DATA))
section('Extracting Image Region Information')
regions = extract_region_information(data)
def complete_evaluation(input_path, output_path, max_results=20):
log = section_logger(0)
log('Initializing data')
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
log('Load dataset')
dataset = AD20kFasterRCNN(
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktrain.csv',
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/training',
transforms=get_transform(train=True))
labels = dataset.labels
dataset_test = AD20kFasterRCNN(
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktest.csv',
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/validation',
transforms=get_transform(train=False),
labels=labels,
is_test=True)
log('Initializing model')
num_classes = len(dataset.labels)
model = load_frcnn(input_path, num_classes, device)
images = []
log('Compute model results')
current_results = 0
for id, objects in enumerate(dataset_test):
image = objects[0].to(device)
predictions = model([image])
clean_data = lambda data: {
k: data[k].tolist()
for k in data.keys() if k in ['boxes', 'labels', 'scores']
}
images.append({
'image': image.cpu().detach().numpy(),
'groundTruth': clean_data(objects[1]),
'predictions': clean_data(predictions[0])
})
del image
del predictions
if current_results > max_results:
break
else:
current_results += 1
log('Perform full evaluation')
full_evaluation(images, output_path, dataset, labels)
def semantic_similarity_evaluation(root_output_path, max_labels_hierarchy=800):
log = section_logger(0)
output_path = os.path.join(root_output_path, 'semantic_similarity')
os.makedirs(output_path, exist_ok=True)
log('Loading the dataset')
dataset = AD20kFasterRCNN(
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktrain.csv',
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/training',
transforms=get_transform(train=True))
nlp = spacy.load("en_core_web_md")
labels = dataset.labels
log('Creating Wordnet tree')
wordnet_tree, synsetmap = create_wordnet_tree(nlp, labels)
log('Drawing Wordnet data')
save_wordnet_data(output_path,
wordnet_tree,
synsetmap,
examples=200,
max_combinations=2000)
draw_full_hierarchy(output_path, wordnet_tree)
log('Generating tuples')
label_tuples = generate_tuples(labels, 10000)
log('Computing NLP similarities')
label_tuples['nlp_cosine'] = label_tuples.apply(
lambda row: nlp_similarity_fn(nlp, str(row['l1']), str(row['l2'])),
axis=1)
log('Computing Wordnet similarities')
label_tuples['wordnet_wup'] = label_tuples.apply(
lambda row: wordnet_similarity(str(row['l1']), str(row['l2']))['wup'],
axis=1)
label_tuples['wordnet_path'] = label_tuples.apply(
lambda row: wordnet_similarity(str(row['l1']), str(row['l2']))['path'],
axis=1)
label_tuples['path_to_ancestor'] = label_tuples.apply(
compute_path_to_ancestor(nlp), axis=1)
label_tuples.to_csv(os.path.join(output_path, 'distances.csv'))
log('Computing tSNE Clustering')
embeddings = np.array([nlp(label).vector for label in labels])
for perplexity in [50, 100, 150, 200]:
tsne_embedding = TSNE(n_components=2,
perplexity=perplexity).fit_transform(embeddings)
show_tsne(tsne_embedding, output_path, "ADE20k_{}".format(perplexity),
labels)
def extract_distributions(data, max_imgs=conf.MAX_LOADED_IMAGES):
log = section_logger(1)
log('Extracting distributions ')
global_objects = ms.Multiset()
occurrences = ms.Multiset()
images = []
for i, image in enumerate(data['objects']):
if i > max_imgs:
break
if not check_image_exists(image['image_id']):
print('Skipping unexistent image: {}.jpg'.format(
image['image_id']))
if i % 25 == 0:
log("Processing image {}".format(i))
image_objs = {}
image_objs['id'] = image['image_id']
objs = ms.Multiset()
objs_pd = {}
total = 0
appeared = set()
for j, obj in enumerate(image['objects']):
if len(obj['names']) > 0:
obj_name = obj['names'][0]
obj_name = simplify(obj_name)
if len(obj_name.strip()) == 0:
continue
total += 1
appeared.add(obj_name)
global_objects.add(obj_name)
objs.add(obj_name)
for obj in appeared:
occurrences.add(obj)
for key in objs.distinct_elements():
objs_pd[key] = objs.get(key, 0) / total
image_objs['pds'] = objs_pd
images.append(image_objs)
return global_objects, occurrences, images
def extract_data_analytics():
section = section_logger()
section('Extracting Visual Genome')
vg = load_visual_genome()
section('Generating DataFrames from the object files')
odf = extract_object_dataframe(vg, int(1e6))
section('Generating DataFrames from the relationship files')
rdf = extract_relationship_dataframe(vg, int(1e6))
section('Saving CSVs')
odf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_objects.csv'))
rdf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_relationships.csv'))
def compute_knowledge_graph():
section = section_logger()
section('Loading JSON files from dataset')
vg = load_visual_genome()
section('Generating Knowledge Graph')
vi_graph, synset_ids = extract_knowledge_graph(vg, limit=1e10)
section('Annotating with Wordnet synsets')
wn_graph, synset_ids = add_wordnet_synsets(vi_graph, synset_ids)
section('Saving graph')
save_graph_info(wn_graph, synset_ids)
return wn_graph
def add_wordnet_synsets(graph, synset_ids, report=2e5):
old_synset_ids = dict(synset_ids)
section = section_logger(1)
counter = 0
for syn_name, syn_id in old_synset_ids.items():
counter += 1
if counter % report == 0:
section("Loaded relationships in {} images".format(counter))
wordnet_syn = wordnet.synset(syn_name)
extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hypernyms', 'generalize.v.01')
extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hyponyms', 'specialize.v.01')
return graph, synset_ids
def create_restricted_genome(input_path, output_path, restrictions):
vg.set_base(input_path)
section = section_logger()
section('Loading Visual Genome')
data = vg.load_visual_genome(os.path.join(input_path, conf.VG_DATA))
section('Creating image dataframes')
image_df = create_images_df(data)
section('Creating object dataframes')
objects_df = get_objects_df(data, image_df)
section('Filter objects')
filtered_df = filter_objects(objects_df, restrictions)
section('Save results')
save_results(output_path, filtered_df)
def generate_opd_from_ad20k(input_path, output_path, top_objects, perc):
section = section_logger()
section('Loading AD20k Dataset')
ad20k_definitions = load_or_dataset(input_path)
section('Creating distributions')
image_df, obj_df = extract_distributions(ad20k_definitions)
section('Saving Raw DataFrame')
save_raw_data(output_path, obj_df, image_df)
section('Filtering objects')
data_df = filter_top_objects(image_df, obj_df, top_objects)
splits = split_distributions(data_df, perc)
section('Saving final distribution')
save_distributions(output_path, splits)
def generate_or_dataset_from_ad20k(dataset_path, output_path):
section = section_logger()
section('Loading Global Mat file')
global_info = load_mat_file(dataset_path)
labels = global_info.iloc[:, 1]
section('Loading Training data')
training_df = load_df_from(os.path.join(dataset_path, TRAINING_PATH),
labels)
section('Loading Test data')
test_df = load_df_from(os.path.join(dataset_path, TEST_PATH), labels)
section('Saving data to CSV')
save_df(output_path, training_df, test_df, global_info)
return None
def generate_vgopd_from_coco(output_path, input_path, top_objects, perc):
section = section_logger()
section('Loading VOC Dataset')
voc_definitions = PascalVOCOR(input_path)
section('Creating distributions')
image_df, obj_df = extract_distributions(voc_definitions)
section('Saving Raw DataFrame')
save_raw_data(output_path, obj_df, image_df)
section('Filtering objects')
data_df = filter_top_objects(image_df, obj_df, top_objects)
splits = split_distributions(data_df, perc)
section('Saving final distribution')
save_distributions(output_path, splits)
def generate_vgopd_from_restrictedvg(output_path, input_path, top_objects,
perc):
section = section_logger()
section('Loading RestrictedVG Dataset')
restricted_definitions = load_dataframe(input_path)
section('Creating distributions')
image_df, obj_df = extract_distributions(restricted_definitions)
section('Saving Raw DataFrame')
save_raw_data(output_path, obj_df, image_df)
section('Filtering objects')
data_df = filter_top_objects(image_df, obj_df, top_objects)
splits = split_distributions(data_df, perc)
section('Saving final distribution')
save_distributions(output_path, splits)
def extract_voc_object_dataframe(voc, limit=10, report=2e5):
section = section_logger(1)
data = []
counter = 0
for img in voc[0]:
if counter > limit:
break
else:
counter += 1
if counter % report == 0:
section("Loaded objects in {} images".format(counter))
rows = map(extract_voc_object_data(img['filename']), img['objects'])
data.extend(rows)
odf = pd.DataFrame(
data,
columns=['object_id', 'synsets', 'names', 'img', 'x', 'y', 'h', 'w'])
return odf
def extract_distributions(raw_data):
log = section_logger(1)
log('Extracting distributions ')
data = raw_data.loc[:, ['image_id', 'name', 'class']]
frequencies = data.pivot_table(index='image_id',
columns='name',
values='class',
aggfunc='count',
fill_value=0.0)
frequencies['sum'] = frequencies.sum(1)
frequencies[frequencies.columns.difference(['image_id', 'sum'])] = \
frequencies[frequencies.columns.difference(['image_id', 'sum'])].div(frequencies["sum"], axis=0)
img_objs = extract_global_objs(frequencies)
obj_df = extract_objects(raw_data)
return img_objs, obj_df
def generate_vgopd_from_vg(output_path, input_path, top_objects, perc):
vg.set_base(input_path)
section = section_logger()
section('Loading Visual Genome')
data = vg.load_visual_genome(os.path.join(input_path, conf.VG_DATA))
section('Creating distributions')
global_objects, occurrences, data_pd = extract_distributions(data)
section('Converting to DataFrame')
obj_df, image_df = convert_to_dataframe(data_pd, global_objects,
occurrences)
save_raw_data(output_path, obj_df, image_df)
section('Filtering objects')
data_df = filter_top_objects(image_df, obj_df, top_objects)
splits = split_distributions(data_df, perc)
section('Saving final distribution')
save_distributions(output_path, splits)
def extract_relationship_dataframe(vg, limit=10, report=2e5):
section = section_logger(1)
data = []
counter = 0
for img in vg['relationships']:
if counter > limit:
break
else:
counter += 1
if counter % report == 0:
section("Loaded relationships in {} images".format(counter))
rows = map(extract_relationship_data(img['image_id']), img['relationships'])
data.extend(list(rows))
rdf = pd.DataFrame(data, columns=['img', 'object', 'subject', 'predicate', 'synsets'])
return rdf
def extract_object_dataframe(vg, limit=10, report=2e5):
section = section_logger(1)
data = []
counter = 0
for img in vg['objects']:
if counter > limit:
break
else:
counter += 1
if counter % report == 0:
section("Loaded objects in {} images".format(counter))
current_image = vg['imageIdMap'][img['image_id']]
assert current_image['image_id'] == img['image_id'], 'Position in the list {} does not correspond to ID {}'.format(img['image_id'], current_image['image_id'])
rows = map(extract_object_data(img['image_id'], current_image['width'], current_image['height']), img['objects'])
data.extend(rows)
odf = pd.DataFrame(data, columns=['object_id', 'synsets', 'names', 'img', 'x', 'y', 'h', 'w'])
return odf
def extract_knowledge_graph(vg, limit=10, report=2e5):
section = section_logger(1)
counter = 0
synset_ids = {}
knowledge_graph = nx.DiGraph()
for img in vg['relationships']:
if counter > limit:
break
else:
counter += 1
if counter % report == 0:
section("Loaded relationships in {} images".format(counter))
for r in img['relationships']:
object = r['object']['synsets']
subject = r['subject']['synsets']
predicate = r['synsets']
for syn_obj in object:
for syn_sub in subject:
id_obj = add_key_to_dict(syn_obj, synset_ids)
id_sub = add_key_to_dict(syn_sub, synset_ids)
if len(predicate) == 1:
predicate_str = predicate[0]
else:
predicate_str = 'relation.n.01'
knowledge_graph.add_edge(id_obj, id_sub, predicate=predicate_str)
return knowledge_graph, synset_ids
def load_df_from(input_path, labels, max_images=1e6):
index_folders = os.listdir(input_path)
section = section_logger(1)
num_images = 0
result_df = None
for index in index_folders:
index_folder = os.path.join(input_path, index)
for category in os.listdir(index_folder):
category_folder = os.path.join(index_folder, category)
category_dfs = []
section('Processing category [{}]'.format(category))
for attribute_file_path in glob(
os.path.join(category_folder, '*.txt')):
attribute_file_name = attribute_file_path[attribute_file_path.
rindex('/') + 1:]
image_id = attribute_file_name[:attribute_file_name.index('.')]
image_id = image_id[:image_id.rindex('_')]
box_df = generate_box_information(category_folder, category,
image_id, labels)
category_dfs.append(box_df)
num_images += 1
if result_df is not None:
category_dfs.append(result_df)
result_df = pd.concat(category_dfs)
if num_images > max_images:
return result_df
return result_df
def train_model(model,
name,
output_path,
lr,
epoch_blocks,
num_epochs,
dataloaders,
optimizer,
scheduler,
loss_criterion,
device,
epoch_loss_history,
epoch_size=conf.STEPS_PER_EPOCH,
batch_log_freq=conf.BATCH_LOG_FREQUENCY,
skip_eval=False):
log = section_logger(1)
step = section_logger(2)
best_model_wts = copy.deepcopy(model.state_dict())
best_eae = float("inf")
for epoch_block in range(epoch_blocks):
for epoch in range(num_epochs):
log('Processing epoch {}'.format(epoch))
for phase in ['train', 'eval']:
log('Starting [{}] phase'.format(phase))
if phase == 'train':
model.train()
elif skip_eval:
continue
else:
model.eval()
running_loss = 0.0
batch_n = 0
log('Starting optimization')
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
loss = loss_criterion(outputs, labels)
if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item()
if batch_n % batch_log_freq == 0:
step('[{}]: {}'.format(batch_n, loss.item()))
if batch_n > epoch_size:
break
batch_n += 1
log('Calculating epoch stats')
epoch_loss = running_loss / batch_n
log('Saving best model')
if phase == 'val' and epoch_loss < best_eae:
best_eae = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
epoch_loss_history[phase].append(epoch_loss)
scheduler.step(epoch)
save_results(name, output_path, model, epoch_loss_history, lr)
log('Finishing epochs block')
model.load_state_dict(best_model_wts)
return model
def alternative_evaluation(input_path, output_path, n, log_step=1):
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
log = section_logger(0)
dataset = AD20kFasterRCNN(
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktrain.csv',
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/training',
transforms=get_transform(train=True))
dataset_test = AD20kFasterRCNN(
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/ade20ktest.csv',
'/home/dani/Documentos/Proyectos/Doctorado/Datasets/ADE20K/ADE20K_2016_07_26/images/validation',
transforms=get_transform(train=False),
labels=dataset.labels,
is_test=True)
nlp = spacy.load("en_core_web_md")
num_classes = len(dataset.labels)
model = load_frcnn(input_path, num_classes, device)
num_examples = 0
results = []
metaparameters = generate_metaparameters(10,
METAPARAMETER_DEF,
static=False)
for id, objects in enumerate(dataset_test):
image = objects[0].to(device)
predictions = model([image])
if id % log_step == 0:
log("Processing image [{}] with {} predictions and {} boxes".
format(id, len(predictions[0]['boxes']),
len(objects[1]['boxes'])))
for meta_id in range(len(metaparameters['iou_threshold'])):
iou_threshold = metaparameters['iou_threshold'][meta_id]
nlp_threshold = metaparameters['iou_threshold'][meta_id]
wnt_threshold = metaparameters['iou_threshold'][meta_id]
partial_results = nlp_evaluation(id,
objects,
predictions[0],
dataset.labels,
nlp,
iou_threshold=iou_threshold,
nlp_threshold=nlp_threshold,
wnt_threshold=wnt_threshold)
results.append(partial_results)
if num_examples < n:
num_examples += 1
else:
break
results_df = pd.concat(results)
results_df.to_csv(os.path.join(output_path, 'FRCNN_ad20k_evaluations.csv'))
def nlp_evaluation(id,
objects,
predictions,
labels,
nlp,
iou_threshold=0.5,
nlp_threshold=0.7,
wnt_threshold=0.5):
_, width, height = objects[0].shape
log = section_logger(2)
map_stats = {
"positives": 0,
"negatives": 0,
"background": 0,
"confidencesPos": [],
"confidencesNeg": []
}
nlp_stats = {
"positives": 0,
"negatives": 0,
"background": 0,
"confidencesPos": [],
"confidencesNeg": [],
"embedding_similarities": []
}
wordnet_stats = {
"positives": 0,
"negatives": 0,
"background": 0,
"confidencesPos": [],
"confidencesNeg": [],
"path_similarities": [],
"wup_similarities": []
}
for pred_id, pred_box in enumerate(predictions['boxes']):
matched_map = False
matched_nlp = False
matched_wordnet = False
confidence = float(
predictions['scores'][pred_id].cpu().detach().numpy())
for gt_id, gt_box in enumerate(objects[1]['boxes']):
adjusted_box = adjust_box(pred_box, width, height)
gt_box = adjust_box(gt_box, width, height)
pred_label_id = predictions['labels'][pred_id]
gt_label_id = objects[1]['labels'][gt_id]
pred_label = labels[pred_label_id]
gt_label = labels[gt_label_id]
iou_pred = IoU(adjusted_box, gt_box)
if iou_pred > iou_threshold and pred_label == gt_label:
map_stats["positives"] += 1
map_stats["confidencesPos"].append(confidence)
else:
map_stats["negatives"] += 1
map_stats["confidencesNeg"].append(confidence)
nlp_similarity = nlp_similarity_fn(nlp, gt_label, pred_label)
nlp_stats["embedding_similarities"].append(nlp_similarity)
if iou_pred > iou_threshold and nlp_similarity > nlp_threshold:
nlp_stats["positives"] += 1
nlp_stats["confidencesPos"].append(confidence)
else:
nlp_stats["confidencesNeg"].append(confidence)
nlp_stats["negatives"] += 1
wnt_similarity = wordnet_similarity(pred_label, gt_label)
wordnet_stats["wup_similarities"].append(wnt_similarity['wup'])
if iou_pred > iou_threshold and wnt_similarity[
'path'] > wnt_threshold:
wordnet_stats["positives"] += 1
wordnet_stats["confidencesPos"].append(confidence)
else:
wordnet_stats["confidencesNeg"].append(confidence)
wordnet_stats["negatives"] += 1
if iou_pred > iou_threshold:
matched_map = True
matched_nlp = True
matched_wordnet = True
if not matched_map:
map_stats["background"] += 1
if not matched_nlp:
nlp_stats["background"] += 1
if not matched_wordnet:
wordnet_stats["background"] += 1
map_stats = clean_stats_for_numpy(map_stats)
wordnet_stats = clean_stats_for_numpy(wordnet_stats)
nlp_stats = clean_stats_for_numpy(nlp_stats)
image_stats = [
id,
len(predictions['boxes']),
len(objects[1]['boxes']),
np.mean(wordnet_stats['wup_similarities']),
np.std(wordnet_stats['wup_similarities']),
np.mean(wordnet_stats['path_similarities']),
np.std(wordnet_stats['path_similarities']),
np.mean(nlp_stats['embedding_similarities']),
np.std(nlp_stats['embedding_similarities']),
np.mean(wordnet_stats['confidencesPos']),
np.std(wordnet_stats['confidencesPos']),
np.mean(wordnet_stats['confidencesNeg']),
np.std(wordnet_stats['confidencesNeg']),
np.mean(nlp_stats['confidencesPos']),
np.std(nlp_stats['confidencesPos']),
np.mean(nlp_stats['confidencesNeg']),
np.std(nlp_stats['confidencesNeg']),
np.mean(map_stats['confidencesPos']),
np.std(map_stats['confidencesPos']),
np.mean(map_stats['confidencesNeg']),
np.std(map_stats['confidencesNeg']), iou_threshold, nlp_threshold,
wnt_threshold, map_stats['positives'], map_stats['negatives'],
map_stats['background'], nlp_stats['positives'],
nlp_stats['negatives'], nlp_stats['background'],
wordnet_stats['positives'], wordnet_stats['negatives'],
wordnet_stats['background']
]
return pd.DataFrame(
[image_stats],
columns=[
'id', 'predictions', 'objects', 'wordnet.mean_wpu',
'wordnet.sd_wpu', 'wordnet.mean_path', 'wordnet.sd_path',
'nlp.mean_embbeding', 'nlp.sd_embedding',
'wordnet.mean_confidence_pos', 'wordnet.sd_confidence_pos',
'wordnet.mean_confidence_neg', 'wordnet.sd_confidence_neg',
'nlp.mean_confidence_pos', 'nlp.sd_confidence_pos',
'nlp.mean_confidence_neg', 'nlp.sd_confidence_neg',
'map.mean_confidence_pos', 'map.sd_confidence_pos',
'map.mean_confidence_neg', 'map.sd_confidence_neg',
'iou_threshold', 'nlp_threshold', 'wnt_threshold', 'map_pos',
'map_neg', 'map_back', 'nlp_pos', 'nlp_neg', 'nlp_back',
'wordnet_pos', 'wordnet_neg', 'wordnet_back'
])
def compute_uncertainty_metrics(predictions_list,
output_path,
labels,
epsilon=1e-2):
# TODOlog = section_logger(0): add logs to detect the slow step
log = section_logger(2)
accumulator_q = 0
accumulator_quot = 0
for pred_id, predictions in enumerate(predictions_list):
log('Processing uncertainty for image {}'.format(pred_id))
_, height, width = predictions['image'].shape
pairs = []
pairs_spatial = []
ground_truth_list = predictions['groundTruth']
detections_list = predictions['predictions']
log('Computing spatial and label Qs')
for gt_id in range(len(ground_truth_list['labels'])):
pair_row = []
pair_spatial_row = []
gt = extract_boxes(ground_truth_list, gt_id, width, height,
epsilon)
for det_id in range(len(detections_list['labels'])):
prediction = extract_boxes(detections_list, det_id, width,
height, epsilon)
spatial_q = compute_spatial_quality(gt, prediction)
label_q = compute_label_quality(gt, prediction)
pPDQ = np.sqrt(spatial_q * label_q)
pair_row.append(pPDQ)
pair_spatial_row.append(spatial_q)
pairs.append(pair_row)
pairs_spatial.append(pair_spatial_row)
log('Finding optimal assignment')
optimal_assignment = find_optimal_assignment(pairs)
optimal_spatial_assignment = find_optimal_assignment(pairs_spatial)
optimally_assigned_predictions = {
key: [
predictions['predictions'][key][assignment[1]]
for assignment in optimal_assignment
]
for key in ['scores', 'labels', 'boxes']
}
optimally_assigned_spatial_predictions = {
key: [
predictions['predictions'][key][assignment[1]]
for assignment in optimal_spatial_assignment
]
for key in ['scores', 'labels', 'boxes']
}
log('Computing confusion metrics')
confusion_metrics = compute_confusion_metrics(predictions,
optimal_assignment)
log('Save image with the assignment')
postfix = "_".join([str(metric) for metric in confusion_metrics])
integral_image = torch.tensor(predictions['image'] *
255).numpy().astype(np.int32)
integral_image = np.transpose(integral_image, (1, 2, 0))
log('Saving images')
show_objects_in_image(output_path,
integral_image,
predictions['groundTruth'],
postfix,
"{}_uncertainty_both".format(pred_id),
labels,
predictions=optimally_assigned_predictions,
prediction_classes=labels)
show_objects_in_image(
output_path,
integral_image,
predictions['groundTruth'],
postfix,
"{}_uncertainty_spatial".format(pred_id),
labels,
predictions=optimally_assigned_spatial_predictions,
prediction_classes=labels)
# Add to accumulators
accumulator_quot += sum(confusion_metrics)
accumulator_q += 1
# Compute global PDQ
return accumulator_q / accumulator_quot, optimal_assignment
