class LabelEmbedding(object):
"""
The main/useful class to implement some of the ideas in : https://hal.inria.fr/hal-00815747v1/document
Given reasonable amout of data with dense predictions and getty tags we learn a mapping that can align them.
arg_max( dense_predictions*W*getty_labels^T )
"""
def __init__(self, config_file, force=False, reduced=False):
self.config_file = config_file
self.init = True
self.force = force
self._read_configuration()
self.init = self._verify_files()
self.is_mat = True
if not reduced:
self.io = EmbeddingIO(self.cfgs['path_logging_dir'])
else:
self.io = EmbeddingIO(None, True, True)
print 'Reduced functionality of Label Embedding'
#if
self.working_order = [(0, 'run_preprocess', self._run_preprocess),
(1, 'run_train', self._run_training),
(2, 'build_embedding', self._build_embedding),
(3, 'run_thresholds', self._run_thresholds),
(4, 'run_test', self._run_testing)]
self.train_method = {
'CFM': self._solve_closed_form,
'CFM_PC': self._solve_closed_form_pc,
'SGD': self._solve_sgd,
'WSABIE': self._solve_wsabie,
'CCA': self._solve_cca
}
lemb_mapping_keys = [d[0] for d in self.cfgs['lemb_mapping_list']]
lemb_mapping_values = [d[1] for d in self.cfgs['lemb_mapping_list']]
self.lemb_mapping = dict(zip(lemb_mapping_keys, lemb_mapping_values))
#def
"""
Check if all files and path exists.
From yaml config file there are variables beginning with path_
"""
def prepare_for_testing(self):
self._read_all_labels()
if not '_PC' in self.cfgs['train_method']:
self.W = self.io.read_weight_matrix(
self.cfgs['npz_files_loc']['weights'])
self.is_mat = True
self.threshold_map, self.black_list = self.io.read_threshold_file(
self.cfgs['npz_files_loc']['thresholds'])
self.threshold_map = self.threshold_map.tolist()
self.io.print_info(
'{0}/{1} concepts are black-listed (not enough validation data)'
.format(len(self.black_list), len(self.getty_labels)))
else:
self.W = self.io.read_all_emb(
self.getty_labels, self.cfgs['train_method'],
self.cfgs['npz_files_loc']['weights'])
self.is_mat = False
if self.cfgs['use_global_threshold']:
self.threshold_map, self.black_list = {
c: 0.5
for c in self.W.keys()
}, []
else:
self.threshold_map, self.black_list = self.io.read_threshold_file(
self.cfgs['npz_files_loc']['thresholds'])
#if
self.not_black_list = self.io.read_getty_labels(
self.cfgs['path_getty_label_file'],
self.cfgs['minimum_samples_for_validation'],
self.cfgs['getty_key_words_types'])
#if
if self.is_mat:
assert self.W.shape[1] == len(self.getty_labels)
self.io.print_info('Found weight {0}x{1} matrix'.format(
self.W.shape[0], self.W.shape[1]))
#if
def init_embedding(self, hedger_file, synset_names, getty_file,
emb_weights_file, emb_threshold_file):
self.all_wn_labels = self.io.read_hedged_labels(hedger_file)
self.leaf_wn_labels = synset_names
self.getty_labels = self.io.read_getty_labels(
getty_file, self.cfgs['getty_stop_list'],
self.cfgs['getty_key_words_types'])
print('Reading label embedding from {0}'.format(emb_weights_file))
if not '_PC' in self.cfgs['train_method']:
self.W = self.io.read_weight_matrix(emb_weights_file)
self.is_mat = True
self.threshold_map, self.black_list = self.io.read_threshold_file(
emb_threshold_file)
self.threshold_map = self.threshold_map.tolist()
self.io.print_info(
'{0}/{1} concepts are black-listed (not enough validation data)'
.format(len(self.black_list), len(self.getty_labels)))
else:
self.W = self.io.read_all_emb(self.getty_labels,
self.cfgs['train_method'],
emb_weights_file)
self.is_mat = False
if self.cfgs['use_global_threshold']:
self.threshold_map, self.black_list = {
c: 0.5
for c in self.W.keys()
}, []
else:
self.threshold_map, self.black_list = self.io.read_threshold_file(
self.cfgs['npz_files_loc']['thresholds'])
#if
self.not_black_list = self.io.read_getty_labels(
getty_file, self.cfgs['minimum_samples_for_validation'],
self.cfgs['getty_key_words_types'])
#if
if self.is_mat:
assert self.W.shape[1] == len(self.getty_labels)
self.io.print_info('Found weight {0}x{1} matrix'.format(
self.W.shape[0], self.W.shape[1]))
#if
self.io.print_info('Testing set-up for label embedding')
#if
def _verify_files(self):
flag = True
paths_to_check = [(k, v) for k, v in self.cfgs.items() if 'path_' in k]
for var, file_path in paths_to_check:
if type(file_path) == dict:
check_files = [(var + '+' + r, j)
for r, j in file_path.items()]
else:
check_files = [(var, file_path)]
#if
file_flag = [os.path.exists(f[1]) for f in check_files]
flag = flag and np.all(file_flag)
for idx, f in enumerate(file_flag):
if not f:
print colored(
'[WARN] Could not find {0} for {1}, no such file or directory'
.format(check_files[idx][1],
check_files[idx][0]), 'blue')
#if
#if
#def
return flag
#def
"""
Simple yaml config file reader
"""
def _read_configuration(self):
pfile = open(self.config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
#def
"""
Read wordnet synset file and getty labels.
"""
def _read_all_labels(self):
self.leaf_wn_labels, self.all_wn_labels = self.io.read_wn_labels(
self.cfgs['path_bet_file'], self.cfgs['path_wn_labels_file'])
self.getty_labels = self.io.read_getty_labels(
self.cfgs['path_getty_label_file'], self.cfgs['getty_stop_list'],
self.cfgs['getty_key_words_types'])
#def
def get_to_work(self, tasks):
for idx, action, action_func in self.working_order:
if tasks[action]:
action_func()
#if
#for
#def
"""
Read all possible labels wordnet and getty
Run converting dense prediction vectors from DL into histogram features.
Run converting getty labels into binary vectors (presense or absence)
Save processed vectors.
"""
def _run_preprocess(self):
self._read_all_labels()
self._read_training_samples()
self._process_vectors()
#def
"""
Second step after pre-processing vectors, depending on training method selected used closed form solution or use SGD
"""
def _run_training(self):
if self.cfgs['train_method'] in self.train_method.keys():
self.train_method[self.cfgs['train_method']]()
else:
self.io.print_error(
'Could not find the specified training method {0}'.format(
self.cfgs['train_method']))
#if
#def
def _build_embedding(self):
self._read_all_labels()
W = self.io.read_all_emb(self.getty_labels, self.cfgs['train_method'],
self.cfgs['path_weight_matrix'])
self.io.write_all_emb(self.cfgs['npz_files_loc']['weights'], W)
self.io.print_info('Done writing {1} embeddings to {0}'.format(
self.cfgs['npz_files_loc']['weights'], len(W.keys())))
#def
"""
Run testing woudl included and instance of FullConvNet and dense predictions.
Mapping predictions using learned weight matrix.
Save Mapped predictions.
TODO : Check code to reflect changes for individual embeddings
"""
def _run_thresholds(self):
self.prepare_for_testing()
self.validation_samples = self.io.read_images(
self.cfgs['path_file_list']['val'])
D_val, G_val, _ = self._read_all_data(self.validation_samples, 'val')
G_val_pred = [
v.predict_proba(D_val)[:,
np.where(v.named_steps['train'].classes_ ==
1)[0][0]]
if type(v) == sklearn.pipeline.Pipeline else
v.predict_proba(D_val)[:, np.where(v.classes_ == 1)[0][0]]
for k, v in self.W.items()
]
G_val_pred = np.array(G_val_pred).transpose()
sample_range = np.array(range(1, D_val.shape[0] + 1), dtype=float)
concepts_threshold = {}
non_trained_concepts = []
for l in range(D_val.shape[1]):
gt_l = np.where(G_val[:, l] > 0)[0].tolist()
if len(gt_l) < self.cfgs['minimum_samples_for_validation']:
self.io.print_warning(
'Skipping {0}, not enough samples {1}'.format(
self.getty_labels[l], len(gt_l)))
non_trained_concepts.append(self.getty_labels[l])
continue
#if
sorted_idx = np.argsort(-G_val_pred[:, l])
good_idx = [1 if idx in gt_l else 0 for idx in sorted_idx.tolist()]
c_sum = np.cumsum(np.array(good_idx)) / sample_range
here = np.where(c_sum > self.cfgs['concept_precision'])[0].tolist()
fhere = here[-1] if not here == [] else np.argmax(c_sum)
concepts_threshold[self.getty_labels[l]] = G_val_pred[
sorted_idx[fhere], l]
self.io.print_info('Concept {0} has threshold {1}'.format(
self.getty_labels[l],
concepts_threshold[self.getty_labels[l]]))
#for
self.io.save_threshold_file(self.cfgs['npz_files_loc']['thresholds'],
concepts_threshold, non_trained_concepts)
#def
def predict(self, dense_pred):
vec_dense = []
for pred in dense_pred:
vec_dense.extend(pred)
#for
predictions = self.dense_to_labels(vec_dense)
predictions = [(p) for p in predictions]
return predictions
#def
def resolve_label_embedding(self, emb_pred):
filtered_pred = [
d[0] for d in emb_pred if d[0] not in self.cfgs['stop_list_lemb']
]
mapped_pred = self.map_lemb_concepts(filtered_pred)
return mapped_pred
#def
def map_lemb_concepts(self, concepts):
return [
self.lemb_mapping[c] if c in self.lemb_mapping.keys() else c
for c in concepts
]
#def
def resolve_antonyms(self, human_predictions):
conflicting_predictions = []
human_dict = {d[0]: d[1] for d in human_predictions}
human_labels = [d[0] for d in human_predictions]
human_scores = [d[1] for d in human_predictions]
for c1, c2 in self.cfgs['antonym_list']:
if not (c1 in human_labels and c2 in human_labels):
continue
#if
s1, s2 = human_dict[c1], human_dict[c2]
idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1)
print 'Supressing {0}:{1}'.format(human_labels[idx],
human_scores[idx])
del human_labels[idx]
del human_scores[idx]
#for
remove_flag = -1
for idx, group in enumerate(self.cfgs['count_order_list']):
_this = np.intersect1d(human_dict.keys(), group)
if len(_this) > 0:
remove_flag = idx + 1
break
#if
#for
if not remove_flag == len(self.cfgs['count_order_list']):
remove_tags = []
for g in self.cfgs['count_order_list'][remove_flag:]:
remove_tags.extend(g)
#for
for t in remove_tags:
if not t in human_labels:
continue
#if
ridx = human_labels.index(t)
del human_labels[ridx]
del human_scores[ridx]
#for
#if
return [(g, s) for g, s in zip(human_labels, human_scores)]
#def
def softmax(self, w, t=1.0):
e = np.exp(w / t)
dist = e / np.sum(e)
return dist
#def
def dense_to_labels(self, vec_dense):
if vec_dense == []:
return []
#def
f_vec_dense = self._build_wn_label_vectors(vec_dense)
if self.is_mat:
getty_like_vec = np.dot(f_vec_dense, self.W)
good = np.argsort(-getty_like_vec)
good_labels = [
(self.getty_labels[g], getty_like_vec[g]) for g in good
if (not self.getty_labels[g] in self.black_list
and self.getty_labels[g] in self.threshold_map.keys())
]
good_labels = [(g, s) for g, s in good_labels
if s >= 0][:self.cfgs['top_k_tags']]
else:
getty_like_vec = [(k, v.predict_proba(f_vec_dense)[0][np.where(
v.named_steps['train'].classes_ == 1)[0][0]])
if type(v) == sklearn.pipeline.Pipeline else
(k, v.predict_proba(f_vec_dense)[0][np.where(
v.classes_ == 1)[0][0]])
for k, v in self.W.items()]
good = np.argsort(-np.array([d[1] for d in getty_like_vec]))
good_labels = [
getty_like_vec[g] for g in good
if getty_like_vec[g][0] in self.not_black_list
]
good_labels = [
(g, s) for g, s in good_labels if s > self.threshold_map[g]
][:self.cfgs['top_k_tags']]
#if
return good_labels
#for
def _run_testing(self):
if not os.path.exists(self.cfgs['npz_files_loc']['weights']):
self.io.print_error(
'Could not find label emdbedding matrix file {}'.format(
self.cfgs['npz_files_loc']['weights']))
return
#if
self.prepare_for_testing()
self.testing_samples = self.io.read_images(
self.cfgs['path_file_list']['test'])
self.io.print_info('Will process {0} test images'.format(
len(self.testing_samples)))
shuffle(self.testing_samples)
#self.testing_samples = self.testing_samples[110:120]
for idx, sample in enumerate(self.testing_samples):
if idx % 1000 == 0:
self.io.print_info('Processing {0}/{1} image'.format(
idx, len(self.testing_samples)))
#if
basename = os.path.basename(sample)
basename, ext = os.path.splitext(basename)
json_file = os.path.join(self.cfgs['path_results_dir']['test'],
basename + '.json')
if os.path.exists(json_file):
if not self.force:
self.io.print_warning(
'Skipping {0}, already exists'.format(json_file))
continue
#if
#if
vec_dense = self.io.read_vector(
os.path.join(self.cfgs['path_dense_pred_loc']['test'],
basename) + self.cfgs['wn_suffix'])
vec_getty = self.io.read_vector(
os.path.join(self.cfgs['path_getty_labels_loc']['test'],
basename) + self.cfgs['getty_suffix'])
good_labels = self.dense_to_labels(vec_dense)
self.io.save_good_labels(json_file, dict(good_labels))
if self.cfgs['show_predicted_labels']:
self.io.print_info('Photo-id {0}'.format(basename))
self.io.print_info('Predicted {0}'.format(good_labels))
self.io.print_warning('GT {0}'.format(vec_getty))
#if
#for
#def
"""
Read images pointed to by training and validation file lists
"""
def _read_training_samples(self):
self.training_data = {}
self.validation_data = {}
self.training_data['samples'] = self.io.read_images(
self.cfgs['path_file_list']['train'])
self.validation_data['samples'] = self.io.read_images(
self.cfgs['path_file_list']['val'])
self.io.print_info(
'Found {0} training samples {1} validation samples'.format(
len(self.training_data['samples']),
len(self.validation_data['samples'])))
#def
"""
Convert dense predictions into vectors
Convert getty labels into binary vectors
"""
def _process_vectors(self):
self._label_to_vectors(self.training_data['samples'], 'train')
self._label_to_vectors(self.validation_data['samples'], 'val')
#def
def _label_to_vectors(self, training_images, tag='train'):
for idx, sample in enumerate(training_images):
if idx % 1000 == 0:
self.io.print_info(
'Processing {0}/{1} vectors for {2}, {3}'.format(
idx, len(training_images), tag, sample))
#if
basename = os.path.basename(sample)
basename, ext = os.path.splitext(basename)
npz_file = os.path.join(self.cfgs['path_processed_vectors'][tag],
basename + '.npz')
if (os.path.exists(npz_file) and not self.force):
self.io.print_info(
'Skipping {0}, already exists'.format(npz_file))
continue
#if
vec_dense = self.io.read_vector(
os.path.join(self.cfgs['path_dense_pred_loc']['train'],
basename) + self.cfgs['wn_suffix'])
vec_getty = self.io.read_vector(
os.path.join(self.cfgs['path_getty_labels_loc']['train'],
basename) + self.cfgs['getty_suffix'])
#f_vec_dense = vec_dense
f_vec_dense = self._build_wn_label_vectors(vec_dense)
f_vec_getty = self._build_getty_label_vectors(vec_getty)
self.io.save_npz_file(f_vec_dense, f_vec_getty, npz_file)
#for
self.io.print_info('Done building {0} vectors'.format(tag))
#def
"""
Convert wn dense predictions to sparse vectors and L2 normalization.
"""
def _build_wn_label_vectors(self, svec):
if svec == []:
return []
#if
vec = np.zeros(len(self.leaf_wn_labels), dtype=float)
idx = np.array([d[0] for d in svec])
values = np.array([d[1] for d in svec])
for d in svec:
vec[d[0]] += d[1]
#if
if not self.cfgs['normalization']['wn_norm'] == 'None':
vec /= (np.linalg.norm(vec) + 0.000001)
#if
return vec
#def
"""
Convert getty label vectors to binary vectors of presense of absence and L2 normalization.
"""
def _build_getty_label_vectors(self, svec):
if svec == []:
return []
#if
vec = np.zeros(len(self.getty_labels), dtype=float)
idx = np.array([
self.getty_labels.index(d) for d in svec if d in self.getty_labels
])
if idx.size == 0:
return []
#if
vec[idx] = 1.0
if not self.cfgs['normalization']['getty_norm'] == 'None':
vec /= (np.linalg.norm(vec) + 0.000001)
#if
return vec
#def
def _init_embedding_matrix(self):
self.W = np.zeros((len(self.leaf_wn_labels), len(self.getty_labels)))
self.io.print_info('Weight matrix set-up {0}x{1}'.format(
self.W.shape[0], self.W.shape[1]))
#def
def _init_identity_matrix(self):
self.I = np.eye(len(self.leaf_wn_labels), len(self.leaf_wn_labels))
#def
def _read_all_data(self, image_list, tag):
D_all_vec = []
G_all_vec = []
all_files = []
for idx, sample in enumerate(image_list):
if idx % 1000 == 0:
self.io.print_info(
'Reading {0}/{1} vectors for {3}, {2}'.format(
idx, len(image_list), sample, tag))
#if
basename = os.path.basename(sample)
basename, ext = os.path.splitext(basename)
npz_file = os.path.join(self.cfgs['path_processed_vectors'][tag],
basename + '.npz')
if not os.path.exists(npz_file):
self.io.print_warning(
'Skipping {0} no such file'.format(npz_file))
continue
#if
D, G = self.io.read_npz_file(npz_file)
D = D.flatten()
if (D.size == 0 or G.size == 0
or not len(D) == self.cfgs['expected_dim']):
self.io.print_warning(
'Skipping {0} mis-match of dimensionality'.format(
npz_file))
continue
#if
D_all_vec.append(D)
G_all_vec.append(G)
all_files.append(basename)
#for
return np.array(D_all_vec), np.array(G_all_vec), all_files
#def
"""
Different solutions to label embedding Closed form solution (CFM) / SGD / WSABIE / CCA
"""
def _solve_closed_form(self):
self._read_all_labels()
self._read_training_samples()
#self._init_embedding_matrix()
self._init_identity_matrix()
D_train, G_train, _ = self._read_all_data(
self.training_data['samples'], 'train')
if self.cfgs['balance_data']:
D_train, G_train = self._balance_training_data(D_train, G_train)
#if
D_val, G_val, _ = self._read_all_data(self.validation_data['samples'],
'val')
train_size = (D_train.nbytes + G_train.nbytes) / (1000.0 * 1024**2)
val_size = (D_val.nbytes + G_val.nbytes) / (1000.0 * 1024**2)
# TODO : compute expected size of data in memory before reading it LOL
# TODO : Future harsimrat will fix this
# Yo future Harsimrat here, this is obsolte now.
self.io.print_info('Train : {0:2f} GB, val : {1:2f} GB'.format(
train_size, val_size))
for l in self.cfgs['lambda']:
self.io.print_info(
'Computing closed form solution (CFM) for lambda = {0}'.format(
l))
self.W = np.dot(
np.linalg.inv(
np.dot(D_train.transpose(), D_train) + l * self.I),
np.dot(D_train.transpose(), G_train))
error = 1 - self._compute_validation_error_on_tags(
self.W, D_val, G_val)
self.io.print_info('(CFM) error for lambda = {0} : {1}'.format(
l, error))
self.io.save_weight_matrix(
self.W,
os.path.join(self.cfgs['path_weight_matrix'],
'CFM-{0}.npz'.format(l)))
#for
#def
"""
Closed form solution per concepts
"""
def _solve_closed_form_pc(self):
self._read_all_labels()
self._read_training_samples()
D_train, G_train, train_files = self._read_all_data(
self.training_data['samples'], 'train')
D_val, G_val, val_files = self._read_all_data(
self.validation_data['samples'], 'val')
train_size = (D_train.nbytes + G_train.nbytes) / (1000.0 * 1024**2)
val_size = (D_val.nbytes + G_val.nbytes) / (1000.0 * 1024**2)
self.io.print_info('Train : {0:2f} GB, val : {1:2f} GB'.format(
train_size, val_size))
self.reset_pipe_line()
for label, label_name in len(self.getty_labels):
svm_file_path = os.path.join(self.cfgs['path_weight_matrix'],
'CFM_PC-{0}.pkl'.format(label_name))
if os.path.exists(svm_file_path):
if not self.force:
self.io.print_warning(
'Skipping {}, already exists'.format(svm_file_path))
continue
#if
#if
tidx = np.sum(G_train[:, label])
vidx = np.sum(G_val[:, label])
if tidx == 0 or vidx == 0:
self.io.print_warning(
'Not enough samples for {0}, train:{1} or val :{2}'.format(
label_name, int(tidx), int(vidx)))
continue
#if
self.io.print_info(
'{1}/{2} Learning embedding for {0} with {3} +samples {4} -ive samples'
.format(label_name, label, len(self.getty_labels),
np.sum(G_train[:, label]),
G_train.shape[0] - np.sum(G_train[:, label])))
self.pipe_line.fit(D_train, G_train[:, label])
error = 1 - self.pipe_line.score(D_val, G_val[:, label])
self.io.print_info('(CFM_PC) error for {1} : {0}'.format(
error, label_name))
self.io.save_emb_file(self.pipe_line, svm_file_path)
#for
#def
def reset_pipe_line(self):
action_list = [
('norm', normalizer_dict[self.cfgs['normalization_type']]),
('scale', scaling_dict[self.cfgs['scaling_type']]),
('train', classifier_dict[self.cfgs['classifier_type']])
]
action_list = [(a, b) for a, b in action_list if not b == None]
self.pipe_line = Pipeline(action_list)
#def
def _solve_sgd(self):
self.io.print_error('SGD not implemented')
#def
def _solve_wsabie(self):
self.io.print_error('WSABIE not implemented')
#fef
def _solve_cca(self):
self.io.print_error('CCA not implemented')
#def
"""
Validation needs to change since we optimize for tag coverage we should compute intersecton over union of predicted tags + getty tags on validation set
"""
def _compute_validation_error(self, W, D, G):
n_samples = G.shape[0]
pG = np.dot(D, W)
for i in range(pG.shape[0]):
pG[i, :] /= (np.linalg.norm(pG[i, :]) + 0.000001)
#for
cG = np.dot(pG, G.transpose())
predictions = np.argmax(cG, axis=0)
gt_match = np.array(range(n_samples))
good = np.bitwise_xor(predictions, gt_match)
correct_match = good == 0
return np.sum(correct_match) / (n_samples + 0.0)
#def
"""
This function needs to change towards what we should optimize currently global predictions of tag is optimized.
"""
def _compute_validation_error_on_tags(self, W, D, G):
pG = np.dot(D, W)
predictions = np.zeros(pG.shape)
for i in range(pG.shape[0]):
pidx = np.argsort(-pG[i, :])[:self.cfgs['top_k_tags']]
predictions[i, pidx] = 1
#for
gt = np.zeros(G.shape)
idx = np.where(G > 0)
gt[idx] = 1
good = predictions * gt
n_samples = np.sum(predictions)
#self.io.print_info('{0} correct out of {1}'.format(np.sum(good),n_samples))
return np.sum(good) / (n_samples + 0.0)
#def
def _compute_validation_error_on_tags_pc(self, W, D, G):
pG = np.dot(D, W)
predictions = np.zeros(pG.shape)
pidx = np.where(pG > 0)[0]
predictions[pidx] = 1
gt = G
good = predictions * gt
n_samples = np.sum(predictions)
# error : (fp + tn)/n_samples
return (np.sum(predictions) + np.sum(gt) -
2 * np.sum(good)) / (n_samples + 0.0)
#def
def _restore_snapshot(self, label_idx):
if label_idx == 0:
self._init_embedding_matrix()
return
#if
label = labe_idx - 1
weight_matrix_path = os.path.join(
self.cfgs['path_weight_matrix'],
'CFM-{1}.npz'.format(l, self.getty_labels[label]))
self.W = self.io.read_weight_matrix(weight_matrix_path)
#def
def _balance_training_data(self, DL_train, G_train):
return DL_train_balance, G_train_balance
#def
#class
class TheanoTester():
def __init__(self, config_file, verbose=False, raw_predictions=False):
if not os.path.exists(config_file):
print 'Error : could not find config file'.format(config_file)
self.init = False
return
self.tester_config_file = config_file
self.verbose = verbose
self.raw_predictions = raw_predictions
self.read_config()
self.io = EmbeddingIO(self.cfgs['log_dir'], 'testing')
self.tester_name = self.cfgs['model_name']
self.model_config_file = self.cfgs['model_config_file']
self.init = False
self.thresholds = None
mapping_keys = [d[0] for d in self.cfgs['mapping_list']]
mapping_values = [d[1] for d in self.cfgs['mapping_list']]
self.mapping = dict(zip(mapping_keys, mapping_values))
self.softpooling_func = f
if self.tester_name not in model_dict.keys():
self.io.print_error(
'Not a valid model type {0} chose among {1}'.format(
self.tester_name, model_dict.keys()))
self.init = False
return
if not os.path.exists(self.tester_config_file):
self.io.print_error('Could not find configuration file {0}'.format(
self.tester_config_file))
self.init = False
return
self.tester = None
self.action_items = {'test': self.run, 'eval': self.eval}
self.test_lmdb = LMDBParser_test(self.cfgs['test_file'],
log_dir=self.cfgs['log_dir'])
self.resize_test = self.cfgs['resize_test']
# self.load_synset_file()
# self.test_lmdb = LMDBParser(self.cfgs['test_file'], log_dir=self.cfgs['log_dir'])
# self.test_lmdb.set_params_test(self.concepts)
def _read_threshold_file(self):
if 'threshold_file' in self.cfgs.keys():
try:
pfile = open(self.cfgs['threshold_file'], 'r')
thresholds = json.load(pfile)
pfile.close()
self.thresholds = {
c: thresholds[c] if c in thresholds.keys() else
self.cfgs['global_threshold']
for c in self.concepts
}
except Exception as err:
self.io.print_error(
'Error parsing threshold file {0},{1}'.format(
self.cfgs['threshold_file'], err))
def read_config(self):
pfile = open(self.tester_config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
def setup(self):
self.tester = model_dict[self.tester_name]()
self.tester.configure(self.model_config_file)
self.tester.define()
#import ipdb; ipdb.set_trace()
self.model_type = self.tester.model.get_config()['name']
self.im_h, self.im_w = self.tester.cfgs[
'image_height'], self.tester.cfgs['image_width']
if not self.tester.init:
self.io.print_error('Error with model definition')
self.init = False
return
self.io.print_info('{0} Model defined from {1}'.format(
self.tester_name, self.model_config_file))
self.compile()
if not self.tester.init:
self.io.print_error('Error with model compilation')
self.init = False
return
self.io.print_info('{0} Model compiled'.format(self.tester_name))
self.init = True
self.load_synset_file()
self.load_mean_file()
self._read_threshold_file()
self.load_confidence_model()
self.read_label_embedding()
def compile(self):
"""
This compile version is to be used for testing only. Since its optimized for predictions
"""
self.output_nodes = self.cfgs['output_node_name']['test']
if self.model_type == 'Sequential':
self.inference = K.function(
[self.tester.model.get_input(train=False)],
[self.tester.model.layers[-1].get_output(train=False)])
elif self.model_type == 'Graph':
#import ipdb; ipdb.set_trace()
self.inference = K.function(
[self.tester.model.get_input(train=False)], [
self.tester.model.nodes[node].get_output(train=False)
for node in self.output_nodes
])
else:
self.io.print_error(
'Say whaaaat ?! This model type is not supported : {}'.format(
self.model_type))
sel.init = False
def read_label_embedding(self):
self.label_embedding = None
self.label_vectors = None
if 'label_embedding_file' in self.cfgs.keys():
self.label_embedding = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'matrix')
self.label_vectors = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'vectors')
self.io.print_info('Read label embedding file {0}'.format(
self.cfgs['label_embedding_file']))
def load_confidence_model(self):
self.confidence_model = None
if 'confidence_model' in self.cfgs.keys():
self.confidence_model = joblib.load(self.cfgs['confidence_model'])
#to be used in
def get_images_frVisualization(self):
self.load_synset_file()
GndTr = dict()
if not self.init:
return
indexes = dict()
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
#filterout images from getty
self.io.print_info(
'Removing getty scrapped images from validation data')
filter_getty = [
contains(img, 'getty-concept-sourcing') for img in self.test_images
]
ind = np.where(np.array(filter_getty) == True)[0]
test_images = np.array(self.test_images)[ind]
#---------------
for image in test_images:
im_basename = os.path.basename(image)
im_basename, img_ext = os.path.splitext(im_basename)
g = self.test_lmdb.get_labels(im_basename)
concepts = np.array(self.concepts)
index = np.where(g == 1)[0]
GndTr[im_basename] = concepts[index]
indexes[im_basename] = index
return (test_images, GndTr, indexes)
def visualize_image(self, beta, pixels):
self.beta = beta
self.load_synset_file()
flag, predictions, confidence, t = self.classify_images(pixels)
#file_paths = [os.path.join(self.cfgs['results_dir'], 'visualization_',str(self.beta)+'_'+im + save_ext) for im in basenames]
def run(self):
self.load_synset_file()
if not self.init:
return
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
image_batches = [
self.test_images[i:i + self.cfgs['batch_size']]
for i in range(0, len(self.test_images), self.cfgs['batch_size'])
]
self.io.print_info('{0} images split into {1} Batches of {2}'.format(
len(self.test_images), len(image_batches),
self.cfgs['batch_size']))
for beta in self.cfgs['betas']:
self.beta = beta
for idx, images in enumerate(image_batches):
if not self.cfgs['force']:
flag = self.check_jsons(images)
if flag:
self.io.print_warning(
'Skipping batch {0} of {1}'.format(
idx, len(image_batches)))
continue
pixels, basenames, dummy = self.fetch_images(images)
dummy = []
flag, predictions, confidence, t = self.classify_images(pixels)
if self.raw_predictions is False:
getty_like_filter = [
self.suppress_stop_list(g) for g in predictions
]
getty_like_safe = [
self.run_thresholds(g) for g in getty_like_filter
]
getty_like_pc = [
self.map_concepts(g) for g in getty_like_safe
]
getty_like_public = [
self.resolve_antonyms(g) for g in getty_like_pc
]
getty_like_unique = [
self.resolve_duplicates(g) for g in getty_like_public
]
predictions_pc = getty_like_unique
save_ext = '.json'
else:
predictions_pc = predictions
save_ext = '.npz'
if not flag:
continue
file_paths = [
os.path.join(self.cfgs['results_dir'],
str(self.beta) + '_' + im + save_ext)
for im in basenames
]
if not self.raw_predictions:
to_save = [dict({'external': p}) for p in predictions_pc]
else:
to_save = predictions_pc
#import ipdb; ipdb.set_trace()
#to_save = [dict({'external': p}) for p in predictions_pc]
#to_save=predictions_pc
#to_save_id = [dict({'external': id_}) for id_ in ids]
assert (len(file_paths) == len(to_save))
self.io.print_info('Done batch {0} of {1} in {2} secs'.format(
idx, len(image_batches), t))
#import ipdb; ipdb.set_trace()
#import ipdb; ipdb.set_trace()
f = map(self.io.save_good_labels, file_paths, to_save)
#f = map(self.io.save_good_labels, file_paths, to_save_id)
def check_jsons(self, images):
basenames = []
for image in images:
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
basenames.append(im_basename)
flags = [
os.path.exists(os.path.join(self.cfgs['results_dir'], b + '.json'))
for b in basenames
]
return np.all(flags)
def compare_various_betas(self):
betas = self.cfgs['betas']
Area_PR = dict()
dir_path = os.path.dirname(os.path.realpath(__file__))
#print dir_path
self.limit_NoofClasses = None
self.load_synset_file()
self.load_synset_file_training()
self.imgHist_Concepts_inTraindata = [
(np.where(np.array(self.concepts) == i)[0][0])
for i in self.trained_concepts
]
#import lmdb
#lmdb_env = lmdb.open("./keywording_test_images_labels", readonly=True, lock=False)
# lmdb_txn = lmdb_env.begin()
# read_label = lmdb_txn.get
# label1test=self.test_lmdb.get_labels(read_label,'10085536')
# label2test=self.test_lmdb.get_labels(read_label,'10085536')
#self.imgname_labels=self.test_lmdb.create_keys_labels_dict()
var = dict()
for beta in betas:
var.update(self.eval_usinglmdb(str(beta)))
#import ipdb; ipdb.set_trace()
if self.limit_NoofClasses == None:
df2 = pd.DataFrame(var, index=self.trained_concepts)
else:
df2 = pd.DataFrame(
var, index=self.trained_concepts[0:self.limit_NoofClasses])
df2.to_csv(dir_path + '/results/' +
self.cfgs['results_dir'].split('/')[-2] + '.csv')
#import ipdb; ipdb.set_trace()
# mAP=[]
# mRec90=[]
# mRec80=[]
# max_f1=[]
# for beta in betas:
# vAP,v90,v80,vf1=Area_PR[beta]
# mAP.append(vAP)
# mRec90.append(v90)
# mRec80.append(v80)
# max_f1.append(vf1)
# #print ("mAP:%.4f,mean Recall:%.4f,beta:%s" % (mAP,mRec,str(beta)))
# #import ipdb; ipdb.set_trace()
# #print(*betas, sep='\t')
# print('\t'.join([str(x) for x in betas]))
# #print(*mRec80, sep='\t')
# print('\t'.join([str(round(x*100,3)) for x in mRec80]))
# #print(*mRec90, sep='\t')
# print('\t'.join([str(round(x*100,3)) for x in mRec90]))
# print('\t'.join([str(round(x*100,3)) for x in mAP]))
# print('\t'.join([str(round(x*100,3)) for x in max_f1]))
#print(*mAP, sep='\t')
#pickle.dump(Area_PR,open('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/result_Area_PR','wb'))
# def generate_csv_file(self):
# betas=self.cfgs['betas']
# Area_PR=dict()
# #import lmdb
# #lmdb_env = lmdb.open("./keywording_test_images_labels", readonly=True, lock=False)
# #import ipdb; ipdb.set_trace()
# # lmdb_txn = lmdb_env.begin()
# # read_label = lmdb_txn.get
# # label1test=self.test_lmdb.get_labels(read_label,'10085536')
# # label2test=self.test_lmdb.get_labels(read_label,'10085536')
# #self.imgname_labels=self.test_lmdb.create_keys_labels_dict()
# for beta in betas:
# Area_PR[beta]=self.eval_usinglmdb(str(beta))
# mAP=[]
# mRec90=[]
# mRec80=[]
# max_f1=[]
# detections=[]
# for beta in betas:
# vAP,v90,v80,max_f1,detections=Area_PR[beta]
# mAP.append(vAP)
# mRec90.append(v90)
# mRec80.append(v80)
#print ("mAP:%.4f,mean Recall:%.4f,beta:%s" % (mAP,mRec,str(beta)))
#import ipdb; ipdb.set_trace()
#print(*betas, sep='\t')
#import ipdb; ipdb.set_trace()
# def compare_various_betas(self):
# import ipdb; ipdb.set_trace()
# betas=self.cfgs['betas']
# for beta in betas:
# self.eval_usinglmdb(str(beta)
def eval_usinglmdb(self, beta):
#import ipdb; ipdb.set_trace()
#import ipdb; ipdb.set_trace()
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
self.predictions = {}
self.gt = {}
valid_keys = []
notgoodGTimgs = []
ImgsSmallNumofTags = []
#import ipdb; ipdb.set_trace()
for idx, image in enumerate(self.test_images):
if idx % 1000 == 0:
self.io.print_info('Reading {0}/{1}'.format(
idx, len(self.test_images)))
im_basename = os.path.basename(image)
im_basename, img_ext = os.path.splitext(im_basename)
#result_json = os.path.join(self.cfgs['results_dir'], im_basename + '.json')
result_json = os.path.join(self.cfgs['results_dir'],
beta + '_' + im_basename + '.npz')
#gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json')
if os.path.exists(result_json):
#p, s,ids = self.io.read_prediction_json_withIDs(result_json)
p = np.load(result_json)['features']
else:
p = []
#g = self.io.read_vector(gt_json)
#import ipdb; ipdb.set_trace()
g = self.test_lmdb.get_labels(im_basename)
#print np.sum(g)
#print p.shape
if len(p) == 0:
print("images with no prediction generated: %s") % im_basename
continue
if len(g) == 0:
#import ipdb; ipdb.set_trace()
print("images with no GT data: %s") % im_basename
notgoodGTimgs.append(im_basename)
continue
if len(g) < 7735 and len(g) != 0:
print("images with small number of tags: %s") % im_basename
ImgsSmallNumofTags.append(im_basename)
continue
#g_filtered=self.parse_gndTruth_withFilter(ids,g)
# if p is [] or g is []:
# continue
self.predictions[im_basename] = p
self.gt[im_basename] = g
valid_keys.append(im_basename)
# if idx>1000:
# break
print 'Number of images: with no GT %d and with small no of tags %d' % (
len(notgoodGTimgs), len(ImgsSmallNumofTags))
#import ipdb; ipdb.set_trace()
y_test = np.zeros((len(valid_keys), len(self.concepts)), dtype='u1')
y_score = np.zeros((len(valid_keys), len(self.concepts)))
for i, key in enumerate(valid_keys):
y_test[i] = self.gt[key]
y_score[i] = self.predictions[key]
gt = dict()
pred = defaultdict(dict)
for i, concept in enumerate(self.concepts):
imgname_ind = np.where(y_test[i] == 1)
np_array_photo_ids = np.array(valid_keys)[imgname_ind]
gt[concept] = list(np_array_photo_ids)
#pred[concept]={key: value for key, value in self.predictions.items() if key in list(np_array_photo_ids)}
# for key in list(np_array_photo_ids):
# pred[concept].update({key: self.predictions[key]})
# pairof_photoid_predictions=self.predictions.items()
# for photo_id, predictions in pairof_photoid_predictions:
# for concept, score in zip(self.concepts,predictions):
# pred[concept][photo_id] = score
# #import ipdb; ipdb.set_trace()
# thresholds=get_thresholds(pred, self.concepts, gt)
# detections=get_detections_per_concept(pred, self.concepts, gt, thresholds)
#class_histogram=y_test.sum(axis=0)
#import ipdb; ipdb.set_trace()
if self.cfgs['percentate_of_classesFrEval'] != None or self.cfgs[
'percentate_of_classesFrEval'] < 1.0:
use_subsetofClsFrEval = True
n_classes = len(self.concepts)
limit_NoofClasses = int(self.cfgs['percentate_of_classesFrEval'] *
n_classes)
subset_indices_FrEval = self.imgHist_Concepts_inTraindata[:
limit_NoofClasses]
self.limit_NoofClasses = limit_NoofClasses
#self.concepts=np.array(self.concepts)[indices]
#y_test_subset=np.zeros((len(valid_keys),len(self.concepts)),dtype='u1')
#y_score_subset=np.zeros((len(valid_keys),len(self.concepts)))
#for i,key in enumerate(valid_keys):
# y_test_subset[i]=self.gt[key][indices]
# y_score_subset[i]=self.predictions[key][indices]
#print "Using %d out of %d rare number of classes for evaluations" % (limit_NoofClasses,n_classes)
else:
use_subsetofClsFrEval = False
subset_indices_FrEval = self.imgHist_Concepts_inTraindata
#subset_indices_FrEval=range(len(self.concepts))
#if use_subsetofClsFrEval:
# y_test=y_test_subset
# y_score=y_score_subset
#plt.plot(np.sort(temp)[::-1])
#plt.savefig('/nas2/praveen/home/rnd-libs/rnd_libs/lib/keras/intermediate/class_histogram')
#fpr = dict()
#tpr = dict()
recAt90 = []
recAt70 = []
ap = []
names = ['ap', 'recAt70', 'recAt90']
#max_f1=[]
for ind in subset_indices_FrEval:
#import ipdb; ipdb.set_trace()
# if y_test[:, i].sum()==0.0:
# continue
#fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
ap.append(
roundoff(
average_precision_score(y_test[:, ind], y_score[:, ind])))
recAt90.append(
roundoff(
recall_at_specified_precision(y_test[:, ind],
y_score[:, ind],
specified_precision=0.9)))
recAt70.append(
roundoff(
recall_at_specified_precision(y_test[:, ind],
y_score[:, ind],
specified_precision=0.7)))
#max_f1[self.concepts[ind]] = max_f1_score(y_test[:, ind], y_score[:, ind])
#import ipdb; ipdb.set_trace()
#return (np.array(ap.values()).mean(), np.array(recAt90.values()).mean(), np.array(recAt80.values()).mean())
#max_f1=[0.0]
#import ipdb; ipdb.set_trace()
returndict = dict()
for name in names:
returndict[beta + '_' + name] = locals()[name]
return (returndict)
#import ipdb; ipdb.set_trace()
# import ipdb; ipdb.set_trace()
# list_thres=[self.cfgs['global_threshold']]
# for t in list_thres:
# self.compute_precision_recall(t)
def eval(self):
self.load_synset_file()
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
self.predictions = {}
self.gt = {}
for idx, image in enumerate(self.test_images):
if idx % 1000 == 0:
self.io.print_info('Reading {0}/{1}'.format(
idx, len(self.test_images)))
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
result_json = os.path.join(self.cfgs['results_dir'],
im_basename + '.json')
gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json')
p, s = self.io.read_prediction_json(result_json)
g = self.io.read_vector(gt_json)
if p is [] or g is []:
continue
self.predictions[im_basename] = zip(p, s)
self.gt[im_basename] = list(set(g).intersection(self.concepts))
for t in [self.cfgs['thresholds']]:
self.compute_precision_recall(t)
def compute_precision_recall(self, thresh):
fps = {}
tps = {}
fns = {}
gt_dict = {}
for c in self.concepts:
fps[c], tps[c], fns[c], gt_dict[c] = [], [], [], []
for key, value in self.predictions.items():
detections = [v[0] for v in value if v[1] >= thresh]
gt = self.gt[key]
for g in gt:
gt_dict[g].append(key)
tp = list(set(gt).intersection(detections))
fp = list(set(detections) - set(gt))
fn = list(set(gt) - set(detections))
for t in tp:
tps[t].append(key)
for f in fp:
fps[f].append(key)
for n in fn:
fns[n].append(key)
for c in self.concepts:
if gt_dict[c] == []:
self.io.print_info(
'Skipping {0}, no samples in ground-truth'.format(c))
continue
p = len(tps[c]) / (len(tps[c]) + len(fps[c]) + 0.00000001)
r = len(tps[c]) / (len(gt_dict[c]) + 0.0)
f1 = self.compute_f1(p, r)
self.io.print_info(
'At {0} : {1} precision {2}, recall {3}, f1-score {4}'.format(
c, thresh, p, r, f1))
def compute_f1(self, p, r):
return 2 * (p * r) / (p + r + 0.000001)
def load_synset_file(self):
pfile = open(self.cfgs['synset_file'], 'r')
concepts = pfile.readlines()
pfile.close()
self.concepts = [p.strip() for p in concepts]
def load_synset_file_training(self):
pfile = open(self.cfgs['synset_file_training'], 'r')
concepts = pfile.readlines()
pfile.close()
self.trained_concepts = [p.strip() for p in concepts]
def load_mean_file(self):
if not self.cfgs['mean_file'] == 'None':
pfile = open(self.cfgs['mean_file'], 'r')
m = np.load(pfile)
pfile.close()
else:
m = [[[128.0]], [[128.0]], [[128.0]]]
m = np.mean(np.mean(m, axis=1), axis=1)
self.mean_pixel_value = np.array([m[0], m[1], m[2]])
self.pixel_scaling = self.cfgs['pixel_scaling'][
'test'] if 'pixel_scaling' in self.cfgs.keys() else None
self.channel_swap = self.cfgs['channel_swap'][
'test'] if 'channel_swap' in self.cfgs.keys() else None
self.image_shuffle = self.cfgs['image_shuffle'][
'test'] if 'image_shuffle' in self.cfgs.keys() else None
def jsonify(self, results):
return [[r[0], str(r[1])] for r in results]
def map_to_synsets(self, results):
results = {r[0]: r[1] for r in results}
return [results[c] for c in self.concepts]
def parse_predictions(self, p):
sort_idx = np.argsort(-p)
concepts = [(self.concepts[idx], float(p[idx]), idx)
for idx in sort_idx[:self.cfgs['top_k']['test']]]
#import ipdb; ipdb.set_trace()
return concepts
def parse_gndTruth_withFilter(self, ids, p):
p = np.array(p)
concepts = np.array(self.concepts)
filtered_gt = p[ids]
filtered_concepts = concepts[ids]
gt_ids = np.where(filtered_gt == 1)[0]
#sort_idx = np.argsort(-p)
#import ipdb; ipdb.set_trace()
return filtered_concepts[gt_ids]
def parse_gndTruth(self, p):
#import ipdb; ipdb.set_trace()
gt = np.array(p)
concepts = np.array(self.concepts)
#filtered_gt=p[ids]
#filtered_concepts=concepts[ids]
gt_ids = np.where(gt == 1)[0]
#sort_idx = np.argsort(-p)
#import ipdb; ipdb.set_trace()
return concepts[gt_ids]
def classify_images(self, images):
confidence = [None]
if images == []:
return False, [(), ()], 0.0
try:
tstart = time.time()
(predictions, features) = self.inference(images)
#import ipdb; ipdb.set_trace()
predictions = f(predictions, self.beta)[0]
#import ipdb; ipdb.set_trace()
#pickle.dump(predictions,open('predictions','wb'))
#test=f(predictions,self.beta)[0]
#predictions=numpy_tensor_mean(predictions,self.beta,1)[0]
#import ipdb; ipdb.set_trace()
# if len(predictions.shape) == 4:
# predictions=predictions[:,:,0,0]
#import ipdb; ipdb.set_trace()
if self.label_vectors is not None:
predictions /= np.linalg.norm(predictions, axis=1)[:,
np.newaxis]
predictions = np.dot(predictions, self.label_vectors)
if self.confidence_model is not None:
confidence = [('Confidence', p)
for p in self.confidence_model.predict(features)]
tend = time.time()
except Exception as err:
self.io.print_error('Error processing image, {0}'.format(err))
return False, [(), ()], None, 0.0
# if not self.raw_predictions:
# results = [self.parse_predictions(p)[1] for p in predictions] , True, [self.parse_predictions(p)[0] for p in predictions], confidence, '{0:.4f}'.format((tend - tstart))
# else:
# results = True, predictions, confidence, '{0:.4f}'.format((tend - tstart))
#results = True, [self.parse_predictions(p) for p in predictions], confidence, '{0:.4f}'.format((tend - tstart))
results = True, predictions, confidence, '{0:.4f}'.format(
(tend - tstart))
return results
def predict(self, image):
image = self.normalize_images([image])
return self.classify_images(image)
def normalize_images(self, images):
normalize_images = []
for image in images:
image *= 255.0
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
try:
image = cv2.resize(image, (self.im_h, self.im_w))
except Exception as err:
self.io.print_error('Could not parse test image {0}'.format(e))
return False, [], 0.0
image -= self.mean_pixel_value
image = image.transpose(2, 0, 1)
# Channel swap since caffe.io.load_image returns RGB image and training was done with BGR
image = image[(2, 1, 0), :, :]
normalize_images.append(image)
return [np.array(normalize_images)]
def scale_image(self, image):
#
# caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way
#
# No more caffe no more scaling down to 0-1 in caffe
# image *= 255.0
tstart = time.time()
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
tend = time.time()
tstart = time.time()
try:
if self.resize_test:
#Tracer()()
#print image.size
image = cv2.resize(image,
(self.im_h, self.im_w)).astype(np.float32)
else:
print 'no resize'
image = image.astype(np.float32)
except Exception as e:
self.io.print_error('Could not parse {0}'.format(e))
return None
tend = time.time()
tstart = time.time()
if self.mean_pixel_value is not None:
image -= self.mean_pixel_value
if self.pixel_scaling is not None:
image /= self.pixel_scaling
if self.channel_swap is not None:
image = image[:, :, self.channel_swap]
# H,W,C to C,H,W
if self.image_shuffle is not None:
image = image.transpose(self.image_shuffle)
tend = time.time()
return image
def fetch_images(self, image_file_names):
images = []
basenames = []
tstart = time.time()
for idx, image_file_name in enumerate(image_file_names):
image_file_name = image_file_name.replace('/nas/', '/nas2/')
im_basename = os.path.basename(image_file_name)
im_basename, _ = os.path.splitext(im_basename)
if not urlparse.urlparse(image_file_name).scheme == "":
url_response = urllib.urlopen(image_file_name)
if url_response.code == 404:
print self.io.print_error(
'[Testing] URL error code : {1} for {0}'.format(
image_file_name, url_response.code))
continue
try:
string_buffer = StringIO.StringIO(url_response.read())
image = np.asarray(bytearray(string_buffer.read()),
dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
image = image[:, :, (2, 1, 0)]
image = image.astype(np.float32)
except Exception as err:
print self.io.print_error(
'[Testing] Error with URL {0} {1}'.format(
image_file_name, err))
continue
elif os.path.exists(image_file_name):
try:
fid = open(image_file_name, 'r')
stream = fid.read()
fid.close()
image = Image.open(cStringIO.StringIO(stream))
image = np.array(image)
image = image.astype(np.float32)
# image = cv2.imread(image_file_name)
# image = image[:, :, (2, 1, 0)]
# image = image.astype(np.float32)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with image file {0} {1}'.format(
image_file_name, err))
continue
else:
try:
image = self.conn.read_image(image_file_name)
if image is None:
raise self.io.print_error(
"[Testing] Image data could not be loaded - %s" %
image_file_name)
continue
image = np.array(image)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with S3 Bucket hash {0} {1}'.format(
image_file_name, err))
continue
# try : do or do not there is not try
#orig_image=image.astype('u1')
image = self.scale_image(image)
if image is None:
continue
if self.verbose:
self.io.print_info('Processing {0} {1}'.format(
image_file_name, image.shape))
images.append(image)
basenames.append(im_basename)
tend = time.time()
self.io.print_info('Fetching {0} images took {1:.4f} secs'.format(
len(image_file_names), tend - tstart))
#return ([np.array(images)], basenames,orig_images)
return ([np.array(images)], basenames)
def fetch_gif_frames(self, inGif):
# Only works for URLs for now
# write frames to disk and pass filenames to fetch_images
outFolder = '/tmp'
frame = Image.open(BytesIO(urllib.urlopen(inGif).read()))
nframes = 0
file_names = []
while frame:
frame_name = '{0}/{1}-{2}.png'.format(outFolder,
os.path.basename(inGif),
nframes)
frame.save(frame_name, 'png')
file_names.append(frame_name)
nframes += 1
try:
frame.seek(nframes)
except EOFError:
break
file_names = [file_names[i] for i in range(0, len(file_names), 10)]
return file_names
def non_max(self, predictions):
concepts = {}
for pred in predictions:
for p in pred:
if p[0] in concepts.keys():
concepts[p[0]].append(float(p[1]))
else:
concepts[p[0]] = [float(p[1])]
for key, value in concepts.items():
concepts[key] = np.max(np.array(value))
labels = concepts.keys()
scores = [concepts[l] for l in labels]
sort_idx = np.argsort(-np.array(scores))
results = [(labels[idx], str(scores[idx])) for idx in sort_idx]
return [results]
def square_it_up(self, image):
edge_size = np.min(image.shape[:2])
cy, cx = np.array(image.shape[:2]) // 2
r_img = image[np.max([0, cy - edge_size //
2]):np.min([cy +
edge_size // 2, image.shape[0]]),
np.max([0, cx - edge_size //
2]):np.min([cx +
edge_size // 2, image.shape[1]]), :]
return r_img
def resolve_duplicates(self, concepts):
seen = set()
unique = []
for c in concepts:
if c[0] in seen:
logger.debug("Suppressing duplicate {}:{}".format(c[0], c[1]))
continue
seen.add(c[0])
unique.append(c)
return unique
def filter_concepts(self, predictions):
return [p[:self.cfgs['store_top_k']] for p in predictions]
def suppress_stop_list(self, predictions):
return [c for c in predictions if c[0] not in self.cfgs['stop_list_']]
def run_thresholds(self, predictions):
return self.threshold_concepts(predictions)
def resolve_antonyms(self, human_predictions):
conflicting_predictions = []
human_dict = {d[0]: d[1] for d in human_predictions}
human_labels = [d[0] for d in human_predictions]
human_scores = [d[1] for d in human_predictions]
for c1, c2 in self.cfgs['antonym_list']:
if not (c1 in human_labels and c2 in human_labels):
continue
# if
s1, s2 = human_dict[c1], human_dict[c2]
idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1)
logger.debug('Suppressing {0}:{1}'.format(human_labels[idx],
human_scores[idx]))
del human_labels[idx]
del human_scores[idx]
# for
remove_flag = -1
for idx, group in enumerate(self.cfgs['count_order_list']):
_this = np.intersect1d(human_dict.keys(), group)
if len(_this) > 0:
remove_flag = idx + 1
break
# if
# for
if not remove_flag == len(self.cfgs['count_order_list']):
remove_tags = []
for g in self.cfgs['count_order_list'][remove_flag:]:
remove_tags.extend(g)
# for
for t in remove_tags:
if t not in human_labels:
continue
# if
ridx = human_labels.index(t)
del human_labels[ridx]
del human_scores[ridx]
# for
# if
return [(g, s) for g, s in zip(human_labels, human_scores)]
# def
def map_concepts(self, concepts):
mapped_predictions = [(self.mapping[c[0]],
c[1]) if c[0] in self.mapping.keys() else c
for c in concepts]
mapped_predictions = self.conditional_mapping(mapped_predictions)
return mapped_predictions
# def
def conditional_mapping(self, predictions):
if 'conditional_list' in self.cfgs.keys():
for condition, mapping in self.cfgs['conditional_list']:
concepts = [p[0] for p in predictions]
holds = list(set(concepts).intersection(set(condition)))
if not len(holds) == 0:
predictions = [
p for p in predictions if not p[0] in mapping
]
# if
# for
# if
return predictions
# def
def threshold_concepts(self, predictions):
if self.thresholds is not None:
predictions = [
p for p in predictions if p[1] >= self.thresholds[p[0]]
]
# if
return predictions
class TheanoTesterCaptions():
def __init__(self, config_file, verbose=False, raw_predictions=False):
if not os.path.exists(config_file):
print 'Error : could not find config file'.format(config_file)
self.init = False
return
self.tester_config_file = config_file
self.verbose = verbose
self.raw_predictions = raw_predictions
self.read_config()
self.io = EmbeddingIO(self.cfgs['log_dir'], 'testing')
self.tester_name = self.cfgs['model_name']
self.model_config_file = self.cfgs['model_config_file']
self.init = False
self.thresholds = None
mapping_keys = [d[0] for d in self.cfgs['mapping_list']]
mapping_values = [d[1] for d in self.cfgs['mapping_list']]
self.mapping = dict(zip(mapping_keys, mapping_values))
if self.tester_name not in model_dict.keys():
self.io.print_error(
'Not a valid model type {0} chose among {1}'.format(
self.tester_name, model_dict.keys()))
self.init = False
return
if not os.path.exists(self.tester_config_file):
self.io.print_error('Could not find configuration file {0}'.format(
self.tester_config_file))
self.init = False
return
self.tester = None
self.action_items = {'test': self.run, 'eval': self.eval}
def _read_threshold_file(self):
if 'threshold_file' in self.cfgs.keys():
try:
pfile = open(self.cfgs['threshold_file'], 'r')
thresholds = json.load(pfile)
pfile.close()
self.thresholds = {
c: thresholds[c] if c in thresholds.keys() else
self.cfgs['global_threshold']
for c in self.concepts
}
except Exception as err:
self.io.print_error(
'Error parsing threshold file {0},{1}'.format(
self.cfgs['threshold_file'], err))
def read_config(self):
pfile = open(self.tester_config_file)
self.cfgs = yaml.load(pfile)
pfile.close()
def setup(self):
self.tester = model_dict[self.tester_name]()
self.tester.configure(self.model_config_file)
self.tester.define()
self.model_type = self.tester.model.get_config()['name']
self.im_h, self.im_w = self.tester.cfgs[
'image_height'], self.tester.cfgs['image_width']
if not self.tester.init:
self.io.print_error('Error with model definition')
self.init = False
return
self.io.print_info('{0} Model defined from {1}'.format(
self.tester_name, self.model_config_file))
self.compile()
if not self.tester.init:
self.io.print_error('Error with model compilation')
self.init = False
return
self.io.print_info('{0} Model compiled'.format(self.tester_name))
self.init = True
self.load_synset_file()
self.load_mean_file()
self._read_threshold_file()
self.load_confidence_model()
self.read_label_embedding()
def compile(self):
"""
This compile version is to be used for testing only. Since its optimized for predictions
"""
self.output_nodes = self.cfgs['output_node_name']['test']
if self.model_type == 'Sequential':
self.inference = K.function(
[self.tester.model.get_input(train=False)],
[self.tester.model.layers[-1].get_output(train=False)])
elif self.model_type == 'Graph':
self.inference = K.function(
[self.tester.model.get_input(train=False)], [
self.tester.model.nodes[node].get_output(train=False)
for node in self.output_nodes
])
else:
self.io.print_error(
'Say whaaaat ?! This model type is not supported : {}'.format(
self.model_type))
sel.init = False
def read_label_embedding(self):
self.label_embedding = None
self.label_vectors = None
if 'label_embedding_file' in self.cfgs.keys():
self.label_embedding = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'matrix')
self.label_vectors = self.io.read_npz_file(
self.cfgs['label_embedding_file'], 'vectors')
self.io.print_info('Read label embedding file {0}'.format(
self.cfgs['label_embedding_file']))
def load_confidence_model(self):
self.confidence_model = None
if 'confidence_model' in self.cfgs.keys():
self.confidence_model = joblib.load(self.cfgs['confidence_model'])
def run(self):
self.load_synset_file()
if not self.init:
return
self.test_images = self.io.read_images(self.cfgs['test_file_list'])
image_batches = [
self.test_images[i:i + self.cfgs['batch_size']]
for i in range(0, len(self.test_images), self.cfgs['batch_size'])
]
self.io.print_info('{0} images split into {1} Batches of {2}'.format(
len(self.test_images), len(image_batches),
self.cfgs['batch_size']))
for idx, images in enumerate(image_batches):
if not self.cfgs['force']:
flag = self.check_jsons(images)
if flag:
self.io.print_warning('Skipping batch {0} of {1}'.format(
idx, len(image_batches)))
continue
pixels, basenames = self.fetch_images(images)
flag, predictions, confidence, t = self.classify_images(pixels)
if self.raw_predictions is False:
getty_like_filter = [
self.suppress_stop_list(g) for g in predictions
]
getty_like_safe = [
self.run_thresholds(g) for g in getty_like_filter
]
getty_like_pc = [self.map_concepts(g) for g in getty_like_safe]
getty_like_public = [
self.resolve_antonyms(g) for g in getty_like_pc
]
getty_like_unique = [
self.resolve_duplicates(g) for g in getty_like_public
]
predictions_pc = getty_like_unique
save_ext = '.json'
else:
predictions_pc = predictions
save_ext = '.npz'
if not flag:
continue
file_paths = [
os.path.join(self.cfgs['results_dir'], im + save_ext)
for im in basenames
]
if not self.raw_predictions:
to_save = [dict({'external': p}) for p in predictions_pc]
else:
to_save = predictions_pc
assert (len(file_paths) == len(to_save))
self.io.print_info('Done batch {0} of {1} in {2} secs'.format(
idx, len(image_batches), t))
f = map(self.io.save_good_labels, file_paths, to_save)
def check_jsons(self, images):
basenames = []
for image in images:
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
basenames.append(im_basename)
flags = [
os.path.exists(os.path.join(self.cfgs['results_dir'], b + '.json'))
for b in basenames
]
return np.all(flags)
#
# Which measure to keep
# Precision/Recall/F1 scores
#
def eval(self):
self.load_synset_file()
self.test_images = self.io.read_images(self.cfgs['val_file_list'])
self.predictions = {}
self.gt = {}
for idx, image in enumerate(self.test_images):
if idx % 1000 == 0:
self.io.print_info('Reading {0}/{1}'.format(
idx, len(self.test_images)))
im_basename = os.path.basename(image)
im_basename, _ = os.path.splitext(im_basename)
result_json = os.path.join(self.cfgs['results_dir'],
im_basename + '.json')
gt_json = os.path.join(self.cfgs['gt_dir'], im_basename + '.json')
p, s = self.io.read_prediction_json(result_json)
g = self.io.read_vector(gt_json)
if p is [] or g is []:
continue
self.predictions[im_basename] = zip(p, s)
self.gt[im_basename] = list(set(g).intersection(self.concepts))
for t in self.cfgs['thresholds']:
self.compute_precision_recall(t)
def compute_precision_recall(self, thresh):
fps = {}
tps = {}
fns = {}
gt_dict = {}
for c in self.concepts:
fps[c], tps[c], fns[c], gt_dict[c] = [], [], [], []
for key, value in self.predictions.items():
detections = [v[0] for v in value if v[1] >= thresh]
gt = self.gt[key]
for g in gt:
gt_dict[g].append(key)
tp = list(set(gt).intersection(detections))
fp = list(set(detections) - set(gt))
fn = list(set(gt) - set(detections))
for t in tp:
tps[t].append(key)
for f in fp:
fps[f].append(key)
for n in fn:
fns[n].append(key)
for c in self.concepts:
if gt_dict[c] == []:
self.io.print_info(
'Skipping {0}, no samples in ground-truth'.format(c))
continue
p = len(tps[c]) / (len(tps[c]) + len(fps[c]) + 0.00000001)
r = len(tps[c]) / (len(gt_dict[c]) + 0.0)
f1 = self.compute_f1(p, r)
self.io.print_info(
'At {0} : {1} precision {2}, recall {3}, f1-score {4}'.format(
c, thresh, p, r, f1))
def compute_f1(self, p, r):
return 2 * (p * r) / (p + r + 0.000001)
def load_synset_file(self):
pfile = open(self.cfgs['synset_file'], 'r')
concepts = pfile.readlines()
pfile.close()
self.concepts = [p.strip() for p in concepts]
def load_mean_file(self):
if not self.cfgs['mean_file'] == 'None':
pfile = open(self.cfgs['mean_file'], 'r')
m = np.load(pfile)
pfile.close()
else:
m = [[[128.0]], [[128.0]], [[128.0]]]
m = np.mean(np.mean(m, axis=1), axis=1)
self.mean_pixel_value = np.array([m[0], m[1], m[2]])
self.pixel_scaling = self.cfgs['pixel_scaling'][
'test'] if 'pixel_scaling' in self.cfgs.keys() else None
self.channel_swap = self.cfgs['channel_swap'][
'test'] if 'channel_swap' in self.cfgs.keys() else None
self.image_shuffle = self.cfgs['image_shuffle'][
'test'] if 'image_shuffle' in self.cfgs.keys() else None
def jsonify(self, results):
return [[r[0], str(r[1])] for r in results]
def map_to_synsets(self, results):
results = {r[0]: r[1] for r in results}
return [results[c] for c in self.concepts]
def parse_predictions(self, p):
sort_idx = np.argsort(-p)
concepts = [(self.concepts[idx], float(p[idx]))
for idx in sort_idx[:self.cfgs['top_k']['test']]]
return concepts
def classify_images(self, images):
confidence = [None]
if images == []:
return False, [(), ()], 0.0
try:
tstart = time.time()
(predictions, features) = self.inference(images)
if self.label_vectors is not None:
predictions /= np.linalg.norm(predictions, axis=1)[:,
np.newaxis]
predictions = np.dot(predictions, self.label_vectors)
if self.confidence_model is not None:
confidence = [('Confidence', p)
for p in self.confidence_model.predict(features)]
tend = time.time()
except Exception as err:
self.io.print_error('Error processing image, {0}'.format(err))
return False, [(), ()], None, 0.0
if not self.raw_predictions:
results = True, [self.parse_predictions(p) for p in predictions
], confidence, '{0:.4f}'.format((tend - tstart))
else:
results = True, predictions, confidence, '{0:.4f}'.format(
(tend - tstart))
return results
def predict(self, image):
image = self.normalize_images([image])
return self.classify_images(image)
def normalize_images(self, images):
normalize_images = []
for image in images:
image *= 255.0
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
try:
image = cv2.resize(image, (self.im_h, self.im_w))
except Exception as err:
self.io.print_error('Could not parse test image {0}'.format(e))
return False, [], 0.0
image -= self.mean_pixel_value
image = image.transpose(2, 0, 1)
# Channel swap since caffe.io.load_image returns RGB image and training was done with BGR
image = image[(2, 1, 0), :, :]
normalize_images.append(image)
return [np.array(normalize_images)]
def scale_image(self, image):
#
# caffe scales image 0-1 in pixel values we bring it back to 255 as keras likes it that way
#
# No more caffe no more scaling down to 0-1 in caffe
# image *= 255.0
tstart = time.time()
if self.cfgs['square_crop']:
if self.verbose:
self.io.print_info(
'Yo ! As you like it - Squaring up the image')
image = self.square_it_up(image)
tend = time.time()
tstart = time.time()
try:
image = cv2.resize(image,
(self.im_h, self.im_w)).astype(np.float32)
except Exception as err:
self.io.print_error('Could not parse {0}'.format(e))
return None
tend = time.time()
tstart = time.time()
if self.mean_pixel_value is not None:
image -= self.mean_pixel_value
if self.pixel_scaling is not None:
image /= self.pixel_scaling
if self.channel_swap is not None:
image = image[:, :, self.channel_swap]
# H,W,C to C,H,W
if self.image_shuffle is not None:
image = image.transpose(self.image_shuffle)
tend = time.time()
return image
def fetch_images(self, image_file_names):
images = []
basenames = []
tstart = time.time()
for idx, image_file_name in enumerate(image_file_names):
im_basename = os.path.basename(image_file_name)
im_basename, _ = os.path.splitext(im_basename)
if not urlparse.urlparse(image_file_name).scheme == "":
url_response = urllib.urlopen(image_file_name)
if url_response.code == 404:
print self.io.print_error(
'[Testing] URL error code : {1} for {0}'.format(
image_file_name, url_response.code))
continue
try:
string_buffer = StringIO.StringIO(url_response.read())
image = np.asarray(bytearray(string_buffer.read()),
dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
image = image[:, :, (2, 1, 0)]
image = image.astype(np.float32)
except Exception as err:
print self.io.print_error(
'[Testing] Error with URL {0} {1}'.format(
image_file_name, err))
continue
elif os.path.exists(image_file_name):
try:
fid = open(image_file_name, 'r')
stream = fid.read()
fid.close()
image = Image.open(cStringIO.StringIO(stream))
image = np.array(image)
image = image.astype(np.float32)
# image = cv2.imread(image_file_name)
# image = image[:, :, (2, 1, 0)]
# image = image.astype(np.float32)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with image file {0} {1}'.format(
image_file_name, err))
continue
else:
try:
image = self.conn.read_image(image_file_name)
if image is None:
raise self.io.print_error(
"[Testing] Image data could not be loaded - %s" %
image_file_name)
continue
image = np.array(image)
if image.ndim == 2:
image = image[:, :, np.newaxis]
image = np.tile(image, (1, 1, 3))
elif image.shape[2] == 4:
image = image[:, :, :3]
except Exception as err:
print self.io.print_error(
'[Testing] Error with S3 Bucket hash {0} {1}'.format(
image_file_name, err))
continue
# try : do or do not there is not try
image = self.scale_image(image)
if image is None:
continue
if self.verbose:
self.io.print_info('Processing {0} {1}'.format(
image_file_name, image.shape))
images.append(image)
basenames.append(im_basename)
tend = time.time()
self.io.print_info('Fetching {0} images took {1:.4f} secs'.format(
len(image_file_names), tend - tstart))
return [np.array(images)], basenames
def fetch_gif_frames(self, inGif):
# Only works for URLs for now
# write frames to disk and pass filenames to fetch_images
outFolder = '/tmp'
frame = Image.open(BytesIO(urllib.urlopen(inGif).read()))
nframes = 0
file_names = []
while frame:
frame_name = '{0}/{1}-{2}.png'.format(outFolder,
os.path.basename(inGif),
nframes)
frame.save(frame_name, 'png')
file_names.append(frame_name)
nframes += 1
try:
frame.seek(nframes)
except EOFError:
break
file_names = [file_names[i] for i in range(0, len(file_names), 10)]
return file_names
def non_max(self, predictions):
concepts = {}
for pred in predictions:
for p in pred:
if p[0] in concepts.keys():
concepts[p[0]].append(float(p[1]))
else:
concepts[p[0]] = [float(p[1])]
for key, value in concepts.items():
concepts[key] = np.max(np.array(value))
labels = concepts.keys()
scores = [concepts[l] for l in labels]
sort_idx = np.argsort(-np.array(scores))
results = [(labels[idx], str(scores[idx])) for idx in sort_idx]
return [results]
def square_it_up(self, image):
edge_size = np.min(image.shape[:2])
cy, cx = np.array(image.shape[:2]) // 2
r_img = image[np.max([0, cy - edge_size //
2]):np.min([cy +
edge_size // 2, image.shape[0]]),
np.max([0, cx - edge_size //
2]):np.min([cx +
edge_size // 2, image.shape[1]]), :]
return r_img
def resolve_duplicates(self, concepts):
seen = set()
unique = []
for c in concepts:
if c[0] in seen:
logger.debug("Suppressing duplicate {}:{}".format(c[0], c[1]))
continue
seen.add(c[0])
unique.append(c)
return unique
def filter_concepts(self, predictions):
return [p[:self.cfgs['store_top_k']] for p in predictions]
def suppress_stop_list(self, predictions):
return [c for c in predictions if c[0] not in self.cfgs['stop_list_']]
def run_thresholds(self, predictions):
return self.threshold_concepts(predictions)
def resolve_antonyms(self, human_predictions):
conflicting_predictions = []
human_dict = {d[0]: d[1] for d in human_predictions}
human_labels = [d[0] for d in human_predictions]
human_scores = [d[1] for d in human_predictions]
for c1, c2 in self.cfgs['antonym_list']:
if not (c1 in human_labels and c2 in human_labels):
continue
# if
s1, s2 = human_dict[c1], human_dict[c2]
idx = human_labels.index(c2) if s1 > s2 else human_labels.index(c1)
logger.debug('Suppressing {0}:{1}'.format(human_labels[idx],
human_scores[idx]))
del human_labels[idx]
del human_scores[idx]
# for
remove_flag = -1
for idx, group in enumerate(self.cfgs['count_order_list']):
_this = np.intersect1d(human_dict.keys(), group)
if len(_this) > 0:
remove_flag = idx + 1
break
# if
# for
if not remove_flag == len(self.cfgs['count_order_list']):
remove_tags = []
for g in self.cfgs['count_order_list'][remove_flag:]:
remove_tags.extend(g)
# for
for t in remove_tags:
if t not in human_labels:
continue
# if
ridx = human_labels.index(t)
del human_labels[ridx]
del human_scores[ridx]
# for
# if
return [(g, s) for g, s in zip(human_labels, human_scores)]
# def
def map_concepts(self, concepts):
mapped_predictions = [(self.mapping[c[0]],
c[1]) if c[0] in self.mapping.keys() else c
for c in concepts]
mapped_predictions = self.conditional_mapping(mapped_predictions)
return mapped_predictions
# def
def conditional_mapping(self, predictions):
if 'conditional_list' in self.cfgs.keys():
for condition, mapping in self.cfgs['conditional_list']:
concepts = [p[0] for p in predictions]
holds = list(set(concepts).intersection(set(condition)))
if not len(holds) == 0:
predictions = [
p for p in predictions if not p[0] in mapping
]
# if
# for
# if
return predictions
# def
def threshold_concepts(self, predictions):
if self.thresholds is not None:
predictions = [
p for p in predictions if p[1] >= self.thresholds[p[0]]
]
# if
return predictions
