def test_sae(self):
"""
Tests if the matrix returned by the sylvester calculation matches all the expected values
"""
s_tr = normalize(self.data['S_tr'], norm='l2', axis=1, copy=False)
x_tr, _ = ZSL.dimension_reduction(self.data['X_tr'], self.data['X_te'], self.labels)
ref = loadmat('mockfiles/sae.mat')['sae']
w = ZSL.sae(x_tr.transpose(), s_tr.transpose(), .2).transpose()
self.assertTrue((np.around(ref, decimals=5) == np.around(w, decimals=5)).all())
def test_zsl_el(self):
"""
Tests if accuracy of Zero Shot Learning computed is the same as the expected one and if
the returned classes of 1NN is the same as the reference
"""
s_tr = normalize(self.data['S_tr'], norm='l2', axis=1, copy=False)
x_tr, x_te = ZSL.dimension_reduction(self.data['X_tr'], self.data['X_te'], self.labels)
w = ZSL.sae(x_tr.transpose(), s_tr.transpose(), .2).transpose()
temp_labels = np.array([int(x) for x in self.data['te_cl_id']])
test_labels = np.array([int(x) for x in self.data['test_labels_cub']])
ref = loadmat('mockfiles/y_hit.mat')['Y_hit5']
acc, y_hit = ZSL.zsl_el(x_te.dot(w), self.data['S_te_pro'], test_labels, temp_labels, 1, z_score=True)
self.assertEqual(0.61405, np.around(acc, decimals=5))
self.assertTrue((ref == y_hit).all())
def test_sub2ind(self):
"""
Tests if the returned value is equal to indexed position of rows and columns combinations
"""
row = np.array([0, 1, 2, 0])
col = np.array([1, 1, 1, 2])
self.assertEqual([3, 4, 5, 6], ZSL.sub2ind((3, 3), row, col))
def _compute_weights(self):
"""
Computes the weights that estimates the latent space using SAE
:return: a 2D numpy array with the matrix of weights computed
"""
return ZSL.sae(self.x_tr.transpose(), self.s_tr.transpose(), self.lambda_).transpose()
def classify_vis_data(self, vis_data, labels, reduce_dim=False):
fold = 0
accuracies = []
skf = StratifiedKFold(n_splits=self.n_folds,
random_state=None,
shuffle=True)
for train_index, test_index in skf.split(vis_data, labels):
logging.info('Running VIS classification for fold %d' % fold)
tr_data, te_data = vis_data[train_index], vis_data[test_index]
tr_labels, te_labels = labels[train_index][:, 0], labels[
test_index][:, 0]
if reduce_dim:
tr_data, te_data = ZSL.dimension_reduction(
tr_data, te_data, list(tr_labels))
clf = make_pipeline(StandardScaler(),
SVC(gamma='auto', C=1.0, kernel='linear'))
clf.fit(tr_data, tr_labels)
prediction = clf.predict(te_data)
fold += 1
accuracies.append(balanced_accuracy_score(te_labels, prediction))
return accuracies
def test_label_matrix(self):
"""
Tests if the matrix of labels returned matches all the expected values
"""
ref = loadmat('mockfiles/label_matrix.mat')['label_matrix']
mat = ZSL.label_matrix(self.labels)
self.assertTrue((mat == ref).all())
def _compute_weights(self):
"""
Computes the weights that estimates the latent space using SAE
:return: a 2D numpy array with the matrix of weights computed
"""
s_tr = normalize(self.s_tr, norm='l2', axis=1, copy=False)
return ZSL.sae(self.x_tr.transpose(), s_tr.transpose(), self.lambda_).transpose()
def test_dimension_reduction(self):
"""
Tests if the x_tr and x_te returned matches all the expected values
"""
x_tr, x_te = ZSL.dimension_reduction(self.data['X_tr'], self.data['X_te'], self.labels)
ref = loadmat('mockfiles/dimension_reduction.mat')['dimension_reduction']
self.assertTrue((np.around(x_tr, decimals=5) == np.around(ref['X_tr'][0][0], decimals=5)).all())
self.assertTrue((np.around(x_te, decimals=5) == np.around(ref['X_te'][0][0], decimals=5)).all())
def test_is_member(self):
"""
Tests if the returned value labels is equal to all available labels in the data set, and if the
indexes correspond to the index of the latest occurrence of a label
"""
ref = loadmat('mockfiles/indexes.mat')['indexes'][0]
indexes, labels = ZSL.is_member(self.labels)
self.assertTrue((list(map(int, ref - 1)) == indexes).all())
self.assertTrue((list(set(self.labels)) == labels).all())
def s2v_projection(self):
"""
Applies zero shot learning in the estimated data, classifies each test sample with the class of the closest
neighbor and computes the accuracy of classification comparing the estimated class with the one stated in the
template array. The projection goes from the semantic space to the feature space (visual features extracted
from a CNN).
:return: float number with the accuracy of the ZSL classification
"""
if self.w is None:
self.w = self._compute_weights()
x_te_pro = self.data['S_te_pro'].dot(self.w.transpose())
acc, _ = ZSL.zsl_el(self.x_te, x_te_pro, self.test_labels, self.temp_labels, self.hit_k, self.z_score)
return acc
def estimate_sae_data(self, tr_vis_data, te_vis_data, tr_sem_data,
tr_labels):
if self.data_type == DataType.CUB:
tr_vis, te_vis = ZSL.dimension_reduction(tr_vis_data, te_vis_data,
tr_labels)
tr_sem = normalize(tr_sem_data, norm='l2', axis=1, copy=True)
sae_w = ZSL.sae(tr_vis.transpose(), tr_sem.transpose(),
self.lambda_).transpose()
tr_sem, te_sem = tr_vis.dot(sae_w), te_vis.dot(sae_w)
else:
tr_vis = normalize(tr_vis_data.transpose(),
norm='l2',
axis=1,
copy=True).transpose()
sae_w = ZSL.sae(tr_vis.transpose(), tr_sem_data.transpose(),
self.lambda_)
tr_sem = tr_vis.dot(
normalize(sae_w, norm='l2', axis=1, copy=True).transpose())
te_sem = te_vis_data.dot(
normalize(sae_w, norm='l2', axis=1, copy=True).transpose())
return tr_sem, te_sem
def __init__(self, data_path):
"""
Defines parameters, loads data and computes weights using SAE
:param data_path: string with path with .mat file with data set
"""
self.hit_k = 1
self.lambda_ = .2
self.z_score = True
self.s_tr = None
self.w = None
self.data = loadmat(data_path)
self.temp_labels = np.array([int(x) for x in self.data['te_cl_id']])
self.test_labels = np.array([int(x) for x in self.data['test_labels_cub']])
labels = list(map(int, self.data['train_labels_cub']))
self.x_tr, self.x_te = ZSL.dimension_reduction(self.data['X_tr'], self.data['X_te'], labels)
def cal_acc(te_est_or):
te_labels = {label: [] for label in set([lb for lb in _data['test_labels']])}
te_est_pt = te_est_or[mask, :]
te_est_st = te_est_or[[not v for v in mask], :]
test_labels_pt = test_labels[mask]
test_labels_st = test_labels[[not v for v in mask]]
for i, sample in enumerate(te_est_pt):
te_labels[test_labels_pt[i]].append(sample)
prototypes = []
for lb in template_labels:
prototypes.append(np.mean(te_labels[lb], axis=0))
prototypes = np.array(prototypes)
acc_value, _ = ZSL.zsl_el(te_est_st, prototypes, test_labels_st, template_labels, 1, True)
return acc_value
def cal_acc_sec(te_est):
acc_value, _ = ZSL.zsl_el(te_est, _data['S_te_pro'], test_labels, template_labels, 1, True)
return acc_value
tr_vis_data = normalize(tr_vis_data, norm='l2', axis=1, copy=True)
te_vis_data = normalize(te_vis_data, norm='l2', axis=1, copy=True)
input_length = output_length = tr_vis_data.shape[1] + tr_sem_data.shape[1]
ae = Encoder(input_length, tr_sem_data.shape[1], output_length,
ModelType.ZSL_AE, 50, res_path)
tr_sem, te_sem = ae.estimate_semantic_data_zsl(tr_vis_data, te_vis_data,
tr_sem_data, False)
return tr_sem, te_sem
# data = loadmat('../Datasets/SEM/cub_demo_data_resnet.mat')
# tr_est, te_est = estimate_sem_data(data['X_tr'], data['X_te'], data['S_tr'], '.')
#
# template_labels = np.array([int(x) for x in data['te_cl_id']])
# test_labels = np.array([int(x) for x in data['test_labels_cub']])
data = loadmat('../Datasets/SEM/awa_demo_data_resnet.mat')
tr_est, te_est = estimate_sem_data(data['X_tr'], data['X_te'], data['S_tr'],
'.')
template_labels = np.array(
[int(x) for x in data['param']['testclasses_id'][0][0]])
test_labels = np.array([int(x) for x in data['param']['test_labels'][0][0]])
acc, _ = ZSL.zsl_el(te_est, data['S_te_pro'], test_labels, template_labels, 1,
False)
print(acc)