def precision_and_recall(actual,predicted,cls): c = (actual == cls) si = sp.argsort(-c) tp = sp.cumsum(sp.single(predicted[si] == cls)) fp = sp.cumsum(sp.single(predicted[si] != cls)) rec = tp /sp.sum(predicted == cls) prec = tp / (fp + tp) return prec,rec
def precision_and_recall(actual, predicted, cls): c = (actual == cls) si = sp.argsort(-c) tp = sp.cumsum(sp.single(predicted[si] == cls)) fp = sp.cumsum(sp.single(predicted[si] != cls)) rec = tp / sp.sum(predicted == cls) prec = tp / (fp + tp) return prec, rec
def proposal_im_detect(conf, caffe_net, im): im = scipy.single(im) im_blob, im_scales = get_image_blob(conf, im) im_size = [scipy.size(im, 0), scipy.size(im, 1)] scaled_im_size = list(np.round(np.array(im_size) * im_scales)) data = np.array(im_blob) r, g, b = data.T im_blob2 = np.array([b, g, r]) im_blob = im_blob2.transpose() im_blob = im_blob.transpose() im_blob = im_blob.transpose(1, 2, 0) im_blob = scipy.single(im_blob) im_blob = np.expand_dims(im_blob, axis=0) # -> (1, 800, 600, 3) im_blob = im_blob.transpose(0, 3, 2, 1) # -> (1, 3, 600, 800) net_inputs = im_blob caffe_net.reshape_as_input(net_inputs) output_blobs = caffe_net.forward(data=net_inputs) proposal_bbox_pred = output_blobs['proposal_bbox_pred'].transpose( 0, 3, 2, 1)[0] box_deltas = proposal_bbox_pred featuremap_size = (scipy.size(box_deltas, 1), scipy.size(box_deltas, 0)) box_deltas = box_deltas.reshape(-1, 4) conf = conf[0][0] anchors, _ = proposal_locate_anchors(conf, im.shape, conf['test_scales'], featuremap_size) pred_boxes = fast_rcnn_bbox_transform_inv(anchors, box_deltas) part1 = pred_boxes - 1 part2 = np.subtract([im_size[1], im_size[0], im_size[1], im_size[0]], 1) / \ np.subtract( [scaled_im_size[1], scaled_im_size[0], scaled_im_size[1], scaled_im_size[0]], 1) pred_boxes = np.multiply(part1, part2) + 1 pred_boxes = clip_boxes(pred_boxes, im.shape[1], im.shape[0]) scores = output_blobs['proposal_cls_prob'][0][-1::][0].T scores = scores.reshape(proposal_bbox_pred.shape[0], proposal_bbox_pred.shape[1], -1, order='F') scores = scores.transpose(2, 1, 0) scores = scores.flatten(1) pred_boxes, scores = filter_boxes(conf['test_min_box_size'], pred_boxes, scores) scores_ind = scores.argsort()[::-1] scores = np.sort(scores)[::-1] pred_boxes = pred_boxes[scores_ind, ] return pred_boxes, scores
def fast_rcnn_conv_feat_detect(conf, caffe_net, im, conv_feat_blob, boxes, max_rois_num_in_gpu): conf = conf[0][0][0] rois_blob, _ = get_blobs(conf, im, boxes) rois_blob = rois_blob - 1 rois_blob = np.expand_dims(np.expand_dims(rois_blob.T, 0), 0) rois_blob = scipy.single(rois_blob) caffe_net.blobs['data'].reshape(*conv_feat_blob.data.shape) np.copyto(caffe_net.blobs['data'].data, conv_feat_blob.data) total_rois = scipy.size(rois_blob, 3) _pseudo_magic = int(math.ceil(float(total_rois) / max_rois_num_in_gpu)) total_scores = [] # np.array(_pseudo_magic) total_box_deltas = [] # np.array(_pseudo_magic) for i in xrange(_pseudo_magic): _i = i + 1 sub_ind_start = 1 + (_i - 1) * max_rois_num_in_gpu sub_ind_end = min(total_rois, _i * max_rois_num_in_gpu) # MATLAB: sub_rois_blob = rois_blob(:, :, :, sub_ind_start:sub_ind_end); sub_rois_blob = rois_blob.T[sub_ind_start - 1:sub_ind_end, ] net_inputs = [np.ndarray(0), sub_rois_blob] caffe_net.reshape_as_input(net_inputs) output_blobs = caffe_net.forward(rois=sub_rois_blob, data=conv_feat_blob.data) if conf['test_binary']: # % simulate binary logistic regression raise Exception('Not implemented') else: # % use softmax estimated probabilities scores = output_blobs['cls_prob'] scores = np.squeeze(scores) # % Apply bounding-box regression deltas box_deltas = output_blobs['bbox_pred'] box_deltas = np.squeeze(box_deltas) total_scores.insert(i, scores) total_box_deltas.insert(i, box_deltas) pred_boxes = fast_rcnn_bbox_transform_inv(boxes, box_deltas) pred_boxes = clip_boxes(pred_boxes, im.shape[1], im.shape[0]) pred_boxes = pred_boxes.T[4:].T scores = scores.T[1:].T return pred_boxes, scores
def map_im_rois_to_feat_rois(conf, im_rois, scales): im_rois = scipy.single(im_rois) levels = scipy.ones(im_rois.shape[0]) levels = np.expand_dims(levels, 1) # FIXME: below __scales = np.array(levels) __scales.fill(scales) bsxfunret = ((im_rois - 1) * __scales) + 1 feat_rois = np.around(bsxfunret) return feat_rois, levels
def prep_im_for_blob(im, im_means, target_size, max_size): im = scipy.single(im) im_means4 = cv2.resize(im_means, (scipy.size(im, 1), scipy.size(im, 0)), interpolation=cv2.INTER_LINEAR) im_means = im_means4 im = im - im_means im_scale = prep_im_for_blob_size((scipy.size(im, 0), scipy.size(im, 1)), target_size, max_size) target_size = (int(scipy.size(im, 1) * im_scale), int(scipy.size(im, 0) * im_scale)) im1 = cv2.resize(im, target_size, interpolation=cv2.INTER_LINEAR) return im1, im_scale
def classify(train_features, train_labels, test_features, test_labels): '''Classify data and return accuracy area under curve average precision and svm raw data in a dictianary''' #mapping labels to 0,1 labels = sp.unique(sp.concatenate((train_labels, test_labels))) assert labels.size == 2 label_to_id = dict([(k,v) for v, k in enumerate(labels)]) train_ys = sp.array([label_to_id[i] for i in train_labels]) test_ys = sp.array([label_to_id[i] for i in test_labels]) #train model = classifier_train(train_features, train_ys, test_features) #test weights = model.coef_.ravel() bias = model.intercept_.ravel() predict = sp.dot(test_features, weights) + bias def_predict = model.predict(test_features) #raw data to be saved for future use cls_data = {'def_prdict' : def_predict, 'predict' : predict, 'test_lables' : test_labels, 'coef' : model.coef_, 'intercept' : model.intercept_} #accuracy hit = 0 for i_ind in range(len(test_labels)): if (predict[i_ind]/abs(predict[i_ind])) * int(test_labels[i_ind]) == 1: hit += 1 accu = sp.single(hit)/len(test_labels) #precison and recall c = predict si = sp.argsort(-c) tp = sp.cumsum(sp.single(test_ys[si] == 1)) fp = sp.cumsum(sp.single(test_ys[si] == 0)) rec = tp /sp.sum(test_ys > 0) prec = tp / (fp + tp) ap = 0 rng = sp.arange(0, 1.1, .1) for th in rng: p = prec[rec>=th].max() if p == []: p =0 ap += p / rng.size #area under curve h = sp.diff(rec) auc = sp.sum(h * (prec[1:] + prec[:-1])) / 2.0 return accu, auc, ap, cls_data
def get_rois_blob(conf, im_rois, im_scale_factors): feat_rois, levels = map_im_rois_to_feat_rois(conf, im_rois, im_scale_factors) rois_blob = np.concatenate((levels.T, feat_rois.T)).T rois_blob = scipy.single(rois_blob) return rois_blob
def kernel_generate_fromcsv(input_csv_fname, input_suffix, output_fname, corrcoef_type = DEFAULT_CORRCOEF_TYPE, nowhiten = DEFAULT_NOWHITEN, variable_name = DEFAULT_VARIABLE_NAME, input_path = DEFAULT_INPUT_PATH, overwrite = DEFAULT_OVERWRITE, ): # add matlab's extension to the output filename if needed if path.splitext(output_fname)[-1] != ".mat": output_fname += ".mat" # can we overwrite ? if path.exists(output_fname) and not overwrite: warnings.warn("not allowed to overwrite %s" % output_fname) return # -------------------------------------------------------------------------- # -- get training and testing filenames from csv print "Processing %s ..." % input_csv_fname csvr = csv.reader(open(input_csv_fname)) rows = [ row for row in csvr ] ori_train_fnames = [ row[0] for row in rows if row[2] == "train" ][:LIMIT] train_fnames = sp.array([ path.join(input_path, fname+input_suffix) for fname in ori_train_fnames ][:LIMIT]) train_labels = sp.array([ row[1] for row in rows if row[2] == "train" ][:LIMIT]) ori_test_fnames = [ row[0] for row in rows if row[2] == "test" ][:LIMIT] test_fnames = sp.array([ path.join(input_path, fname+input_suffix) for fname in ori_test_fnames ][:LIMIT]) test_labels = sp.array([ row[1] for row in rows if row[2] == "test" ][:LIMIT]) ntrain = len(train_fnames) ntest = len(test_fnames) # -------------------------------------------------------------------------- # -- load features from train filenames # set up progress bar print "Loading training data ..." pbar = ProgressBar(widgets=widgets, maxval=ntrain) pbar.start() fvector0 = io.loadmat(train_fnames[0])[variable_name].ravel() featshape = fvector0.shape featsize = fvector0.size # go train_features = sp.empty((ntrain,) + featshape, dtype='float32') error = False for i, fname in enumerate(train_fnames): try: fvector = io.loadmat(fname)[variable_name].ravel() except TypeError: print "[ERROR] couldn't open", fname, "deleting it!" os.unlink(fname) error = True except: print "[ERROR] unkwon with", fname raise assert(not sp.isnan(fvector).any()) assert(not sp.isinf(fvector).any()) train_features[i] = fvector.reshape(fvector0.shape) pbar.update(i+1) pbar.finish() print "-"*80 if error: raise RuntimeError("An error occured (load train). Exiting.") # -- preprocess train print "Preprocessing train features ..." if nowhiten: whiten_vectors = None else: fshape = train_features.shape train_features.shape = fshape[0], -1 npoints, ndims = train_features.shape if npoints < MEAN_MAX_NPOINTS: fmean = train_features.mean(0) else: # - try to optimize memory usage... sel = train_features[:MEAN_MAX_NPOINTS] fmean = sp.empty_like(sel[0,:]) sp.add.reduce(sel, axis=0, dtype="float32", out=fmean) curr = sp.empty_like(fmean) npoints_done = MEAN_MAX_NPOINTS while npoints_done < npoints: sel = train_features[npoints_done:npoints_done+MEAN_MAX_NPOINTS] sp.add.reduce(sel, axis=0, dtype="float32", out=curr) sp.add(fmean, curr, fmean) npoints_done += MEAN_MAX_NPOINTS fmean /= npoints if npoints < STD_MAX_NPOINTS: fstd = train_features.std(0) else: # - try to optimize memory usage... sel = train_features[:MEAN_MAX_NPOINTS] mem = sp.empty_like(sel) curr = sp.empty_like(mem[0,:]) seln = sel.shape[0] sp.subtract(sel, fmean, mem[:seln]) sp.multiply(mem[:seln], mem[:seln], mem[:seln]) fstd = sp.add.reduce(mem[:seln], axis=0, dtype="float32") npoints_done = MEAN_MAX_NPOINTS while npoints_done < npoints: sel = train_features[npoints_done:npoints_done+MEAN_MAX_NPOINTS] seln = sel.shape[0] sp.subtract(sel, fmean, mem[:seln]) sp.multiply(mem[:seln], mem[:seln], mem[:seln]) sp.add.reduce(mem[:seln], axis=0, dtype="float32", out=curr) sp.add(fstd, curr, fstd) npoints_done += MEAN_MAX_NPOINTS fstd = sp.sqrt(fstd/npoints) fstd[fstd==0] = 1 whiten_vectors = (fmean, fstd) train_features.shape = fshape train_features = preprocess_features(train_features, whiten_vectors = whiten_vectors) assert(not sp.isnan(sp.ravel(train_features)).any()) assert(not sp.isinf(sp.ravel(train_features)).any()) # -- train categories = sp.unique(train_labels) #if categories.size == 2: # categories = [categories[0]] #else: # raise NotImplementedError("not sure if it works with ncats > 2") corrcoef_kernels = {} cat_index = {} for icat, cat in enumerate(categories): if corrcoef_type == 'pos_neg_mean': #print train_features.shape #print train_features[train_labels == cat].shape #print train_features[train_labels != cat].shape corrcoef_ker = train_features[train_labels == cat].sum(0) \ - train_features[train_labels != cat].sum(0) corrcoef_ker /= ntrain elif corrcoef_type == 'pos_mean_neg_mean': corrcoef_ker = train_features[train_labels == cat].mean(0) \ - train_features[train_labels != cat].mean(0) elif corrcoef_type == 'pos_mean': corrcoef_ker = train_features[train_labels == cat].mean(0) else: raise ValueError("corrcoef_type '%s' not understood" % corrcoef_type) corrcoef_ker -= corrcoef_ker.mean() corrcoef_ker_mag = sp.linalg.norm(corrcoef_ker) assert corrcoef_ker_mag > 0 corrcoef_ker /= sp.linalg.norm(corrcoef_ker) assert(not sp.isnan(corrcoef_ker).any()) assert(not sp.isinf(corrcoef_ker).any()) corrcoef_kernels[cat] = corrcoef_ker cat_index[cat] = icat # -------------------------------------------------------------------------- # -- load features from test filenames # set up progress bar print "Testing (on the fly) ..." pbar = ProgressBar(widgets=widgets, maxval=ntest) pbar.start() # -- test # XXX: code adapted from beta svm_ova_fromfilenames (to review!) pred = sp.zeros((ntest)) distances = sp.zeros((ntest, len(categories))) for itest, fname in enumerate(test_fnames): try: fvector = io.loadmat(fname)[variable_name].ravel() except TypeError: print "[ERROR] couldn't open", fname, "deleting it" os.unlink(fname) error = True except: print "[ERROR] unkwon with", fname raise assert(not sp.isnan(fvector).any()) assert(not sp.isinf(fvector).any()) # whiten if needed if whiten_vectors is not None: fmean, fstd = whiten_vectors fvector -= fmean assert((fstd!=0).all()) fvector /= fstd assert(not sp.isnan(fvector).any()) assert(not sp.isinf(fvector).any()) # corrcoef testv = fvector testv -= testv.mean() testv_mag = sp.linalg.norm(testv) assert testv_mag > 0 testv /= testv_mag for icat, cat in enumerate(categories): corrcoef_ker = corrcoef_kernels[cat] resp = sp.dot(testv, corrcoef_ker) distances[itest, icat] = resp pbar.update(itest+1) pbar.finish() print "-"*80 if error: raise RuntimeError("An error occured (load test). Exiting.") if len(categories) > 1: pred = distances.argmax(1) #print sp.array([cat_index[e] for e in test_labels]).astype('int') gt = sp.array([cat_index[e] for e in test_labels]).astype("int") perf = (pred == gt) accuracy = 100.*perf.sum() / ntest else: pred = sp.sign(distances).ravel() gt = sp.array(test_labels) cat = categories[0] gt[gt != cat] = -1 gt[gt == cat] = +1 gt = gt.astype("int") perf = (pred == gt) accuracy = 100.*perf.sum() / ntest print distances.shape print "Classification accuracy on test data (%):", accuracy svm_labels = gt # -- average precision # XXX: redo this part to handle other labels than +1 / -1 ap = 0 if distances.shape[1] == 1: distances = distances.ravel() assert test_labels.ndim == 1 assert svm_labels.ndim == 1 # -- inverse predictions if needed # (that is when svm was trained with flipped +1/-1 labels) # -- convert test_labels to +1/-1 (int) try: test_labels = array([int(elt) for elt in test_labels]) if (test_labels != svm_labels).any(): distances = -distances #if not ((test_labels==-1).any() and (test_labels==1).any()): # test_labels[test_labels!=test_labels[0]] = +1 # test_labels[test_labels==test_labels[0]] = -1 #print test_labels # -- convert test_labels to +1/-1 (int) test_labels = array([int(elt) for elt in test_labels]) # -- get average precision c = distances #print c si = sp.argsort(-c) tp = sp.cumsum(sp.single(test_labels[si]>0)) fp = sp.cumsum(sp.single(test_labels[si]<0)) rec = tp/sp.sum(test_labels>0) prec = tp/(fp+tp) #print prec, rec #from pylab import * #plot(prec, rec) #show() ap = 0 rng = sp.arange(0,1.1,.1) for th in rng: p = prec[rec>=th].max() if p == []: p = 0 ap += p / rng.size print "Average Precision:", ap except ValueError: ap = 0 # XXX: clean this test_y = sp.array([svm_labels.ravel()==lab for lab in sp.unique(svm_labels.ravel())] )*2-1 test_y = test_y.T print distances # -------------------------------------------------------------------------- # -- write output file if output_fname is not None: print "Writing %s ..." % (output_fname) # TODO: save more stuff (alphas, etc.) data = {"accuracy": accuracy, "average_precision":ap, "test_distances": distances, "test_labels": test_labels, "test_y": test_y, "svm_labels": svm_labels, } io.savemat(output_fname, data, format='4') return accuracy
def classify(train_features, train_labels, test_features, test_labels, sphere=True): '''Classify data and return accuracy area under curve average precision and svm raw data in a dictianary''' #mapping labels to 0,1 labels = sp.unique(sp.concatenate((train_labels, test_labels))) assert labels.size == 2 label_to_id = dict([(k,v) for v, k in enumerate(labels)]) train_ys = sp.array([label_to_id[i] for i in train_labels]) test_ys = sp.array([label_to_id[i] for i in test_labels]) #train model = classifier_train(train_features, train_ys, test_features,sphere=sphere) #test weights = model.coef_.ravel() bias = model.intercept_.ravel() test_predictor = sp.dot(test_features, weights) + bias test_prediction = model.predict(test_features) train_prediction = model.predict(train_features) #raw data to be saved for future use cls_data = {'test_prediction' : test_prediction, 'test_lables' : test_labels, 'coef' : model.coef_, 'intercept' : model.intercept_ } #accuracy test_accuracy = 100*(test_prediction == test_ys).sum()/float(len(test_ys)) train_accuracy = 100*(train_prediction == train_ys).sum()/float(len(train_ys)) #precison and recall c = test_predictor si = sp.argsort(-c) tp = sp.cumsum(sp.single(test_ys[si] == 1)) fp = sp.cumsum(sp.single(test_ys[si] == 0)) rec = tp /sp.sum(test_ys > 0) prec = tp / (fp + tp) ap = 0 rng = sp.arange(0, 1.1, .1) for th in rng: p = prec[rec>=th].max() if p == []: p =0 ap += p / rng.size #area under curve h = sp.diff(rec) auc = sp.sum(h * (prec[1:] + prec[:-1])) / 2.0 return {'auc':auc, 'ap':ap, 'train_accuracy': train_accuracy, 'test_accuracy' : test_accuracy, 'cls_data':cls_data }
def svm_ova_fromfilenames(input_filenames, weights = DEFAULT_WEIGHTS, # -- regularization = DEFAULT_REGULARIZATION, no_trace_normalization = DEFAULT_NO_TRACE_NORMALIZATION, # -- output_filename = DEFAULT_OUTPUT_FILENAME, overwrite = DEFAULT_OVERWRITE, ): """ TODO: docstring """ assert len(weights) <= len(input_filenames) if output_filename is not None: # add matlab's extension to the output filename if needed if path.splitext(output_filename)[-1] != ".mat": output_filename += ".mat" # can we overwrite ? if path.exists(output_filename) and not overwrite: warnings.warn("not allowed to overwrite %s" % output_filename) return # -- lw = len(weights) lf = len(input_filenames) if lw <= lf: weights += [1. for _ in xrange(lf-lw)] print "Using weights =", weights # -- check # -- kernel fusion distance_fusion = True # -- kernel_train = None kernel_test = None train_fnames = None test_fnames = None for fname, weight in zip(input_filenames, weights): print "Loading %s (weight=%f) ..." % (fname, weight) if weight == 0: continue kernel_mat = io.loadmat(fname) #print kernel_mat.keys() #continue #raise # -- check that kernels come from the same "source" if train_fnames is None: train_fnames = kernel_mat["train_fnames"] test_fnames = kernel_mat["test_fnames"] else: assert (train_fnames == kernel_mat["train_fnames"]).all() assert (test_fnames == kernel_mat["test_fnames"]).all() if distance_fusion: ktrn = kernel_mat['distance_traintrain'] ktst = kernel_mat['distance_traintest'] else: ktrn = kernel_mat['kernel_traintrain'] ktst = kernel_mat['kernel_traintest'] #if not no_trace_normalization: # ktrn_trace = ktrn.trace() # ktrn /= ktrn_trace # ktst /= ktrn_trace if kernel_train is None: kernel_train = weight * ktrn kernel_test = weight * ktst else: kernel_train += weight * ktrn kernel_test += weight * ktst if distance_fusion: mu = kernel_train.mean() kernel_train = sp.exp(-kernel_train/mu) kernel_test = sp.exp(-kernel_test/mu) train_labels = array([str(elt) for elt in kernel_mat["train_labels"]]) test_labels = array([str(elt) for elt in kernel_mat["test_labels"]]) # XXX: clean this!!! n_categories = len(unique(train_labels)) #if not no_trace_normalization: # kernel_train_trace = kernel_train.trace() # kernel_train /= kernel_train_trace # kernel_test /= kernel_train_trace kernel_train = kernel_train.astype(double) kernel_test = kernel_test.astype(double) n_test = kernel_test.shape[1] alphas = {} support_vectors = {} biases = {} customkernel = Kernel.CustomKernel() customkernel.set_full_kernel_matrix_from_full(kernel_train) cat_index = {} # -- train categories = unique(train_labels) if categories.size == 2: categories = [categories[0]] for icat, cat in enumerate(categories): ltrain = zeros((train_labels.size)) ltrain[train_labels != cat] = -1 ltrain[train_labels == cat] = +1 ltrain = ltrain.astype(double) current_labels = Features.Labels(ltrain) svm = Classifier.LibSVM(regularization, customkernel, current_labels) #print svm assert(svm.train()) #print svm alphas[cat] = svm.get_alphas() svs = svm.get_support_vectors() support_vectors[cat] = svs biases[cat] = svm.get_bias() cat_index[cat] = icat #print "ok" # -- test pred = zeros((n_test)) distances = zeros((n_test, len(categories))) for icat, cat in enumerate(categories): for point in xrange(n_test): index_sv = support_vectors[cat] resp = dot(alphas[cat], kernel_test[index_sv, point]) \ + biases[cat] distances[point, icat] = resp if len(categories) > 1: pred = distances.argmax(1) gt = array([cat_index[e] for e in test_labels]).astype("int") perf = (pred == gt) accuracy = 100.*perf.sum() / n_test else: pred = sign(distances).ravel() gt = array(test_labels) cat = categories[0] gt[gt != cat] = -1 gt[gt == cat] = +1 gt = gt.astype("int") perf = (pred == gt) accuracy = 100.*perf.sum() / n_test print distances.shape print "Classification accuracy on test data (%):", accuracy svm_labels = gt # -- average precision # XXX: redo this part to handle other labels than +1 / -1 ap = 0 if distances.shape[1] == 1: distances = distances.ravel() assert test_labels.ndim == 1 assert svm_labels.ndim == 1 # -- inverse predictions if needed # (that is when svm was trained with flipped +1/-1 labels) # -- convert test_labels to +1/-1 (int) try: test_labels = array([int(elt) for elt in test_labels]) if (test_labels != svm_labels).any(): distances = -distances # -- convert test_labels to +1/-1 (int) test_labels = array([int(elt) for elt in test_labels]) # -- get average precision c = distances si = sp.argsort(-c) tp = sp.cumsum(sp.single(test_labels[si]>0)) fp = sp.cumsum(sp.single(test_labels[si]<0)) rec = tp/sp.sum(test_labels>0) prec = tp/(fp+tp) ap = 0 rng = sp.arange(0,1.1,.1) for th in rng: p = prec[rec>=th].max() if p == []: p = 0 ap += p / rng.size print "Average Precision:", ap except ValueError: ap = 0 # -------------------------------------------------------------------------- # -- write output file if output_filename is not None: print "Writing %s ..." % (output_filename) # TODO: save more stuff (alphas, etc.) data = {"accuracy": accuracy, "average_precision":ap, "test_distances": distances, "test_labels": test_labels, "svm_labels": svm_labels, } io.savemat(output_filename, data) return accuracy