#%%
with h5py.File(preprocessed_data_path, 'r') as in_file:
model = ks.models.load_model(
'%s/%s' % (checkpoint_path, os.listdir(checkpoint_path)[-1]))
clf = classification.create_model(model, 12, np.array(in_file['x_train']),
np.array(in_file['y_train']),
labeled_samples_per_class)
classification.save_model(clf, classifier_path)
####################################################################################################
# Test classifier
####################################################################################################
with h5py.File(preprocessed_data_path, 'r') as in_file:
clf = classification.load_model(classifier_path)
class_names = bytes(in_file['class_names']).decode('UTF-8').splitlines()
classification.test_model(clf, np.array(in_file['x_test']),
np.array(in_file['y_test']), class_names,
results_out_path)
####################################################################################################
# Visualize results
####################################################################################################
with h5py.File(preprocessed_data_path, 'r') as in_file:
yp_test = clf.predict(np.array(in_file['x_test']))
visualization.visualize_confusion_matrix(np.array(in_file['x_test']),
np.array(in_file['y_test']),
yp_test, figure_out_path)
# train classifiers on original corpus and all augmented corpi
classifier_orig = cl.train_classifier_bayes(X_orig, y_train_orig)
classifier_method_1 = cl.train_classifier_bayes(X_method_1, y_train_method_1)
classifier_method_2 = cl.train_classifier_bayes(X_method_2, y_train_method_2)
classifier_method_3 = cl.train_classifier_bayes(X_method_3, y_train_method_3)
# test all classifiers with respective training data
acc_orig_t, conf_mat_orig_t = t.test_classifier_bayes(classifier_orig, X_orig, y_train_orig)
acc_method_1_t, conf_mat_method_1_t = t.test_classifier_bayes(classifier_method_1, X_method_1, y_train_method_1)
acc_method_2_t, conf_mat_method_2_t = t.test_classifier_bayes(classifier_method_2, X_method_2, y_train_method_2)
acc_method_3_t, conf_mat_method_3_t = t.test_classifier_bayes(classifier_method_3, X_method_3, y_train_method_3)
# test all classifiers with test data
acc_orig, conf_mat_orig = t.test_classifier_bayes(classifier_orig, X_test, y_test_orig)
acc_method_1, conf_mat_method_1 = t.test_classifier_bayes(classifier_method_1, X_test, y_test_orig)
acc_method_2, conf_mat_method_2 = t.test_classifier_bayes(classifier_method_2, X_test, y_test_orig)
acc_method_3, conf_mat_method_3 = t.test_classifier_bayes(classifier_method_3, X_test, y_test_orig)
# visualize and export results
vis.acc_bar_plot([acc_orig_t, acc_method_1_t, acc_method_2_t, acc_method_3_t],
['Original', 'Method 1', 'Method 2', 'Method 3'],
'Model Performance on Training Data')
vis.acc_bar_plot([acc_orig, acc_method_1, acc_method_2, acc_method_3],
['Original', 'Method 1', 'Method 2', 'Method 3'],
'Model Performance on Test Data')
vis.visualize_confusion_matrix(conf_mat_orig, 'Original')
vis.visualize_confusion_matrix(conf_mat_method_1, 'Method 1')
vis.visualize_confusion_matrix(conf_mat_method_2, 'Method 2')
vis.visualize_confusion_matrix(conf_mat_method_3, 'Method 3')
def export_visualizations(gmm, log_dir):
"""
Visualizes and saves the images of the confusion matrix and fv representations
:param gmm: instance of sklearn GaussianMixture (GMM) object Gauassian mixture model
:param log_dir: path to the trained model
:return None (exports images)
"""
# load the model
model_checkpoint = os.path.join(log_dir, "model.ckpt")
if not (os.path.isfile(model_checkpoint + ".meta")):
raise ValueError("No log folder availabe with name " + str(log_dir))
# reBuild Graph
with tf.Graph().as_default():
with tf.device('/gpu:' + str(GPU_INDEX)):
points_pl, labels_pl, w_pl, mu_pl, sigma_pl, = MODEL.placeholder_inputs(
BATCH_SIZE,
NUM_POINT,
gmm,
)
is_training_pl = tf.placeholder(tf.bool, shape=())
# Get model and loss
pred, fv = MODEL.get_model(points_pl,
w_pl,
mu_pl,
sigma_pl,
is_training_pl,
num_classes=NUM_CLASSES)
ops = {
'points_pl': points_pl,
'labels_pl': labels_pl,
'w_pl': w_pl,
'mu_pl': mu_pl,
'sigma_pl': sigma_pl,
'is_training_pl': is_training_pl,
'pred': pred,
'fv': fv
}
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# Create a session
sess = tf_util.get_session(GPU_INDEX, limit_gpu=LIMIT_GPU)
# Restore variables from disk.
saver.restore(sess, model_checkpoint)
print("Model restored.")
# Load the test data
for fn in range(len(TEST_FILES)):
log_string('----' + str(fn) + '-----')
current_data, current_label = provider.loadDataFile(
TEST_FILES[fn])
current_data = current_data[:, 0:NUM_POINT, :]
current_label = np.squeeze(current_label)
file_size = current_data.shape[0]
num_batches = file_size / BATCH_SIZE
for batch_idx in range(num_batches):
start_idx = batch_idx * BATCH_SIZE
end_idx = (batch_idx + 1) * BATCH_SIZE
feed_dict = {
ops['points_pl']:
current_data[start_idx:end_idx, :, :],
ops['labels_pl']: current_label[start_idx:end_idx],
ops['w_pl']: gmm.weights_,
ops['mu_pl']: gmm.means_,
ops['sigma_pl']: np.sqrt(gmm.covariances_),
ops['is_training_pl']: False
}
pred_label, fv_data = sess.run([ops['pred'], ops['fv']],
feed_dict=feed_dict)
pred_label = np.argmax(pred_label, 1)
all_fv_data = fv_data if (
fn == 0 and batch_idx == 0) else np.concatenate(
[all_fv_data, fv_data], axis=0)
true_labels = current_label[start_idx:end_idx] if (
fn == 0 and batch_idx == 0) else np.concatenate(
[true_labels, current_label[start_idx:end_idx]],
axis=0)
all_pred_labels = pred_label if (
fn == 0 and batch_idx == 0) else np.concatenate(
[all_pred_labels, pred_label], axis=0)
# Export Confusion Matrix
visualization.visualize_confusion_matrix(true_labels,
all_pred_labels,
classes=LABEL_MAP,
normalize=False,
export=True,
display=False,
filename=os.path.join(
log_dir, 'confusion_mat'),
n_classes=NUM_CLASSES)
# Export Fishre Vector Visualization
label_tags = [LABEL_MAP[i] for i in true_labels]
visualization.visualize_fv(all_fv_data,
gmm,
label_tags,
export=True,
display=False,
filename=os.path.join(log_dir,
'fisher_vectors'))
# plt.show() #uncomment this to see the images in addition to saving them
print("Confusion matrix and Fisher vectores were saved to /" +
str(log_dir))
# Create classifier
####################################################################################################
#%%
model = mnist_model.get_encoder(weights=True)
clf = classification.create_model(model, mnist_model.get_embeddings_index(),
train_images.reshape((-1, 28, 28, 1)),
train_labels, labeled_samples_per_class)
classification.save_model(clf, classifier_path)
#%%
####################################################################################################
# Test classifier
####################################################################################################
clf = classification.load_model(classifier_path)
class_names = [
'T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt',
'Sneaker', 'Bag', 'Boot'
]
classification.test_model(clf, test_images.reshape((-1, 28, 28, 1)),
test_labels, class_names, results_out_path)
####################################################################################################
# Visualize results
####################################################################################################
yp_test = clf.predict(test_images.reshape((-1, 28, 28, 1)))
visualization.visualize_confusion_matrix(test_images.reshape((-1, 28, 28, 1)),
test_labels, yp_test, figure_out_path)
with open('%s/history.pkl' % checkpoint_path, 'wb') as hist_file:
hist_file.write(pickle.dumps(history.history))
####################################################################################################
# Create classifier
####################################################################################################
#%%
# Create embedding model
model = ks.models.load_model(
'%s/%s' % (checkpoint_path, os.listdir(checkpoint_path)[-1]))
clf = classification.create_model(model, 6, data['x_train'], data['y_train'],
sample_size)
classification.save_model(clf, classifier_path)
####################################################################################################
# Test classifier
####################################################################################################
clf = classification.load_model(classifier_path)
classification.test_model(clf, data['x_test'], data['y_test'],
data['class_names'])
####################################################################################################
# Visualize results
####################################################################################################
yp_test = clf.predict(data['x_test'])
visualization.visualize_confusion_matrix(data['x_test'], data['y_test'],
yp_test, figure_out_path)
def evaluate(self, data=None, export=False):
print("Running evaluation....................")
is_training = False
total_correct = 0
total_seen = 0
loss_sum = 0
total_seen_class = [0] * self.data_config.num_classes
total_correct_class = [0] * self.data_config.num_classes
points_idx = range(self.data_config.num_points)
current_step = 0
pointcloud_data = self.test_data['pointcloud_data']
labels = self.test_data['labels']
true_labels = []
all_pred_labels = []
file_size = pointcloud_data.shape[0]
num_batches = file_size / self.train_config.batch_size
for batch_idx in range(int(num_batches)):
start_idx = batch_idx * self.train_config.batch_size
end_idx = (batch_idx + 1) * self.train_config.batch_size
points_batch = pointcloud_data[start_idx:end_idx, ...]
points_batch = utils.scale_to_unit_sphere(points_batch)
label_batch = labels[start_idx:end_idx]
xforms_np, rotations_np = pf.get_xforms(
self.train_config.batch_size,
rotation_range=setting.rotation_range,
scaling_range=setting.scaling_range,
order=setting.rotation_order)
feed_dict = {
self.ops['points_pl']:
points_batch,
self.ops['labels_pl']:
label_batch,
self.ops['w_pl']:
self.gmm.weights_ if self.model == "3DmFV" else [1],
self.ops['mu_pl']:
self.gmm.means_ if self.model == "3DmFV" else [1],
self.ops['sigma_pl']:
np.sqrt(self.gmm.covariances_ if self.model ==
"3DmFV" else [1]),
self.ops['is_training_pl']:
is_training,
self.ops['xforms']:
xforms_np if self.model == "PointCNN" else [1],
self.ops['rotations']:
rotations_np if self.model == "PointCNN" else [1],
self.ops['jitter_range']:
np.array([setting.jitter])
if self.model == "PointCNN" else [1]
}
summary, _, loss_val, pred_val, probs, accuracy = self.sess.run(
[
self.ops['summary_op'], self.ops['metrics_op'],
self.ops['loss'], self.ops['predictions'],
self.ops['probabilities'], self.ops['accuracy']
],
feed_dict=feed_dict)
self.sv.summary_computed(self.sess, summary)
correct = np.sum(pred_val == label_batch)
true_labels.extend(label_batch)
all_pred_labels.extend(pred_val)
total_correct += correct
total_seen += self.train_config.batch_size
loss_sum += (loss_val * self.train_config.batch_size)
for i in range(start_idx, end_idx):
l = labels[i]
total_seen_class[l] += 1
total_correct_class[l] += (
pred_val[i - start_idx]
== l)[0] if self.model == "PointCNN" else (
pred_val[i - start_idx] == l)
current_step += 1
total_correct_class
acc = total_correct / float(total_seen)
acc_per_class = np.asarray(total_correct_class) / np.asarray(
total_seen_class)
acc_avg_cls = np.mean(
np.array(total_correct_class) /
np.array(total_seen_class, dtype=np.float))
print("current eval accuracy: ", accuracy)
print("correct ", total_correct_class)
print("seen ", total_seen_class)
print("acc per class ", acc_per_class)
print('eval mean loss: %f' % (loss_sum / float(total_seen)))
print('eval accuracy: %f' % (acc))
print('eval avg class acc: %f' % (acc_avg_cls))
if export:
true_labels = np.asarray(true_labels)
all_pred_labels = np.asarray(all_pred_labels)
label_map = []
for i, label in enumerate(self.label_map.values()):
if "_" in label:
label_split = label.split("_")
label_map.append("-".join(label_split))
else:
label_map.append(label)
label_map = np.asarray(label_map)
visualization.visualize_confusion_matrix(
true_labels,
all_pred_labels,
classes=label_map,
normalize=True,
export=True,
display=False,
filename=os.path.join(
'./log/images/',
'confusion_mat_{}_{}'.format(self.data_config.dataset_name,
self.model)),
n_classes=self.data_config.num_classes,
acc_fontsize=10,
label_fontsize=12)
visualization.visualize_histogram(
total_correct_class,
total_seen_class,
label_map,
self.model,
filepath=os.path.abspath(
'./log/images/histogram_{}_{}.pdf'.format(
self.data_config.dataset_name, self.model)))
return (acc, acc_avg_cls)