def binarize_images(train_images, val_images, test_images, random_seed):
train_images = utils.binarize(train_images, random_seed)
val_images = utils.binarize(val_images, random_seed)
test_images = utils.binarize(test_images, random_seed)
return train_images, val_images, test_images
def prepare_testing_images(self, S, I=None, M=None, label=1.0):
S256 = self.edgeSmooth1(S)
S256_in = self.edgedilate(S256, label)
S128_in = F.interpolate(S256_in, mode='bilinear', scale_factor=0.5, align_corners=True)
S64_in = F.interpolate(S256_in, mode='bilinear', scale_factor=0.25, align_corners=True)
# for synthesis
if I is None:
return S256_in, S128_in, S64_in
# for editing
assert(M is not None)
I128 = F.interpolate(I, mode='bilinear', scale_factor=0.5, align_corners=True)
I64 = F.interpolate(I, mode='bilinear', scale_factor=0.25, align_corners=True)
M128 = binarize(F.interpolate(M, mode='bilinear', scale_factor=0.5,
align_corners=True), threshold=-1)
M64 = binarize(F.interpolate(M, mode='bilinear', scale_factor=0.25,
align_corners=True), threshold=-1)
I256_in = I * (1-M)/2
S256_in = S256_in * (1+M)/2
I128_in = I128 * (1-M128)/2
S128_in = S128_in * (1+M128)/2
I64_in = I64 * (1-M64)/2
S64_in = S64_in * (1+M64)/2
return [torch.cat((S256_in, I256_in, M), dim=1),
torch.cat((S128_in, I128_in, M128), dim=1),
torch.cat((S64_in, I64_in, M64), dim=1)]
def train(dataset_name, max_epochs=10000, test_period=1):
# Load dataset
dataset, image_height, image_width, num_channels, next_train_batch, next_test_batch = load_images(
dataset_name)
# setup train, test
train = dataset.train
test = dataset.test
num_train_batches = train.num_examples / BATCH_SIZE
num_test_batches = test.num_examples / BATCH_SIZE
with tf.Session() as sess:
network = Network(sess, image_height, image_width, num_channels)
# tf.initialize_all_variables().run()
# TODO make more general
stat = Statistic(sess, 'mnist', 'train', tf.trainable_variables(),
test_period)
stat.load_model()
SAMPLE_DIR = os.path.join('samples', 'mnist', 'train')
initial_step = stat.get_t() if stat else 0
sampled_images = []
for epoch in xrange(max_epochs):
print('Current epoch: %i' % epoch)
training_costs = []
for i in xrange(num_train_batches):
images = binarize(next_train_batch(BATCH_SIZE)).reshape(
[BATCH_SIZE, image_height, image_width, num_channels])
cost = network.test(images, with_update=True)
training_costs.append(cost)
# test
if epoch % test_period == 0:
print('Running tests...')
testing_costs = []
for i in xrange(num_test_batches):
images = binarize(next_test_batch(BATCH_SIZE)).reshape(
[BATCH_SIZE, image_height, image_width, num_channels])
cost = network.test(images, with_update=False)
testing_costs.append(cost)
avg_train_cost = np.average(training_costs)
avg_test_cost = np.average(testing_costs)
print('Test cost at epoch %d: %04f' % (epoch, avg_test_cost))
stat.on_step(avg_train_cost, avg_test_cost)
samples = network.generate_images(100)
save_images(samples,
image_height,
image_width,
10,
10,
directory=SAMPLE_DIR)
def inner_func(x, y):
if binarize_x:
x = binarize(x)
if len(x.shape) == 1:
x = x.reshape((-1, 1))
assert len(x.shape) == 2
unique_y = np.unique(y)
has_one_class = (len(unique_y) < 2)
if has_one_class:
pred = y
scores = [0]
else:
clf.fit(x, y)
pred = clf.predict(x)
clf_feat = []
for metric in CLASSIFICATION_METRICS:
metric_name = metric.func_name
if not has_one_class:
scores = []
for cls in unique_y:
scores.append(metric((y == cls) + 0.0,
(pred == cls) + 0.0))
clf_feat.append((metric_name + "_avg", np.mean(scores)))
clf_feat.append((metric_name + "_sum", sum(scores)))
clf_feat.append((metric_name + "_max", max(scores)))
return clf_feat
def _construct_model_from_theta(self, theta):
# create new model
model_new = self.model.new()
model_dict = self.model.state_dict()
# calculate new weight dict(params)
params, offset = {}, 0
for k, v in self.model.named_parameters():
v_length = np.prod(v.size())
params[k] = theta[offset:offset + v_length].view(v.size())
offset += v_length
assert offset == len(theta)
# update new param and load updated model
model_dict.update(params)
model_new.load_state_dict(model_dict)
# mask alpha only select two path
new_alpha = []
for alpha in self.model.arch_parameters():
new_step = []
for step in F.softmax(alpha, dim=-1).data:
# select two path
new_step.append(utils.binarize(step, 2))
new_alpha.append(
Variable(torch.stack(new_step), requires_grad=True))
self.model._alphas_parameters = new_alpha
return model_new.cuda()
def prepare_training_images128(self, S, I, l, label, level=2, M=None):
# ground truth fine sketches at resolution 128 and 64
# edgeSmooth to make sure that each line is of about 3 pixel width in each resolution
S128 = self.edgeSmooth1(S)
S64 = self.edgeSmooth2(F.interpolate(self.edgeSmooth2(S), mode='bilinear', scale_factor=0.5, align_corners=True))
# synthesized rough sketches at resolution 128 and 64
S128_in = self.edgedilate(random_discard(self.deform(S128, label-1)), label)
S64_in = F.interpolate(S128_in, mode='bilinear', scale_factor=0.5, align_corners=True)
I64 = F.interpolate(I, mode='bilinear', scale_factor=0.5, align_corners=True)
# for synthesis
if M is None:
if level == 1:
return S64_in, S64, I64, None
with torch.no_grad():
feature64 = self.G64.forward_feature(S64_in, l).detach()
return S128_in, S128, I, feature64
# for editing
M64 = binarize(F.interpolate(M, mode='bilinear', scale_factor=0.5,
align_corners=True), threshold=-1)
I128_in = I * (1-M)/2
S128 = S128 * (1+M)/2
S128_in = S128_in * (1+M)/2
I64_in = I64 * (1-M64)/2
S64 = S64 * (1+M64)/2
S64_in = S64_in * (1+M64)/2
if level == 1:
return S64_in, I64_in, S64, I64, M64, None
with torch.no_grad():
feature64 = self.G64.forward_feature(torch.cat((S64_in, I64_in, M64), dim=1), l).detach()
return S128_in, I128_in, S128, I, M128, feature64
def fit_hopfield(params):
# get params
n_label = params.get('n_label', None)
n_sample = params['n_sample']
fit_mode = params['fit_mode']
# load dataset
dataset = load_alphabet()
# target_name
target_names = params.get('target_names', None)
if target_names is None:
target_names = dataset.target_names[:n_label]
# transform data
dataset.data = binarize(dataset.data, binary_values=(-1,1))
# create train data
X, y = create_train_data(data=dataset.data,
target=dataset.target,
target_names=target_names,
n_sample=n_sample)
print_train_data(X, y, target_names)
# fit hopfield
hf = Hopfield(mode=fit_mode)
hf.fit(X, y, watch_weight=False)
# set params
params['img_shape'] = dataset.image_shape
return hf, X, y, target_names, params
def inner_func(x, y):
if binarize_x:
x = binarize(x)
if len(x.shape) == 1:
x = x.reshape((-1, 1))
assert len(x.shape) == 2
unique_y = np.unique(y)
has_one_class = (len(unique_y) < 2)
if has_one_class:
pred = y
scores = [0]
else:
clf.fit(x, y)
pred = clf.predict(x)
clf_feat = []
for metric in CLASSIFICATION_METRICS:
metric_name = metric.func_name
if not has_one_class:
scores = []
for cls in unique_y:
scores.append(metric((y == cls) + 0.0, (pred == cls) + 0.0))
clf_feat.append((metric_name + "_avg", np.mean(scores)))
clf_feat.append((metric_name + "_sum", sum(scores)))
clf_feat.append((metric_name + "_max", max(scores)))
return clf_feat
def get_mlp_model(n_in, n_out, n_layers=2, n_hidden=50):
assert n_layers >= 2, '`n_layers` should be greater than 1 (otherwise it is just an mlp)'
# initialize weights
weights = [utils.get_weights('w_1', n_in, n_hidden)]
weights += [utils.get_weights('w_%d' % i, n_hidden, n_hidden) for i in range(2, n_layers)]
weights += [utils.get_weights('w_%d' % n_layers, n_hidden, n_out)]
# initialize biases
biases = [utils.get_weights('b_%d' % i, n_hidden) for i in range(1, n_layers)]
biases += [utils.get_weights('b_%d' % n_layers, n_out)]
# binarized versions
deterministic_binary_weights = [utils.binarize(w, mode='deterministic') for w in weights]
stochastic_binary_weights = [utils.binarize(w, mode='stochastic') for w in weights]
# variables
lr = T.scalar(name='learning_rate')
X = T.matrix(name='X', dtype=theano.config.floatX)
y = T.matrix(name='y', dtype=theano.config.floatX)
# generate outputs of mlps
d_outs = [utils.hard_sigmoid(T.dot(X, deterministic_binary_weights[0]) + biases[0])]
for w, b in zip(deterministic_binary_weights[1:], biases[1:]):
d_outs.append(utils.hard_sigmoid(T.dot(d_outs[-1], w) + b))
s_outs = [utils.hard_sigmoid(T.dot(X, stochastic_binary_weights[0]) + biases[0])]
for w, b in zip(stochastic_binary_weights[1:], biases[1:]):
s_outs.append(utils.hard_sigmoid(T.dot(s_outs[-1], w) + b))
# cost function (see utils)
cost = utils.get_cost((s_outs[-1]+1.)/2., (y+1.)/2., mode='mse')
# get the update functions
params = weights + biases
grads = [T.grad(cost, p) for p in stochastic_binary_weights + biases]
updates = [(p, T.clip(p - lr * g, -1, 1)) for p, g in zip(params, grads)]
# generate training and testing functions
train_func = theano.function([X, y, lr], [cost], updates=updates)
test_func = theano.function([X], [d_outs[-1]])
grads_func = theano.function([X, y], grads)
int_output_func = theano.function([X], s_outs + d_outs)
return train_func, test_func, grads_func, weights + biases, int_output_func
def process_pipeline(frame, keep_state=True):
"""
Apply whole lane detection pipeline to an input color frame.
:param frame: input color frame
:param keep_state: if True, lane-line state is conserved (this permits to average results)
:return: output blend with detected lane overlaid
"""
global line_lt, line_rt, processed_frames
# undistort the image using coefficients found in calibration
undistorted_img = undistort(frame, mtx, dist)
# binarize the frame and highlight lane lines
binarized_img = binarize(undistorted_img)
# perspective transform to obtain bird's eye view
birdeye_img, matrix, inversed_matrix = birdeye(binarized_img,
visualise=False)
# 2 order polynomial curve fit onto lane lines found
if processed_frames > 0 and keep_state and line_lt.detected and line_rt.detected:
find_lane_by_previous_fits(birdeye_img,
line_lt,
line_rt,
visualise=False)
else:
find_lane_by_sliding_windows(birdeye_img,
line_lt,
line_rt,
n_windows=9,
visualise=False)
# compute offset in meter from center of the lane
offset_meter = offset_from_lane_center(line_lt,
line_rt,
frame_width=frame.shape[1])
# draw the surface enclosed by lane lines back onto the original frame
blend_on_road = draw_back_onto_the_road(undistorted_img, inversed_matrix,
line_lt, line_rt, keep_state)
mean_curvature_meter = np.mean(
[line_lt.curvature_meter, line_rt.curvature_meter])
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(blend_on_road,
'Curvature radius: {:.02f}m'.format(mean_curvature_meter),
(60, 60), font, 1, (255, 255, 255), 2)
cv2.putText(blend_on_road,
'Offset from center: {:.02f}m'.format(offset_meter), (60, 90),
font, 1, (255, 255, 255), 2)
processed_frames += 1
return blend_on_road
def recall_with_noise(clf, X, noise_amount=0.05):
X = X.astype(float)
X_noise = random_noise(X, mode='s&p', amount=noise_amount)
X_noise = binarize(X_noise, binary_values=(-1,1))
X_recall = []
for x in X_noise:
recall = clf.recall(x=x, n_times=10).reshape(-1)
recall[recall < 0] = -1
recall[recall >= 0] = 1
X_recall.append(recall)
X_recall = np.array(X_recall)
return X, X_noise, X_recall
def convert_to_numerical(data, data_type):
assert isinstance(data, np.ndarray)
if data_type == "Binary":
return [data]
elif data_type == "Numerical":
ss = StandardScaler()
return [ss.fit_transform(data)]
elif data_type == "Categorical":
binarized = binarize(data)
assert binarized.shape[0] == data.shape[0]
return list(binarized.T)
else:
raise Exception
def sample_imgs(self):
samples = np.zeros((self.flags.sample_batch, *self.img_size),
dtype=np.float32)
for i in range(self.img_size[0]):
for j in range(self.img_size[1]):
for k in range(self.img_size[2]):
next_sample = utils.binarize(
self.sess.run(self.output,
feed_dict={self.input_tf: samples}))
samples[:, i, j, k] = next_sample[:, i, j, k]
return samples
def call(self, inputs):
if self.hyst == 0:
bin_gamma = binarize(self.gamma, cf.threshold)
pruned_kernel = prune_mul(self.kernel, bin_gamma)
elif self.hyst == 1:
bin_alpha_a = binarize(self.gamma, cf.threshold)
bin_alpha_b = binarize(self.gamma, cf.threshold + cf.epsilon)
bin_alpha = tf.add(
tf.multiply(self.alpha, bin_alpha_a),
tf.multiply(
tf.constant(1.0, shape=[1,
max_dil(self.kernel_size[-1])]) -
self.alpha, bin_alpha_b))
#self.add_update((self.alpha, bin_alpha), inputs)
self._assign_new_value(self.alpha, bin_alpha)
pruned_kernel = prune_mul(self.kernel, bin_alpha)
outputs = self._convolution_op(inputs, pruned_kernel)
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NCHW')
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
def experiment(num_linear, h_dim, phase, binary=True):
trX, teX, _, _ = load_mnist_2d('../models/data/mnist/raw')
if binary:
trX = binarize(trX)
teX = binarize(teX)
valX = trX[50000:]
trX = trX[:50000]
tr_dataset = QuickDataset(trX)
val_dataset = QuickDataset(valX)
te_dataset = QuickDataset(teX)
dataset = (tr_dataset, val_dataset, te_dataset)
config = VAEBernoulli_config(X_dim=(28 * 28,), h_dim=h_dim, Z_dim=20, height=28, width=28, channel=1, sigma=0.5,
epoch=50, max_epoch=100,
sample_dir='exp_result/VAEBernoulliLong/%d_layers' % num_linear,
model_dir='models/VAEBernoulliLong/%d_layers' % num_linear,
model_name='VAEBernoulliLong_%dlayers' % num_linear,
retrain=False, phase=phase)
config['num_linear'] = num_linear
config['bn'] = True
model = VAEBernoulliLinearLong(config)
elbo = model.build(dataset)
print("[num_linear {}] [phase {}] [elbo {:.4f}]".format(num_linear, phase, elbo))
def test(self, testset):
ranking = {}
y_real, y_pred = [], []
for i, qid in enumerate(testset):
ranking[qid] = []
percentage = round(float(i + 1) / len(testset), 2)
print('Progress: ', percentage, sep='\t', end='\r')
q1 = testset[qid]
q1 = utils.binarize(utils.parse_tree(q1['tree']))
q1_vec, _ = self.expr_for_tree(q1['root'], q1)
duplicates = testset[qid]['duplicates']
for duplicate in duplicates:
rel_question = duplicate['rel_question']
rel_question_id = rel_question['id']
q2 = rel_question['tree']
q2 = utils.binarize(utils.parse_tree(q2))
q2_vec, _ = self.expr_for_tree(q2['root'], q2)
x = dy.concatenate([q1_vec, q2_vec])
probs = dy.softmax(self.W * x + self.bW).vec_value()
score = probs.index(max(probs))
y_pred.append(score)
if rel_question['relevance'] != 'Irrelevant':
y_real.append(1)
else:
y_real.append(0)
ranking[qid].append((score, score, rel_question_id))
dy.renew_cg()
gold = utils.prepare_gold(GOLD_PATH)
map_baseline, map_model = utils.evaluate(gold, ranking)
f1score = f1_score(y_real, y_pred)
return map_baseline, map_model, f1score
def predict_one_sample(self, sess, input_data, threshold=None):
pred_labels = np.array([], dtype=np.intp)
sequence, seq_length, unique_count = input_data
feed_dict = self.create_feed_dict(sequence,
seq_length,
unique_count,
dropout=self.config.dropout)
pred = sess.run(self.pred, feed_dict)
pred = softmax(pred, -1)[:, 1]
if threshold is not None:
pred = utils.binarize(pred, self.threshold)
pred_labels = np.concatenate((pred_labels, pred))
return pred_labels.astype(np.int8)
def predict(self, sess, data_gen, threshold=None):
"""Predicts labels on unlabeled test set. If threshold is provided,
it binarizes predictions, otherwise returns softmax output.
"""
pred_labels = np.array([], dtype=np.intp)
for inputs, seq_length in data_gen:
feed_dict = self.create_feed_dict(inputs,
seq_length,
dropout=self.config.dropout)
pred = sess.run(self.pred, feed_dict)
pred = sigmoid(pred)
if threshold is not None:
pred = utils.binarize(pred, self.threshold)
pred_labels = np.concatenate((pred_labels, pred))
return pred_labels.astype(np.int8)
def train_next_batch(self):
batch_data = self.sess.run(self.next_batch_train)
batch_imgs = batch_data[0]["image"]
batch_labels = batch_data[1]
imgs_array = np.reshape(batch_imgs,
[self.flags.batch_size, *self.img_size])
imgs_array = imgs_array / 255.
imgs_array = utils.binarize(
imgs_array
) # input of the pixelrnn for mnist should be binarized data
# one-hot representations
labels_array = np.zeros((batch_labels.shape[0], 10))
labels_array[range(batch_labels.shape[0]), batch_labels] = 1
return imgs_array, labels_array
def predict(self, X):
'''
Make prediction by the trained model.
Parameters
----------
X: ndarray of shape (m, n)
data to be predicted, the same shape as trainning data
Returns
-------
C: ndarray of shape (m,)
Predicted class label per sample.
'''
if self.w is None:
raise Exception("Model haven't been trained!")
# X = np.hstack([X, np.ones([len(X), 1])])
return binarize(sigmoid(X.dot(self.w)+self.b))
def inner_func(x, y):
if binarize_x:
x = binarize(x)
if len(x.shape) == 1:
x = x.reshape((-1, 1))
assert len(x.shape) == 2
clf.fit(x, y)
pred = clf.predict(x)
clf_feat = []
for metric in REGRESSION_METRICS:
metric_name = metric.func_name
score = metric(y, pred)
clf_feat.append((metric_name, score))
for f in ALL_BINARY_FEATURES + NN_BINARY_FEATURES:
clf_feat += binary_feature_wrapper(f)(y, pred)
for f in ALL_UNARY_FEATURES + NN_UNARY_FEATURES:
clf_feat += unary_feature_wrapper(f)(y - pred, None)
return clf_feat
def inner_func(x, y):
if binarize_x:
x = binarize(x)
if len(x.shape) == 1:
x = x.reshape((-1, 1))
assert len(x.shape) == 2
clf.fit(x, y)
pred = clf.predict(x)
clf_feat = []
for metric in REGRESSION_METRICS:
metric_name = metric.func_name
score = metric(y, pred)
clf_feat.append((metric_name, score))
for f in ALL_BINARY_FEATURES + NN_BINARY_FEATURES:
clf_feat += binary_feature_wrapper(f)(y, pred)
for f in ALL_UNARY_FEATURES + NN_UNARY_FEATURES:
clf_feat += unary_feature_wrapper(f)(y - pred, None)
return clf_feat
def test_next_batch(self):
# idxs = np.random.randint(low=0, high=self.num_tests, size=self.flags.batch_size)
# batch_imgs, batch_labels = self.test_x[idxs], self.test_y[idxs]
batch_data = self.sess.run(self.next_batch_test)
batch_imgs = batch_data[0]["image"]
batch_labels = batch_data[1]
imgs_array = np.reshape(batch_imgs,
[self.flags.batch_size, *self.img_size])
imgs_array = imgs_array / 255.
imgs_array = utils.binarize(
imgs_array
) # input of the pixelrnn for mnist should be binarized data
# one-hot representations
labels_array = np.zeros((batch_labels.shape[0], 10))
labels_array[range(batch_labels.shape[0]), batch_labels] = 1
return imgs_array, labels_array
def evaluate(self, sess, data_gen, threshold=None):
"""Evaluates model on dev dataset and returns f1_score and predicted
labels. If threshold is provided, it binarizes predictions, otherwise
returns softmax output.
"""
score = None
pred_labels = np.array([], dtype=np.intp)
labels = np.array([], dtype=np.intp)
for inputs, seq_length, batch_labels in data_gen:
feed_dict = self.create_feed_dict(inputs, seq_length, batch_labels,
self.config.dropout)
pred = sess.run(self.pred, feed_dict)
pred = sigmoid(pred)
if threshold is not None:
pred = utils.binarize(pred, threshold)
pred_labels = np.concatenate((pred_labels, pred))
labels = np.concatenate((labels, batch_labels))
if threshold is not None:
score = f1_score(labels, pred_labels)
return score, pred_labels, labels
import numpy as np
from matplotlib import pyplot as plt
from utils import sigmoid, get_cmap, binarize
a = np.array([[0.2, 0.5, 0.8], [0.6, 0.3, 0.2]])
print a
b = binarize(a)
print b
x = np.random.uniform(size=10000)
x = np.random.normal(size=10000)
# the histogram of the data
n, bins, patches = plt.hist(x, 100, normed=1, facecolor='green', alpha=0.75)
plt.xlabel('Pre-activation')
plt.ylabel('Probability')
plt.title('Histogram of Z - Prior Noise 4')
plt.grid(True)
plt.show()
# the histogram of the data
#sigmoidX = sigmoid(8*(x-0.5)+3)
sample_names.add(sample_name)
if sample_name in predictions:
p_weights_list=predictions[sample_name]
p_new_weights=scores_np[index]
p_updated_weights=np.max(np.vstack((p_weights_list,p_new_weights)),axis=0)
else:
p_updated_weights = scores_np[index]
predictions[sample_name]=p_updated_weights
if sample_name not in truelabels:
truelabels[sample_name]=shuffled_labels[index]
np.savez('resnet_results',
sample_names=list(sample_names),
truelabels=truelabels,
predictions=predictions)
results=np.load('resnet_results.npz')
truelabels=results['truelabels'].item()
sample_names=results['sample_names']
predictions=results['predictions'].item()
weak_labels={}
for sample in list(sample_names):
truelabel=truelabels[sample]
prediction=predictions[sample]
binarized_prediction=binarize(prediction,BINARIZATION_THRESHOLD)
weak_labels[sample]=binarized_prediction
np.save('weak_labels',weak_labels)
fpath = '../datasets/monks/'
names = ['monks-1_train',
'monks-1_test',
'monks-2_train',
'monks-2_test',
'monks-3_train',
'monks-3_test']
datasets = {name: read_monk(fpath+name+'.csv') for name in names}
# monk1_train = read_monk('../datasets/monks/monks-1_train.csv')
# monk1_test = read_monk('../datasets/monks/monks-1_test.csv')
categories_sizes = [3, 3, 2, 3, 4, 2]
for name in names:
""" split each monk set, binarize, merge again, export to csv """
monk = datasets[name]
y, X = np.hsplit(monk, [1])
X_bin = u.binarize(X, categories_sizes)
d_bin = np.hstack((y, X_bin))
with open(fpath + name + '_bin.csv', 'w') as f:
csv_writer = csv.writer(f, delimiter=',')
for row in range(d_bin.shape[0]):
csv_writer.writerow(d_bin[row, :])
def test_on_mixed_samples(model,
test_loader,
loss_op,
writer,
results_folder,
n_saved_results=5,
epoch=0):
"""
Perform evaluation on the test set
Returns: average MSE error on the whole test set
"""
print("Testing on mixed set...")
model.eval()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
test_epoch_loss = 0
test_images = None
test_images2 = None
if len(test_loader) > n_saved_results:
chosen_sample_i = torch.multinomial(torch.Tensor(
range(len(test_loader))),
num_samples=n_saved_results,
replacement=False)
else:
chosen_sample_i = range(len(test_loader))
n_output_channels = 3
with torch.no_grad():
for index, (img, gt) in enumerate(tqdm(test_loader)):
# img, _ = data
img = img.to(device)
n_output_channels = img.shape[1]
gt = gt.to(device)
output = model(img)
# diff = torch.abs(img - output)
# Grayscaled diff image (average of 3 channels)
diff_avg = 1 - SSIM(img, output)[1]
diff_avg = torch.mean(diff_avg, dim=1, keepdim=True)
diff_avg = channelwised_normalize(diff_avg)
# diff = channelwised_normalize(diff)
th_diff, gth_diff, otsu_diff = binarize(diff_avg,
n_output_channels)
# Make the grayscale image 3-channeled
# diff_avg = diff_avg
loss = 1 - loss_op(diff_avg, gt)
test_epoch_loss += loss.item()
# Save the results if requested
if index in chosen_sample_i:
io_pair = torch.cat((img, output), dim=3)
gt_pair = torch.cat((gt, diff_avg), dim=3)
gt_pair = gt_pair.squeeze(0)
gt_pair = transforms.ToPILImage()(gt_pair.cpu())
draw = ImageDraw.Draw(gt_pair)
font = ImageFont.truetype(font="BebasNeue-Regular.ttf",
size=150)
# font = ImageFont.truetype("sans-serif.ttf", 16)
draw.text((0, 0), f"{loss.item():.3f}", (0), font=font)
draw.text((0, 25), f"{loss.item():.3f}", (255), font=font)
gt_pair = transforms.ToTensor()(gt_pair).unsqueeze(0).expand(
-1, n_output_channels, -1, -1).to(device)
image = torch.cat((io_pair.to(device), gt_pair.to(device),
th_diff.to(device), gth_diff.to(device),
otsu_diff.to(device)), 0)
if test_images is None:
test_images = image
else:
test_images = torch.cat((test_images, image), dim=0)
#####DIRTY ADDITION DIFF MAP RESULT#######
# diff = torch.abs(img - output)
# # Grayscaled diff image (average of 3 channels)
# diff_avg = torch.mean(diff, dim=1, keepdim=True)
# diff_avg = channelwised_normalize(diff_avg)
# # diff = channelwised_normalize(diff)
# th_diff, gth_diff, otsu_diff = binarize(diff_avg, n_output_channels)
# # Make the grayscale image 3-channeled
# # diff_avg = diff_avg
# loss = nn.MSELoss()(diff_avg, gt)
# # Save the results if requested
# if index in chosen_sample_i:
# gt_pair = torch.cat((gt, diff_avg), dim=3)
# gt_pair = gt_pair.squeeze(0)
# gt_pair = transforms.ToPILImage()(gt_pair.cpu())
# draw = ImageDraw.Draw(gt_pair)
# font = ImageFont.truetype(font="BebasNeue-Regular.ttf", size=150)
# # font = ImageFont.truetype("sans-serif.ttf", 16)
# draw.text((0,0),f"{loss.item():.3f}", (0), font=font)
# draw.text((0,25),f"{loss.item():.3f}",(255), font=font)
# gt_pair = transforms.ToTensor()(gt_pair).unsqueeze(0).expand(-1, n_output_channels, -1, -1).to(device)
# image = torch.cat((io_pair.to(device), gt_pair.to(device), th_diff.to(device), gth_diff.to(device), otsu_diff.to(device)), 0)
# if test_images2 is None:
# test_images2 = image
# else:
# test_images2 = torch.cat((test_images2, image), dim=0)
test_epoch_loss = test_epoch_loss / len(test_loader)
test_images = torchvision.utils.make_grid(test_images, nrow=5)
test_images = test_images.unsqueeze(0)
test_images = F.interpolate(test_images, scale_factor=0.1)
result_image = os.path.join(results_folder, f"val_{epoch}.png")
torchvision.utils.save_image(test_images, result_image)
print(f"Test images saved at {results_folder}")
# test_images2 = torchvision.utils.make_grid(test_images2, nrow=5)
# test_images2 = test_images2.unsqueeze(0)
# test_images2 = F.interpolate(test_images2, scale_factor=0.1)
# result_image = os.path.join(results_folder, f"val_{epoch}_more.png")
# torchvision.utils.save_image(test_images2, result_image)
# print(f"Additional diff map images saved at {results_folder}")
# write to tensorboard
if writer:
test_images = test_images.squeeze(0)
writer.add_image('Test images', test_images, global_step=epoch)
writer.add_scalar(f"{loss_op.__class__.__name__}/Test",
test_epoch_loss,
global_step=epoch)
return test_epoch_loss
def main(args):
"""
This function returns the statistics reported on the PanNuke dataset, reported in the paper below:
Gamper, Jevgenij, Navid Alemi Koohbanani, Simon Graham, Mostafa Jahanifar, Syed Ali Khurram,
Ayesha Azam, Katherine Hewitt, and Nasir Rajpoot.
"PanNuke Dataset Extension, Insights and Baselines." arXiv preprint arXiv:2003.10778 (2020).
Args:
Root path to the ground-truth
Root path to the predictions
Path where results will be saved
Output:
Terminal output of bPQ and mPQ results for each class and across tissues
Saved CSV files for bPQ and mPQ results for each class and across tissues
"""
true_root = args['--true_path']
pred_root = args['--pred_path']
save_path = args['--save_path']
if not os.path.exists(save_path):
os.mkdir(save_path)
true_path = os.path.join(true_root,'masks.npy') # path to the GT for a specific split
pred_path = os.path.join(pred_root, 'masks.npy') # path to the predictions for a specific split
types_path = os.path.join(true_root,'types.npy') # path to the nuclei types
# load the data
true = np.load(true_path)
pred = np.load(pred_path)
types = np.load(types_path)
mPQ_all = []
bPQ_all = []
# loop over the images
for i in range(true.shape[0]):
pq = []
pred_bin = binarize(pred[i,:,:,:5])
true_bin = binarize(true[i,:,:,:5])
if len(np.unique(true_bin)) == 1:
pq_bin = np.nan # if ground truth is empty for that class, skip from calculation
else:
[_, _, pq_bin], _ = get_fast_pq(true_bin, pred_bin) # compute PQ
# loop over the classes
for j in range(5):
pred_tmp = pred[i,:,:,j]
pred_tmp = pred_tmp.astype('int32')
true_tmp = true[i,:,:,j]
true_tmp = true_tmp.astype('int32')
pred_tmp = remap_label(pred_tmp)
true_tmp = remap_label(true_tmp)
if len(np.unique(true_tmp)) == 1:
pq_tmp = np.nan # if ground truth is empty for that class, skip from calculation
else:
[_, _, pq_tmp] , _ = get_fast_pq(true_tmp, pred_tmp) # compute PQ
pq.append(pq_tmp)
mPQ_all.append(pq)
bPQ_all.append([pq_bin])
# using np.nanmean skips values with nan from the mean calculation
mPQ_each_image = [np.nanmean(pq) for pq in mPQ_all]
bPQ_each_image = [np.nanmean(pq_bin) for pq_bin in bPQ_all]
# class metric
neo_PQ = np.nanmean([pq[0] for pq in mPQ_all])
inflam_PQ = np.nanmean([pq[1] for pq in mPQ_all])
conn_PQ = np.nanmean([pq[2] for pq in mPQ_all])
dead_PQ = np.nanmean([pq[3] for pq in mPQ_all])
nonneo_PQ = np.nanmean([pq[4] for pq in mPQ_all])
# Print for each class
print('Printing calculated metrics on a single split')
print('-'*40)
print('Neoplastic PQ: {}'.format(neo_PQ))
print('Inflammatory PQ: {}'.format(inflam_PQ))
print('Connective PQ: {}'.format(conn_PQ))
print('Dead PQ: {}'.format(dead_PQ))
print('Non-Neoplastic PQ: {}'.format(nonneo_PQ))
print('-' * 40)
# Save per-class metrics as a csv file
for_dataframe = {'Class Name': ['Neoplastic', 'Inflam', 'Connective', 'Dead', 'Non-Neoplastic'],
'PQ': [neo_PQ, conn_PQ, conn_PQ, dead_PQ, nonneo_PQ]}
df = pd.DataFrame(for_dataframe, columns=['Tissue name', 'PQ'])
df.to_csv(save_path + '/class_stats.csv')
# Print for each tissue
all_tissue_mPQ = []
all_tissue_bPQ = []
for tissue_name in tissue_types:
indices = [i for i, x in enumerate(types) if x == tissue_name]
tissue_PQ = [mPQ_each_image[i] for i in indices]
print('{} PQ: {} '.format(tissue_name, np.nanmean(tissue_PQ)))
tissue_PQ_bin = [bPQ_each_image[i] for i in indices]
print('{} PQ binary: {} '.format(tissue_name, np.nanmean(tissue_PQ_bin)))
all_tissue_mPQ.append(np.nanmean(tissue_PQ))
all_tissue_bPQ.append(np.nanmean(tissue_PQ_bin))
# Save per-tissue metrics as a csv file
for_dataframe = {'Tissue name': tissue_types + ['mean'],
'PQ': all_tissue_mPQ + [np.nanmean(all_tissue_mPQ)] , 'PQ bin': all_tissue_bPQ + [np.nanmean(all_tissue_bPQ)]}
df = pd.DataFrame(for_dataframe, columns=['Tissue name', 'PQ', 'PQ bin'])
df.to_csv(save_path + '/tissue_stats.csv')
# Show overall metrics - mPQ is average PQ over the classes and the tissues, bPQ is average binary PQ over the tissues
print('-' * 40)
print('Average mPQ:{}'.format(np.nanmean(all_tissue_mPQ)))
print('Average bPQ:{}'.format(np.nanmean(all_tissue_bPQ)))
print("Finding mask")
mask = utils.findMask(image)
if options.images > 1:
utils.showImage(mask, "mask")
print("Applying local normalization")
image = np.where(mask == 1.0, utils.localNormalize(image), image)
if options.images > 1:
utils.showImage(image, "locally normalized")
print("Estimating orientations")
orientations = np.where(mask == 1.0, utils.estimateOrientations(image, interpolate=False), -1.0)
if options.images > 0:
utils.showOrientations(image, orientations, "orientations", 8)
print("Filtering")
image = np.where(mask == 1.0, wahabFilter(image, orientations), 1.0)
if options.images > 0:
utils.showImage(image, "filtered")
if options.binarize:
print("Binarizing")
image = np.where(mask == 1.0, utils.binarize(image), 1.0)
if options.images > 0:
utils.showImage(image, "binarized")
if options.images > 0:
plt.show()
if not options.dryrun:
misc.imsave(destinationImage, image)
def sample(self, arr):
return np.random.binomial(n=1, p=arr)
def sample_h_given_v(self, v):
probs = self.propup(v)
states = self.sample(probs)
return probs, states
def sample_v_given_h(self, h):
probs = self.propdown(h)
states = self.sample(probs)
return probs, states
if __name__ == '__main__':
data, target = get_mnist(random=True, num=1000)
data = binarize(data, 0., 1.)
dim = 28
h = 100
x = 10
y = 10
rbm = BernoulliRBM(
dim * dim, h, learning_rate=0.1,
sparsity_target=0.05,
regulatization_param=0.01
)
rbm.train(data, epochs=1000)
for i in xrange(h):
plt.subplot(x, y, i)
train_mat, test_mat = load_data("coat")
x_train, y_train = rating_mat_to_sample(train_mat)
x_test, y_test = rating_mat_to_sample(test_mat)
num_user = train_mat.shape[0]
num_item = train_mat.shape[1]
elif dataset_name == "yahoo":
x_train, y_train, x_test, y_test = load_data("yahoo")
x_train, y_train = shuffle(x_train, y_train)
num_user = x_train[:, 0].max() + 1
num_item = x_train[:, 1].max() + 1
print("# user: {}, # item: {}".format(num_user, num_item))
# binarize
y_train = binarize(y_train)
y_test = binarize(y_test)
def setup_seed(seed):
np.random.seed(seed)
torch.manual_seed(seed)
"MF-CVIB parameter invariance"
output_prefix = "./demo/"
# alpha_list = [0.1,1e-2,1e-3,1e-4]
# gamma_list = [1,0.1,1e-2,1e-3,1e-4]
alpha_list = [2, 1, 0.5, 0.1]
gamma_list = [1, 0.1, 1e-2, 1e-3]
def train(self):
dy.renew_cg()
trainer = dy.AdamTrainer(self.model)
early = 0.0
best = -1
for epoch in range(self.EPOCH):
print('\n')
dy.renew_cg()
losses = []
closs = 0
batch_timing = []
for i, trainrow in enumerate(self.traindata):
start = time.time()
q1 = utils.binarize(utils.parse_tree(trainrow['q1_tree']))
q2 = utils.binarize(utils.parse_tree(trainrow['q2_tree']))
label = trainrow['label']
loss = self.get_loss(q1, q2, label)
losses.append(loss)
if len(losses) == self.BATCH:
loss = dy.esum(losses)
# loss += self.regularization_loss()
_loss = loss.value()
closs += _loss
loss.backward()
trainer.update()
dy.renew_cg()
# percentage of trainset processed
percentage = str(
round((float(i + 1) / len(self.traindata)) * 100,
2)) + '%'
# time of epoch processing
time_epoch = round(
sum(batch_timing) / float(len(batch_timing)), 2)
print(
"Epoch: {0} \t\t Loss: {1} \t\t Epoch time: {2} \t\t Trainset: {3}"
.format(epoch + 1, round(_loss, 2), time_epoch,
percentage),
end=' \r')
losses = []
batch_timing = []
end = time.time()
t = (end - start)
batch_timing.append(t)
log = "Epoch: {0} \t\t Loss: {1} \t\t Best: {2}".format(
epoch + 1, round(closs / self.BATCH, 2), round(best, 2))
print('\n' + log)
log = 'Dev evaluation...'
print(log)
map_baseline, map_model, f1score = self.test(self.devset)
print('MAP Model: ',
round(map_model, 2),
'MAP baseline: ',
round(map_baseline, 2),
'F1 score: ',
str(round(f1score, 2)),
sep='\t',
end='\n')
trainer.learning_rate *= 0.95
if map_model > best:
best = copy.copy(map_model)
early = 0
# path = self.fname() + '.dy'
# self.model.save(os.path.join(EVALUATION_PATH, path))
else:
early += 1
if early == self.EARLY_STOP:
break
if word in w2v.vocab:
embedding_matrix[i] = w2v.word_vec(word)
else:
#print (word)
pass
#target using Average Embeddings
y = {}
tfidf = tokenizer.sequences_to_matrix(full_seq, mode = 'tfidf')
denom = 1 + np.sum(tfidf, axis =1)[:, None]
normed_tfidf = tfidf/denom
average_embeddings = np.dot(normed_tfidf, embedding_matrix)
y['ae'] = average_embeddings
print (f"Shape of the Average Embeddings: {y['ae'].shape}")
B = utils.binarize(y["ae"])
target_dim = B.shape[1]
print (f'The Shape of Binarized Average Embeddings is {B.shape}')
print (f'The shape of the train_inp_data is {inp_data.shape}')
train_inp_data = model.TextData(inp_data, B)
train_dataloader = DataLoader(train_inp_data, shuffle = True, batch_size = 100)
MODEL = model.TextCluster(embedding_matrix, target_dim)
crit = nn.MSELoss()
optimizer = optim.Adam(MODEL.parameters(), lr = 1e-3, betas = [0.9, 0.999], eps = 1e-08)
for epoch in range(1, EPOCHS_NUM + 1):
print (f'EPOCH-{epoch}')
for batch in tqdm(train_dataloader):
#################################################
# Preparing target using Average embeddings (AE)
#################################################
Y = {}
tfidf = tokenizer.sequences_to_matrix(sequences_full, mode='tfidf')
denom = 1 + np.sum(tfidf, axis=1)[:, None]
normed_tfidf = tfidf/denom
average_embeddings = np.dot(normed_tfidf, embedding_matrix)
Y["ae"] = average_embeddings
print("Shape of average embedding: ", Y['ae'].shape)
# binary Y
from utils import binarize
reduction_name = "ae"
B = binarize(Y[reduction_name])
# Last dimension in the CNN
TARGET_DIM = B.shape[1]
# Example of binarized target vector
print(B.shape)
print(B[0])
################################################
# train model
################################################
from keras.layers import Input, Embedding, Flatten, Reshape
from keras.layers import Dense, Conv1D, Dropout, merge
def evaluate(
self,
experiment_path: str,
pred_file='hard_predictions_{}.txt',
tag_file='tagging_predictions_{}.txt',
event_file='event_{}.txt',
segment_file='segment_{}.txt',
class_result_file='class_result_{}.txt',
time_ratio=10. / 500,
postprocessing='double',
threshold=None,
window_size=None,
save_seq=False,
sed_eval=True, # Do evaluation on sound event detection ( time stamps, segemtn/evaluation based)
**kwargs):
"""evaluate
:param experiment_path: Path to already trained model using train
:type experiment_path: str
:param pred_file: Prediction output file, put into experiment dir
:param time_resolution: Resolution in time (1. represents the model resolution)
:param **kwargs: Overwrite standard args, please pass `data` and `label`
"""
# Update config parameters with new kwargs
config = torch.load(list(Path(f'{experiment_path}').glob("run_config*"))[0], map_location='cpu')
# Use previous config, but update data such as kwargs
config_parameters = dict(config, **kwargs)
# Default columns to search for in data
config_parameters.setdefault('colname', ('filename', 'encoded'))
model_parameters = torch.load(
glob.glob("{}/run_model*".format(experiment_path))[0],
map_location=lambda storage, loc: storage)
encoder = torch.load(glob.glob(
'{}/run_encoder*'.format(experiment_path))[0],
map_location=lambda storage, loc: storage)
strong_labels_df = pd.read_csv(config_parameters['label'], sep='\t')
# Evaluation is done via the filenames, not full paths
if not np.issubdtype(strong_labels_df['filename'].dtype, np.number):
strong_labels_df['filename'] = strong_labels_df['filename'].apply(
os.path.basename)
if 'audiofilepath' in strong_labels_df.columns: # In case of ave dataset, the audiofilepath column is the main column
strong_labels_df['audiofilepath'] = strong_labels_df[
'audiofilepath'].apply(os.path.basename)
colname = 'audiofilepath' # AVE
else:
colname = 'filename' # Dcase etc.
# Problem is that we iterate over the strong_labels_df, which is ambigious
# In order to conserve some time and resources just reduce strong_label to weak_label format
weak_labels_df = strong_labels_df.groupby(
colname)['event_label'].unique().apply(
tuple).to_frame().reset_index()
if "event_labels" in strong_labels_df.columns:
assert False, "Data with the column event_labels are used to train not to evaluate"
weak_labels_array, encoder = utils.encode_labels(
labels=weak_labels_df['event_label'], encoder=encoder)
dataloader = dataset.getdataloader(
{
'filename': weak_labels_df['filename'].values,
'encoded': weak_labels_array,
},
config_parameters['data'],
batch_size=1,
shuffle=False,
colname=config_parameters[
'colname'] # For other datasets with different key names
)
model = getattr(models, config_parameters['model'])(
inputdim=dataloader.dataset.datadim,
outputdim=len(encoder.classes_),
**config_parameters['model_args'])
model.load_state_dict(model_parameters)
model = model.to(DEVICE).eval()
time_predictions, clip_predictions = [], []
sequences_to_save = []
mAP_pred, mAP_tar = [], []
with torch.no_grad():
for batch in tqdm(dataloader, unit='file', leave=False):
_, target, filenames = batch
clip_pred, pred, _ = self._forward(model, batch)
clip_pred = clip_pred.cpu().detach().numpy()
mAP_tar.append(target.numpy().squeeze(0))
mAP_pred.append(clip_pred.squeeze(0))
pred = pred.cpu().detach().numpy()
if postprocessing == 'median':
if threshold is None:
thres = 0.5
else:
thres = threshold
if window_size is None:
window_size = 1
filtered_pred = utils.median_filter(
pred, window_size=window_size, threshold=thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
elif postprocessing == 'cATP-SDS':
# cATP-SDS postprocessing uses an "Optimal" configurations, assumes we have a prior
# Values are taken from the Surface Disentange paper
# Classes are (DCASE2018 only)
# ['Alarm_bell_ringing' 'Blender' 'Cat' 'Dishes' 'Dog'
# 'Electric_shaver_toothbrush' 'Frying' 'Running_water' 'Speech'
# 'Vacuum_cleaner']
assert pred.shape[
-1] == 10, "Only supporting DCASE2018 for now"
if threshold is None:
thres = 0.5
else:
thres = threshold
if window_size is None:
window_size = [17, 42, 17, 9, 16, 74, 85, 64, 18, 87]
# P(y|x) > alpha
clip_pred = utils.binarize(clip_pred, threshold=thres)
pred = pred * clip_pred
filtered_pred = np.zeros_like(pred)
# class specific filtering via median filter
for cl in range(pred.shape[-1]):
# Median filtering also applies thresholding
filtered_pred[..., cl] = utils.median_filter(
pred[..., cl],
window_size=window_size[cl],
threshold=thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
elif postprocessing == 'double':
# Double thresholding as described in
# https://arxiv.org/abs/1904.03841
if threshold is None:
hi_thres, low_thres = (0.75, 0.2)
else:
hi_thres, low_thres = threshold
filtered_pred = utils.double_threshold(pred,
high_thres=hi_thres,
low_thres=low_thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
elif postprocessing == 'triple':
# Triple thresholding as described in
# Using frame level + clip level predictions
if threshold is None:
clip_thres, hi_thres, low_thres = (0.5, 0.75, 0.2)
else:
clip_thres, hi_thres, low_thres = threshold
clip_pred = utils.binarize(clip_pred, threshold=clip_thres)
# Apply threshold to
pred = clip_pred * pred
filtered_pred = utils.double_threshold(pred,
high_thres=hi_thres,
low_thres=low_thres)
decoded_pred = utils.decode_with_timestamps(
encoder, filtered_pred)
for num_batch in range(len(decoded_pred)):
filename = filenames[num_batch]
cur_pred = pred[num_batch]
cur_clip = clip_pred[num_batch].reshape(1, -1)
# Clip predictions, independent of per-frame predictions
bin_clips = utils.binarize(cur_clip)
# Binarize with default threshold 0.5 For clips
bin_clips = encoder.inverse_transform(
bin_clips.reshape(1,
-1))[0] # 0 since only single sample
# Add each label individually into list
for clip_label in bin_clips:
clip_predictions.append({
'filename': filename,
'event_label': clip_label,
})
# Save each frame output, for later visualization
if save_seq:
labels = weak_labels_df.loc[weak_labels_df['filename']
== filename]['event_label']
to_save_df = pd.DataFrame(pred[num_batch],
columns=encoder.classes_)
# True labels
to_save_df.rename({'variable': 'event'},
axis='columns',
inplace=True)
to_save_df['filename'] = filename
to_save_df['pred_labels'] = np.array(labels).repeat(
len(to_save_df))
sequences_to_save.append(to_save_df)
label_prediction = decoded_pred[num_batch]
for event_label, onset, offset in label_prediction:
time_predictions.append({
'filename': filename,
'event_label': event_label,
'onset': onset,
'offset': offset
})
assert len(time_predictions) > 0, "No outputs, lower threshold?"
pred_df = pd.DataFrame(
time_predictions,
columns=['filename', 'event_label', 'onset', 'offset'])
clip_pred_df = pd.DataFrame(
clip_predictions,
columns=['filename', 'event_label', 'probability'])
test_data_filename = os.path.splitext(
os.path.basename(config_parameters['label']))[0]
if save_seq:
pd.concat(sequences_to_save).to_csv(os.path.join(
experiment_path, 'probabilities.csv'),
index=False,
sep='\t',
float_format="%.4f")
pred_df = utils.predictions_to_time(pred_df, ratio=time_ratio)
if pred_file:
pred_df.to_csv(os.path.join(experiment_path,
pred_file.format(test_data_filename)),
index=False,
sep="\t")
tagging_df = metrics.audio_tagging_results(strong_labels_df, pred_df)
clip_tagging_df = metrics.audio_tagging_results(
strong_labels_df, clip_pred_df)
print("Tagging Classwise Result: \n{}".format(
tabulate(clip_tagging_df,
headers='keys',
showindex=False,
tablefmt='github')))
print("mAP: {}".format(
metrics.mAP(np.array(mAP_tar), np.array(mAP_pred))))
if tag_file:
clip_tagging_df.to_csv(os.path.join(
experiment_path, tag_file.format(test_data_filename)),
index=False,
sep='\t')
if sed_eval:
event_result, segment_result = metrics.compute_metrics(
strong_labels_df, pred_df, time_resolution=1.0)
print("Event Based Results:\n{}".format(event_result))
event_results_dict = event_result.results_class_wise_metrics()
class_wise_results_df = pd.DataFrame().from_dict({
f: event_results_dict[f]['f_measure']
for f in event_results_dict.keys()
}).T
class_wise_results_df.to_csv(os.path.join(
experiment_path, class_result_file.format(test_data_filename)),
sep='\t')
print("Class wise F1-Macro:\n{}".format(
tabulate(class_wise_results_df,
headers='keys',
tablefmt='github')))
if event_file:
with open(
os.path.join(experiment_path,
event_file.format(test_data_filename)),
'w') as wp:
wp.write(event_result.__str__())
print("=" * 100)
print(segment_result)
if segment_file:
with open(
os.path.join(experiment_path,
segment_file.format(test_data_filename)),
'w') as wp:
wp.write(segment_result.__str__())
event_based_results = pd.DataFrame(
event_result.results_class_wise_average_metrics()['f_measure'],
index=['event_based'])
segment_based_results = pd.DataFrame(
segment_result.results_class_wise_average_metrics()
['f_measure'],
index=['segment_based'])
result_quick_report = pd.concat((
event_based_results,
segment_based_results,
))
# Add two columns
tagging_macro_f1, tagging_macro_pre, tagging_macro_rec = tagging_df.loc[
tagging_df['label'] == 'macro'].values[0][1:]
static_tagging_macro_f1, static_tagging_macro_pre, static_tagging_macro_rec = clip_tagging_df.loc[
clip_tagging_df['label'] == 'macro'].values[0][1:]
result_quick_report.loc['Time Tagging'] = [
tagging_macro_f1, tagging_macro_pre, tagging_macro_rec
]
result_quick_report.loc['Clip Tagging'] = [
static_tagging_macro_f1, static_tagging_macro_pre,
static_tagging_macro_rec
]
with open(
os.path.join(
experiment_path,
'quick_report_{}.md'.format(test_data_filename)),
'w') as wp:
print(tabulate(result_quick_report,
headers='keys',
tablefmt='github'),
file=wp)
print("Quick Report: \n{}".format(
tabulate(result_quick_report,
headers='keys',
tablefmt='github')))
