def layerwise_pretrain(self,
X,
batch_size,
n_iter,
optimizer,
l_rate,
decay,
lr_scheduler=None):
def l2_norm(label, pred):
return np.mean(np.square(label - pred)) / 2.0
solver = Solver('sgd',
momentum=0.9,
wd=decay,
learning_rate=l_rate,
lr_scheduler=lr_scheduler)
solver.set_metric(mx.metric.CustomMetric(l2_norm))
solver.set_monitor(Monitor(1000))
data_iter = mx.io.NDArrayIter([X],
batch_size=batch_size,
shuffle=False,
last_batch_handle='roll_over')
for i in range(self.N):
if i == 0:
data_iter_i = data_iter
else:
X_i = model.extract_feature(self.internals[i - 1], self.args,
['data'], data_iter, X.shape[0],
self.xpu).values()[0]
data_iter_i = mx.io.NDArrayIter([X_i],
batch_size=batch_size,
last_batch_handle='roll_over')
solver.solve(self.xpu, self.stacks[i], self.args, self.args_grad,
['data'], data_iter_i, 0, n_iter, self.args_mult)
def finetune(self,
X,
batch_size,
n_iter,
optimizer,
l_rate,
decay,
lr_scheduler=None,
print_every=1000):
def l2_norm(label, pred):
return np.mean(np.square(label - pred)) / 2.0
solver = Solver(optimizer,
momentum=0.9,
wd=decay,
learning_rate=l_rate,
lr_scheduler=lr_scheduler)
solver.set_metric(mx.metric.CustomMetric(l2_norm))
solver.set_monitor(Monitor(print_every))
data_iter = mx.io.NDArrayIter({'data': X},
batch_size=batch_size,
shuffle=True,
last_batch_handle='roll_over')
logging.info('Fine tuning...')
solver.solve(self.xpu, self.loss, self.args, self.args_grad, self.auxs,
data_iter, 0, n_iter, {}, False)
def cluster(self, X, y=None, update_interval=None):
N = X.shape[0]
if not update_interval:
update_interval = N
batch_size = 256
test_iter = mx.io.NDArrayIter({'data': X}, batch_size=batch_size, shuffle=False,
last_batch_handle='pad')
args = {k: mx.nd.array(v.asnumpy(), ctx=self.xpu) for k, v in self.args.items()}
z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0]
kmeans = KMeans(self.num_centers, n_init=20)
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
solver = Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.01)
def ce(label, pred):
return np.sum(label * np.log(label / (pred + 0.000001))) / label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
label_buff = np.zeros((X.shape[0], self.num_centers))
train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff}, batch_size=batch_size,
shuffle=False, last_batch_handle='roll_over')
self.y_pred = np.zeros((X.shape[0]))
def refresh(i):
if i % update_interval == 0:
z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0]
p = np.zeros((z.shape[0], self.num_centers))
self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p])
y_pred = p.argmax(axis=1)
print(np.std(np.bincount(y_pred)), np.bincount(y_pred))
print(np.std(np.bincount(y.astype(np.int))), np.bincount(y.astype(np.int)))
if y is not None:
print(cluster_acc(y_pred, y)[0])
weight = 1.0 / p.sum(axis=0)
weight *= self.num_centers / weight.sum()
p = (p ** 2) * weight
train_iter.data_list[1][:] = (p.T / p.sum(axis=1)).T
print(np.sum(y_pred != self.y_pred), 0.001 * y_pred.shape[0])
if np.sum(y_pred != self.y_pred) < 0.001 * y_pred.shape[0]:
self.y_pred = y_pred
return True
self.y_pred = y_pred
solver.set_iter_start_callback(refresh)
solver.set_monitor(Monitor(50))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 1000000000, {}, False)
self.end_args = args
if y is not None:
return cluster_acc(self.y_pred, y)[0]
else:
return -1
def finetune(self,
X,
R,
V,
lambda_v_rt,
lambda_u,
lambda_v,
dir_save,
batch_size,
n_iter,
optimizer,
l_rate,
decay,
lr_scheduler=None):
def l2_norm(label, pred):
print(type(label))
print(type(pred))
print(np.shape(label))
print(np.shape(pred))
return np.mean(np.square(label - pred)) / 2.0
solver = Solver(optimizer,
momentum=0.9,
wd=decay,
learning_rate=l_rate,
lr_scheduler=lr_scheduler)
solver.set_metric(mx.metric.CustomMetric(l2_norm))
solver.set_monitor(Monitor(1000))
data_iter = mx.io.NDArrayIter(
{
'data': X,
'V': V,
'lambda_v_rt': lambda_v_rt
},
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad')
logging.info('Fine tuning...')
# self.loss is the net
U, V, theta, BCD_loss = solver.solve(X, R, V, lambda_v_rt, lambda_u,
lambda_v, dir_save, batch_size,
self.xpu, self.loss, self.args,
self.args_grad, self.auxs,
data_iter, 0, n_iter, {}, False)
return U, V, theta, BCD_loss
def layerwise_pretrain(self, X, batch_size, n_iter, optimizer, l_rate, decay,
lr_scheduler=None, print_every=1000):
def l2_norm(label, pred):
return np.mean(np.square(label-pred))/2.0
solver = Solver(optimizer, momentum=0.9, wd=decay, learning_rate=l_rate,
lr_scheduler=lr_scheduler)
solver.set_metric(mx.metric.CustomMetric(l2_norm))
solver.set_monitor(Monitor(print_every))
data_iter = mx.io.NDArrayIter({'data': X}, batch_size=batch_size, shuffle=True,
last_batch_handle='roll_over')
for i in range(self.N):
if i == 0:
data_iter_i = data_iter
else:
X_i = list(model.extract_feature(
self.internals[i-1], self.args, self.auxs, data_iter, X.shape[0],
self.xpu).values())[0]
data_iter_i = mx.io.NDArrayIter({'data': X_i}, batch_size=batch_size,
last_batch_handle='roll_over')
logging.info('Pre-training layer %d...', i)
solver.solve(self.xpu, self.stacks[i], self.args, self.args_grad, self.auxs,
data_iter_i, 0, n_iter, {}, False)
def layerwise_pretrain(self, X, b_size, total_iter, opti, l_rate, decay, scheduler_lr=None):
def l2_norm(label, pred):
return np.mean(np.square(label-pred))/2.0
solver = Solver(opti, momentum=0.9, wd=decay, learning_rate=l_rate, scheduler_lr=scheduler_lr)
# procedding solver.set_metric
solver.set_metric(mx.metric.CustomMetric(l2_norm))
# procedding solver.set_monitor
solver.set_monitor(Monitor(1000))
# procedding solver. data_iter
data_iter = mx.io.NDArrayIter({'data': X}, b_size=b_size, shuffle=True,
last_batch_handle='roll_over')
# processing all is in range
for i in range(self.N):
if i == 0:
data_iter_i = data_iter
else:
X_i = model.extract_feature(self.internals[i-1], self.args, self.auxs,
data_iter, X.shape[0], self.xpu).values()[0]
data_iter_i = mx.io.NDArrayIter({'data': X_i}, b_size=b_size,
last_batch_handle='roll_over')
logging.info('Pre-training layer %d...'%i)
solver.solve(self.xpu, self.stacks[i], self.args, self.args_grad, self.auxs, data_iter_i,
0, total_iter, {}, False)
def finetune(self,
X,
batch_size,
n_iter,
optimizer,
l_rate,
decay,
lr_scheduler=None):
def l2_norm(label, pred):
return np.mean(np.square(label - pred)) / 2.0
solver = Solver('sgd',
momentum=0.9,
wd=decay,
learning_rate=l_rate,
lr_scheduler=lr_scheduler)
solver.set_metric(mx.metric.CustomMetric(l2_norm))
solver.set_monitor(Monitor(1000))
data_iter = mx.io.NDArrayIter([X],
batch_size=batch_size,
shuffle=False,
last_batch_handle='roll_over')
solver.solve(self.xpu, self.loss, self.args, self.args_grad, ['data'],
data_iter, 0, n_iter, self.args_mult)
def cluster_varying_mu(self,
X,
y_dec,
roi_labels,
classes,
batch_size,
save_to,
labeltype,
update_interval=None):
# y = y_dec X, y_dec, roi_labels, classes, batch_size, save_to, labeltype, update_in
N = X.shape[0]
self.best_args['update_interval'] = update_interval
self.best_args['y_dec'] = y_dec
self.best_args['roi_labels'] = roi_labels
self.best_args['classes'] = classes
self.best_args['batch_size'] = batch_size
# selecting batch size
# [42*t for t in range(42)] will produce 16 train epochs
# [0, 42, 84, 126, 168, 210, 252, 294, 336, 378, 420, 462, 504, 546, 588, 630]
test_iter = mx.io.NDArrayIter({'data': X},
batch_size=N,
shuffle=False,
last_batch_handle='pad')
args = {
k: mx.nd.array(v.asnumpy(), ctx=self.xpu)
for k, v in self.args.items()
}
## embedded point zi
z = model.extract_feature(self.feature, args, None, test_iter, N,
self.xpu).values()[0]
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
self.perplexity = 25
self.learning_rate = 200
# reconstruct wordy labels list(Y)==named_y
named_y = [classes[kc] for kc in y_dec]
self.best_args['named_y'] = named_y
# To initialize the cluster centers, we pass the data through
# the initialized DNN to get embedded data points and then
# perform standard k-means clustering in the feature space Z
# to obtain k initial centroids {mu j}
kmeans = KMeans(self.best_args['num_centers'], n_init=20)
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
### KL DIVERGENCE MINIMIZATION. eq(2)
# our model is trained by matching the soft assignment to the target distribution.
# To this end, we define our objective as a KL divergence loss between
# the soft assignments qi (pred) and the auxiliary distribution pi (label)
solver = Solver(
'sgd', momentum=0.9, wd=0.0, learning_rate=0.01
) # , lr_scheduler=mx.misc.FactorScheduler(20*update_interval,0.4)) #0.01
def ce(label, pred):
return np.sum(label * np.log(label /
(pred + 0.000001))) / label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
label_buff = np.zeros((X.shape[0], self.best_args['num_centers']))
train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff},
batch_size=self.best_args['batch_size'],
shuffle=False,
last_batch_handle='roll_over')
self.best_args['y_pred'] = np.zeros((X.shape[0]))
self.best_args['acci'] = []
self.best_args['bestacci'] = []
self.ploti = 0
figprogress = plt.figure(figsize=(20, 15))
print 'Batch_size = %f' % self.best_args['batch_size']
print 'update_interval = %f' % update_interval
self.best_args['plot_interval'] = int(5 * update_interval)
print 'plot_interval = %f' % self.best_args['plot_interval']
self.maxAcc = 0.0
def refresh(i): # i=3, a full epoch occurs every i=798/48
if i % self.best_args['update_interval'] == 0:
z = list(
model.extract_feature(self.feature, args, None, test_iter,
N, self.xpu).values())[0]
p = np.zeros((z.shape[0], self.best_args['num_centers']))
self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p])
# the soft assignments qi (pred)
y_pred = p.argmax(axis=1)
print np.std(np.bincount(y_pred)), np.bincount(y_pred)
# use a y that only considers the filledbyBC examples
# compare soft assignments with known labels
print '\n... Updating i = %f' % i
allL = np.asarray(roi_labels)
data = z[allL != 'K', :]
datalabels = allL[allL != 'K']
RFmodel = RandomForestClassifier(n_jobs=2,
n_estimators=200,
random_state=0,
verbose=0)
# Evaluate a score by cross-validation
# integer=5, to specify the number of folds in a (Stratified)KFold,
scores = cross_val_score(RFmodel, data, datalabels, cv=5)
# do for overall class B and M
labels = np.asarray(self.best_args['named_y'])
Z_embedding_tree = sklearn.neighbors.BallTree(z, leaf_size=6)
# This finds the indices of 5 closest neighbors
nme_dist = [
'Diffuse', 'Focal', 'Linear', 'MultipleRegions',
'Regional', 'Segmental', 'N/A'
]
nme_intenh = [
'Clumped', 'ClusteredRing', 'Heterogeneous', 'Homogeneous',
'Stippled or punctate', 'N/A'
]
wnme_dist = np.zeros((len(nme_dist), len(nme_dist)),
dtype=np.int64)
wnme_intenh = np.zeros((len(nme_intenh), len(nme_intenh)),
dtype=np.int64)
NME_descriptorate = []
kznme = 0
for k in range(z.shape[0]):
iclass = labels[k]
dist, ind = Z_embedding_tree.query([z[k]], k=6)
dist5nn, ind5nn = dist[k != ind], ind[k != ind]
class5nn = labels[ind5nn]
if (len(class5nn) > 0
and (iclass.split('_')[1] != 'N/A'
or iclass.split('_')[1] != 'N/A')):
# increment detections for final NME descriptor accuracy
kznme += 1
prednmed = []
prednmed.append(
sum([
lab.split('_')[1] == 'Diffuse'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Focal'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Linear'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'MultipleRegions'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Regional'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Segmental'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'N/A' for lab in class5nn
]))
prednmeie = []
prednmeie.append(
sum([
lab.split('_')[2] == 'Clumped'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'ClusteredRing'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'Heterogeneous'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'Homogeneous'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'Stippled or punctate'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'N/A' for lab in class5nn
]))
# predicion based on majority
pred_nme_dist = [
nme_dist[l] for l, pc in enumerate(prednmed)
if pc >= max(prednmed) and max(prednmed) > 0
]
pred_nme_intenh = [
nme_intenh[l] for l, pc in enumerate(prednmeie)
if pc >= max(prednmeie) and max(prednmeie) > 0
]
# allow a second kind of detection rate - a nme similar local neighborhood
if (iclass.split('_')[1] in pred_nme_dist
or iclass.split('_')[2] in pred_nme_intenh):
NME_descriptorate.append(1)
# for nme_dist
label_nme_dist = [
l for l, pc in enumerate(nme_dist)
if iclass.split('_')[1] == pc
]
labelpred_nme_dist = [
l for l, pc in enumerate(prednmed)
if pc >= max(prednmed) and max(prednmed) > 0
]
for u in label_nme_dist:
for v in labelpred_nme_dist:
wnme_dist[v, u] += 1
# for nme_intenh
label_nme_intenh = [
l for l, pc in enumerate(nme_intenh)
if iclass.split('_')[2] == pc
]
labelpred_intenh = [
l for l, pc in enumerate(prednmeie)
if pc >= max(prednmeie) and max(prednmeie) > 0
]
for u in label_nme_intenh:
for v in labelpred_intenh:
wnme_intenh[v, u] += 1
# compute Z-space Accuracy
Acc = scores.mean()
print "cvRF Z-space Accuracy = %f " % Acc
print scores.tolist()
NME_Acc = sum(np.asarray(NME_descriptorate)) / float(kznme)
print "NME decriptor agreenment (NMErate) = %f " % NME_Acc
indwnme_dist = linear_assignment(wnme_dist.max() - wnme_dist)
indwnme_intenh = linear_assignment(wnme_intenh.max() -
wnme_intenh)
Acc5nn_nme_dist = sum(
[wnme_dist[v, u] for v, u in indwnme_dist]) / float(kznme)
Acc5nn_nme_intenh = sum(
[wnme_intenh[v, u]
for v, u in indwnme_intenh]) / float(kznme)
print "Acc5nn_nme_dist (DIST) = %f " % Acc5nn_nme_dist
print "Acc5nn_nme_intenh (INT_ENH) = %f " % Acc5nn_nme_intenh
if (i == 0):
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
self.best_args['initAcc'] = Acc
# plot initial z
figinint = plt.figure()
axinint = figinint.add_subplot(1, 1, 1)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
axinint,
title=
'kmeans init tsne: Acc={}\n NME_Acc={}\n Acc5nn_nme_dist={}\n Acc5nn_nme_intenh={}'
.format(Acc, NME_Acc, Acc5nn_nme_dist,
Acc5nn_nme_intenh),
legend=True)
figinint.savefig('{}//tsne_init_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'],
self.best_args['num_centers'], labeltype),
bbox_inches='tight')
plt.close()
# save best args
self.best_args['acci'].append(Acc)
if (Acc >= self.maxAcc):
print 'Improving maxAcc = {}'.format(Acc)
for key, v in args.items():
self.best_args[key] = args[key]
self.maxAcc = Acc
self.best_args['zbestacci'] = z
self.best_args['bestacci'].append(Acc)
self.best_args['dec_mu'][:] = args['dec_mu'].asnumpy()
self.best_args['NME_Acc'] = NME_Acc
self.best_args['Acc5nn_nme_dist'] = Acc5nn_nme_dist
self.best_args['Acc5nn_nme_intenh'] = Acc5nn_nme_intenh
if (i > 0 and i % self.best_args['plot_interval'] == 0
and self.ploti <= 15):
# Visualize the progression of the embedded representation in a subsample of data
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
axprogress = figprogress.add_subplot(4, 4, 1 + self.ploti)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
axprogress,
title="Epoch %d z_tsne iter (%d)" % (self.ploti, i),
legend=False)
self.ploti = self.ploti + 1
## COMPUTING target distributions P
## we compute pi by first raising qi to the second power and then normalizing by frequency per cluster:
weight = 1.0 / p.sum(axis=0) # p.sum provides fj
weight *= self.best_args['num_centers'] / weight.sum()
p = (p**2) * weight
train_iter.data_list[1][:] = (p.T / p.sum(axis=1)).T
print np.sum(y_pred != self.best_args['y_pred']
), 0.001 * y_pred.shape[0]
# For the purpose of discovering cluster assignments, we stop our procedure when less than tol% of points change cluster assignment between two consecutive iterations.
# tol% = 0.001
if i == self.best_args[
'update_interval'] * 100: # performs 1epoch = 615/3 = 205*1000epochs
self.best_args['y_pred'] = y_pred
self.best_args['p'] = p
self.best_args['z'] = z
self.best_args['acci'].append(Acc)
return True
self.best_args['y_pred'] = y_pred
self.best_args['p'] = p
self.best_args['z'] = z
# start solver
solver.set_iter_start_callback(refresh)
solver.set_monitor(Monitor(50))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 1000000000, {}, False)
self.end_args = args
self.best_args['end_args'] = args
# finish
figprogress = plt.gcf()
figprogress.savefig('{}\\tsne_progress_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'], self.best_args['num_centers'],
labeltype),
bbox_inches='tight')
plt.close()
# plot final z
figfinal = plt.figure()
axfinal = figfinal.add_subplot(1, 1, 1)
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(self.best_args['zbestacci'])
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
self.best_args['named_y'],
axfinal,
title=
'final tsne: Acc={}\n NME_Acc={} \n Acc5nn_nme_dist={}\n Acc5nn_nme_intenh={}'
.format(self.best_args['bestacci'][-1], self.best_args['NME_Acc'],
self.best_args['Acc5nn_nme_dist'],
self.best_args['Acc5nn_nme_intenh']),
legend=True)
figfinal.savefig('{}\\tsne_final_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'], self.best_args['num_centers'],
labeltype),
bbox_inches='tight')
plt.close()
outdict = {
'acc': self.best_args['acci'],
'bestacc': self.best_args['bestacci'],
'NME_Acc': self.best_args['NME_Acc'],
'Acc5nn_nme_dist': self.best_args['Acc5nn_nme_dist'],
'Acc5nn_nme_intenh': self.best_args['Acc5nn_nme_intenh'],
'p': self.best_args['p'],
'z': self.best_args['z'],
'y_pred': self.best_args['y_pred'],
'named_y': self.best_args['named_y'],
'num_centers': self.best_args['num_centers']
}
return outdict
def cluster(self, X_train, y_dec_train, y_train, classes, batch_size, save_to, labeltype, update_interval, logger):
N = X_train.shape[0]
self.best_args['update_interval'] = update_interval
self.best_args['y_dec'] = y_dec_train
self.best_args['roi_labels'] = y_train
self.best_args['classes'] = classes
self.best_args['batch_size'] = batch_size
self.logger = logger
# selecting batch size
# [42*t for t in range(42)] will produce 16 train epochs
# [0, 42, 84, 126, 168, 210, 252, 294, 336, 378, 420, 462, 504, 546, 588, 630]
test_iter = mx.io.NDArrayIter({'data': X_train},
batch_size=N, shuffle=False,
last_batch_handle='pad')
args = {k: mx.nd.array(v.asnumpy(), ctx=self.xpu) for k, v in self.args.items()}
## embedded point zi
self.z = model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values()[0]
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
self.perplexity = 5
self.learning_rate = 125
# reconstruct wordy labels list(Y)==named_y
named_y = [classes[kc] for kc in y_dec_train]
self.best_args['named_y'] = named_y
# To initialize the cluster centers, we pass the data through
# the initialized DNN to get embedded data points and then
# perform standard k-means clustering in the feature space Z
# to obtain k initial centroids {mu j}
kmeans = KMeans(self.best_args['num_centers'], n_init=20)
kmeans.fit(self.z)
args['dec_mu'][:] = kmeans.cluster_centers_
figprogress = plt.figure(figsize=(20, 15))
print 'Batch_size = %f'% self.best_args['batch_size']
print 'update_interval = %f'% update_interval
self.best_args['plot_interval'] = int(8*update_interval)
print 'plot_interval = %f'% self.best_args['plot_interval']
self.best_args['y_pred'] = np.zeros((X_train.shape[0]))
self.best_args['meanAuc_cv'] = []
self.best_args['std_auc'] = []
self.best_args['auc_val'] = []
self.best_args['overall_metric'] = []
self.ploti = 0
self.maxAUC = 10000.0
### Define DEC training varialbes
label_buff = np.zeros((X_train.shape[0], self.best_args['num_centers']))
train_iter = mx.io.NDArrayIter({'data': X_train},
{'label': label_buff},
batch_size=self.best_args['batch_size'],
shuffle=True, last_batch_handle='roll_over')
### KL DIVERGENCE MINIMIZATION. eq(2)
# our model is trained by matching the soft assignment to the target distribution.
# To this end, we define our objective as a KL divergence loss between
# the soft assignments qi (pred) and the auxiliary distribution pi (label)
solver = Solver('sgd',learning_rate=0.1,lr_scheduler=mx.misc.FactorScheduler(100,0.1)) ### original: 0.01, try1: Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.000125, lr_scheduler=mx.misc.FactorScheduler(20*update_interval,0.5)) try 2: Solver('sgd', momentum=0.6, wd=0.05, learning_rate=0.00125, lr_scheduler=mx.misc.FactorScheduler(20*update_interval,0.5))
#solver = Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.01)
def ce(label, pred):
DECmetric = np.sum(label*np.log(label/(pred+0.000001)))/label.shape[0]
print("DECmetric = {}".format(DECmetric))
#####################
# Z-space MLP fully coneected layer for classification
#####################
batch_size = 50
# Run classifier with cross-validation and plot ROC curves
cv = StratifiedKFold(n_splits=5, random_state=3)
# Evaluate a score by cross-validation
tprs = []; aucs = []
mean_fpr = np.linspace(0, 1, 100)
cvi = 0
for train, test in cv.split(self.Z_train, self.yZ_train):
# Multilayer Perceptron
MLP_train_iter = mx.io.NDArrayIter(self.Z_train[train], self.yZ_train[train], batch_size, shuffle=True)
MLP_val_iter = mx.io.NDArrayIter(self.Z_train[test], self.yZ_train[test], batch_size)
# We’ll define the MLP using MXNet’s symbolic interface
dataMLP = mx.sym.Variable('data')
#The following code declares two fully connected layers with 128 and 64 neurons each.
#Furthermore, these FC layers are sandwiched between ReLU activation layers each
#one responsible for performing an element-wise ReLU transformation on the FC layer output.
# The first fully-connected layer and the corresponding activation function
fc1 = mx.sym.FullyConnected(data=dataMLP, num_hidden = 128)
act1 = mx.sym.Activation(data=fc1, act_type="relu")
fc2 = mx.sym.FullyConnected(data=act1, num_hidden = 32)
act2 = mx.sym.Activation(data=fc2, act_type="relu")
# data has 2 classes
fc3 = mx.sym.FullyConnected(data=act2, num_hidden=2)
# Softmax with cross entropy loss
mlp = mx.sym.SoftmaxOutput(data=fc3, name='softmax')
# create a trainable module on CPU
#mon = mx.mon.Monitor(interval=100, pattern='.*', sort=True); # Defaults to mean absolute value |x|/size(x)
#checkpoint = mx.callback.do_checkpoint('mlp_model_params_z{}_mu{}.arg'.format(self.best_args['znum'],self.best_args['num_centers']))
self.mlp_model = mx.mod.Module(symbol=mlp, context=mx.cpu())
self.mlp_model.fit(MLP_train_iter, # train data
monitor=None,
optimizer='sgd', # use SGD to train
optimizer_params={'learning_rate':0.1}, # use fixed learning rate
eval_metric= 'acc', #MLPacc(yZ_val, Z_val), # report accuracy during trainin
num_epoch=100)
#epoch_end_callbackcheckpoint) # train for at most 10 dataset passes. extras: #monitor=mon,
#After the above training completes, we can evaluate the trained model by running predictions on validation data.
#The following source code computes the prediction probability scores for each validation data.
# prob[i][j] is the probability that the i-th validation contains the j-th output class.
prob_val = self.mlp_model.predict(MLP_val_iter)
# Compute ROC curve and area the curve
fpr, tpr, thresholds = roc_curve(self.yZ_train[test], prob_val.asnumpy()[:,1])
# to create an ROC with 100 pts
tprs.append(interp(mean_fpr, fpr, tpr))
tprs[-1][0] = 0.0
roc_auc = auc(fpr, tpr)
print roc_auc
aucs.append(roc_auc)
cvi += 1
# compute across all cvs
mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
std_auc = np.std(aucs)
print r'cv meanROC (AUC = {0:.4f} $\pm$ {0:.4f})'.format(mean_auc, std_auc)
Z_test_iter = mx.io.NDArrayIter(self.Z_test, None, batch_size)
prob_test = self.mlp_model.predict(Z_test_iter)
# Compute ROC curve and area the curve
fpr_val, tpr_val, thresholds_val = roc_curve(self.yZ_test, prob_test.asnumpy()[:,1])
self.auc_val = auc(fpr_val, tpr_val)
print r'cv test (AUC = {0:.4f})'.format(self.auc_val)
# compute Z-space metric
overall_metric = -np.log(mean_auc) -np.log(1-DECmetric) #np.log(1-mean_auc) + np.log(DECmetric)
print("overall_metric: DEC+MLP = {}".format(overall_metric))
self.best_args['overall_metric'].append(overall_metric)
if(overall_metric <= self.maxAUC):
print '================== Improving auc_val = {}'.format(self.auc_val)
for key, v in args.items():
self.best_args[key] = args[key]
self.best_args['meanAuc_cv'].append(mean_auc)
self.best_args['std_auc'].append(std_auc)
self.best_args['auc_val'].append(self.auc_val)
self.best_args['pbestacci'] = self.p
self.best_args['zbestacci'] = self.z
self.best_args['dec_mu'][:] = args['dec_mu'].asnumpy()
#self.best_args['mlp_model'] = self.mlp_model
self.mlp_model.save_params(os.path.join(save_to,'mlp_model_params_z{}_mu{}.arg'.format(self.best_args['znum'],self.best_args['num_centers'])))
self.maxAUC = overall_metric
return overall_metric
def refresh(i): # i=3, a full epoch occurs every i=798/48
if i%self.best_args['update_interval'] == 0:
self.z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0]
self.p = np.zeros((self.z.shape[0], self.best_args['num_centers']))
self.dec_op.forward([self.z, args['dec_mu'].asnumpy()], [self.p])
self.best_args['dec_mu'] = args['dec_mu']
# the soft assignments qi (pred)
y_pred = self.p.argmax(axis=1)
print np.std(np.bincount(y_pred)), np.bincount(y_pred)
## COMPUTING target distributions P
## we compute pi by first raising qi to the second power and then normalizing by frequency per cluster:
print '\n... Updating i = %f' % i
weight = 1.0/self.p.sum(axis=0) # p.sum provides fj
weight *= self.best_args['num_centers']/weight.sum()
self.p = (self.p**2)*weight
train_iter.data_list[1][:] = (self.p.T/self.p.sum(axis=1)).T
#print np.sum(y_pred != self.best_args['y_pred']), 0.001*y_pred.shape[0]
#####################
# prep Z-space MLP fully coneected layer for classification
#####################
# compare soft assignments with known labels (only B or M)
print '\n... Updating MLP fully coneected layer i = %f' % i
sep = int(self.z.shape[0]*0.10)
print(self.z.shape)
datalabels = np.asarray(self.best_args['roi_labels'])
dataZspace = np.concatenate((self.z, self.p), axis=1) #zbestacci #dec_model['zbestacci']
Z = dataZspace[datalabels!='K',:]
y = datalabels[datalabels!='K']
print(Z)
# Do a 5 fold cross-validation
self.Z_test = Z[:sep]
self.yZ_test = np.asanyarray(y[:sep]=='M').astype(int)
self.Z_train = Z[sep:]
self.yZ_train = np.asanyarray(y[sep:]=='M').astype(int)
print(self.Z_test.shape)
print(self.Z_train.shape)
if(i==0):
self.tsne = TSNE(n_components=2, perplexity=self.perplexity, learning_rate=self.learning_rate,
init='pca', random_state=0, verbose=2, method='exact')
self.Z_tsne = self.tsne.fit_transform(dataZspace)
# plot initial z
figinint = plt.figure()
axinint = figinint.add_subplot(1,1,1)
plot_embedding_unsuper_NMEdist_intenh(self.Z_tsne, named_y, axinint, title='kmeans init tsne:\n', legend=True)
figinint.savefig('{}//tsne_init_z{}_mu{}_{}.pdf'.format(save_to,self.best_args['znum'],self.best_args['num_centers'],labeltype), bbox_inches='tight')
plt.close()
if(i>0 and i%self.best_args['plot_interval']==0 and self.ploti<=15):
# Visualize the progression of the embedded representation in a subsample of data
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
tsne = TSNE(n_components=2, perplexity=self.perplexity, learning_rate=self.learning_rate,
init='pca', random_state=0, verbose=2, method='exact')
Z_tsne = tsne.fit_transform(dataZspace)
axprogress = figprogress.add_subplot(4,4,1+self.ploti)
plot_embedding_unsuper_NMEdist_intenh(Z_tsne, named_y, axprogress, title="iter %d z_tsne" % (i), legend=False)
self.ploti = self.ploti+1
# For the purpose of discovering cluster assignments, we stop our procedure when less than tol% of points change cluster assignment between two consecutive iterations.
# tol% = 0.001
if i == self.best_args['update_interval']*120: # performs 1epoch = 615/3 = 205*1000epochs
return True
# Deeo learning metrics to minimize
solver.set_metric(mx.metric.CustomMetric(ce))
# start solver
solver.set_iter_start_callback(refresh)
solver.set_monitor(Monitor(self.best_args['update_interval']))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 1000000000, {}, False)
self.end_args = args
self.best_args['end_args'] = args
# finish
figprogress = plt.gcf()
figprogress.savefig('{}\\tsne_progress_z{}_mu{}_{}.pdf'.format(save_to,self.best_args['znum'],self.best_args['num_centers'],labeltype), bbox_inches='tight')
plt.close()
# plot final z
figfinal = plt.figure()
axfinal = figfinal.add_subplot(1,1,1)
tsne = TSNE(n_components=2, perplexity=self.perplexity, learning_rate=self.learning_rate,
init='pca', random_state=0, verbose=2, method='exact')
Z_tsne = tsne.fit_transform(self.z)
plot_embedding_unsuper_NMEdist_intenh(Z_tsne, self.best_args['named_y'], axfinal, title='final tsne', legend=True)
figfinal.savefig('{}\\tsne_final_z{}_mu{}_{}.pdf'.format(save_to,self.best_args['znum'],self.best_args['num_centers'],labeltype), bbox_inches='tight')
plt.close()
outdict = {'meanAuc_cv':self.best_args['meanAuc_cv'],
'std_auc':self.best_args['std_auc'],
'auc_val':self.best_args['auc_val'],
'overall_metric':self.best_args['overall_metric'],
'dec_mu':self.best_args['dec_mu'],
'y_pred': self.best_args['y_pred'],
'named_y': self.best_args['named_y'],
'classes':self.best_args['classes'],
'num_centers': self.best_args['num_centers'],
'znum':self.best_args['znum'],
'update_interval':self.best_args['update_interval'],
'batch_size':self.best_args['batch_size']}
return outdict
def cluster(self,
X_train,
y_dec_train,
y_train,
classes,
batch_size,
save_to,
labeltype,
update_interval=None):
N = X_train.shape[0]
self.best_args['update_interval'] = update_interval
self.best_args['y_dec'] = y_dec_train
self.best_args['roi_labels'] = y_train
self.best_args['classes'] = classes
self.best_args['batch_size'] = batch_size
# selecting batch size
# [42*t for t in range(42)] will produce 16 train epochs
# [0, 42, 84, 126, 168, 210, 252, 294, 336, 378, 420, 462, 504, 546, 588, 630]
test_iter = mx.io.NDArrayIter({'data': X_train},
batch_size=N,
shuffle=False,
last_batch_handle='pad')
args = {
k: mx.nd.array(v.asnumpy(), ctx=self.xpu)
for k, v in self.args.items()
}
## embedded point zi
z = model.extract_feature(self.feature, args, None, test_iter, N,
self.xpu).values()[0]
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
self.perplexity = 15
self.learning_rate = 125
# reconstruct wordy labels list(Y)==named_y
named_y = [classes[kc] for kc in y_dec_train]
self.best_args['named_y'] = named_y
# To initialize the cluster centers, we pass the data through
# the initialized DNN to get embedded data points and then
# perform standard k-means clustering in the feature space Z
# to obtain k initial centroids {mu j}
kmeans = KMeans(self.best_args['num_centers'], n_init=20)
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
### KL DIVERGENCE MINIMIZATION. eq(2)
# our model is trained by matching the soft assignment to the target distribution.
# To this end, we define our objective as a KL divergence loss between
# the soft assignments qi (pred) and the auxiliary distribution pi (label)
solver = Solver(
'sgd',
momentum=0.9,
wd=0.0,
learning_rate=0.1,
lr_scheduler=mx.misc.FactorScheduler(20 * update_interval, 0.5)
) # , lr_scheduler=mx.misc.FactorScheduler(20*update_interval,0.4)) #0.01
def ce(label, pred):
return np.sum(label * np.log(label /
(pred + 0.000001))) / label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
label_buff = np.zeros(
(X_train.shape[0], self.best_args['num_centers']))
train_iter = mx.io.NDArrayIter({'data': X_train},
{'label': label_buff},
batch_size=self.best_args['batch_size'],
shuffle=False,
last_batch_handle='roll_over')
self.best_args['y_pred'] = np.zeros((X_train.shape[0]))
self.best_args['acci'] = []
self.best_args['bestacci'] = []
self.ploti = 0
figprogress = plt.figure(figsize=(20, 15))
print 'Batch_size = %f' % self.best_args['batch_size']
print 'update_interval = %f' % update_interval
self.best_args['plot_interval'] = int(20 * update_interval)
print 'plot_interval = %f' % self.best_args['plot_interval']
self.maxAcc = 0.0
def refresh(i): # i=3, a full epoch occurs every i=798/48
if i % self.best_args['update_interval'] == 0:
z = list(
model.extract_feature(self.feature, args, None, test_iter,
N, self.xpu).values())[0]
p = np.zeros((z.shape[0], self.best_args['num_centers']))
self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p])
# the soft assignments qi (pred)
y_pred = p.argmax(axis=1)
#print np.std(np.bincount(y_dec_train)), np.bincount(y_dec_train)
print np.std(np.bincount(y_pred)), np.bincount(y_pred)
#####################
# Z-space CV RF classfier METRICS
#####################
# compare soft assignments with known labels (only B or M)
print '\n... Updating i = %f' % i
allL = np.asarray(y_train)
dataZspace = np.concatenate(
(z[allL != 'K', :], np.reshape(y_pred[allL != 'K'],
(-1, 1))),
axis=1)
ydatalabels = np.asarray(allL[allL != 'K'] == 'M').astype(
int) # malignant is positive class
cv = StratifiedKFold(n_splits=5)
RFmodel = RandomForestClassifier(n_jobs=2,
n_estimators=500,
random_state=0,
verbose=0)
# Evaluate a score by cross-validation
tprs = []
aucs = []
mean_fpr = np.linspace(0, 1, 100)
cvi = 0
for train, test in cv.split(dataZspace, ydatalabels):
probas = RFmodel.fit(dataZspace[train],
ydatalabels[train]).predict_proba(
dataZspace[test])
# Compute ROC curve and area the curve
fpr, tpr, thresholds = roc_curve(ydatalabels[test],
probas[:, 1])
# to create an ROC with 100 pts
tprs.append(interp(mean_fpr, fpr, tpr))
tprs[-1][0] = 0.0
roc_auc = auc(fpr, tpr)
aucs.append(roc_auc)
cvi += 1
mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
# integer=5, to specify the number of folds in a (Stratified)KFold,
#scores_BorM = cross_val_score(RFmodel, data, datalabels, cv=5)
# compute Z-space Accuracy
#Acc = scores_BorM.mean()
Acc = mean_auc
print "cvRF BorM mean_auc = %f " % Acc
#print scores_BorM.tolist()
if (i == 0):
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
self.best_args['initAcc'] = Acc
# plot initial z
figinint = plt.figure()
axinint = figinint.add_subplot(1, 1, 1)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
axinint,
title='kmeans init tsne: Acc={}'.format(Acc),
legend=True)
figinint.savefig('{}//tsne_init_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'],
self.best_args['num_centers'], labeltype),
bbox_inches='tight')
plt.close()
# save best args
self.best_args['acci'].append(Acc)
if (Acc >= self.maxAcc):
print 'Improving mean_auc = {}'.format(Acc)
for key, v in args.items():
self.best_args[key] = args[key]
self.maxAcc = Acc
self.best_args['pbestacci'] = p
self.best_args['zbestacci'] = z
self.best_args['bestacci'].append(Acc)
self.best_args['dec_mu'][:] = args['dec_mu'].asnumpy()
if (i > 0 and i % self.best_args['plot_interval'] == 0
and self.ploti <= 15):
# Visualize the progression of the embedded representation in a subsample of data
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
axprogress = figprogress.add_subplot(4, 4, 1 + self.ploti)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
axprogress,
title="Epoch %d z_tsne Acc (%f)" % (i, Acc),
legend=False)
self.ploti = self.ploti + 1
## COMPUTING target distributions P
## we compute pi by first raising qi to the second power and then normalizing by frequency per cluster:
weight = 1.0 / p.sum(axis=0) # p.sum provides fj
weight *= self.best_args['num_centers'] / weight.sum()
p = (p**2) * weight
train_iter.data_list[1][:] = (p.T / p.sum(axis=1)).T
print np.sum(y_pred != self.best_args['y_pred']
), 0.001 * y_pred.shape[0]
# For the purpose of discovering cluster assignments, we stop our procedure when less than tol% of points change cluster assignment between two consecutive iterations.
# tol% = 0.001
if i == self.best_args[
'update_interval'] * 200: # performs 1epoch = 615/3 = 205*1000epochs
self.best_args['y_pred'] = y_pred
self.best_args['acci'].append(Acc)
return True
self.best_args['y_pred'] = y_pred
# start solver
solver.set_iter_start_callback(refresh)
solver.set_monitor(Monitor(20))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 1000000000, {}, False)
self.end_args = args
self.best_args['end_args'] = args
# finish
figprogress = plt.gcf()
figprogress.savefig('{}\\tsne_progress_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'], self.best_args['num_centers'],
labeltype),
bbox_inches='tight')
plt.close()
# plot final z
figfinal = plt.figure()
axfinal = figfinal.add_subplot(1, 1, 1)
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(self.best_args['zbestacci'])
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
self.best_args['named_y'],
axfinal,
title='final tsne: Acc={}'.format(self.best_args['bestacci'][-1]),
legend=True)
figfinal.savefig('{}\\tsne_final_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'], self.best_args['num_centers'],
labeltype),
bbox_inches='tight')
plt.close()
outdict = {
'initAcc': self.best_args['initAcc'],
'acci': self.best_args['acci'],
'bestacci': self.best_args['bestacci'],
'pbestacci': self.best_args['pbestacci'],
'zbestacci': self.best_args['zbestacci'],
'dec_mubestacci': self.best_args['dec_mu'],
'y_pred': self.best_args['y_pred'],
'named_y': self.best_args['named_y'],
'classes': self.best_args['classes'],
'num_centers': self.best_args['num_centers'],
'znum': self.best_args['znum'],
'update_interval': self.best_args['update_interval'],
'batch_size': self.best_args['batch_size']
}
return outdict
def cluster_varying_mu(self,
X,
y_dec,
roi_labels,
classes,
save_to,
labeltype,
update_interval=None):
# y = y_dec
N = X.shape[0]
self.best_args['update_interval'] = update_interval
self.best_args['y_dec'] = y_dec
self.best_args['roi_labels'] = roi_labels
self.best_args['classes'] = classes
# selecting batch size
# [42*t for t in range(42)] will produce 16 train epochs
# [0, 42, 84, 126, 168, 210, 252, 294, 336, 378, 420, 462, 504, 546, 588, 630]
batch_size = self.best_args['batch_size'] #615/3 42 #256
test_iter = mx.io.NDArrayIter({'data': X},
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad')
args = {
k: mx.nd.array(v.asnumpy(), ctx=self.xpu)
for k, v in self.args.items()
}
## embedded point zi
z = model.extract_feature(self.feature, args, None, test_iter, N,
self.xpu).values()[0]
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
self.perplexity = 15
self.learning_rate = 200
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
# plot initial z
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
# reconstruct wordy labels list(Y)==named_y
named_y = [classes[kc] for kc in y_dec]
self.best_args['named_y'] = named_y
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
ax,
title='{} tsne with perplexity {}'.format(labeltype,
self.perplexity),
legend=True)
fig.savefig(save_to + os.sep + 'iter1_tsne_init_z' +
str(self.best_args['znum']) + '_varying_mu' +
str(self.best_args['num_centers']) + '.pdf',
bbox_inches='tight')
plt.close()
# To initialize the cluster centers, we pass the data through
# the initialized DNN to get embedded data points and then
# perform standard k-means clustering in the feature space Z
# to obtain k initial centroids {mu j}
kmeans = KMeans(self.best_args['num_centers'], n_init=20)
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
### KL DIVERGENCE MINIMIZATION. eq(2)
# our model is trained by matching the soft assignment to the target distribution.
# To this end, we define our objective as a KL divergence loss between
# the soft assignments qi (pred) and the auxiliary distribution pi (label)
solver = Solver('sgd',
momentum=0.9,
wd=0.0,
learning_rate=0.01,
lr_scheduler=mx.misc.FactorScheduler(
100 * update_interval, 0.5)) #0.01
def ce(label, pred):
return np.sum(label * np.log(label /
(pred + 0.000001))) / label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
label_buff = np.zeros((X.shape[0], self.best_args['num_centers']))
train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff},
batch_size=N,
shuffle=False,
last_batch_handle='roll_over')
self.best_args['y_pred'] = np.zeros((X.shape[0]))
self.best_args['acci'] = []
self.best_args['bestacci'] = []
self.ploti = 0
fig = plt.figure(figsize=(20, 15))
print 'Batch_size = %f' % batch_size
print 'update_interval = %f' % update_interval
self.best_args['plot_interval'] = int(25 * update_interval)
print 'plot_interval = %f' % self.best_args['plot_interval']
self.maxAcc = 0.0
def refresh(i): # i=3, a full epoch occurs every i=798/48
if i % self.best_args['update_interval'] == 0:
z = model.extract_feature(self.feature, args, None, test_iter,
N, self.xpu).values()[0]
p = np.zeros((z.shape[0], self.best_args['num_centers']))
self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p])
# the soft assignments qi (pred)
y_pred = p.argmax(axis=1)
print np.std(np.bincount(y_pred)), np.bincount(y_pred)
# use a y that only considers the filledbyBC examples
# compare soft assignments with known labels
print '\n... Updating i = %f' % i
# do for overall class B and M
Z_embedding_tree = sklearn.neighbors.BallTree(z, leaf_size=5)
# This finds the indices of 5 closest neighbors
labels = np.asarray(self.best_args['roi_labels'])
Neg = sum(labels == np.unique(labels)[0]) #for B
Pos = sum(labels == np.unique(labels)[2]) #for M
TP = []
TN = []
for k in range(z.shape[0]):
iclass = labels[k]
dist, ind = Z_embedding_tree.query([z[k]], k=6)
dist5nn, ind5nn = dist[k != ind], ind[k != ind]
class5nn = labels[ind5nn]
# exlcude U class
class5nn = class5nn[class5nn != 'K']
if (len(class5nn) > 0):
predc = []
for c in np.unique(class5nn):
predc.append(sum(class5nn == c))
# predicion based on majority
predclass = np.unique(class5nn)[predc == max(predc)]
if (len(predclass) == 1):
# compute TP if M
if (iclass == 'M'):
TP.append(predclass[0] == iclass)
# compute TN if B
if (iclass == 'B'):
TN.append(predclass[0] == iclass)
if (len(predclass) == 2):
# compute TP if M
if (iclass == 'M'):
TP.append(predclass[1] == iclass)
# compute TN if B
if (iclass == 'B'):
TN.append(predclass[0] == iclass)
# compute TPR and TNR
TPR = sum(TP) / float(Pos)
TNR = sum(TN) / float(Neg)
Acc = sum(TP + TN) / float(Pos + Neg)
print "True Posite Rate (TPR) = %f " % TPR
print "True Negative Rate (TNR) = %f " % TNR
print "Accuracy (Acc) = %f " % Acc
# save best args
self.best_args['acci'].append(Acc)
if (Acc >= self.maxAcc):
print 'Improving maxAcc = {}'.format(Acc)
for key, v in args.items():
self.best_args[key] = args[key]
self.maxAcc = Acc
self.best_args['bestacci'].append(Acc)
self.best_args['dec_mu'][:] = args['dec_mu'].asnumpy()
if (i % self.best_args['plot_interval'] == 0
and self.ploti <= 15):
# Visualize the progression of the embedded representation in a subsample of data
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
ax = fig.add_subplot(4, 4, 1 + self.ploti)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
ax,
title="Epoch %d z_tsne iter (%d)" % (self.ploti, i),
legend=False)
self.ploti = self.ploti + 1
## COMPUTING target distributions P
## we compute pi by first raising qi to the second power and then normalizing by frequency per cluster:
weight = 1.0 / p.sum(axis=0) # p.sum provides fj
weight *= self.best_args['num_centers'] / weight.sum()
p = (p**2) * weight
train_iter.data_list[1][:] = (p.T / p.sum(axis=1)).T
print np.sum(y_pred != self.best_args['y_pred']
), 0.001 * y_pred.shape[0]
# For the purpose of discovering cluster assignments, we stop our procedure when less than tol% of points change cluster assignment between two consecutive iterations.
# tol% = 0.001
if i == self.best_args[
'update_interval'] * 600: # performs 1epoch = 615/3 = 205*1000epochs
self.best_args['y_pred'] = y_pred
self.best_args['p'] = p
self.best_args['z'] = z
self.best_args['acci'].append(Acc)
return True
self.best_args['y_pred'] = y_pred
self.best_args['p'] = p
self.best_args['z'] = z
# start solver
solver.set_iter_start_callback(refresh)
solver.set_monitor(Monitor(100))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 1000000000, {}, False)
self.end_args = args
self.best_args['end_args'] = args
# finish
fig = plt.gcf()
fig.savefig(save_to + os.sep + 'iter1_tsne_progress_z' +
str(self.best_args['znum']) + '_varying_mu' +
str(self.best_args['num_centers']) + '.pdf',
bbox_inches='tight')
plt.close()
# plot final z
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(self.best_args['z'])
plot_embedding_unsuper_NMEdist_intenh(Z_tsne,
named_y,
ax,
title="tsne with perplexity %d" %
self.perplexity,
legend=True)
fig.savefig(save_to + os.sep + 'tsne_final_z' +
str(self.best_args['znum']) + '_varying_mu' +
str(self.best_args['num_centers']) + '.pdf',
bbox_inches='tight')
plt.close()
outdict = {
'acc': self.best_args['acci'],
'bestacc': self.best_args['bestacci'],
'p': self.best_args['p'],
'z': self.best_args['z'],
'y_pred': self.best_args['y_pred'],
'named_y': self.best_args['named_y'],
'num_centers': self.best_args['num_centers']
}
return outdict
def cluster_unsuperv(self, X, y, y_tsne, fighome, update_interval=None):
N = X.shape[0]
plotting_interval = N
if not update_interval:
update_interval = int(self.batch_size / 5.0)
# selecting batch size
# [42*t for t in range(42)] will produce 16 train epochs
# [0, 42, 84, 126, 168, 210, 252, 294, 336, 378, 420, 462, 504, 546, 588, 630]
batch_size = self.batch_size #615/3 42 #256
test_iter = mx.io.NDArrayIter({'data': X},
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad')
args = {
k: mx.nd.array(v.asnumpy(), ctx=self.xpu)
for k, v in self.args.items()
}
## embedded point zi
z = model.extract_feature(self.feature, args, None, test_iter, N,
self.xpu).values()[0]
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
pp = 15
tsne = TSNE(n_components=2,
perplexity=pp,
learning_rate=275,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
# plot initial z
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
plot_embedding(Z_tsne,
y_tsne,
ax,
title="tsne with perplexity %d" % pp,
legend=True,
plotcolor=True)
fig.savefig(fighome + os.sep + 'tsne_init_z' + str(self.znum) + '.pdf',
bbox_inches='tight')
plt.close()
# To initialize the cluster centers, we pass the data through
# the initialized DNN to get embedded data points and then
# perform standard k-means clustering in the feature space Z
# to obtain k initial centroids {mu j}
kmeans = KMeans(self.num_centers, n_init=20)
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
### KL DIVERGENCE MINIMIZATION. eq(2)
# our model is trained by matching the soft assignment to the target distribution.
# To this end, we define our objective as a KL divergence loss between
# the soft assignments qi (pred) and the auxiliary distribution pi (label)
solver = Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.01)
def ce(label, pred):
return np.sum(label * np.log(label /
(pred + 0.000001))) / label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
label_buff = np.zeros((X.shape[0], self.num_centers))
train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff},
batch_size=batch_size,
shuffle=False,
last_batch_handle='roll_over')
self.y_pred = np.zeros((X.shape[0]))
self.solvermetric = []
self.ploti = 0
fig = plt.figure(figsize=(20, 15))
print 'Batch_size = %f' % self.batch_size
print 'update_interval = %f' % update_interval
print 'tolernace = len(ypred)/1000 = %f' % float(
0.001 * self.y_pred.shape[0])
def refresh_unsuperv(i):
if i % update_interval == 0:
print '... Updating i = %f' % i
z = model.extract_feature(self.feature, args, None, test_iter,
N, self.xpu).values()[0]
p = np.zeros((z.shape[0], self.num_centers))
self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p])
# the soft assignments qi (pred)
y_pred = p.argmax(axis=1)
print np.std(np.bincount(y_pred)), np.bincount(y_pred)
if y is not None:
# compare soft assignments with known labels (unused)
print np.std(np.bincount(y.astype(np.int))), np.bincount(
y.astype(np.int))
print y_pred[0:5], y.astype(np.int)[0:5]
print 'Clustering Acc = %f' % cluster_acc(y_pred, y)[0]
self.acci.append(cluster_acc(y_pred, y)[0])
## COMPUTING target distributions P
## we compute pi by first raising qi to the second power and then normalizing by frequency per cluster:
weight = 1.0 / p.sum(axis=0) # p.sum provides fj
weight *= self.num_centers / weight.sum()
p = (p**2) * weight
train_iter.data_list[1][:] = (p.T / p.sum(axis=1)).T
print "sum(I(y'-1!=y) = %f" % np.sum(y_pred != self.y_pred)
self.solvermetric.append(solver.metric.get()[1])
print "solver.metric = %f" % solver.metric.get()[1]
# For the purpose of discovering cluster assignments, we stop our procedure when less than tol% of points change cluster assignment between two consecutive iterations.
# tol% = 0.001
# if np.sum(y_pred != self.y_pred) < 0.001*y_pred.shape[0]: # performs 1epoch = 615/3 = 205*1000epochs #
# self.y_pred = y_pred
# return True
self.y_pred = y_pred
self.p = p
self.z = z
# to plot
if i % plotting_interval == 0:
if (self.ploti <= 15):
# Visualize the progression of the embedded representation in a subsample of data
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
tsne = TSNE(n_components=2,
perplexity=15,
learning_rate=275,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(self.z)
ax = fig.add_subplot(3, 4, 1 + self.ploti)
plot_embedding(Z_tsne,
y_tsne,
ax,
title="Epoch %d z_tsne iter (%d)" %
(self.ploti, i),
legend=False,
plotcolor=True)
self.ploti = self.ploti + 1
# start solver
solver.set_iter_start_callback(refresh_unsuperv)
# monitor every self.batch_size
solver.set_monitor(Monitor(self.batch_size))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 12 * N, {}, False)
# finish
fig = plt.gcf()
fig.savefig(fighome + os.sep + 'tsne_progress_k' +
str(self.num_centers) + '_z' + str(self.znum) + '.pdf',
bbox_inches='tight')
plt.close()
# plot progression of clustering loss
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(range(len(self.solvermetric)), self.solvermetric, '-.')
ax.set_xlabel("iter")
ax.set_ylabel("L loss for num_centers =" + str(self.num_centers))
fig.savefig(fighome + os.sep + 'clustering_loss_numcenters' +
str(self.num_centers) + '_z' + str(self.znum) + '.pdf',
bbox_inches='tight')
plt.close()
self.end_args = args
outdict = {'p': self.p, 'z': self.z, 'y_pred': self.y_pred}
return outdict
def cluster(self,
X_train,
y_dec_train,
y_train,
classes,
batch_size,
save_to,
labeltype,
update_interval=None):
N = X_train.shape[0]
self.best_args['update_interval'] = update_interval
self.best_args['y_dec'] = y_dec_train
self.best_args['roi_labels'] = y_train
self.best_args['classes'] = classes
self.best_args['batch_size'] = batch_size
# selecting batch size
# [42*t for t in range(42)] will produce 16 train epochs
# [0, 42, 84, 126, 168, 210, 252, 294, 336, 378, 420, 462, 504, 546, 588, 630]
test_iter = mx.io.NDArrayIter({'data': X_train},
batch_size=N,
shuffle=False,
last_batch_handle='pad')
args = {
k: mx.nd.array(v.asnumpy(), ctx=self.xpu)
for k, v in self.args.items()
}
## embedded point zi
z = model.extract_feature(self.feature, args, None, test_iter, N,
self.xpu).values()[0]
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
self.perplexity = 15
self.learning_rate = 125
# reconstruct wordy labels list(Y)==named_y
named_y = [classes[kc] for kc in y_dec_train]
self.best_args['named_y'] = named_y
# To initialize the cluster centers, we pass the data through
# the initialized DNN to get embedded data points and then
# perform standard k-means clustering in the feature space Z
# to obtain k initial centroids {mu j}
kmeans = KMeans(self.best_args['num_centers'], n_init=20)
kmeans.fit(z)
args['dec_mu'][:] = kmeans.cluster_centers_
### KL DIVERGENCE MINIMIZATION. eq(2)
# our model is trained by matching the soft assignment to the target distribution.
# To this end, we define our objective as a KL divergence loss between
# the soft assignments qi (pred) and the auxiliary distribution pi (label)
solver = Solver(
'sgd', momentum=0.9, wd=0.0, learning_rate=0.01
) # , lr_scheduler=mx.misc.FactorScheduler(20*update_interval,0.4)) #0.01
def ce(label, pred):
return np.sum(label * np.log(label /
(pred + 0.000001))) / label.shape[0]
solver.set_metric(mx.metric.CustomMetric(ce))
label_buff = np.zeros(
(X_train.shape[0], self.best_args['num_centers']))
train_iter = mx.io.NDArrayIter({'data': X_train},
{'label': label_buff},
batch_size=self.best_args['batch_size'],
shuffle=False,
last_batch_handle='roll_over')
self.best_args['y_pred'] = np.zeros((X_train.shape[0]))
self.best_args['acci'] = []
self.best_args['bestacci'] = []
self.ploti = 0
figprogress = plt.figure(figsize=(20, 15))
print 'Batch_size = %f' % self.best_args['batch_size']
print 'update_interval = %f' % update_interval
self.best_args['plot_interval'] = int(20 * update_interval)
print 'plot_interval = %f' % self.best_args['plot_interval']
self.maxAcc = 0.0
def refresh(i): # i=3, a full epoch occurs every i=798/48
if i % self.best_args['update_interval'] == 0:
z = list(
model.extract_feature(self.feature, args, None, test_iter,
N, self.xpu).values())[0]
p = np.zeros((z.shape[0], self.best_args['num_centers']))
self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p])
# the soft assignments qi (pred)
y_pred = p.argmax(axis=1)
print np.std(np.bincount(y_dec)), np.bincount(y_dec)
print np.std(np.bincount(y_pred)), np.bincount(y_pred)
#####################
# Z-space CV RF classfier METRICS
#####################
# compare soft assignments with known labels (only B or M)
print '\n... Updating i = %f' % i
datalabels = np.asarray(y_train)
dataZspace = np.concatenate(
(z, p), axis=1) #zbestacci #dec_model['zbestacci']
Xdata = dataZspace[datalabels != 'K', :]
ydatalabels = datalabels[datalabels != 'K']
RFmodel = RandomForestClassifier(n_jobs=2,
n_estimators=500,
random_state=0,
verbose=0)
# Evaluate a score by cross-validation
# integer=5, to specify the number of folds in a (Stratified)KFold,
scores_BorM = cross_val_score(RFmodel,
Xdata,
ydatalabels,
cv=5)
# compute Z-space Accuracy
Acc = scores_BorM.mean()
print "cvRF BorM Accuracy = %f " % Acc
print scores_BorM.tolist()
# use only the filledbyBC examples (first 0-202 exaples)
nme_dist_label = [
lab.split('_')[1]
for lab in self.best_args['named_y'][0:202]
]
nme_intenh_label = [
lab.split('_')[2]
for lab in self.best_args['named_y'][0:202]
]
# compute Z-space Accuracy
scores_dist = cross_val_score(RFmodel,
z[0:202],
nme_dist_label,
cv=5)
print "cvRF nme_dist Accuracy = %f " % scores_dist.mean()
scores_intenh = cross_val_score(RFmodel,
z[0:202],
nme_intenh_label,
cv=5)
print "cvRF nme_intenh Accuracy = %f " % scores_intenh.mean()
#####################
# CALCULATE 5nn METRICS
#####################
labels = np.asarray(self.best_args['named_y'])
Z_embedding_tree = sklearn.neighbors.BallTree(z, leaf_size=4)
# This finds the indices of 5 closest neighbors
nme_dist = [
'Diffuse', 'Focal', 'Linear', 'MultipleRegions',
'Regional', 'Segmental'
]
nme_intenh = [
'Clumped', 'ClusteredRing', 'Heterogeneous', 'Homogeneous',
'Stippled or punctate'
]
wnme_dist = np.zeros((len(nme_dist), len(nme_dist)),
dtype=np.int64)
wnme_intenh = np.zeros((len(nme_intenh), len(nme_intenh)),
dtype=np.int64)
BorM_diag = []
TP = []
TN = []
FP = []
FN = []
missed = []
NME_descript_dist = []
NME_descript_intenh = []
# count stattistics
for k in range(z.shape[0]):
iclass = labels[k]
dist, ind = Z_embedding_tree.query([z[k]], k=2)
dist5nn, ind5nn = dist[k != ind], ind[k != ind]
class5nn = labels[ind5nn]
# compute DIAGNOSTIC ACC based on nme similar local neighborhood
if (iclass.split('_')[0] != 'K'):
predBorM = []
predBorM.append(
sum([lab.split('_')[0] == 'M'
for lab in class5nn]))
predBorM.append(
sum([lab.split('_')[0] == 'B'
for lab in class5nn]))
predBorM.append(
sum([lab.split('_')[0] == 'K'
for lab in class5nn]))
pred_BorM = [
['M', 'B', 'K'][l] for l, pc in enumerate(predBorM)
if pc >= max(predBorM) and max(predBorM) > 0
][0]
# compute TP if M
if (pred_BorM != 'K'):
if (iclass.split('_')[0] == pred_BorM):
BorM_diag.append(1)
# confusino table
if (iclass.split('_')[0] == 'M'):
TP.append(1)
if (iclass.split('_')[0] == 'B'):
TN.append(1)
if (iclass.split('_')[0] != pred_BorM):
if (iclass.split('_')[0] == 'M'):
FN.append(1)
if (iclass.split('_')[0] == 'B'):
FP.append(1)
else:
missed.append(1)
if (k <= 202 and iclass.split('_')[1] != 'N/A'):
# increment detections for final NME descriptor accuracy
prednmed = []
prednmed.append(
sum([
lab.split('_')[1] == 'Diffuse'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Focal'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Linear'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'MultipleRegions'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Regional'
for lab in class5nn
]))
prednmed.append(
sum([
lab.split('_')[1] == 'Segmental'
for lab in class5nn
]))
# predicion based on majority voting
pred_nme_dist = [
nme_dist[l] for l, pc in enumerate(prednmed)
if pc >= max(prednmed) and max(prednmed) > 0
]
# compute NME ACC based on nme similar local neighborhood
if (iclass.split('_')[1] in pred_nme_dist):
NME_descript_dist.append(1)
if (k <= 202 and iclass.split('_')[2] != 'N/A'):
prednmeie = []
prednmeie.append(
sum([
lab.split('_')[2] == 'Clumped'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'ClusteredRing'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'Heterogeneous'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'Homogeneous'
for lab in class5nn
]))
prednmeie.append(
sum([
lab.split('_')[2] == 'Stippled or punctate'
for lab in class5nn
]))
# predicion based on majority voting
pred_nme_intenh = [
nme_intenh[l] for l, pc in enumerate(prednmeie)
if pc >= max(prednmeie) and max(prednmeie) > 0
]
# compute NME ACC based on nme similar local neighborhoo
if (iclass.split('_')[2] in pred_nme_intenh):
NME_descript_intenh.append(1)
#####################
# collect stats
#####################
BorM_diag_Acc = sum(BorM_diag) / float(len(datalabels))
#Acc = BorM_diag_Acc
print "BorM_diag_Acc = %f " % BorM_diag_Acc
## good if high
TPR = sum(TP) / float(sum(datalabels == 'M'))
print "TPR = %f " % TPR
TNR = sum(TN) / float(sum(datalabels == 'B'))
print "TNR = %f " % TNR
## bad if high
FPR = sum(FP) / float(sum(datalabels == 'B'))
print "FPR = %f " % FPR
FNR = sum(FN) / float(sum(datalabels == 'M'))
print "FNR = %f " % FNR
# good if reduces
missedR = sum(np.asarray(missed)) / float(len(datalabels))
print "missedR = %f " % missedR
Acc5nn_nme_dist = sum(NME_descript_dist) / 202.0
Acc5nn_nme_intenh = sum(NME_descript_intenh) / 202.0
print "Acc5nn_nme_dist (DIST) = %f " % Acc5nn_nme_dist
print "Acc5nn_nme_intenh (INT_ENH) = %f " % Acc5nn_nme_intenh
if (i == 0):
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
self.best_args['initAcc'] = Acc
# plot initial z
figinint = plt.figure()
axinint = figinint.add_subplot(1, 1, 1)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
axinint,
title=
'kmeans init tsne: Acc={}\n RF_nme_dist={}\n RF_intenh={}'
.format(Acc, scores_dist.mean(), scores_intenh.mean()),
legend=True)
figinint.savefig('{}//tsne_init_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'],
self.best_args['num_centers'], labeltype),
bbox_inches='tight')
plt.close()
# save best args
self.best_args['acci'].append(Acc)
if (Acc >= self.maxAcc):
print 'Improving maxAcc = {}'.format(Acc)
for key, v in args.items():
self.best_args[key] = args[key]
self.maxAcc = Acc
self.best_args['pbestacci'] = p
self.best_args['zbestacci'] = z
self.best_args['bestacci'].append(Acc)
self.best_args['cvRF_nme_dist'] = scores_dist.mean()
self.best_args['cvRF_nme_intenh'] = scores_intenh.mean()
self.best_args['dec_mu'][:] = args['dec_mu'].asnumpy()
self.best_args['BorM_diag_Acc'] = BorM_diag_Acc
self.best_args['TPR'] = TPR
self.best_args['TNR'] = TNR
self.best_args['FPR'] = FPR
self.best_args['FNR'] = FNR
self.best_args['missedR'] = missedR
self.best_args['Acc5nn_nme_dist'] = Acc5nn_nme_dist
self.best_args['Acc5nn_nme_intenh'] = Acc5nn_nme_intenh
if (i > 0 and i % self.best_args['plot_interval'] == 0
and self.ploti <= 15):
# Visualize the progression of the embedded representation in a subsample of data
# For visualization we use t-SNE (van der Maaten & Hinton, 2008) applied to the embedded points zi. It
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(z)
axprogress = figprogress.add_subplot(4, 4, 1 + self.ploti)
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
named_y,
axprogress,
title="Epoch %d z_tsne Acc (%f)" % (i, Acc),
legend=False)
self.ploti = self.ploti + 1
## COMPUTING target distributions P
## we compute pi by first raising qi to the second power and then normalizing by frequency per cluster:
weight = 1.0 / p.sum(axis=0) # p.sum provides fj
weight *= self.best_args['num_centers'] / weight.sum()
p = (p**2) * weight
train_iter.data_list[1][:] = (p.T / p.sum(axis=1)).T
print np.sum(y_pred != self.best_args['y_pred']
), 0.001 * y_pred.shape[0]
# For the purpose of discovering cluster assignments, we stop our procedure when less than tol% of points change cluster assignment between two consecutive iterations.
# tol% = 0.001
if i == self.best_args[
'update_interval'] * 100: # performs 1epoch = 615/3 = 205*1000epochs
self.best_args['y_pred'] = y_pred
self.best_args['acci'].append(Acc)
return True
self.best_args['y_pred'] = y_pred
# start solver
solver.set_iter_start_callback(refresh)
solver.set_monitor(Monitor(20))
solver.solve(self.xpu, self.loss, args, self.args_grad, None,
train_iter, 0, 1000000000, {}, False)
self.end_args = args
self.best_args['end_args'] = args
# finish
figprogress = plt.gcf()
figprogress.savefig('{}\\tsne_progress_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'], self.best_args['num_centers'],
labeltype),
bbox_inches='tight')
plt.close()
# plot final z
figfinal = plt.figure()
axfinal = figfinal.add_subplot(1, 1, 1)
tsne = TSNE(n_components=2,
perplexity=self.perplexity,
learning_rate=self.learning_rate,
init='pca',
random_state=0,
verbose=2,
method='exact')
Z_tsne = tsne.fit_transform(self.best_args['zbestacci'])
plot_embedding_unsuper_NMEdist_intenh(
Z_tsne,
self.best_args['named_y'],
axfinal,
title='final tsne: Acc={}\n RF_nme_dist={}\n RF_intenh={}'.format(
self.best_args['bestacci'][-1],
self.best_args['cvRF_nme_dist'],
self.best_args['cvRF_nme_intenh']),
legend=True)
figfinal.savefig('{}\\tsne_final_z{}_mu{}_{}.pdf'.format(
save_to, self.best_args['znum'], self.best_args['num_centers'],
labeltype),
bbox_inches='tight')
plt.close()
outdict = {
'initAcc': self.best_args['initAcc'],
'acci': self.best_args['acci'],
'bestacci': self.best_args['bestacci'],
'pbestacci': self.best_args['pbestacci'],
'zbestacci': self.best_args['zbestacci'],
'dec_mubestacci': self.best_args['dec_mu'],
'cvRF_nme_dist': self.best_args['cvRF_nme_dist'],
'cvRF_nme_intenh': self.best_args['cvRF_nme_intenh'],
'BorM_diag_Acc': self.best_args['BorM_diag_Acc'],
'TPR': self.best_args['TPR'],
'TNR': self.best_args['TNR'],
'FPR': self.best_args['FPR'],
'FNR': self.best_args['FNR'],
'missedR': self.best_args['missedR'],
'Acc5nn_nme_dist': self.best_args['Acc5nn_nme_dist'],
'Acc5nn_nme_intenh': self.best_args['Acc5nn_nme_intenh'],
'y_pred': self.best_args['y_pred'],
'named_y': self.best_args['named_y'],
'classes': self.best_args['classes'],
'num_centers': self.best_args['num_centers'],
'znum': self.best_args['znum'],
'update_interval': self.best_args['update_interval'],
'batch_size': self.best_args['batch_size']
}
return outdict