def initialize_model(self, ae_weights=None, gamma=0.1, optimizer='adam'):
if ae_weights is not None:
self.autoencoder.load_weights(ae_weights)
print
'Pretrained AE weights are loaded successfully.'
else:
print
'ae_weights must be given. E.g.'
print
' python IDEC.py mnist --ae_weights weights.h5'
exit()
hidden = self.autoencoder.get_layer(name='encoder_%d' %
(self.n_stacks - 1)).output
self.encoder = Model(inputs=self.autoencoder.input, outputs=hidden)
# prepare IDEC model
clustering_layer = ClusteringLayer(self.n_clusters,
name='clustering')(hidden)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
self.model.compile(loss={
'clustering': 'kld',
'decoder_0': 'mse'
},
loss_weights=[gamma, 1],
optimizer=optimizer)
def __init__(self,
dims,
gamma=0.1,
n_clusters=10,
alpha=1.0,
bn=True,
init='glorot_uniform'):
self.dims = dims
self.input_dim = dims[0]
self.n_stacks = len(self.dims) - 1
self.n_clusters = n_clusters
self.alpha = alpha
if bn:
self.autoencoder, self.encoder = autoencoder_bn(self.dims,
init=init)
else:
self.autoencoder, self.encoder = autoencoder(self.dims, init=init)
# prepare DEC model
clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(
self.encoder.output)
# reconstruct = Add()(self.autoencoder.output)
# clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(reconstruct)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer] +
self.autoencoder.output)
def __init__(self,
dims,
gamma=0.1,
n_clusters=10,
alpha=1.0,
bn=False,
init='glorot_uniform'):
self.dims = dims
self.input_dim = dims[0]
self.n_stacks = len(self.dims) - 1
self.n_clusters = n_clusters
self.alpha = alpha
# Fine-tuning
print('Fine-tuning')
self.autoencoder = Sequential()
self.encoder = Sequential()
for encoder in trained_encoders:
self.autoencoder.add(encoder)
self.encoder.add(encoder)
for decoder in trained_decoders:
self.autoencoder.add(decoder)
# prepare DEC model
clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(
self.encoder.output)
# reconstruct = Add()(self.autoencoder.output)
# clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(reconstruct)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
def initialize(self, X, save_autoencoder=False, **kwargs):
if self.pretrained_weights is None:
print 'Training autoencoder.'
self.autoencoder.fit(X, X, batch_size=self.batch_size, **kwargs)
if save_autoencoder:
self.autoencoder.save_weights('autoencoder.h5')
else:
print 'Loading pretrained weights for autoencoder.'
self.autoencoder.load_weights(self.pretrained_weights)
# update encoder, decoder
for i in range(len(self.encoder.layers)):
self.encoder.layers[i].set_weights(
self.autoencoder.layers[i].get_weights())
# initialize cluster centres using k-means
print 'Initializing cluster centres with k-means.'
if self.cluster_centres is None:
kmeans = KMeans(n_clusters=self.n_clusters, n_init=50)
self.y_pred = kmeans.fit_predict(self.encoder.predict(X))
self.cluster_centres = kmeans.cluster_centers_
# prepare DEC model
self.DEC = Model(input=self.input_layer,
output=ClusteringLayer(self.n_clusters,
weights=self.cluster_centres,
name='clustering')(
self.encoded))
self.DEC.compile(loss='kullback_leibler_divergence',
optimizer='adadelta')
return
def __init__(self, dims, n_clusters=4, alpha=1.0):
super(IDEC, self).__init__()
self.dims = dims
self.input_dim = dims[0]
self.n_stacks = len(self.dims) - 1
self.n_clusters = n_clusters
self.alpha = alpha
self.autoencoder = autoencoder(self.dims)
hidden = self.autoencoder.get_layer(name='encoder_%d' %
(self.n_stacks - 1)).output
self.encoder = Model(inputs=self.autoencoder.input, outputs=hidden)
# prepare IDEC model
clustering_layer = ClusteringLayer(self.n_clusters,
alpha=self.alpha,
name='clustering')(hidden)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
self.pretrained = False
self.centers = []
self.y_pred = []
def initialize_model(self, x, gamma=0.1, optimizer='adam'):
self.autoencoder.compile(loss='mse', optimizer=optimizer) # SGD(lr=0.01, momentum=0.9),
self.autoencoder.fit(x, x, batch_size=self.batch_size, epochs=self.pretrain_epochs)
hidden = self.autoencoder.get_layer(name='encoder_%d' % (self.n_stacks - 1)).output
self.encoder = Model(inputs=self.autoencoder.input, outputs=hidden)
# prepare IDEC model
clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(hidden)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
self.model.compile(loss={'clustering': 'kld', 'decoder_0': 'mse'},
loss_weights=[gamma, 1],
optimizer=optimizer)
def initialize_model(self, ae_weights=None, gamma=0.1, optimizer='adam'):
if ae_weights is not None:
self.autoencoder.load_weights(ae_weights)
print('Pretrained AE weights are loaded successfully.')
else:
print(
'ae_weights must be given by adding path to arguments e.g. : --ae_weights weights.h5'
)
exit()
ml_penalty, cl_penalty = 0.1, 1
cst_optimizer = 'adam'
if self.classifier_name == 'mlp':
hidden = self.autoencoder.get_layer(name='encoder_%d' %
(self.n_stacks - 1)).output
decoder_layer = 'decoder_0'
else:
hidden = self.autoencoder.get_layer(name='encoder_d_%d' %
(self.d_stacks)).output
decoder_layer = 'lambda_o_0'
self.encoder = Model(inputs=self.autoencoder.input, outputs=hidden)
clustering_layer = ClusteringLayer(self.n_clusters,
name='clustering')(hidden)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
self.model.compile(loss={
'clustering': 'kld',
decoder_layer: 'mse'
},
loss_weights=[gamma, 1],
optimizer=optimizer)
for layer in self.model.layers:
print(layer, layer.trainable)
self.ml_model = Model(
inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
self.ml_model.compile(loss={
'clustering': ml_loss(),
decoder_layer: 'mse'
},
loss_weights=[ml_penalty, 1],
optimizer=cst_optimizer)
self.cl_model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer])
self.cl_model.compile(loss={'clustering': cl_loss()},
optimizer=cst_optimizer)
def __init__(self,
dims,
gamma=0.1,
n_clusters=10,
alpha=1.0,
init='glorot_uniform'):
# super(IDEC,self).__init__(dims,
# n_clusters=10,
# alpha=1.0,
# init='glorot_uniform')
self.dims = dims
self.input_dim = dims[0]
self.n_stacks = len(self.dims) - 1
self.n_clusters = n_clusters
self.alpha = alpha
self.autoencoder, self.encoder = autoencoder(self.dims, init=init)
# prepare DEC model
clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(
self.encoder.output)
self.model = Model(inputs=self.autoencoder.input,
outputs=[clustering_layer, self.autoencoder.output])
def clustering(self,
x,
y=None,
train_dev_data=None,
validation_data=None,
tol=1e-3,
update_interval=140,
maxiter=2e4,
save_dir='./results/dec',
pretrained_weights=None,
alpha=K.variable(1.0),
beta=K.variable(0.0),
gamma=K.variable(0.0),
loss_weight_decay=True,
loss=None,
loss_weights=None):
print('Update interval', update_interval)
save_interval = x.shape[0] / self.batch_size * 5 # 5 epochs
print('Save interval', save_interval)
try:
self.load_weights(pretrained_weights)
except AttributeError:
# initialize cluster centers using k-means
print('Initializing cluster centers with k-means.')
kmeans = KMeans(n_clusters=self.n_clusters, n_init=20)
y_pred = kmeans.fit_predict(self.encoder.predict(x))
y_pred_last = y_pred
self.model.get_layer(name='clustering').set_weights(
[kmeans.cluster_centers_])
y_p = self.predict_clusters(x)
cluster_to_label_mapping, n_assigned_list, majority_class_fractions = \
get_cluster_to_label_mapping_safe(y[:,1], y_p, n_classes, n_clusters)
print(
np.argmax((1 - np.array(majority_class_fractions)) *
np.array(n_assigned_list)))
cluster_to_label_mapping[np.argmax(
(1 - np.array(majority_class_fractions)) *
np.array(n_assigned_list))] = 1
cluster_weights = self.model.get_layer(name='clustering').get_weights()
#self.tmp_model = Model(inputs=self.model.input, outputs=self.model.layers[-2].output)
a = Input(shape=(400, )) # input layer
#self.model.summary()
#self.model.layers[1].trainable = False
#self.model.layers[2].trainable = False
#self.model.layers[3].trainable = False
self.model.layers[1].kernel_regularizer = regularizers.l2(0.5)
self.model.layers[2].kernel_regularizer = regularizers.l2(0.5)
self.model.layers[3].kernel_regularizer = regularizers.l2(0.5)
self.model.layers[4].kernel_regularizer = regularizers.l2(0.5)
#self.model.summary()
#self.autoencoder.summary()
#self.encoder.summary()
#exit()
#q_out = self.model(b)
hidden = self.encoder(a)
q_out = ClusteringLayer(self.n_clusters, name='clustering')(hidden)
#d = Dropout(0.2)(a)
e_out = self.autoencoder(a)
#pred = MappingLayer(cluster_to_label_mapping, output_dim=n_classes, \
# name='mapping', kernel_initializer=MapInitializer(cluster_to_label_mapping, n_classes))(q_out)
#embed = self.encoder(a)
#d_out = Dropout(0.4)(embed)
#self.encoder.summary()
#d_out = Dropout(0.7)(hidden)
#pred = MappingLayer(cluster_to_label_mapping, output_dim=n_classes, \
# name='mapping', kernel_initializer=MapInitializer(cluster_to_label_mapping, n_classes))(d_out)
#pred = Dense(2, activation='softmax')(hidden)
pred = Dense(2, activation='softmax')(q_out)
#pred = Dense(2, activation='softmax')(d_out)
self.model = Model(inputs=a, outputs=[pred, q_out, e_out])
#self.model = Model(inputs=a, outputs=[pred, q_out])
self.model.get_layer(name='clustering').set_weights(cluster_weights)
#optimizer = SGD(lr=1e-2)
optimizer = 'adam'
#self.model.compile(optimizer=optimizer, \
# loss={'dense_1': 'categorical_crossentropy', 'clustering': 'kld'}, \
# loss_weights={'dense_1': alpha, 'clustering': beta})
if loss is None:
self.model.compile(optimizer=optimizer, \
loss={'dense_1': 'categorical_crossentropy', 'clustering': 'kld', 'model_1': 'mse'},
loss_weights={'dense_1': alpha, 'clustering': beta, 'model_1': gamma})
else:
self.model.compile(optimizer=optimizer, \
loss=loss,
loss_weights=loss_weights)
#self.model.compile(optimizer=optimizer, \
# loss={'mapping': 'categorical_crossentropy', 'model_3': 'kld'}, \
# loss_weights={'mapping': alpha, 'model_3': beta})
import csv, os
if not os.path.exists(save_dir):
os.makedirs(save_dir)
#logfile = open(save_dir + '/dec_log.csv', 'w')
#logwriter = csv.DictWriter(logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'L'])
#logwriter.writeheader()
#tmp = self.encoder.predict(x)
loss = [0, 0, 0]
index = 0
q = self.model.predict(x, verbose=0)[1]
y_pred_last = q.argmax(1)
metrics_dict = {
'iteration': [],
'train_fom': [],
'train_f1': [],
'train_f1c': [],
'train_h': [],
'train_nmi': [],
'valid_fom': [],
'valid_f1': [],
'valid_f1c': [],
'valid_h': [],
'valid_nmi': []
}
best_train_dev_loss = [np.inf, np.inf, np.inf]
for ite in range(int(maxiter)):
if ite % update_interval == 0:
q = self.model.predict(x, verbose=0)[1]
valid_p = self.target_distribution(
self.model.predict(validation_data[0], verbose=0)[1])
p = self.target_distribution(
q) # update the auxiliary target distribution p
#print(p)
#print(np.sum(self.encoder.predict(x) - tmp))
#print(np.sum(self.encoder.predict(x) - self.tmp_model.predict(x)))
# evaluate the clustering performance
y_pred = q.argmax(1)
delta_label = np.sum(y_pred != y_pred_last).astype(
np.float32) / y_pred.shape[0]
y_pred_last = y_pred
y_pred = self.model.predict(x)[0]
if y is not None:
#acc = np.round(cluster_acc(y, y_pred), 5)
#ari = np.round(metrics.adjusted_rand_score(y, y_pred), 5)
loss = np.round(loss, 5)
#val_loss = np.round(self.model.test_on_batch(validation_data[0], [validation_data[1], valid_p]), 5)
c_map, _, _ = \
get_cluster_to_label_mapping_safe(y[:,1], q.argmax(1), n_classes, n_clusters, toprint=False)
f, _, _, _ = one_percent_fpr(y[:, 1], y_pred[:, 1], 0.01)
f = np.round(f, 5)
#print(y[:20,1], np.argmax(y_pred, axis=1)[:20])
train_cluster_pred = self.model.predict(
x, verbose=0)[1].argmax(1)
f1 = np.round(f1_score(y[:, 1], np.argmax(y_pred, axis=1)),
5)
f1c = np.round(
calc_f1_score(y[:, 1], train_cluster_pred, c_map), 5)
h = np.round(
homogeneity_score(y[:, 1], train_cluster_pred), 5)
nmi = np.round(
metrics.normalized_mutual_info_score(
y[:, 1], train_cluster_pred), 5)
val_loss = np.round(
self.model.test_on_batch(
validation_data[0],
[validation_data[1], valid_p, validation_data[0]]),
5)
y_pred_valid = self.model.predict(validation_data[0])[0]
valid_cluster_pred = self.model.predict(
validation_data[0], verbose=0)[1].argmax(1)
f_valid, _, _, _ = one_percent_fpr(
validation_data[1][:, 1], y_pred_valid[:, 1], 0.01)
f_valid = np.round(f_valid, 5)
f1_valid = np.round(
f1_score(validation_data[1][:, 1],
np.argmax(y_pred_valid, axis=1)), 5)
f1c_valid = np.round(
calc_f1_score(validation_data[1][:, 1],
valid_cluster_pred, c_map), 5)
h_valid = np.round(
homogeneity_score(validation_data[1][:, 1],
valid_cluster_pred), 5)
nmi_valid = np.round(
metrics.normalized_mutual_info_score(
validation_data[1][:, 1], valid_cluster_pred), 5)
#logdict = dict(iter=ite, acc=acc, nmi=nmi, ari=ari, L=loss)
#logwriter.writerow(logdict)
#print('Iter', ite, ': Acc', acc, ', nmi', nmi, ', ari', ari, '; loss=', loss)
#_, _, _ = \
# get_cluster_to_label_mapping_safe(validation_data[1][:,1], self.model.predict(validation_data[0], verbose=0)[1].argmax(1), n_classes, n_clusters)
print('Iter', ite, ' :1% fpr', f, ', F1=', f1, ', F1c=',
f1c, 'h=', h, 'nmi=', nmi, '; loss=', loss,
';\n v 1% fpr=,', f_valid, ', vF1=', f1_valid,
', vF1c=', f1c_valid, 'vh=', h_valid, 'vnmi=',
nmi_valid, '; vloss=,', val_loss)
metrics_dict['iteration'].append(ite)
metrics_dict['train_fom'].append(f)
metrics_dict['train_f1'].append(f1)
metrics_dict['train_f1c'].append(f1c)
metrics_dict['train_h'].append(h)
metrics_dict['train_nmi'].append(nmi)
metrics_dict['valid_fom'].append(f_valid)
metrics_dict['valid_f1'].append(f1_valid)
metrics_dict['valid_f1c'].append(f1c_valid)
metrics_dict['valid_h'].append(h_valid)
metrics_dict['valid_nmi'].append(nmi_valid)
train_dev_p = self.target_distribution(
self.model.predict(train_dev_data[0], verbose=0)[1])
train_dev_loss = np.round(
self.model.test_on_batch(train_dev_data[0], [
train_dev_data[1], train_dev_p, train_dev_data[0]
]), 5)
if train_dev_loss[1] < best_train_dev_loss[
1] and train_dev_loss[-1] < best_train_dev_loss[
-1]: # only interested in classification improvements
print('saving model: ', best_train_dev_loss, ' -> ',
train_dev_loss)
self.model.save_weights('best_train_dev_loss.hf')
best_train_dev_loss = train_dev_loss
best_ite = ite
# check stop criterion
if ite > 0 and delta_label < tol:
print('delta_label ', delta_label, '< tol ', tol)
print('Reached tolerance threshold. Stopping training.')
logfile.close()
break
# train on batch
"""
if (index + 1) * self.batch_size > x.shape[0]:
loss = self.model.train_on_batch(x=x[index * self.batch_size::],
y=[y[index * self.batch_size::], \
p[index * self.batch_size::]])
index = 0
else:
loss = self.model.train_on_batch(x=x[index * self.batch_size:(index + 1) * self.batch_size],
y=[y[index * self.batch_size:(index + 1) * self.batch_size], \
p[index * self.batch_size:(index + 1) * self.batch_size]])
index += 1
"""
if loss_weight_decay:
"""
if ite < 50:
alpha = K.variable(1.0)
beta = K.variable(0.0)
gamma = K.variable(1.0)
elif ite >= 50:
#alpha = K.variable(1.0)
alpha = K.variable(0.0)
#beta = K.variable(0.0)
beta = K.variable(1.0)
gamma = K.variable(1.0)
update_interval = 140
self.model.optimizer = SGD(lr=0.01, momentum=0.9)
"""
"""
elif ite >= 200 and ite < 300:
#alpha = K.variable(1.0*(1 - ((ite - 200)/100.)))
alpha = K.variable(1.0)
beta = K.variable(1.0)
gamma = K.variable(1.0)
print(K.eval(alpha), K.eval(beta))
"""
#alpha = K.variable(1.0*(1 - ((ite - 200)/100.)))
"""
if ite < 40:
alpha = K.variable((1 - ite/maxiter))
beta = K.variable(1-alpha)
gamma = K.variable(1.0)
print(K.eval(alpha), K.eval(beta), K.eval(gamma))
if ite == 40:
print('Initializing cluster centers with k-means.')
kmeans = KMeans(n_clusters=self.n_clusters, n_init=20)
y_pred = kmeans.fit_predict(self.encoder.predict(x))
self.model.get_layer(name='clustering').set_weights([kmeans.cluster_centers_])
if ite >= 40:
alpha = K.variable(0.0)
beta = K.variable(1.0)
gamma = K.variable(1.0)
update_interval=140
self.model.optimizer = SGD(lr=0.01, momentum=0.9)
"""
alpha = K.variable((1 - ite / maxiter))
beta = K.variable(1 - alpha)
gamma = K.variable(1.0)
print(K.eval(alpha), K.eval(beta), K.eval(gamma))
history = self.model.fit(x=x, y=[y,p,x], \
validation_data=(validation_data[0], [validation_data[1], valid_p, validation_data[0]]), \
callbacks=[MyLossWeightCallback(alpha, beta, gamma)], verbose=0)
else:
print(K.eval(alpha), K.eval(beta), K.eval(gamma))
history = self.model.fit(x=x, y=[y,p,x], \
validation_data=(validation_data[0], [validation_data[1], valid_p, validation_data[0]]), \
verbose=0)
#history = self.model.fit(x=x, y=[y,p], callbacks=[MyLossWeightCallback(alpha, beta)], verbose=0)
#print(history.history)
loss = [
history.history[k][0] for k in history.history.keys()
if 'val' not in k
]
# save intermediate model
if ite % save_interval == 0:
# save IDEC model checkpoints
print('saving model to:',
save_dir + '/DEC_model_' + str(ite) + '.h5')
self.model.save_weights(save_dir + '/DEC_model_' + str(ite) +
'.h5')
ite += 1
# save the trained model
#logfile.close()
print('saving model to:', save_dir + '/DEC_model_final.h5')
self.model.save_weights(save_dir + '/DEC_model_final.h5')
y_p = self.model.predict(x, verbose=0)[1].argmax(1)
cluster_to_label_mapping, n_assigned_list, majority_class_fractions = \
get_cluster_to_label_mapping_safe(y[:,1], y_p, n_classes, n_clusters)
return y_pred, metrics_dict, best_ite
def clustering(self,
x,
y,
validation_data,
tol=1e-3,
update_interval=140,
maxiter=2e4,
save_dir='./results/dec',
pretrained_weights=None,
alpha=K.variable(0.9),
beta=K.variable(0.1)):
print('Update interval', update_interval)
save_interval = x.shape[0] / self.batch_size * 5 # 5 epochs
print('Save interval', save_interval)
try:
self.load_weights(pretrained_weights)
except AttributeError:
# initialize cluster centers using k-means
print('Initializing cluster centers with k-means.')
kmeans = KMeans(n_clusters=self.n_clusters, n_init=20)
y_pred = kmeans.fit_predict(self.encoder.predict(x))
y_pred_last = y_pred
self.model.get_layer(name='clustering').set_weights(
[kmeans.cluster_centers_])
# using only the labelled data set calculate the cluster label assignments
y_pred = self.predict_clusters(x)
self.n_classes = len(np.unique(y))
cluster_to_label_mapping, n_assigned_list, majority_class_fractions = \
get_cluster_to_label_mapping_safe(y[:,1], y_pred, self.n_classes, self.n_clusters)
# ensure that the cluster with the most real detections is assigned the real class
print(
np.argmax((1 - np.array(majority_class_fractions)) *
np.array(n_assigned_list)))
cluster_to_label_mapping[np.argmax(
(1 - np.array(majority_class_fractions)) *
np.array(n_assigned_list))] = 1
cluster_centres = np.squeeze(
np.array(self.model.get_layer(name='clustering').get_weights()))
# build training set based on euclidean distances
"""
x_encoded = self.encoder.predict(x_l)
x_encoded_tiled = np.tile(x_encoded[:,:,np.newaxis], (1,1,self.n_clusters))
cluster_centres_tiled = np.tile(cluster_centres[np.newaxis,:,:], (x_l.shape[0],1,1))
euclidean_distances = np.squeeze(K.eval(euclidean_distance((x_encoded_tiled, cluster_centres_tiled))))
cluster_preds = y_p
y_c = []
for i in range(x_l.shape[0]):
l = np.argmax(y_l[i])
c = cluster_preds[i]
cl = cluster_to_label_mapping[c]
if l == cl:
y_c.append(c)
else:
ed = euclidean_distances[i][[np.where(cluster_to_label_mapping == l)]]
ac = int(np.array(cluster_to_label_mapping)[np.where(cluster_to_label_mapping == l)][np.argmin(ed)])
y_c.append(ac)
# one hot encode these cluster assignements
y_c = np_utils.to_categorical(y_c, self.n_clusters)
"""
"""
# build the model
a = Input(shape=(400,)) # input layer
q_out = self.model(a)
embed = self.encoder(a)
pred = Dense(self.n_clusters, activation='softmax')(embed)
self.model = Model(inputs=a, outputs=[q_out, pred])
optimizer = SGD(lr=1e-1)
#optimizer = 'adam'
self.model.compile(optimizer=optimizer, \
loss={'dense_1': 'categorical_crossentropy', 'model_3': 'kld'}, \
loss_weights={'dense_1': alpha, 'model_3': beta})
"""
a = Input(shape=(400, )) # input layer
self.model.layers[1].kernel_regularizer = regularizers.l2(0.5)
self.model.layers[2].kernel_regularizer = regularizers.l2(0.5)
self.model.layers[3].kernel_regularizer = regularizers.l2(0.5)
self.model.layers[4].kernel_regularizer = regularizers.l2(0.5)
hidden = self.encoder(a)
q_out = ClusteringLayer(self.n_clusters, name='clustering')(hidden)
e_out = self.autoencoder(a)
pred = Dense(self.n_clusters, activation='softmax')(hidden)
self.model = Model(inputs=a, outputs=[pred, q_out, e_out])
self.model.get_layer(name='clustering').set_weights(cluster_weights)
optimizer = SGD(lr=1e-1)
self.model.compile(optimizer=optimizer, \
loss={'dense_1': 'categorical_crossentropy', 'clustering': 'kld', 'model_1': 'mse'}, \
loss_weights={'dense_1': alpha, 'clustering': beta, 'model_1': 1})
loss = [0, 0, 0]
index = 0
q = self.model.predict(x, verbose=0)[1]
y_pred_last = q.argmax(1)
best_val_loss = [np.inf, np.inf, np.inf]
for ite in range(int(maxiter)):
if ite % update_interval == 0:
q = self.model.predict(x, verbose=0)[1]
p = self.target_distribution(
q) # update the auxiliary target distribution p
# evaluate the clustering performance
y_pred = q.argmax(1)
delta_label = np.sum(y_pred != y_pred_last).astype(
np.float32) / y_pred.shape[0]
y_pred_last = y_pred
y_pred = self.model.predict(x)[0]
#print(self.model.get_layer(name='model_3').layers[-1].get_weights()[0])
y_c = self.cluster_assignment(
x, y, p.argmax(1),
self.model.get_layer(
name='model_3').layers[-1].get_weights()[0],
np.array(cluster_to_label_mapping))
if y is not None:
loss = np.round(loss, 5)
valid_p = self.target_distribution(
self.model.predict(validation_data[0], verbose=0)[1])
y_c_valid = self.cluster_assignment(
validation_data[0], validation_data[1],
valid_p.argmax(1),
self.model.get_layer(
name='model_3').layers[-1].get_weights()[0],
np.array(cluster_to_label_mapping))
val_loss = np.round(
self.model.test_on_batch(validation_data[0],
[y_c_valid, valid_p]), 5)
#val_loss = np.round(self.model.test_on_batch(validation_data[0], [validation_data[1], valid_p, validation_data[0]]), 5)
#f, _, _, _ = one_percent_fpr(y[:,1], y_pred[:,1], 0.01)
#f = np.round(f, 5)
#print(y[:20,1], np.argmax(y_pred, axis=1)[:20])
#f1 = np.round(f1_score(y[:,1], np.argmax(y_pred, axis=1)), 5)
#f1 = np.round(calc_f1_score(y, q.argmax(1), cluster_to_label_mapping), 5)
h = np.round(
homogeneity_score(
y[:, 1],
self.model.predict(x, verbose=0)[1].argmax(1)), 5)
#y_pred_valid = self.model.predict(validation_data[0])[0]
#f_valid, _, _, _ = one_percent_fpr(validation_data[1][:,1], y_pred_valid[:,1], 0.01)
#f_valid = np.round(f_valid, 5)
#f1_valid = np.round(f1_score(validation_data[1][:,1], np.argmax(y_pred_valid, axis=1)), 5)
#f1_valid = np.round(calc_f1_score(validation_data[1], self.model.predict(validation_data[0], verbose=0)[1], cluster_to_label_mapping), 5)
h_valid = np.round(
homogeneity_score(
validation_data[1][:, 1],
self.model.predict(validation_data[0],
verbose=0)[1].argmax(1)), 5)
#logdict = dict(iter=ite, acc=acc, nmi=nmi, ari=ari, L=loss)
#logwriter.writerow(logdict)
#print('Iter', ite, ': Acc', acc, ', nmi', nmi, ', ari', ari, '; loss=', loss)
_, _, _ = \
get_cluster_to_label_mapping_safe(y[:,1], q.argmax(1), n_classes, n_clusters)
_, _, _ = \
get_cluster_to_label_mapping_safe(validation_data[1][:,1], self.model.predict(validation_data[0], verbose=0)[1].argmax(1), n_classes, n_clusters)
#print('Iter', ite, ' :1% fpr', f, ', F1=', f1, 'h=', h, '; loss=', loss, \
# ';\n\t\t valid 1% fpr=,', f_valid, ', valid F1=', f1_valid, 'h_valid=', h_valid, '; valid_loss=,', val_loss, )
print('Iter', ite, 'h=', h, '; loss=', loss, 'h_valid=',
h_valid, '; valid_loss=,', val_loss)
if val_loss[1] < best_val_loss[
1]: # only interested in classification improvements
print('saving model: ', best_val_loss, ' -> ',
val_loss)
self.model.save_weights('best_val_loss.hf')
best_val_loss = val_loss
#alpha = K.variable(0.9*(1 - (ite/maxiter)))
#beta = K.variable(1 - alpha)
print(K.eval(alpha), K.eval(beta))
history = self.model.fit(
x=x,
y=[y_c, p],
callbacks=[MyLossWeightCallback(alpha, beta)],
verbose=0)
#history = self.model.fit(x=x, y=[y,p,x], callbacks=[MyLossWeightCallback(alpha, beta)], verbose=0)
print(history.history)
loss = [history.history[k][0] for k in history.history.keys()]
# save intermediate model
if ite % save_interval == 0:
# save IDEC model checkpoints
print('saving model to:',
save_dir + '/DEC_model_' + str(ite) + '.h5')
self.model.save_weights(save_dir + '/DEC_model_' +
str(ite) + '.h5')
ite += 1
# save the trained model
#logfile.close()
print('saving model to:', save_dir + '/DEC_model_final.h5')
self.model.save_weights(save_dir + '/DEC_model_final.h5')
return y_pred
if args.bn:
h = Dense(dims[i + 1], name='encoder_%d' % i)(h)
h = normalization.BatchNormalization()(h)
h = Activation(activation='relu')(h)
else:
h = Dense(dims[i + 1], activation='relu', name='encoder_%d' % i)(h)
# hidden layer
z_mu = Dense(dims[-1], name='encoder_mu')(
h) # hidden layer, features are extracted from here
z_log_var = Dense(dims[-1], activation='softplus',
name='encoder_logvar')(h)
z = Lambda(sampling, output_shape=(dims[-1], ),
name='z')([z_mu, z_log_var])
clustering_layer = ClusteringLayer(n_clusters, name='clustering')(z_mu)
# instantiate encoder model
# latent_inputs = Input(shape=(dims[-1],), name='z_sampling')
h = z
# internal layers in decoder
for i in range(n_stacks - 1, 0, -1):
if args.bn:
h = Dense(dims[i], name='decoder_%d' % i)(h)
h = normalization.BatchNormalization()(h)
h = Activation(activation='relu')(h)
else:
h = Dense(dims[i], activation='relu', name='decoder_%d' % i)(h)
# output
decoder_out = Dense(dims[0], activation='linear', name='decoder_out')(h)
