def bkm(fn, prefix, k, b, n_classes, maxiter=10): with open(fn) as fx: content = fx.readlines() glen1 = len(content) seq1 = range(glen1) rn.shuffle(seq1) ls = seq1[0:k] cidx = itemgetter(*ls)(content) centers, c_lab = get_batch(cidx, prefix, n_classes) print "Get centers' feature!" cfeat = afc.getFeat(centers, True, o_layer="fcb") print(cfeat.shape) rn.shuffle(seq1) predicts = np.zeros((glen1)) groundtruths = np.zeros((glen1)) iters = 0 step = (glen1 / b) / 10 print("Step is " + str(step)) preg = 0 while (maxiter > iters): count1 = np.zeros((n_classes)) for i in range(0, glen1, b): ls = seq1[i:i + b] X = itemgetter(*ls)(content) data, lab1 = get_batch(X, prefix, n_classes) groundtruths[i:i + b] = lab1.argmax() # find nearest center and record the count fea1 = afc.getFeat(data, False, o_layer="fcb") p_lab = np.zeros((b)) for j in range(b): diff1 = pow(pow(fea1[j, :] - cfeat, 2.0), 0.5) diff1 = diff1.sum(axis=1) p_lab[j] = diff1.argmin() count1[int(p_lab[j])] += 1 predicts[i:i + b] = p_lab # update the center by SGD for j in range(b): idx1 = int(p_lab[j]) lr = 1.0 / count1[idx1] cfeat[idx1, :] = (1 - lr) * cfeat[idx1, :] + lr * (fea1[j, :]) print(".") iters += 1 nmi = mi(predicts, groundtruths) print("Iter " + str(iters) + " where NMI is " + str(nmi)) return nmi
def bkm(fn, prefix, k, b, n_classes, maxiter=10): with open(fn) as fx: content = fx.readlines() glen1 = len(content) seq1 = range(glen1) rn.shuffle(seq1) ls = seq1[0:k] cidx = itemgetter(*ls)(content) centers, c_lab = get_batch(cidx, prefix, n_classes) print "Get centers' feature!" cfeat = afc.getFeat(centers, True, o_layer="out") print(cfeat.shape) rn.shuffle(seq1) predicts = np.zeros((glen1)) groundtruths = np.zeros((glen1)) iters = 0 step = (glen1 / b) / 10 print("Step is " + str(step)) preg = 0 count1 = np.zeros((n_classes)) allfea = np.zeros((glen1, 1000)) print "Extracting features...." for i in range(0, glen1, b): ls = seq1[i:i + b] X = itemgetter(*ls)(content) data, lab1 = get_batch(X, prefix, n_classes) groundtruths[i:i + b] = lab1.argmax() # find nearest center and record the count fea1 = afc.getFeat(data, False, o_layer="out") allfea[i:i + b] = fea1 if (i % round(glen1 / 100) < b): print "Progress in extracting feature: " + str( i / float(glen1) * 100) + "%" np.save("kmeans_places_val.fea", allfea) #~ allfea=np.load("kmeans_places_val.fea.npy") print "Start to cluster images..." k_means = KMeans(init=cfeat, n_clusters=n_classes, n_init=1) k_means.fit(allfea) predicts = k_means.labels_ nmi = mi(predicts, groundtruths) return nmi
def trainer(): global load_thread global seq1 global step global data_queue global lr tii = time.time() config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.5 X = tf.placeholder("float", [None, 224, 224, 3]) sX = tf.placeholder("float", [None, 7, 7]) Y = tf.placeholder("float", [None, n_classes]) X2 = tf.placeholder(tf.float32, [None, 224, 224, 3]) # 构建模型 oldpred = al.getalex(X, keep_prob) sess2 = tf.Session(config=config) sess2.run(tf.initialize_all_variables()) saver = tf.train.Saver() saver.restore(sess2, "/Data/jess/tf_pretrained.ckpt") xx = tf.trainable_variables() sess2.close() import alexccnnmodel as al2 biasesInd = 44 weightsInd = 34 B1 = tf.placeholder("float", al2.biases['bc9'].get_shape().as_list()) updateB = al2.biases['bc9'].assign(B1) W1 = tf.placeholder("float", al2.weights['wc9'].get_shape().as_list()) updateW = al2.weights['wc9'].assign(W1) sess2 = tf.Session(config=config) sess2.run(tf.initialize_all_variables()) fc8 = al2.getccnn(X, keep_prob, xx, xx) saver.restore(sess2, "/Data/jess/tf_alex_pre_ccnn_model_iter285001.ckpt-285001") xx2 = tf.trainable_variables() sess2.close() with tf.Session(config=config) as sess: w10 = tf.Variable(tf.random_normal([1000, n_classes], stddev=0.01)) b10 = tf.Variable(tf.random_normal([n_classes], stddev=0.01)) pred = al2.alex_ccnn10(X, xx2, xx2, keep_prob, w10, b10, o_layer="out") fcb = al2.alex_ccnn10(X, xx2, xx2, keep_prob, w10, b10, o_layer="fcb") cost = -tf.reduce_sum(Y * tf.log(pred)) optimizer = tf.train.GradientDescentOptimizer( learning_rate=learning_rate).minimize(cost) correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(Y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) #==========Get Init Center====================@ #~ centers, lab1=val_batch(n_classes, ss=0) #~ rn.shuffle(seq1) saver = tf.train.Saver() gtlab = np.zeros((vlen1)) predicts = np.zeros((vlen1)) lr = 1e-3 step = 1 len2 = len(xx2) myvarlist = [xx2[weightsInd], xx2[biasesInd]] myvar = tf.trainable_variables() for kk in range(len2): print( str(kk) + "th data is " + str(xx2[kk].get_shape().as_list()) + " with " + str(type(xx2[kk]))) varxx = tf.gradients(cost, myvarlist) print(varxx) sess.run(tf.initialize_all_variables()) lastgrad = [] # Total epochs we have to reach. allRuntime = 0 saver.restore(sess, '/Data/tf_place_val_step_454791.ckpt') step = 454791 vst = int(np.floor((step * batch_size) / vlen1)) centers, lab1 = val_batch(n_classes, ss=0) for v in range(vst, 1000): ts = time.time() rn.shuffle(seq1) load_thread = threading.Thread(target=load_data) load_thread.deamon = True load_thread.start() cfeat = sess.run(fcb, feed_dict={X: centers, keep_prob: 1.}) featLen = cfeat.shape[1] count1 = np.zeros((n_classes)) acc_batch_x = np.zeros((batch_size, 224, 224, 3)) acc_batch_y = np.zeros((batch_size, n_classes)) bct = 0 for k in range(0, vlen1, batch_size): epcho1 = np.floor((step * batch_size) / vlen1) #============Extract feature================== batch_sx, batch_ys = data_queue.get() fea1 = sess.run(fcb, feed_dict={X: batch_sx, keep_prob: 1.}) gtlab[k:k + batch_size] = batch_ys.argmax(axis=1) b = batch_size b2 = 2 p_lab = np.zeros((b)) p_lab2 = np.zeros((b, n_classes)) p_err = np.zeros((b)) for j in range(b): diff1 = pow(pow(fea1[j, :] - cfeat, 2.0), 0.5) #~ pdb.set_trace() diff1 = diff1.sum(axis=1) p_lab[j] = diff1.argmin() p_lab2[j, int(p_lab[j])] = 1 count1[int(p_lab[j])] += 1 p_err[j] = min(diff1) predicts[k:k + b] = p_lab # Use the most realible set to update network parameters acc_ind = sorted(range(len(p_err)), key=lambda k: p_err[k]) myidn = acc_ind[0:b2] acc_batch_x[bct:bct + b2, :, :, :] = batch_sx[myidn, :, :, :] acc_batch_y[bct:bct + b2, :] = p_lab2[myidn, :] #~ acc_batch_x = batch_sx #~ acc_batch_y = p_lab2 bct = bct + b2 perrave = p_err.sum(axis=0) / batch_size #When the size of the collected training sample is larger than batch size, performing network updating -*- if (bct >= batch_size): bct = 0 #===========Update network==================== sess.run(optimizer, feed_dict={ X: acc_batch_x, Y: acc_batch_y, keep_prob: dropout, learning_rate: lr }) #===========Update center====================== #~ hh=input("Input any word to continue...") var_grad = sess.run(varxx, feed_dict={ X: acc_batch_x, Y: acc_batch_y, keep_prob: 1. }) allvar = sess.run(myvar, feed_dict={ X: acc_batch_x, Y: acc_batch_y, keep_prob: 1. }) if (lastgrad != []): bct = 0 sess.run(updateB, feed_dict={ B1: allvar[biasesInd] + lr * lastgrad[1] }) sess.run(updateW, feed_dict={ W1: allvar[weightsInd] + lr * lastgrad[0] }) fea1 = sess.run(fcb, feed_dict={ X: acc_batch_x, keep_prob: 1. }) sess.run(updateB, feed_dict={B1: allvar[biasesInd]}) sess.run(updateW, feed_dict={W1: allvar[weightsInd]}) #print str(k) + "th ==> " + str(lastgrad[1][1:5]) lastgrad = var_grad for j in myidn: idx1 = int(p_lab[j]) lr2 = 1.0 / count1[idx1] cfeat[idx1, :] = (1 - lr2) * cfeat[idx1, :] + lr2 * fea1[j, :] #for j in range(n_classes): #cfeat[j, :] = cfeat[j, :] + lr*var_grad if step % display_step == 1: # 计算损失值 loss = sess.run(cost, feed_dict={ X: batch_sx, Y: batch_ys, keep_prob: 1. }) acc = sess.run(accuracy, feed_dict={ X: batch_sx, Y: batch_ys, keep_prob: 1. }) print("Iter=" + str(step * batch_size) + "/epcho=" + str(np.floor((step * batch_size) / vlen1)) + ", Loss= " + "{:.6f}".format(loss) + ", Training Accuracy=" + "{:.5f}".format(acc) + ", lr=" + str(lr) + ", Sec.=" + "{:.5f}".format(time.time() - tii)) #print "This stage has " + str(acc_batch_y.shape[0]) + " samples to be used to update the network." tii = time.time() step += 1 #input() nmi = mi(predicts, gtlab) td = time.time() - ts allRuntime = allRuntime + td print( str(v) + "th epoch's NMI to mini-batch kmeans is " + str(nmi) + ". Time: " + str(td) + " sec.") save_path = saver.save( sess, "/Data/tf_place_val_step_" + str(step) + ".ckpt") print("Model saved in file: %s" % save_path) threading.Thread._Thread__stop(load_thread) threading.Thread._Thread__stop(load_thread)
if step % display_step == 1: # 计算损失值 loss = sess.run(cost, feed_dict={X: acc_batch_x[0:bct,:,:,:], Y: acc_batch_y[0:bct], keep_prob: 1.}) acc = sess.run(accuracy, feed_dict={X: acc_batch_x[0:bct,:,:,:], Y: acc_batch_y[0:bct], keep_prob: 1.}) print "Iter=" + str(step*batch_size) + "/epcho=" + str(np.floor((step*batch_size)/vlen1)) + ", Loss= " + "{:.6f}".format(loss) + ", Training Accuracy=" + "{:.5f}".format(acc) + ", lr=" + str(lr) + ",time"+str(time.time()-ttk) ttk = time.time() print "there are " + str(copp) +"/"+str(copp_cc) + " samples are estimated as corrected!!" losshist[lossct] = loss lossct = lossct+1 copp=0 copp_cc=0 pred_str1="" gt_str1="" step += 1 if (v>300): if (v%300)==0 | (v%3000)==0: lr=lr/10 if (v % 100)==0: nmi = mi(predicts[0:k+b], gtlab[0:k+b]) td = time.time()-ts allRuntime=allRuntime+td print("\033[1;31m"+str(v)+"th epoch's NMI to mini-batch kmeans is " + str(nmi) + ". Time: " + str(td) + " sec.\033[0m\n") save_path = saver.save(sess, "Data/tf_mnist_" + str(step) + ".ckpt") threading.Thread._Thread__stop(load_thread)
import scipy.cluster.hierarchy as sch from sklearn.cluster import AgglomerativeClustering X = cluster1 points = X[:,[0,1]] # create dendrogram dendrogram = sch.dendrogram(sch.linkage(points, method='ward')) # create clusters hc = AgglomerativeClustering(n_clusters=2, affinity = 'euclidean', linkage = 'ward') # save clusters for chart y_hc = hc.fit_predict(points) plt.scatter(points[y_hc ==0,0], points[y_hc == 0,1], s=100, c='red') plt.scatter(points[y_hc==1,0], points[y_hc == 1,1], s=100, c='black') plt.scatter(points[y_hc ==2,0], points[y_hc == 2,1], s=100, c='blue') #plt.scatter(points[y_hc ==3,0], points[y_hc == 3,1], s=100, c='cyan') ##Evaluation methods for different clustering techniques ###!!VARIABLE y_km changes for each of the existing techniques, y_km is example variable ###for KMeans. Respective variable for ###DBSCAN is labels ###Hierarchical clustering is y_hc from sklearn.metrics import silhouette_score f'Silhouette Score(n=2): {silhouette_score(X, y_km)}' ##X[:,2] is base data column with true class labels from sklearn.metrics.cluster import normalized_mutual_info_score as mi mi(X[:,2],y_km)