def experimentPlot(dataset, sess, d_feat, d_in, batch_size, outFolder): pointsToSample = 10000 #First do the PCA2 of the real data cVectorTransform = lambda x: transformFeature_Norm(x, sess, d_feat, d_in) trainX, rlbs = trainsetTransform(cVectorTransform, dataset) rlbs = OneHotToInt(rlbs) pca = PCA(n_components=2) showDimRed(trainX, rlbs, str('PCA_Real'), pca, outFolder) #Generate PCA2 with saved model and C vector #Create generated dataset points = 0 pointsToSample = pointsToSample - ( pointsToSample % batch_size ) #Make pointsSample divisible by batch_size. To exactly fit the array generatedSample = np.empty((pointsToSample, 2)) while (points != pointsToSample): generatedBatch = sess.run( d_in )[0: batch_size] #Input is not neaded to generate random sample (random block included inside) #Use C transform cTransformed = cVectorTransform(generatedBatch) #Do the same pca2 sample = pca.transform(cTransformed) generatedSample[points:points + batch_size] = sample points += batch_size #plot sublot both images plt.scatter(generatedSample[:, 0], generatedSample[:, 1]) plt.savefig(os.path.join(outFolder, 'generatedSamplePCA.png'))
def main(configPath): expName = "clasifyExp" res = {} with open(configPath, 'r') as f: res = json.load(f) dataFolder = res['train_Folder'] modelPath = res['modelPath'] batch_size = res['batch_size'] imageSize = res['imageSize'] if res.has_key('outFolder'): outp = res['outFolder'] else: assert (len(configPath.split(".")) == 2) expName = configPath.split(".")[0] outp = os.path.join("imagenesTest", expName) outFolder = outp batch_size = res['batch_size'] if res.has_key('semiSup'): semiSup = res['semiSup'] else: semiSup = False if res.has_key('trainSplit'): trainSplit = res['trainSplit'] else: trainSplit = 0.7 if not os.path.exists(outFolder): os.makedirs(outFolder) # Define dataset #We define here TRAIN 50% AND VAL 50% dataset = DataFolder(dataFolder, batch_size, testProp=0.01, validation_proportion=0.5, out_size=imageSize) # Load saved GAN with tf.Session() as sess: new_saver = tf.train.import_meta_graph(modelPath + '.meta') new_saver.restore(sess, modelPath) # Define dtransform to use (raw,cond) transformList = loadDatatransform(res, sess, addEnc=True) transformList.append(('rawImageFlat', flatImage)) result = {} for name, datatransform in transformList: #Transform dataseet trainX, realLabels = trainsetTransform(datatransform, dataset) tX, ty = testsetTransform( datatransform, dataset) #Here we ask for validation set in reality realLabels = np.where(realLabels)[1] ty = np.where(ty)[1] # do LSVM clasify clf = SGDClassifier() clf.fit(trainX, realLabels) pred = clf.predict(tX) fullRes = classification_report(ty, pred) acc = accuracy_score(ty, pred) result[name] = {"acc": acc, "full": fullRes} # If we train semi supervised test the discriminator %class if semiSup: acc, fullRes = classifyWithDiscriminator(sess, dataFolder, batch_size, imageSize) result['Discriminator semi sup'] = {"acc": acc, "full": fullRes} tf.reset_default_graph() # Do supervised training with same number of points as GAN if trainSplit != 0.7: acc, fullRes = trainCNNsupervised(dataFolder, batch_size, trainSplit=trainSplit) result['CNN-alt2 lessPoints'] = {"acc": acc, "full": fullRes} # Do supervised training with 70% train split acc, fullRes = trainCNNsupervised(dataFolder, batch_size, trainSplit=0.7) result['CNN-alt2'] = {"acc": acc, "full": fullRes} data = pd.DataFrame.from_dict(result) writer = pd.ExcelWriter(os.path.join(outFolder, expName + '.xlsx')) data.to_excel(writer, 'Sheet1') writer.save() pass
def main(configFile, layerName, labelNames=None): # -----Cargar parametros print "Loading config file ", configFile limitPoints = 2000 res = {} with open(configFile, 'r') as f: res = json.load(f) dataFolder = res['train_Folder'] if res.has_key('exp_name'): exp_name = res['exp_name'] else: filename = os.path.split(configFile)[-1] assert (len(filename.split(".")) == 2) exp_name = filename.split(".")[0] if res.has_key('modelPath'): modelPath = res['modelPath'] else: train_dataset = res['train_dataset'] modelPath = os.path.join("ckt", train_dataset, exp_name, "last.ckpt") batchSize = res['batch_size'] imageSize = res['imageSize'] discrInputName = res['discrInputName'] useDataset = DataFolder(dataFolder, batchSize, testProp=0.6, validation_proportion=0.5, out_size=imageSize) outFolder = os.path.join("ShowAct", exp_name) if not os.path.exists(outFolder): os.makedirs(outFolder) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.05) #config=tf.ConfigProto(gpu_options=gpu_options) with tf.device('/cpu:0'): with tf.Session() as sess: new_saver = tf.train.import_meta_graph(modelPath + '.meta') new_saver.restore(sess, modelPath) d_in = sess.graph.get_tensor_by_name(discrInputName) # Crear lista de capas a usar automaticamente mediante navegacion del grafo if not ":0" in layerName: layerName = layerName + ":0" #Define layer to use layerFunction = getActLayer(sess, layerName, d_in, norm=False) #Get layer activations of entire train set trainX, realLabels = trainsetTransform(layerFunction, useDataset) print "Procesando " + str(layerName) print trainX.shape outFolder = os.path.join("ShowAct", exp_name) if not os.path.exists(outFolder): os.makedirs(outFolder) if saveAct: #Export the activations to a .mat file save the layer name, the dataset folder and the number of points (data is a matrix N,D ) data_dict = { 'layerName': layerName, 'data': trainX, 'dataset': useDataset.getFolderName(), 'n_points': trainX.shape[0] } savemat( os.path.join(outFolder, layerName.replace('/', '_') + '.mat'), data_dict) #DO TSNE if doTSNE: model = TSNE(n_components=2) #Save image expName_layer_TSNE.png outName = layerName.replace('/', '_') + str(' TSNE_Real') showDimRed(trainX[0:limitPoints], realLabels[0:limitPoints], outName, model, outFolder, labelsName=labelNames)
def evaluate(self,outName,show = 3,nClustersTofind=3): outImageShape = self.imageShape[:-1] if (len(self.imageShape) == 3 and self.imageShape[2] == 1) else self.imageShape images, labeles = self.dataset.next_batch(self.batchSize) batchRecons = self.reconstruction(images) # Check dataset format (min - max values) minDataset = images.min() maxDataset = images.max() # Check reconstruction format (min - max values) minRec = batchRecons.min() maxRec = batchRecons.max() formatImageDataset = lambda x : x formatImageRec = lambda x : x if (minDataset < 0 or maxDataset > 1): formatImageDataset = lambda image: (image - minDataset) / (maxDataset-minDataset) print("Dataset image not in 0-1 range. Range ({0} / {1})".format(minDataset,maxDataset)) if (minRec < 0 or maxRec > 1): formatImageRec = lambda image: (image - minRec) / (maxRec-minRec) print("Rec image not in 0-1 range. Range ({0} / {1})".format(minRec,maxRec)) for i in range(show): original = images[i].reshape(outImageShape) recon = batchRecons[i].reshape(outImageShape) plt.figure('original') plt.imshow(formatImageDataset(original)) plt.figure('Reconstruction') plt.imshow(formatImageRec(recon)) plt.show() transformName = outName labNames={} showBlokeh=True # Definir transformacion como activaciones en salida encoder transfFun = lambda x: self.getLatentRepresentation(x) # Tomar activaciones trainX, rlbs = trainsetTransform(transfFun, self.dataset) rlbs = OneHotToInt(rlbs) # Crear carpeta de resultados if not os.path.exists(outName): os.makedirs(outName) # Mostrar labels reales con PCA 2 pca = PCA(n_components=2) transformed = showDimRed(trainX, rlbs, 'latentRep PCA_Real', pca, outName) kmeans = cluster.KMeans(n_clusters=nClustersTofind) spectral = cluster.SpectralClustering(n_clusters=nClustersTofind, eigen_solver='arpack', affinity="nearest_neighbors") algorithms = [kmeans,spectral] for clusterAlg in algorithms: # Categorizar con K means o spectral points, predClust, realsLab = clusterLabeling(self.dataset, transfFun, clusterAlg, trainX) name = clusterAlg.__class__.__name__ print "Showing results for Cluster ", name showResults(self.dataset, points, predClust, np.array(realsLab), transformName + " " + 'Cluster ' + str(name), outName, labelNames=labNames)
def main(configFile, labelNames=None): # -----Cargar parametros print "Loading config file ", configFile limitPoints = 2000 res = {} with open(configFile, 'r') as f: res = json.load(f) dataFolder = res['train_Folder'] if res.has_key('exp_name'): exp_name = res['exp_name'] else: filename = os.path.split(configFile)[-1] assert (len(filename.split(".")) == 2) exp_name = filename.split(".")[0] if res.has_key('modelPath'): modelPath = res['modelPath'] else: train_dataset = res['train_dataset'] modelPath = os.path.join("ckt", train_dataset, exp_name, "last.ckpt") batchSize = res['batch_size'] imageSize = res['imageSize'] discrInputName = res['discrInputName'] useDataset = DataFolder(dataFolder, batchSize, testProp=0.6, validation_proportion=0.5, out_size=imageSize) outFolder = os.path.join("ShowAct", exp_name) if not os.path.exists(outFolder): os.makedirs(outFolder) with tf.Session() as sess: new_saver = tf.train.import_meta_graph(modelPath + '.meta') new_saver.restore(sess, modelPath) d_in = sess.graph.get_tensor_by_name(discrInputName) # Crear lista de capas a usar automaticamente mediante navegacion del grafo capas = [] current = d_in.op.outputs[0] it = 100 while not ('OutDiscriminator:0' in current.name): nextNames = [elem.name for elem in current.consumers()] if any([('batchnorm/mul_1' in name) for name in nextNames]): for ind, way in enumerate(current.consumers()): if 'mul_1' in way.name: current = current.consumers()[ind].outputs[0] else: if len(current.consumers()) > 0: current = current.consumers()[0].outputs[0] else: capas.append(current.name) break #todo this stop somethimes where it shouldnt capas.append(current.name) it = it - 1 if it == 0: raise Exception( "Error en busqueda de grafo entro a ciclo maxima profundidad 100" ) #Add FC endoder (las FC before encoder) if (res['doEncoderLabel']) and (not (res['discrEncoderName'] is None)): finalEncoder = sess.graph.get_tensor_by_name( res['discrEncoderName']) lastFcEncoder = finalEncoder.op.inputs[0].op.inputs[0].op.inputs[ 0].op.inputs[0] capas.append(lastFcEncoder.name) #Filter capas like resize capas = filter(lambda x: not "Reshape" in x, capas) #Do the TSNE of the raw image layerFunction = lambda x: sess.run(d_in, { d_in: x }).reshape(batchSize, -1) trainX, realLabels = trainsetTransform(layerFunction, useDataset) if doTSNE: model = TSNE(n_components=2) outName = "ImageRaw" + str(' TSNE_Real') showDimRed(trainX[0:limitPoints], realLabels[0:limitPoints], outName, model, outFolder, labelsName=labelNames) for layerName in capas: if not ":0" in layerName: layerName = layerName + ":0" #Define layer to use layerFunction = getActLayer(sess, layerName, d_in, norm=False) #Get layer activations of entire train set trainX, realLabels = trainsetTransform(layerFunction, useDataset) print "Procesando " + str(layerName) print trainX.shape outFolder = os.path.join("ShowAct", exp_name) if not os.path.exists(outFolder): os.makedirs(outFolder) if saveAct: #Export the activations to a .mat file save the layer name, the dataset folder and the number of points (data is a matrix N,D ) data_dict = { 'layerName': layerName, 'data': trainX, 'dataset': useDataset.getFolderName(), 'n_points': trainX.shape[0] } savemat( os.path.join(outFolder, layerName.replace('/', '_') + '.mat'), data_dict) #DO TSNE if doTSNE: model = TSNE(n_components=2) #Save image expName_layer_TSNE.png outName = layerName.replace('/', '_') + str(' TSNE_Real') showDimRed(trainX[0:limitPoints], realLabels[0:limitPoints], outName, model, outFolder, labelsName=labelNames)