def experimentPlot(dataset, sess, d_feat, d_in, batch_size, outFolder):
pointsToSample = 10000
#First do the PCA2 of the real data
cVectorTransform = lambda x: transformFeature_Norm(x, sess, d_feat, d_in)
trainX, rlbs = trainsetTransform(cVectorTransform, dataset)
rlbs = OneHotToInt(rlbs)
pca = PCA(n_components=2)
showDimRed(trainX, rlbs, str('PCA_Real'), pca, outFolder)
#Generate PCA2 with saved model and C vector
#Create generated dataset
points = 0
pointsToSample = pointsToSample - (
pointsToSample % batch_size
) #Make pointsSample divisible by batch_size. To exactly fit the array
generatedSample = np.empty((pointsToSample, 2))
while (points != pointsToSample):
generatedBatch = sess.run(
d_in
)[0:
batch_size] #Input is not neaded to generate random sample (random block included inside)
#Use C transform
cTransformed = cVectorTransform(generatedBatch)
#Do the same pca2
sample = pca.transform(cTransformed)
generatedSample[points:points + batch_size] = sample
points += batch_size
#plot sublot both images
plt.scatter(generatedSample[:, 0], generatedSample[:, 1])
plt.savefig(os.path.join(outFolder, 'generatedSamplePCA.png'))
def main(configPath):
expName = "clasifyExp"
res = {}
with open(configPath, 'r') as f:
res = json.load(f)
dataFolder = res['train_Folder']
modelPath = res['modelPath']
batch_size = res['batch_size']
imageSize = res['imageSize']
if res.has_key('outFolder'):
outp = res['outFolder']
else:
assert (len(configPath.split(".")) == 2)
expName = configPath.split(".")[0]
outp = os.path.join("imagenesTest", expName)
outFolder = outp
batch_size = res['batch_size']
if res.has_key('semiSup'):
semiSup = res['semiSup']
else:
semiSup = False
if res.has_key('trainSplit'):
trainSplit = res['trainSplit']
else:
trainSplit = 0.7
if not os.path.exists(outFolder):
os.makedirs(outFolder)
# Define dataset
#We define here TRAIN 50% AND VAL 50%
dataset = DataFolder(dataFolder,
batch_size,
testProp=0.01,
validation_proportion=0.5,
out_size=imageSize)
# Load saved GAN
with tf.Session() as sess:
new_saver = tf.train.import_meta_graph(modelPath + '.meta')
new_saver.restore(sess, modelPath)
# Define dtransform to use (raw,cond)
transformList = loadDatatransform(res, sess, addEnc=True)
transformList.append(('rawImageFlat', flatImage))
result = {}
for name, datatransform in transformList:
#Transform dataseet
trainX, realLabels = trainsetTransform(datatransform, dataset)
tX, ty = testsetTransform(
datatransform,
dataset) #Here we ask for validation set in reality
realLabels = np.where(realLabels)[1]
ty = np.where(ty)[1]
# do LSVM clasify
clf = SGDClassifier()
clf.fit(trainX, realLabels)
pred = clf.predict(tX)
fullRes = classification_report(ty, pred)
acc = accuracy_score(ty, pred)
result[name] = {"acc": acc, "full": fullRes}
# If we train semi supervised test the discriminator %class
if semiSup:
acc, fullRes = classifyWithDiscriminator(sess, dataFolder,
batch_size, imageSize)
result['Discriminator semi sup'] = {"acc": acc, "full": fullRes}
tf.reset_default_graph()
# Do supervised training with same number of points as GAN
if trainSplit != 0.7:
acc, fullRes = trainCNNsupervised(dataFolder,
batch_size,
trainSplit=trainSplit)
result['CNN-alt2 lessPoints'] = {"acc": acc, "full": fullRes}
# Do supervised training with 70% train split
acc, fullRes = trainCNNsupervised(dataFolder, batch_size, trainSplit=0.7)
result['CNN-alt2'] = {"acc": acc, "full": fullRes}
data = pd.DataFrame.from_dict(result)
writer = pd.ExcelWriter(os.path.join(outFolder, expName + '.xlsx'))
data.to_excel(writer, 'Sheet1')
writer.save()
pass
def main(configFile, layerName, labelNames=None):
# -----Cargar parametros
print "Loading config file ", configFile
limitPoints = 2000
res = {}
with open(configFile, 'r') as f:
res = json.load(f)
dataFolder = res['train_Folder']
if res.has_key('exp_name'):
exp_name = res['exp_name']
else:
filename = os.path.split(configFile)[-1]
assert (len(filename.split(".")) == 2)
exp_name = filename.split(".")[0]
if res.has_key('modelPath'):
modelPath = res['modelPath']
else:
train_dataset = res['train_dataset']
modelPath = os.path.join("ckt", train_dataset, exp_name, "last.ckpt")
batchSize = res['batch_size']
imageSize = res['imageSize']
discrInputName = res['discrInputName']
useDataset = DataFolder(dataFolder,
batchSize,
testProp=0.6,
validation_proportion=0.5,
out_size=imageSize)
outFolder = os.path.join("ShowAct", exp_name)
if not os.path.exists(outFolder):
os.makedirs(outFolder)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.05)
#config=tf.ConfigProto(gpu_options=gpu_options)
with tf.device('/cpu:0'):
with tf.Session() as sess:
new_saver = tf.train.import_meta_graph(modelPath + '.meta')
new_saver.restore(sess, modelPath)
d_in = sess.graph.get_tensor_by_name(discrInputName)
# Crear lista de capas a usar automaticamente mediante navegacion del grafo
if not ":0" in layerName:
layerName = layerName + ":0"
#Define layer to use
layerFunction = getActLayer(sess, layerName, d_in, norm=False)
#Get layer activations of entire train set
trainX, realLabels = trainsetTransform(layerFunction, useDataset)
print "Procesando " + str(layerName)
print trainX.shape
outFolder = os.path.join("ShowAct", exp_name)
if not os.path.exists(outFolder):
os.makedirs(outFolder)
if saveAct:
#Export the activations to a .mat file save the layer name, the dataset folder and the number of points (data is a matrix N,D )
data_dict = {
'layerName': layerName,
'data': trainX,
'dataset': useDataset.getFolderName(),
'n_points': trainX.shape[0]
}
savemat(
os.path.join(outFolder,
layerName.replace('/', '_') + '.mat'),
data_dict)
#DO TSNE
if doTSNE:
model = TSNE(n_components=2)
#Save image expName_layer_TSNE.png
outName = layerName.replace('/', '_') + str(' TSNE_Real')
showDimRed(trainX[0:limitPoints],
realLabels[0:limitPoints],
outName,
model,
outFolder,
labelsName=labelNames)
def evaluate(self,outName,show = 3,nClustersTofind=3):
outImageShape = self.imageShape[:-1] if (len(self.imageShape) == 3 and self.imageShape[2] == 1) else self.imageShape
images, labeles = self.dataset.next_batch(self.batchSize)
batchRecons = self.reconstruction(images)
# Check dataset format (min - max values)
minDataset = images.min()
maxDataset = images.max()
# Check reconstruction format (min - max values)
minRec = batchRecons.min()
maxRec = batchRecons.max()
formatImageDataset = lambda x : x
formatImageRec = lambda x : x
if (minDataset < 0 or maxDataset > 1):
formatImageDataset = lambda image: (image - minDataset) / (maxDataset-minDataset)
print("Dataset image not in 0-1 range. Range ({0} / {1})".format(minDataset,maxDataset))
if (minRec < 0 or maxRec > 1):
formatImageRec = lambda image: (image - minRec) / (maxRec-minRec)
print("Rec image not in 0-1 range. Range ({0} / {1})".format(minRec,maxRec))
for i in range(show):
original = images[i].reshape(outImageShape)
recon = batchRecons[i].reshape(outImageShape)
plt.figure('original')
plt.imshow(formatImageDataset(original))
plt.figure('Reconstruction')
plt.imshow(formatImageRec(recon))
plt.show()
transformName = outName
labNames={}
showBlokeh=True
# Definir transformacion como activaciones en salida encoder
transfFun = lambda x: self.getLatentRepresentation(x)
# Tomar activaciones
trainX, rlbs = trainsetTransform(transfFun, self.dataset)
rlbs = OneHotToInt(rlbs)
# Crear carpeta de resultados
if not os.path.exists(outName):
os.makedirs(outName)
# Mostrar labels reales con PCA 2
pca = PCA(n_components=2)
transformed = showDimRed(trainX, rlbs, 'latentRep PCA_Real',
pca, outName)
kmeans = cluster.KMeans(n_clusters=nClustersTofind)
spectral = cluster.SpectralClustering(n_clusters=nClustersTofind,
eigen_solver='arpack',
affinity="nearest_neighbors")
algorithms = [kmeans,spectral]
for clusterAlg in algorithms:
# Categorizar con K means o spectral
points, predClust, realsLab = clusterLabeling(self.dataset,
transfFun, clusterAlg,
trainX)
name = clusterAlg.__class__.__name__
print "Showing results for Cluster ", name
showResults(self.dataset, points, predClust, np.array(realsLab),
transformName + " " + 'Cluster ' + str(name), outName,
labelNames=labNames)
def main(configFile, labelNames=None):
# -----Cargar parametros
print "Loading config file ", configFile
limitPoints = 2000
res = {}
with open(configFile, 'r') as f:
res = json.load(f)
dataFolder = res['train_Folder']
if res.has_key('exp_name'):
exp_name = res['exp_name']
else:
filename = os.path.split(configFile)[-1]
assert (len(filename.split(".")) == 2)
exp_name = filename.split(".")[0]
if res.has_key('modelPath'):
modelPath = res['modelPath']
else:
train_dataset = res['train_dataset']
modelPath = os.path.join("ckt", train_dataset, exp_name, "last.ckpt")
batchSize = res['batch_size']
imageSize = res['imageSize']
discrInputName = res['discrInputName']
useDataset = DataFolder(dataFolder,
batchSize,
testProp=0.6,
validation_proportion=0.5,
out_size=imageSize)
outFolder = os.path.join("ShowAct", exp_name)
if not os.path.exists(outFolder):
os.makedirs(outFolder)
with tf.Session() as sess:
new_saver = tf.train.import_meta_graph(modelPath + '.meta')
new_saver.restore(sess, modelPath)
d_in = sess.graph.get_tensor_by_name(discrInputName)
# Crear lista de capas a usar automaticamente mediante navegacion del grafo
capas = []
current = d_in.op.outputs[0]
it = 100
while not ('OutDiscriminator:0' in current.name):
nextNames = [elem.name for elem in current.consumers()]
if any([('batchnorm/mul_1' in name) for name in nextNames]):
for ind, way in enumerate(current.consumers()):
if 'mul_1' in way.name:
current = current.consumers()[ind].outputs[0]
else:
if len(current.consumers()) > 0:
current = current.consumers()[0].outputs[0]
else:
capas.append(current.name)
break #todo this stop somethimes where it shouldnt
capas.append(current.name)
it = it - 1
if it == 0:
raise Exception(
"Error en busqueda de grafo entro a ciclo maxima profundidad 100"
)
#Add FC endoder (las FC before encoder)
if (res['doEncoderLabel']) and (not (res['discrEncoderName'] is None)):
finalEncoder = sess.graph.get_tensor_by_name(
res['discrEncoderName'])
lastFcEncoder = finalEncoder.op.inputs[0].op.inputs[0].op.inputs[
0].op.inputs[0]
capas.append(lastFcEncoder.name)
#Filter capas like resize
capas = filter(lambda x: not "Reshape" in x, capas)
#Do the TSNE of the raw image
layerFunction = lambda x: sess.run(d_in, {
d_in: x
}).reshape(batchSize, -1)
trainX, realLabels = trainsetTransform(layerFunction, useDataset)
if doTSNE:
model = TSNE(n_components=2)
outName = "ImageRaw" + str(' TSNE_Real')
showDimRed(trainX[0:limitPoints],
realLabels[0:limitPoints],
outName,
model,
outFolder,
labelsName=labelNames)
for layerName in capas:
if not ":0" in layerName:
layerName = layerName + ":0"
#Define layer to use
layerFunction = getActLayer(sess, layerName, d_in, norm=False)
#Get layer activations of entire train set
trainX, realLabels = trainsetTransform(layerFunction, useDataset)
print "Procesando " + str(layerName)
print trainX.shape
outFolder = os.path.join("ShowAct", exp_name)
if not os.path.exists(outFolder):
os.makedirs(outFolder)
if saveAct:
#Export the activations to a .mat file save the layer name, the dataset folder and the number of points (data is a matrix N,D )
data_dict = {
'layerName': layerName,
'data': trainX,
'dataset': useDataset.getFolderName(),
'n_points': trainX.shape[0]
}
savemat(
os.path.join(outFolder,
layerName.replace('/', '_') + '.mat'),
data_dict)
#DO TSNE
if doTSNE:
model = TSNE(n_components=2)
#Save image expName_layer_TSNE.png
outName = layerName.replace('/', '_') + str(' TSNE_Real')
showDimRed(trainX[0:limitPoints],
realLabels[0:limitPoints],
outName,
model,
outFolder,
labelsName=labelNames)