def main(_):
gan1Name = FLAGS.gan1.replace("checkpoint/", "").replace(
"/media/simen/PLEX/Simens-GAN/checkpoints_augmented_ikea/", "")
gan1Name = gan1Name.replace("/", "").replace(
"mediasimenPLEXSimens-GANcheckpoints",
"").replace("mediasimenPLEXSimens-GANcheckpoints_02.06",
"").replace("_02.06", "")
gan2Name = FLAGS.gan2.replace("checkpoint/", "").replace(
"/media/simen/PLEX/Simens-GAN/checkpoints_augmented_ikea/", "")
gan2Name = gan2Name.replace("/", "").replace(
"mediasimenPLEXSimens-GANcheckpoints",
"").replace("mediasimenPLEXSimens-GANcheckpoints_02.06",
"").replace("_02.06", "")
maxTrainingEpocs = -1
useTrainingNoiseIfStaticNoise = True
useTrainingNoiseIfImprovedNoise = True
if ("static_z" in str(gan2Name)):
if (useTrainingNoiseIfStaticNoise):
gan2Name += "witTrainingNoise"
else:
gan2Name += "randomNoise"
if ("improved_z_noise" in str(gan2Name)):
if (useTrainingNoiseIfImprovedNoise):
gan2Name += "witTrainingNoise"
else:
gan2Name += "randomNoise"
#maxTrainingEpocs = -1
maxTrainingEpocs = -1
if (maxTrainingEpocs > 0):
print("WARNING; Only testing on the first", maxTrainingEpocs,
" checkpoints!!")
print("WARNING; Only testing on the first", maxTrainingEpocs,
" checkpoints!!")
print("WARNING; Only testing on the first", maxTrainingEpocs,
" checkpoints!!")
print("WARNING; Only testing on the first", maxTrainingEpocs,
" checkpoints!!")
print("WARNING; Only testing on the first", maxTrainingEpocs,
" checkpoints!!")
#gan1Name += "_epoch_0_to" + maxTrainingEpocs + "_only"
gan2Name += "_epoch_0_to" + str(maxTrainingEpocs) + "_only"
if (len(gan1Name) > len(gan2Name)):
print("Are the models in correct order????")
print("Are the models in correct order????")
print("Are the models in correct order????")
print("Are the models in correct order????")
print("The shorter on is GAN2 but should probalby be GAN1? amarigt?")
# print("Aborting to be safe")
# return
print(" ")
print("GAN 1 -", gan1Name)
print("GAN 2 -", gan2Name)
print(" ")
gamFolder = "gamResults/"
if not os.path.exists(gamFolder):
os.makedirs(gamFolder)
print("Folder not found. Creaintg it!")
fileName = gan1Name + "__vs__" + gan2Name
fileName = fileName.replace("/", "")
if os.path.exists(gamFolder + fileName + ".csv"):
print(" ")
print("Results allready excsists!")
print(fileName)
print(fileName)
print(fileName)
print("Are you sure?")
print("Are you sure?")
print("Are you sure?")
print("Are you sure?")
print("Aborting to be safe")
return
if FLAGS.input_width is None:
FLAGS.input_width = FLAGS.input_height
if FLAGS.output_width is None:
FLAGS.output_width = FLAGS.output_height
print("Processing test data")
testDatasetSize = 200 * FLAGS.batch_size
sampleDatasetSize = 200 #*FLAGS.batch_size
if (FLAGS.dataset == "cifar"):
data = geCifar()
else:
if (FLAGS.dataset == "cat"):
data = glob(os.path.join("./data", "cat/*", "*.jpg"))
else:
data = glob(os.path.join("./data", FLAGS.dataset, "*.jpeg"))
if (len(data) == 0):
print("Did not find any photos with extension .jpeg")
data = glob(os.path.join("./data", FLAGS.dataset, "*.jpg"))
print("Trying extension .jpg - Found", len(data), "images :)")
FLAGS.input_fname_pattern = "*.jpg"
np.random.shuffle(data)
# testDatasetSize = len(data)
batch_files = data[:min(len(data), testDatasetSize)]
np.random.shuffle(data)
sample_files = data[:int(len(data))]
if (useTrainingNoiseIfStaticNoise):
np.random.seed(seed=1337)
#np.random.seed(seed=456456546)
static_z = np.random.uniform(-1, 1, [len(data), 100]).astype(np.float32)
if (FLAGS.dataset == "cifar"):
testDataset = data
else:
testDataset = [
get_image(batch_file,
input_height=FLAGS.input_height,
input_width=FLAGS.input_width,
resize_height=FLAGS.output_height,
resize_width=FLAGS.output_width,
is_crop=FLAGS.is_crop,
is_grayscale=False) for batch_file in batch_files
]
print("Loaded dataset", FLAGS.dataset, "Actaull size:", len(data),
". Test size:", len(testDataset))
# pp.pprint(flags.FLAGS.__flags)
# pp.pprint(flags.FLAGS.__parse)
# newUtils.createFolderName(FLAGS)
# print(FLAGS.sample_dir)
#newUtils.createConfingCSV(FLAGS.sample_dir,FLAGS)
# return
# pp.pprint(flags.FLAGS.__flags)
# print(FLAGS.sample_dir.default_value)
# if not os.path.exists(path):
# print("path", path, "not found. Creating new folder")
# os.makedirs(path)
# if not os.path.exists(FLAGS.checkpoint_dir):
# os.makedirs(FLAGS.checkpoint_dir)
# if not os.path.exists(FLAGS.sample_dir):
# os.makedirs(FLAGS.sample_dir)
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
print("Starting to load dcgan")
with tf.Session(config=run_config) as sess:
dcgan = DCGAN(sess,
input_width=FLAGS.input_width,
input_height=FLAGS.input_height,
output_width=FLAGS.output_width,
output_height=FLAGS.output_height,
batch_size=FLAGS.batch_size,
c_dim=FLAGS.c_dim,
dataset_name=FLAGS.dataset,
input_fname_pattern=FLAGS.input_fname_pattern,
is_crop=FLAGS.is_crop,
checkpoint_dir=FLAGS.checkpoint_dir,
sample_dir=FLAGS.sample_dir,
evalSize=1)
minIterations = min(dcgan.loadTrainingITerations(FLAGS.gan1),
dcgan.loadTrainingITerations(FLAGS.gan2))
minEpochs = min(dcgan.getNumberOfCheckpoints(FLAGS.gan1),
dcgan.getNumberOfCheckpoints(FLAGS.gan2))
if (maxTrainingEpocs > 0):
minEpochs = maxTrainingEpocs
print("The least trained model has", minEpochs, "checkpoints")
print(" ")
print("GAN 1 -", gan1Name, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan1), "iterations")
print("GAN 2 -", gan2Name, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan2), "iterations")
print(" ")
averageTestScoreGAN_1 = 0
averageTestScoreGAN_2 = 0
averageSampleScoreGAN_1 = 0
averageSampleScoreGAN_2 = 0
listTestScoreGAN_1 = []
listTestScoreGAN_2 = []
listSampleScoreGAN_1 = []
listSampleScoreGAN_2 = []
resultsThroughout = []
gan1Wins = 0
gan2Wins = 0
ties = 0
#minEpochs = 2
for n in range(0, minEpochs):
print(" ")
print(" ")
print(" ")
print(" ")
print(" ")
print("Epoch", n)
print("GAN 1 -", gan1Name, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan1), "iterations")
print("GAN 2 -", gan2Name, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan2), "iterations")
#Gan 1
dcgan.loadCloestsCheckpointNumber(FLAGS.gan1, minIterations, n)
testScoreGAN_1 = dcgan.evalImages(testDataset, FLAGS, True)
averageTestScoreGAN_1 += testScoreGAN_1
listTestScoreGAN_1.append(testScoreGAN_1)
print(" ")
print("GAN 1 test error rate:",
convertDecimalToPerctage(testScoreGAN_1))
samplesGAN_1 = dcgan.getGeneratorSamples(sampleDatasetSize)
# GAN 2
dcgan.loadCloestsCheckpointNumber(FLAGS.gan2, minIterations, n)
testScoreGAN_2 = dcgan.evalImages(testDataset, FLAGS, True)
averageTestScoreGAN_2 += testScoreGAN_2
listTestScoreGAN_2.append(testScoreGAN_2)
print(
"GAN 2 test error rate:",
convertDecimalToPerctage(testScoreGAN_2) + "",
)
# if (testScoreGAN_2 == 0 and testScoreGAN_1 == 0):
# testratio = 1
if (testScoreGAN_2 == 0):
testScoreGAN_2 = 0.00000001
if (testScoreGAN_1 == 0):
testScoreGAN_1 = 0.00000001
#testratio = float(testScoreGAN_1) / float(testScoreGAN_2)
testratio = float(min(testScoreGAN_1, testScoreGAN_2)) / float(
max(testScoreGAN_1, testScoreGAN_2))
print("Test ratio =", testratio)
sampleScoreGAN_2 = dcgan.evalImages(samplesGAN_1, FLAGS, False)
averageSampleScoreGAN_2 += sampleScoreGAN_2
listSampleScoreGAN_2.append(sampleScoreGAN_2)
print(
"GAN 2 error rate on GAN 1's samples:",
convertDecimalToPerctage(sampleScoreGAN_2) + "",
)
if ("improved_z_noise" in str(gan2Name)):
cifarDataset = False
if (FLAGS.dataset == "cifar"):
cifarDataset = True
print("improved_z_noise!!!")
if (useTrainingNoiseIfImprovedNoise):
samplesGAN_2 = dcgan.getGeneratorSamples(
sampleDatasetSize,
sample_files,
improved_z_noise=True,
cifarDataset=cifarDataset)
np.random.shuffle(data)
sample_files = data[:int(len(data))]
else:
samplesGAN_2 = dcgan.getGeneratorSamples(
sampleDatasetSize, cifarDataset=cifarDataset)
elif ("static_z" in str(gan2Name)):
print("static_z!!!")
samplesGAN_2 = dcgan.getGeneratorSamples(sampleDatasetSize,
static_z=static_z)
#np.random.shuffle(static_z)
else:
samplesGAN_2 = dcgan.getGeneratorSamples(sampleDatasetSize)
#GAN1 again
dcgan.loadCloestsCheckpointNumber(FLAGS.gan1, minIterations, n)
sampleScoreGAN_1 = dcgan.evalImages(samplesGAN_2, FLAGS, False)
averageSampleScoreGAN_1 += sampleScoreGAN_1
listSampleScoreGAN_1.append(sampleScoreGAN_1)
print(" ")
print(
"GAN 1 test error on GAN 2's samples:",
convertDecimalToPerctage(sampleScoreGAN_1) + "",
)
print(" ")
print("testRatio = GAN1", float(testScoreGAN_1), " / GAN2",
float(testScoreGAN_2), " = ", testratio)
if (sampleScoreGAN_1 == 0 and sampleScoreGAN_2 == 0):
sampleRatio = 1
elif (sampleScoreGAN_2 == 0):
sampleRatio = 0
else:
sampleRatio = float(sampleScoreGAN_1) / float(sampleScoreGAN_2)
# print("sampleRatio = ", float(sampleScoreGAN_1), " / ", float(sampleScoreGAN_2), " = ", sampleRatio)
print("sampleRatio = GAN1", float(sampleScoreGAN_1), " / GAN2",
float(sampleScoreGAN_2), " = ", sampleRatio)
print(" ")
if (round(testratio) == 1):
print("Test ratio PASSED - round(", testratio, ") = 1.0")
else:
print("Test ratio FAILED - round(", testratio, ") =",
round(testratio))
print(" ")
print("testRatio =", testratio, " ~ ", round(testratio))
print("sampleRatio =", sampleRatio, " ~ ", round(sampleRatio, 3))
sameplRatioRound = round(sampleRatio, 3)
if (round(testratio) != 1):
print("Tie (test failed)")
ties += 1
resultsThroughout.append("tie_test_failed")
elif (sameplRatioRound > 1.0):
# if (sampleRatio < 0.999):
print("WINNER GAN 1 with score", sampleRatio, " - ", gan1Name)
gan1Wins += 1
resultsThroughout.append("GAN_1")
#print(gan1Name+ " score +"+ str(sampleRatioDiff) +" against " + gan1Name+ " score -"+ str(sampleRatioDiff))
elif (sameplRatioRound < 1):
print("WINNER GAN 2 with score", sampleRatio, " - ", gan2Name)
gan2Wins += 1
resultsThroughout.append("GAN_2")
#print(gan1Name + " score +" + str(sampleRatioDiff) + " against " + gan2Name + " score -" + str(sampleRatioDiff))
elif (sampleRatio == 1.00):
print("TIE - Sample score is 1.0")
ties += 1
resultsThroughout.append("TIE")
else:
print("TIE - Sample score of", sampleRatio,
"is too close to 1.0")
resultsThroughout.append("TIE")
ties += 1
print(" ")
print(" ")
print("gan1Wins:", gan1Wins)
print("gan2Wins:", gan2Wins)
print("ties:", ties)
print("")
averageTestScoreGAN_1 = float(averageTestScoreGAN_1) / float(minEpochs)
averageTestScoreGAN_2 = float(averageTestScoreGAN_2) / float(minEpochs)
averageSampleScoreGAN_1 = float(averageSampleScoreGAN_1) / float(
minEpochs)
averageSampleScoreGAN_2 = float(averageSampleScoreGAN_2) / float(
minEpochs)
print("averageTestScoreGAN_1:", averageTestScoreGAN_1)
print("averageTestScoreGAN_2:", averageTestScoreGAN_2)
print("averageSampleScoreGAN_1:", averageSampleScoreGAN_1)
print("averageSampleScoreGAN_2:", averageSampleScoreGAN_2)
if (testScoreGAN_2 == 0):
testScoreGAN_2 = 0.00000001
if (testScoreGAN_1 == 0):
testScoreGAN_1 = 0.00000001
#testratio = float(averageTestScoreGAN_1) / float(averageTestScoreGAN_2)
testratio = float(min(testScoreGAN_1, testScoreGAN_2)) / float(
max(testScoreGAN_1, testScoreGAN_2))
print(" ")
print("testRatio = GAN1", float(averageTestScoreGAN_1), " / GAN2",
float(averageTestScoreGAN_2), " = ", testratio)
if (averageSampleScoreGAN_1 == 0 and averageSampleScoreGAN_2 == 0):
sampleRatio = 1
elif (averageSampleScoreGAN_2 == 0):
sampleRatio = 0
else:
sampleRatio = float(averageSampleScoreGAN_1) / float(
averageSampleScoreGAN_2)
# print("sampleRatio = ", float(averageSampleScoreGAN_1), " / ", float(averageSampleScoreGAN_2), " = ", sampleRatio)
print("sampleRatio = GAN1", float(averageSampleScoreGAN_1), " / GAN2",
float(averageSampleScoreGAN_2), " = ", sampleRatio)
print(" ")
if (round(testratio) == 1):
print("Test ratio PASSED - round(", testratio, ") = 1.0")
else:
print("Test ratio FAILED - round(", testratio, ") =",
round(testratio))
print(" ")
print("testRatio =", testratio, " ~ ", round(testratio))
print("sampleRatio =", sampleRatio, " ~ ", round(sampleRatio, 3))
sameplRatioRound = round(sampleRatio, 3)
winningGan = "tie"
if (round(testratio) != 1):
print("Tie (test failed)")
# ties += 1
winningGan = "tie_test_failed"
elif (sameplRatioRound > 1.0):
# if (sampleRatio < 0.999):
print("WINNER GAN 1 with score", sampleRatio, " - ", gan1Name)
# gan1Wins += 1
winningGan = "GAN_1"
# print(gan1Name+ " score +"+ str(sampleRatioDiff) +" against " + gan1Name+ " score -"+ str(sampleRatioDiff))
elif (sameplRatioRound < 1):
print("WINNER GAN 2 with score", sampleRatio, " - ", gan2Name)
# gan2Wins += 1
winningGan = "GAN_2"
# print(gan1Name + " score +" + str(sampleRatioDiff) + " against " + gan2Name + " score -" + str(sampleRatioDiff))
elif (sampleRatio == 1.00):
print("TIE - Sample score is 1.0")
# ties += 1
else:
print("TIE - Sample score of", sampleRatio, "is too close to 1.0")
# ties += 1
print(" ")
newUtils.createGAMCSV(resultsThroughout, gan1Name, gan2Name,
listTestScoreGAN_1, listTestScoreGAN_2,
listSampleScoreGAN_1, listSampleScoreGAN_2,
winningGan, sampleRatio, testratio, gan1Wins,
gan2Wins, ties, testDatasetSize,
sampleDatasetSize * FLAGS.batch_size)
print(" ")
return
#####################################
gan1Name = FLAGS.gan1.replace("checkpoint/", "")
gan2Name = FLAGS.gan2.replace("checkpoint/", "")
minIterations = min(dcgan.loadTrainingITerations(FLAGS.gan1),
dcgan.loadTrainingITerations(FLAGS.gan2))
#minIterations = 53947
print("Minimum training iterations are", minIterations)
print(" ")
print(" ")
print("GAN 1 -", gan1Name, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan1), "iterations")
print("GAN 2 -", gan2Name, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan2), "iterations")
print(" ")
# print("Loading checkpoint at: ",FLAGS.gan1)
if not dcgan.loadCloestsCheckpoint(FLAGS.gan1, minIterations):
raise Exception("[!] Train a model first, then run test mode")
testScoreGAN_1 = dcgan.evalImages(testDataset, FLAGS, True)
print(" ")
print("GAN 1 test error rate:",
convertDecimalToPerctage(testScoreGAN_1))
samplesGAN_1 = dcgan.getGeneratorSamples(sampleDatasetSize,
dataset=sample_files,
improved_z_noise=False)
privateSampleScoreGAN_1 = dcgan.evalImages(samplesGAN_1, FLAGS, False)
print("GAN 1 error rate on its own samples:",
convertDecimalToPerctage(privateSampleScoreGAN_1))
# print("Loading checkpoint at: ",FLAGS.gan2)
if not dcgan.loadCloestsCheckpoint(FLAGS.gan2, minIterations):
raise Exception("[!] Train a model first, then run test mode")
testScoreGAN_2 = dcgan.evalImages(testDataset, FLAGS, True)
print(" ")
print(
"GAN 2 test error rate:",
convertDecimalToPerctage(testScoreGAN_2) + "",
)
sampleScoreGAN_2 = dcgan.evalImages(samplesGAN_1, FLAGS, False)
print(
"GAN 2 error rate on GAN 1's samples:",
convertDecimalToPerctage(sampleScoreGAN_2) + "",
)
samplesGAN_2 = dcgan.getGeneratorSamples(sampleDatasetSize,
dataset=sample_files,
improved_z_noise=True)
privateSampleScoreGAN_2 = dcgan.evalImages(samplesGAN_2, FLAGS, False)
print(
"GAN 2 error rate on its own samples:",
convertDecimalToPerctage(privateSampleScoreGAN_2) + "",
)
# print(" ")
# print("testRatio = GAN1",float(testScoreGAN_1) ," / GAN2", float(testScoreGAN_2), " = ", testratio )
# print(" ")
# print("GAN 1 -", FLAGS.gan1,"again")
# print(" ")
# print("Loading checkpoint at: ",FLAGS.gan1)
if not dcgan.loadCloestsCheckpoint(FLAGS.gan1, minIterations):
raise Exception("[!] Train a model first, then run test mode")
# print(" ")
sampleScoreGAN_1 = dcgan.evalImages(samplesGAN_2, FLAGS, False)
print(" ")
print(
"GAN 1 test error on GAN 2's samples:",
convertDecimalToPerctage(sampleScoreGAN_1) + "",
)
print(" ")
print(" ")
print(" ")
if (testScoreGAN_2 == 0):
testratio = 0
else:
testratio = float(testScoreGAN_1) / float(testScoreGAN_2)
print(" ")
print("testRatio = GAN1", float(testScoreGAN_1), " / GAN2",
float(testScoreGAN_2), " = ", testratio)
sampleRatio = float(sampleScoreGAN_1) / float(sampleScoreGAN_2)
# print("sampleRatio = ", float(sampleScoreGAN_1), " / ", float(sampleScoreGAN_2), " = ", sampleRatio)
print("sampleRatio = GAN1", float(sampleScoreGAN_1), " / GAN2",
float(sampleScoreGAN_2), " = ", sampleRatio)
print(" ")
if (round(testratio) == 1):
print("Test ratio PASSED - round(", testratio, ") = 1.0")
else:
print("Test ratio FAILED - round(", testratio, ") =",
round(testratio))
print(" ")
print("testRatio =", round(testratio, 3))
print("sampleRatio =", round(sampleRatio, 3))
sampleRatioDiff = abs(
float(sampleScoreGAN_1) - float(sampleScoreGAN_2))
if (sampleRatio < 0.999):
print("WINNER GAN 1 with score", sampleRatio, " - ", gan1Name)
#print(gan1Name+ " score +"+ str(sampleRatioDiff) +" against " + gan1Name+ " score -"+ str(sampleRatioDiff))
elif (sampleRatio > 1.001):
print("WINNER GAN 2 with score", sampleRatio, " - ", gan2Name)
#print(gan1Name + " score +" + str(sampleRatioDiff) + " against " + gan2Name + " score -" + str(sampleRatioDiff))
elif (sampleRatio == 1.00):
print("TIE - Sample score is 1.0")
else:
print("TIE - Sample score of", sampleRatio, "is too close to 1.0")
print(" ")
print(" ")
print(" ")
print("GAN 1 -", FLAGS.gan1, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan1), "iterations")
print("GAN 2 -", FLAGS.gan2, "trained for",
dcgan.loadTrainingITerations(FLAGS.gan2), "iterations")
print(" ")
print("Testing past checkpoints of GAN 1 on the samples from GAN 2")
sampleScoreGAN_1, testScoreGAN_1 = dcgan.evalPastCheckpoints(
FLAGS.gan1,
testDataset,
samplesGAN_2,
FLAGS,
maxIterationNumber=minIterations)
print(" ")
print(" ")
print("Testing past checkpoints of GAN 2 on the samples from GAN 1")
sampleScoreGAN_2, testScoreGAN_2 = dcgan.evalPastCheckpoints(
FLAGS.gan2,
testDataset,
samplesGAN_1,
FLAGS,
maxIterationNumber=minIterations)
####
if (testScoreGAN_2 == 0):
testratio = 0
else:
testratio = float(testScoreGAN_1) / float(testScoreGAN_2)
print(" ")
print("testRatio = GAN1", float(testScoreGAN_1), " / GAN2",
float(testScoreGAN_2), " = ", testratio)
sampleRatio = float(sampleScoreGAN_1) / float(sampleScoreGAN_2)
# print("sampleRatio = ", float(sampleScoreGAN_1), " / ", float(sampleScoreGAN_2), " = ", sampleRatio)
print("sampleRatio = GAN1", float(sampleScoreGAN_1), " / GAN2",
float(sampleScoreGAN_2), " = ", sampleRatio)
print(" ")
if (round(testratio) == 1):
print("Test ratio PASSED - round(", testratio, ") = 1.0")
else:
print("Test ratio FAILED - round(", testratio, ") =",
round(testratio))
print(" ")
print("testRatio =", round(testratio, 3))
print("sampleRatio =", round(sampleRatio, 3))
sampleRatioDiff = abs(
float(sampleScoreGAN_1) - float(sampleScoreGAN_2))
if (sampleRatio < 0.999):
print("WINNER GAN 1 with score", sampleRatio, " - ", gan1Name)
#print(gan1Name+ " score +"+ str(sampleRatioDiff) +" against " + gan1Name+ " score -"+ str(sampleRatioDiff))
elif (sampleRatio > 1.001):
print("WINNER GAN 2 with score", sampleRatio, " - ", gan2Name)
#print(gan1Name + " score +" + str(sampleRatioDiff) + " against " + gan2Name + " score -" + str(sampleRatioDiff))
elif (sampleRatio == 1.00):
print("TIE - Sample score is 1.0")
else:
print("TIE - Sample score of", sampleRatio, "is too close to 1.0")
print(" ")
print(" ")
