perGPUTestData[gpu].append(split[gpu])
for gpu in range(numGpus):
with tf.name_scope('tower_%d' % (gpu)) as scope:
with tf.device('/gpu:%d' % gpu):
print("Defining tower " + str(gpu))
print(perGPUTrainData[gpu][0].get_shape())
print(len(perGPUTrainData[gpu]))
trainCode = models.inference(perGPUTrainData[gpu][:-1],
first=(gpu == 0),
useType="train",
modelType=modelType)
print(trainCode.get_shape())
predictions = models.predictForces(trainCode,
batch_size // numGpus,
log,
useType="train",
first=(gpu == 0),
addToCollection=True)
# l2_loss = tf.nn.l2_loss(predictions - perGPUTrainData[gpu][-1], name="l2_loss_gpu_"+str(gpu))
weights = numpy.array([[10, 1]])
l2_loss = tf.reduce_sum(tf.squared_difference(
predictions * weights, perGPUTrainData[gpu][-1] * weights),
name="l2_loss_gpu_" + str(gpu))
tf.add_to_collection('l2_losses', l2_loss)
total_l2_loss = tf.reduce_mean(tf.get_collection('l2_losses'))
tf.add_to_collection('losses', total_l2_loss)
print("All towers defined.")
netOut = []
for gpu in range(numGpus):
True, dtype=tf.bool),
trainable=False)
global_step = tf.Variable(0, trainable=False)
dataset = tf.data.Dataset.from_tensor_slices(
(tfIdfTrain.astype(numpy.float32), trainLabels))
dataset = dataset.shuffle(buffer_size=15000)
dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(10)
iterator = dataset.make_one_shot_iterator()
trainCode, batchTrainLabels = iterator.get_next()
predictions = models.predictForces(trainCode,
batch_size,
log,
useType="train",
first=True)
weights = numpy.array([[10, 1]])
l2_loss = tf.reduce_sum(tf.squared_difference(predictions * weights,
batchTrainLabels * weights),
name="l2_loss")
tf.add_to_collection('losses', l2_loss)
valDataset = tf.data.Dataset.from_tensor_slices(
(tfIdfVal.astype(numpy.float32), valLabels))
valDataset = valDataset.batch(batch_size)
valIterator = valDataset.make_initializable_iterator()
valCode, batchValLabels = valIterator.get_next()
valPredictions = models.predictForces(valCode,
batch_size,
perGPUValData[gpu].append(split[gpu])
netOut = []
for gpu in range(numGpus):
with tf.name_scope('tower_%d' % (gpu)) as scope:
with tf.device('/gpu:%d' % gpu):
print(perGPUValData[gpu][0].get_shape())
print(len(perGPUValData[gpu]))
valCode = models.inference(perGPUValData[gpu],
first=(gpu == 0),
useType="test",
modelType=modelType)
print(valCode.get_shape())
gpuValPredictions = models.predictForces(valCode,
batch_size // numGpus,
log,
useType="test",
first=(gpu == 0))
netOut.append(gpuValPredictions)
allOutputs = tf.concat(netOut, axis=0)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
myconfig = tf.ConfigProto()
myconfig.gpu_options.allow_growth = True
sess = tf.Session(config=myconfig)
ckpt = tf.train.get_checkpoint_state(train_dir)
if ckpt and ckpt.model_checkpoint_path:
perGPUTestDataSmall[gpu].append(split[0])
perGPUTestLabelsSmall[gpu].append(split[1])
for gpu in range(numGpus):
perGPUTrainLabelsSmall[gpu] = tf.concat(perGPUTrainLabelsSmall[gpu], axis=-1)
perGPUTestLabelsSmall[gpu] = tf.concat(perGPUTestLabelsSmall[gpu], axis=-1)
testLabelsSmall = tf.concat(perGPUTestLabelsSmall, axis=0)
for gpu in range(numGpus):
with tf.name_scope('tower_%d' % (gpu)) as scope:
with tf.device('/gpu:%d' % gpu):
print("Defining tower "+str(gpu))
trainCodeBig = models.inference(perGPUTrainDataBig[gpu], first=(gpu==0), useType="train", modelType=modelType)
trainCodeSmall = models.inference(perGPUTrainDataSmall[gpu], first=False, useType="train", modelType=modelType)
flowPredictions = models.predictFlow(trainCodeSmall, batch_size//numGpus, log, useType="train", first=(gpu==0))
forcePredictions = models.predictForces(trainCodeBig, batch_size//numGpus, log, useType="train", first=(gpu==0))
# flowReconBig = models.reconstruct(trainCodeBig, batch_size//numGpus, log, useType="train", first=(gpu==0))
# flowReconSmall = models.reconstruct(trainCodeSmall, batch_size//numGpus, log, useType="train", first=(gpu==0))
# l2_loss = tf.nn.l2_loss(predictions - perGPUTrainData[gpu][-1], name="l2_loss_gpu_"+str(gpu))
# print(perGPUTrainLabels[gpu].get_shape())
weights = numpy.array([[10,1]])
l2_loss_forces = tf.reduce_sum(tf.squared_difference(forcePredictions*weights, perGPUTrainDataBig[gpu][-1]*weights), name="l2_force_loss_gpu_"+str(gpu))
l2_loss_flow = tf.reduce_sum(tf.squared_difference(flowPredictions, perGPUTrainLabelsSmall[gpu]), name="l2_flow_loss_gpu_"+str(gpu))
# l2_flow_recon_loss = tf.reduce_sum(tf.squared_difference(flowReconSmall, tf.concat(perGPUTrainDataSmall[gpu], axis=-1)), name="l2_flow_recon_loss_gpu_"+str(gpu))
# l2_force_recon_loss = tf.reduce_sum(tf.squared_difference(flowReconBig, tf.concat(perGPUTrainDataBig[gpu][:-1], axis=-1)), name="l2_force_recon_loss_gpu_"+str(gpu))
tf.add_to_collection('l2_losses', l2_loss_forces)
tf.add_to_collection('l2_losses', l2_loss_flow)
# tf.add_to_collection('l2_losses', l2_flow_recon_loss)
# tf.add_to_collection('l2_losses', l2_force_recon_loss)
total_l2_loss = tf.reduce_mean(tf.get_collection('l2_losses'))
count += 1
print("Test to tf.idf")
tfTest = wrTest/numpy.sum(wrTest, axis=1, keepdims=True)
tfIdfTest = tfTest*idf
numpy.save(descriptor+"PerModality_tfIdfTest_"+str(nWords), tfIdfTest)
log = open(train_dir+"Testing.txt", "w", 1)
global_step = tf.Variable(0, trainable=False)
testDataset = tf.data.Dataset.from_tensor_slices((tfIdfTest.astype(numpy.float32), testLabels))
testDataset = testDataset.batch(batch_size)
testIterator = testDataset.make_initializable_iterator()
testCode, batchTestLabels = testIterator.get_next()
testPredictions = models.predictForces(testCode, batch_size, log, useType="test", first=True)
testError = tf.reduce_mean(tf.square(tf.subtract(testPredictions, batchTestLabels)), axis=0)
print("Towers defined.")
# Track the moving averages of all trainable variables.
model_vars = tf.trainable_variables()
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(model_vars)
to_test_op = to_testing(variable_averages)
saver = tf.train.Saver(tf.global_variables())
saverMax = tf.train.Saver(tf.global_variables())
myconfig = tf.ConfigProto(log_device_placement=False)
