perGPUTestData[gpu].append(split[gpu]) for gpu in range(numGpus): with tf.name_scope('tower_%d' % (gpu)) as scope: with tf.device('/gpu:%d' % gpu): print("Defining tower " + str(gpu)) print(perGPUTrainData[gpu][0].get_shape()) print(len(perGPUTrainData[gpu])) trainCode = models.inference(perGPUTrainData[gpu][:-1], first=(gpu == 0), useType="train", modelType=modelType) print(trainCode.get_shape()) predictions = models.predictForces(trainCode, batch_size // numGpus, log, useType="train", first=(gpu == 0), addToCollection=True) # l2_loss = tf.nn.l2_loss(predictions - perGPUTrainData[gpu][-1], name="l2_loss_gpu_"+str(gpu)) weights = numpy.array([[10, 1]]) l2_loss = tf.reduce_sum(tf.squared_difference( predictions * weights, perGPUTrainData[gpu][-1] * weights), name="l2_loss_gpu_" + str(gpu)) tf.add_to_collection('l2_losses', l2_loss) total_l2_loss = tf.reduce_mean(tf.get_collection('l2_losses')) tf.add_to_collection('losses', total_l2_loss) print("All towers defined.") netOut = [] for gpu in range(numGpus):
True, dtype=tf.bool), trainable=False) global_step = tf.Variable(0, trainable=False) dataset = tf.data.Dataset.from_tensor_slices( (tfIdfTrain.astype(numpy.float32), trainLabels)) dataset = dataset.shuffle(buffer_size=15000) dataset = dataset.repeat() dataset = dataset.batch(batch_size) dataset = dataset.prefetch(10) iterator = dataset.make_one_shot_iterator() trainCode, batchTrainLabels = iterator.get_next() predictions = models.predictForces(trainCode, batch_size, log, useType="train", first=True) weights = numpy.array([[10, 1]]) l2_loss = tf.reduce_sum(tf.squared_difference(predictions * weights, batchTrainLabels * weights), name="l2_loss") tf.add_to_collection('losses', l2_loss) valDataset = tf.data.Dataset.from_tensor_slices( (tfIdfVal.astype(numpy.float32), valLabels)) valDataset = valDataset.batch(batch_size) valIterator = valDataset.make_initializable_iterator() valCode, batchValLabels = valIterator.get_next() valPredictions = models.predictForces(valCode, batch_size,
perGPUValData[gpu].append(split[gpu]) netOut = [] for gpu in range(numGpus): with tf.name_scope('tower_%d' % (gpu)) as scope: with tf.device('/gpu:%d' % gpu): print(perGPUValData[gpu][0].get_shape()) print(len(perGPUValData[gpu])) valCode = models.inference(perGPUValData[gpu], first=(gpu == 0), useType="test", modelType=modelType) print(valCode.get_shape()) gpuValPredictions = models.predictForces(valCode, batch_size // numGpus, log, useType="test", first=(gpu == 0)) netOut.append(gpuValPredictions) allOutputs = tf.concat(netOut, axis=0) variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) myconfig = tf.ConfigProto() myconfig.gpu_options.allow_growth = True sess = tf.Session(config=myconfig) ckpt = tf.train.get_checkpoint_state(train_dir) if ckpt and ckpt.model_checkpoint_path:
perGPUTestDataSmall[gpu].append(split[0]) perGPUTestLabelsSmall[gpu].append(split[1]) for gpu in range(numGpus): perGPUTrainLabelsSmall[gpu] = tf.concat(perGPUTrainLabelsSmall[gpu], axis=-1) perGPUTestLabelsSmall[gpu] = tf.concat(perGPUTestLabelsSmall[gpu], axis=-1) testLabelsSmall = tf.concat(perGPUTestLabelsSmall, axis=0) for gpu in range(numGpus): with tf.name_scope('tower_%d' % (gpu)) as scope: with tf.device('/gpu:%d' % gpu): print("Defining tower "+str(gpu)) trainCodeBig = models.inference(perGPUTrainDataBig[gpu], first=(gpu==0), useType="train", modelType=modelType) trainCodeSmall = models.inference(perGPUTrainDataSmall[gpu], first=False, useType="train", modelType=modelType) flowPredictions = models.predictFlow(trainCodeSmall, batch_size//numGpus, log, useType="train", first=(gpu==0)) forcePredictions = models.predictForces(trainCodeBig, batch_size//numGpus, log, useType="train", first=(gpu==0)) # flowReconBig = models.reconstruct(trainCodeBig, batch_size//numGpus, log, useType="train", first=(gpu==0)) # flowReconSmall = models.reconstruct(trainCodeSmall, batch_size//numGpus, log, useType="train", first=(gpu==0)) # l2_loss = tf.nn.l2_loss(predictions - perGPUTrainData[gpu][-1], name="l2_loss_gpu_"+str(gpu)) # print(perGPUTrainLabels[gpu].get_shape()) weights = numpy.array([[10,1]]) l2_loss_forces = tf.reduce_sum(tf.squared_difference(forcePredictions*weights, perGPUTrainDataBig[gpu][-1]*weights), name="l2_force_loss_gpu_"+str(gpu)) l2_loss_flow = tf.reduce_sum(tf.squared_difference(flowPredictions, perGPUTrainLabelsSmall[gpu]), name="l2_flow_loss_gpu_"+str(gpu)) # l2_flow_recon_loss = tf.reduce_sum(tf.squared_difference(flowReconSmall, tf.concat(perGPUTrainDataSmall[gpu], axis=-1)), name="l2_flow_recon_loss_gpu_"+str(gpu)) # l2_force_recon_loss = tf.reduce_sum(tf.squared_difference(flowReconBig, tf.concat(perGPUTrainDataBig[gpu][:-1], axis=-1)), name="l2_force_recon_loss_gpu_"+str(gpu)) tf.add_to_collection('l2_losses', l2_loss_forces) tf.add_to_collection('l2_losses', l2_loss_flow) # tf.add_to_collection('l2_losses', l2_flow_recon_loss) # tf.add_to_collection('l2_losses', l2_force_recon_loss) total_l2_loss = tf.reduce_mean(tf.get_collection('l2_losses'))
count += 1 print("Test to tf.idf") tfTest = wrTest/numpy.sum(wrTest, axis=1, keepdims=True) tfIdfTest = tfTest*idf numpy.save(descriptor+"PerModality_tfIdfTest_"+str(nWords), tfIdfTest) log = open(train_dir+"Testing.txt", "w", 1) global_step = tf.Variable(0, trainable=False) testDataset = tf.data.Dataset.from_tensor_slices((tfIdfTest.astype(numpy.float32), testLabels)) testDataset = testDataset.batch(batch_size) testIterator = testDataset.make_initializable_iterator() testCode, batchTestLabels = testIterator.get_next() testPredictions = models.predictForces(testCode, batch_size, log, useType="test", first=True) testError = tf.reduce_mean(tf.square(tf.subtract(testPredictions, batchTestLabels)), axis=0) print("Towers defined.") # Track the moving averages of all trainable variables. model_vars = tf.trainable_variables() variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) variables_averages_op = variable_averages.apply(model_vars) to_test_op = to_testing(variable_averages) saver = tf.train.Saver(tf.global_variables()) saverMax = tf.train.Saver(tf.global_variables()) myconfig = tf.ConfigProto(log_device_placement=False)